Scrambler
1
|
00001 !######################################################################### 00002 ! 00003 ! Copyright (C) 2003-2012 Department of Physics and Astronomy, 00004 ! University of Rochester, 00005 ! Rochester, NY 00006 ! 00007 ! io_chombo.f90 is part of AstroBEAR. 00008 ! 00009 ! AstroBEAR is free software: you can redistribute it and/or modify 00010 ! it under the terms of the GNU General Public License as published by 00011 ! the Free Software Foundation, either version 3 of the License, or 00012 ! (at your option) any later version. 00013 ! 00014 ! AstroBEAR is distributed in the hope that it will be useful, 00015 ! but WITHOUT ANY WARRANTY; without even the implied warranty of 00016 ! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 00017 ! GNU General Public License for more details. 00018 ! 00019 ! You should have received a copy of the GNU General Public License 00020 ! along with AstroBEAR. If not, see <http://www.gnu.org/licenses/>. 00021 ! 00022 !######################################################################### 00026 00029 00033 00038 MODULE IOChombo 00039 00040 USE HDF5 00041 USE HDF5Declarations 00042 USE ChomboDeclarations 00043 USE GlobalDeclarations 00044 USE PhysicsDeclarations 00045 USE ProcessingDeclarations 00046 USE TreeDeclarations 00047 USE DataDeclarations 00048 USE MessageDeclarations 00049 USE MpiTransmission 00050 USE EllipticDeclarations 00051 USE CommunicationDeclarations 00052 USE MpiPacking 00053 USE IOComms 00054 USE IOParsing 00055 USE DistributionControl 00056 USE TimeStep, ONLY : olddt, overall_maxspeed,maxcfl 00057 IMPLICIT NONE 00058 PRIVATE 00059 00060 PUBLIC MakeChomboFile, ChomboRestartInit, ChomboRestartFinalize, ChomboReloadLevel !ChomboRestart 00061 00062 CONTAINS 00063 00064 00068 SUBROUTINE MakeChomboFile(nframe) 00069 00070 USE ParticleControl, ONLY: lParticles, Particle_WriteData 00071 USE CoolingSrc, ONLY: CoolingCheck 00072 00073 ! input declarations 00074 INTEGER, INTENT(IN) :: nframe 00075 INTEGER :: FinestLevel 00076 00077 ! Variable declarations 00078 CHARACTER(LEN=23) :: s_filename 00079 00080 INTEGER(HID_T) :: hid_property_id 00081 INTEGER(HID_T) :: hid_file_id 00082 00083 INTEGER(HID_T) :: hid_group_id 00084 INTEGER(HID_T) :: hid_chombo_global_group_id 00085 INTEGER(HID_T) :: hid_level_group_id 00086 00087 INTEGER :: i_err 00088 00089 INTEGER(HID_T) :: hid_boxes_property_id 00090 INTEGER(HID_T) :: hid_boxes_dataspace_id 00091 INTEGER(HID_T) :: hid_boxes_dataset_id 00092 00093 INTEGER(HID_T) :: hid_data_attributes_group_id 00094 00095 INTEGER(HSIZE_T), DIMENSION(1) :: ia_dataset_dims ! Holds the size of the dataset. 00096 INTEGER, PARAMETER :: MaxNodesPerLevel=200 00097 INTEGER (HSIZE_T):: i_index ! Used for any incidental loops along the way. 00098 INTEGER (HSIZE_T):: i_mhd_index ! Used when looping through aux data. 00099 00100 ! Stores the cumulative offset of the chombo dataset hyperslab that we are writing to. 00101 INTEGER(HSIZE_T) :: i_qvar_offset 00102 00103 ! Stores the cumulative offset of the chombo dataset hyperslab that we are writing to. 00104 INTEGER(HSIZE_T) :: i_aux_offset 00105 00106 INTEGER :: i, j, k ! indices to use when looping along spatial dimensions. 00107 00108 REAL(KIND = qPrec), ALLOCATABLE, DIMENSION(:) :: dbla_chombo_qvar 00109 00110 INTEGER :: i_level ! Tracks the level in several loops. 00111 INTEGER :: i_dimension ! Used in the loop that assembles the boxes data. 00112 INTEGER :: i_variable ! Used in the loop that assembles chombo data. 00113 00114 00115 CHARACTER(LEN = I_MAX_CNAME_LENGTH), DIMENSION(0:I_MAX_COMPONENTS) :: sa_component_names 00116 CHARACTER(LEN = I_MAX_CNAME_LENGTH) :: s_tracer 00117 CHARACTER(LEN = I_MAX_CNAME_LENGTH) :: s_component_tag 00118 00119 CHARACTER(LEN = I_MAX_CNAME_LENGTH) :: s_level_name 00120 00121 INTEGER, DIMENSION(I_CHOMBO_BOX_SIZE) :: ia_box_global ! array for holding box data. 00122 00123 INTEGER :: i_level_node_index 00124 00125 INTEGER :: NrCons 00126 INTEGER :: mx, my, mz 00127 00128 REAL(KIND = qPrec) :: dbl_domain_offset 00129 00130 INTEGER :: i_grid_offset 00131 INTEGER, DIMENSION(I_CHOMBO_BOX_SIZE) :: box_array 00132 INTEGER, ALLOCATABLE, DIMENSION(:,:) :: ia_box_data 00133 00134 INTEGER(HSIZE_T) :: i_box_offset 00135 INTEGER :: i_grid_data_size 00136 00137 ! Used for iterating over the nodes on a level. 00138 TYPE(NodedefList), POINTER :: nodelist 00139 TYPE(InfoDef), POINTER :: Info 00140 00141 ! MHD (auxiliary) variable handling. 00142 INTEGER :: i_aux_data_size 00143 INTEGER, DIMENSION(3) :: ia_aux_extension 00144 REAL(KIND = qPrec), ALLOCATABLE, DIMENSION(:) :: dbla_chombo_aux 00145 00146 ! Additional communication-related variables. 00147 TYPE(PackedMessage), POINTER :: message 00148 TYPE(PackedMessage), POINTER :: next_message 00149 INTEGER, DIMENSION(4) :: data_sizes 00150 00151 INTEGER :: i_box_data_size 00152 INTEGER :: i_counter 00153 INTEGER :: remote_grid_count 00154 INTEGER, DIMENSION(3) :: local_mX 00155 INTEGER :: proc_id 00156 TYPE(Nodedef), POINTER :: node 00157 00158 TYPE(ChomboHandle), POINTER :: chandle 00159 TYPE(NodedefList), POINTER :: child_list 00160 INTEGER :: child_count 00161 INTEGER :: i_childbox_data_size, i_acc_childbox_data_size 00162 LOGICAL :: lChildDatasetsInitialized 00163 INTEGER :: DummyInt, iErr 00164 TYPE(DiagnosticListDef), POINTER :: Diagnostic 00165 00166 NULLIFY(Info) 00167 00168 WRITE(s_filename, '(A10,I5.5,A4)') 'out/chombo', nframe, '.hdf' 00169 00170 CALL CreateChomboHandle(s_filename, chandle, CHOMBO_HANDLE_WRITE) 00171 00172 ! Start out by assuming that the finest level is the maximum allowable level. 00173 ! If it turns out there is no data on the max level, then FinestLevel could change. 00174 FinestLevel = MaxLevel 00175 00176 !!!!! Add required attributes for Chombo HDF specification. New attributes are 00177 !!!!! added to the root group usingAdd_ChomboHDF_Attribute() subroutine, which 00178 !!!!! is described below. 00179 00180 ! time attribute (uses Info's current time). 00181 CALL Add_HDF5_Attribute_Double("time", chandle%root_group_id, levels(0)%tnow) 00182 CALL Add_HDF5_Attribute_Double("olddt", chandle%root_group_id, olddt) 00183 CALL Add_HDF5_Attribute_Double("overall_maxspeed", chandle%root_group_id, overall_maxspeed) 00184 CALL Add_HDF5_Attribute_Double("maxcfl", chandle%root_group_id, maxcfl) 00185 00186 ! iteration attribute (uses Info's current frame number). 00187 CALL Add_HDF5_Attribute_Int("iteration", chandle%root_group_id, nFrame) 00188 00189 ! num_components attribute (use Info's number of output variables). 00190 CALL Add_HDF5_Attribute_Int("num_components", chandle%root_group_id, NrVars+NrDiagnosticVars) 00191 00192 ! node centering (fixes contour maps in visit) 00193 CALL Add_HDF5_Attribute_Int("data_centering", chandle%root_group_id, 0) ! cell centered 00194 00195 00196 !!!This pro_lo attribute causes pseudocolor slices to not work with multiple root patches in visit 00197 !CALL Add_HDF5_Attribute_FloatVector("prob_lo", chandle%root_group_id, GxBounds(:,1)) 00198 00199 00200 00201 00202 !!!!! The scaling factors below are not part of the Chombo HDF specification, but they're 00203 !!!!! useful enough that we're including them anyway. 00204 00205 ! Number density. 00206 CALL Add_HDF5_Attribute_Double("n_density_scale", chandle%root_group_id, nScale) 00207 00208 ! Mass density. 00209 CALL Add_HDF5_Attribute_Double("rho_scale", chandle%root_group_id, rScale) 00210 00211 CALL Add_HDF5_Attribute_Double("length_scale", chandle%root_group_id, lScale) 00212 00213 CALL Add_HDF5_Attribute_Double("velocity_scale", chandle%root_group_id, velScale) 00214 00215 CALL Add_HDF5_Attribute_Double("pressure_scale", chandle%root_group_id, pScale) 00216 00217 CALL Add_HDF5_Attribute_Double("temperature_scale", chandle%root_group_id, TempScale) 00218 00219 !Uses var_index to keep track of what variables are in what slots 00220 NrCons = NrHydroVars - NrTracerVars 00221 00222 ! Create tags for conservative variables using indices initialized in 00223 ! PhysicsControl::SetVariableIndices(). 00224 DO i=1, NrCons 00225 sa_component_names(i-1)= trim(FieldName(i)) 00226 END DO 00227 00228 ! Creates tags for tracer variables. 00229 DO i = 1, NrTracerVars !NrCons+1,NrHydroVars 00230 sa_component_names(nTracerLo-1+i-1) = trim(TracerName(i)) 00231 END DO 00232 00233 ! Creates tags for Elliptic variables. 00234 DO i = 1, NrEllipticVars !NrHydroVars+1,NrHydroVars+NrEllipticVars 00235 sa_component_names(nEllipticLo-1+i-1) = trim(EllipticName(i)) 00236 ! IF (i == iPhi) sa_component_names(i-1)="Phi" 00237 ! IF (i == iPhiSinks) sa_component_names(i-1)="PhiSinks" 00238 ! IF (i == iPhiGas) sa_component_names(i-1)="PhiGas" 00239 ! IF (i == iPhiDot) sa_component_names(i-1)="PhiDot" 00240 END DO 00241 00242 Diagnostic=>FirstDiagnostic 00243 i=1 00244 DO WHILE (ASSOCIATED(Diagnostic)) 00245 sa_component_names(NrHydroVars+NrEllipticVars+i-1) = trim(Diagnostic%Field%Name) 00246 i=i+1 00247 Diagnostic=>Diagnostic%next 00248 END DO 00249 00250 ! If MaintainAuxArrays is true, then the current problem uses MHD. 00251 IF (MaintainAuxArrays) THEN 00252 00253 ! All MHD problems are three-dimensional, so you have to include all three 00254 ! directional momentum and magnetic field components. 00255 ! Add derived scaling factors. 00256 CALL Add_HDF5_Attribute_Double("px_scale", chandle%root_group_id, rScale * velScale) 00257 CALL Add_HDF5_Attribute_Double("py_scale", chandle%root_group_id, rScale * velScale) 00258 CALL Add_HDF5_Attribute_Double("pz_scale", chandle%root_group_id, rScale * velScale) 00259 00260 IF (iE .ne. 0) CALL Add_HDF5_Attribute_Double("E_scale", chandle%root_group_id, pScale) 00261 00262 CALL Add_HDF5_Attribute_Double("Bx_scale", chandle%root_group_id, pScale) 00263 CALL Add_HDF5_Attribute_Double("By_scale", chandle%root_group_id, pScale) 00264 CALL Add_HDF5_Attribute_Double("Bz_scale", chandle%root_group_id, pScale) 00265 00266 ELSE 00267 00268 ! If the project is three-dimensional, then the pz component must be added; 00269 ! otherwise, begin adding tracer components. 00270 IF (nDim == 2) THEN 00271 ! ! Add derived scaling factors. 00272 CALL Add_HDF5_Attribute_Double("px_scale", chandle%root_group_id, rScale * velScale) 00273 CALL Add_HDF5_Attribute_Double("py_scale", chandle%root_group_id, rScale * velScale) 00274 IF (iE .ne. 0) CALL Add_HDF5_Attribute_Double("E_scale", chandle%root_group_id, pScale) 00275 ELSE 00276 ! Add derived scaling factors. 00277 CALL Add_HDF5_Attribute_Double("px_scale", chandle%root_group_id, rScale * velScale) 00278 CALL Add_HDF5_Attribute_Double("py_scale", chandle%root_group_id, rScale * velScale) 00279 CALL Add_HDF5_Attribute_Double("pz_scale", chandle%root_group_id, rScale * velScale) 00280 IF (iE .ne. 0) CALL Add_HDF5_Attribute_Double("E_scale", chandle%root_group_id, pScale) 00281 END IF 00282 END IF 00283 00284 00285 ! This part is separated out so that changing the assembly of the components 00286 ! is easy, but the addition of attributes may eventually have to be done 00287 ! inline in order to reduce overhead. 00288 ! 00289 ! Construct new component_n attributes for each sa_component_name(n). 00290 DO i_index = 0, NrVars + NrDiagnosticVars - 1 00291 00292 ! Create component tag (attribute name). 00293 IF (i_index <= 9) THEN 00294 WRITE(s_component_tag,'(A10,I1.1)') 'component_', i_index 00295 ELSE 00296 WRITE(s_component_tag,'(A10,I2.2)') 'component_', i_index 00297 END IF 00298 00299 CALL Add_HDF5_Attribute_String(s_component_tag, & 00300 chandle%root_group_id, TRIM(sa_component_names(i_index))) 00301 END DO 00302 00303 !!! End Assembling Component Array. 00304 00305 !!! Create Group Chombo_global And Its Attributes 00306 00307 ! Create Chombo_global group. 00308 CALL h5gcreate_f(chandle%file_handle, "Chombo_global", hid_chombo_global_group_id, i_err) 00309 CALL CatchHDF5Error("h5gcreate_f ('Chombo_global')", i_err) 00310 00311 ! Add testReal attribute to Chombo_global group (value is a literal). 00312 CALL Add_HDF5_Attribute_Double("testReal", hid_chombo_global_group_id, & 00313 DBL_TEST_REAL_DAT) 00314 00315 ! Add SpaceDim attribute to Chombo_global group. We can just use Info%nDim 00316 ! because we don't generally do 4D problems. 00317 IF (nDim == 1) THEN 00318 CALL Add_HDF5_Attribute_Int("SpaceDim", hid_chombo_global_group_id, 2) 00319 ELSE 00320 CALL Add_HDF5_Attribute_Int("SpaceDim", hid_chombo_global_group_id, nDim) 00321 END IF 00322 ! Close the Chombo_global group. 00323 CALL h5gclose_f(hid_chombo_global_group_id, i_err) 00324 CALL CatchHDF5Error("h5gclose_f ('Chombo_global')", i_err) 00325 00326 !!! End Create Chombo_global group. 00327 00328 !!! Storing the domain boundaries in the Chombo file. 00329 CALL Add_HDF5_Attribute_FloatVector("lower_bound", chandle%root_group_id, GxBounds(:,1)) 00330 CALL Add_HDF5_Attribute_FloatVector("upper_bound", chandle%root_group_id, GxBounds(:,2)) 00331 00332 00333 i_acc_childbox_data_size = 1 ! The initial value for i_childbox_data_size; after level -2 00334 ! it will have more meaningful values, but we want to make 00335 ! sure that the loop below functions properly. 00336 00337 ! Clear the next_level_cost array before using it. 00338 chandle%next_level_cost = 0 00339 00340 !!! Assembling the group_level(n) groups. 00341 00342 DO i_level = -2, FinestLevel 00343 00344 ! PRINT "('Level ', i2, ' begun.')", i_level 00345 00346 ! The size of the current level's boxes dataset is equal to the number of child boxes on the 00347 ! previous level. Therefore, rather than recalculate it, we just move over the accumulated 00348 ! childbox count. 00349 i_box_data_size = i_acc_childbox_data_size 00350 00351 ! Clear level data counters. 00352 i_grid_data_size = 0 00353 i_acc_childbox_data_size = 0 00354 IF (MaintainAuxArrays) i_aux_data_size = 0 00355 lChildDatasetsInitialized = .FALSE. 00356 00357 IF (i_level >= 0) THEN 00358 00359 ! [BDS] Select the youngest node on level i_level. 00360 nodelist => Nodes(i_level)%p 00361 00362 DO proc_id = 0, MPI_np - 1 00363 00364 i_counter = 0 00365 00366 IF (proc_id > Master) THEN 00367 00368 ! Tell worker to send data 00369 CALL MPI_SEND(DummyInt, 1, MPI_INTEGER, proc_id, TRANSMIT_IO_WORKER_GRIDS, MPI_COMM_WORLD, iErr) 00370 ! Create a message to receive data from this process. 00371 CALL CreatePackedMessage(i_level, & 00372 proc_id, & 00373 TRANSMIT_FRAME_DATA, & 00374 STAGE_RECV, & 00375 message, & 00376 chandle%next_level_cost(proc_id)) 00377 00378 ! Retrieve the level statistics from the remote processor. 00379 CALL IO_UnparseLevelStatistics(message, data_sizes) 00380 00381 chandle%next_level_cost(message%remote_proc) = TERMINATION_BOX_BYTES + & 00382 IO_LEVEL_STAT_BYTES + & 00383 data_sizes(IO_NEXTLEVELCOST) 00384 00385 !Stores the number of child grids on this level. 00386 i_childbox_data_size = data_sizes(IO_CHILDCOUNT) 00387 00388 IF (i_level >= 0) THEN 00389 ! Initializes the size of the data:datatype=0 array. 00390 i_grid_data_size = data_sizes(IO_Q_SIZE) 00391 00392 00393 ! Initializes the size of the MHD_data array. If there is no MHD in this problem 00394 ! (i.e., if lMHD is false), then this will be 0. 00395 i_aux_data_size = data_sizes(IO_AUX_SIZE) 00396 END IF 00397 00398 ELSE 00399 ! Obtain level statistics for the local processor. 00400 CALL IO_GetDatasetSizes(i_level, FinestLevel, data_sizes) 00401 00402 00403 !Stores the number of child grids on this level. 00404 i_childbox_data_size = data_sizes(IO_CHILDCOUNT) 00405 00406 IF (i_level >= 0) THEN 00407 00408 ! Initializes the size of the data:datatype=0 array. 00409 i_grid_data_size = data_sizes(IO_Q_SIZE) 00410 00411 ! Initializes the size of the MHD_data array. If there is no MHD in this problem 00412 ! (i.e., if lMHD is false), then this will be 0. 00413 i_aux_data_size = data_sizes(IO_AUX_SIZE) 00414 00415 ! [BDS] Select the youngest node on level i_level. 00416 nodelist => Nodes(i_level)%p 00417 00418 END IF 00419 00420 END IF 00421 00422 i_acc_childbox_data_size = i_acc_childbox_data_size + i_childbox_data_size 00423 00424 DO ! Loop over nodes on i_level. 00425 00426 IF (proc_id > Master) THEN 00427 00428 NULLIFY(node, Info) 00429 00430 ! Extract the next node from the message. This will return null on 00431 ! a termination box. 00432 CALL IO_UnparseRemoteNode(message, node) 00433 00434 IF (ASSOCIATED(node)) THEN 00435 ! If the level is less than the finest level, then unparse any 00436 ! child nodes that have been sent along with the current node. 00437 IF (i_level < FinestLevel) CALL IO_UnparseNodeChildren(message, child_count, node) 00438 00439 ! Unparse the grid data associated with the current node. 00440 IF (i_level > -1) CALL IO_UnparseRemoteGrid(message, Info, FinestLevel) 00441 ELSE 00442 EXIT 00443 END IF 00444 00445 ELSE 00446 00447 ! If node_ptr is now null, then there are no nodes 00448 ! left on level i_level and we can exit the loop. 00449 IF (.NOT. ASSOCIATED(nodelist)) EXIT 00450 IF (.NOT. ASSOCIATED(nodelist%self)) EXIT 00451 00452 ! Retrieve the InfoDef structure of node_ptr. 00453 node => nodelist%self 00454 child_count = NodeCount(node%children) 00455 Info => nodelist%self%Info 00456 00457 ! Die if unable to obtain node_ptr's Info structure. 00458 IF (.NOT. ASSOCIATED(Info)) THEN 00459 PRINT *, "MakeChomboFile error ", i_err, & 00460 ": unable to obtain node info structure." 00461 STOP 00462 END IF 00463 00464 END IF 00465 00466 ! If there is cell-centered data to be stored in the dataset and the offset 00467 ! is empty, then assume the level has not been created and create it. If 00468 ! there is no grid data, then do nothing. If there is data to store and the 00469 ! offset is not 0, then the dataset extension code will fix the problem when 00470 ! this routine goes to write the data to the dataset. 00471 00472 ! q data is never sent without costmap data, so an empty q offset will be accompanied 00473 ! by an empty costmap offset as well. 00474 00475 IF (i_grid_data_size > 0) THEN 00476 IF (.NOT. HandleIsValid(chandle%level_id)) THEN 00477 00478 ! Open a chombo handle for this level. 00479 CALL IO_OpenCurrentLevel(chandle, i_level) 00480 00481 ! Create the dx attribute 00482 CALL Add_HDF5_Attribute_Double("dx", chandle%level_id, levels(i_level)%dx) 00483 ! Create the anisotropic attribute (optional chomobo attrib for anisotropic spacing) 00484 ! [BDS][20100712]: The design does not currently support anisotropic systems. 00485 CALL Add_HDF5_Attribute_FloatVector("anisotropic", chandle%level_id, (/1d0,1d0,1d0/))!Info%dX/Info%dX(1)) 00486 00487 ! Create the ref_ratio attribute. 00488 CALL Add_HDF5_Attribute_Int("ref_ratio", chandle%level_id, levels(i_level)%CoarsenRatio) 00489 00490 ! loop over the three spatial dimensions to construct the box object.. 00491 DO i_index = 1,3 00492 00493 ! Set the lower-bounds of the box object. This will be the 00494 ! lower-bound value in mGlobal, minus one as an offset for 00495 ! Chombo's field-space. 00496 00497 ia_box_global(i_index) = 0 00498 00499 ! ia_box_global = #cells along each dimension, adjusted by the 00500 ! refinement level. This can be found by taking 00501 ! the difference between the max and min values 00502 ! for each dimension and dividing by dX(dim). 00503 00504 IF (levels(i_level)%dX /= zero) THEN 00505 ia_box_global(i_index+3)=Gmx(i_index) * product(levels(0:i_level-1)%coarsenratio)-1 !(GxBounds(i_index,2) - GxBounds(i_index,1)) / levels(i_level)%dX 00506 END IF 00507 00508 ! If there is a fractional part to the domain offset, then 00509 ! add another cell to the global box to accomodate it. 00510 ! ia_box_global(i_index + 3) = MAX(dbl_domain_offset - 1, 0) 00511 00512 END DO 00513 00514 ! Create prob_domain attribute using a box object. 00515 CALL Add_HDF5_Attribute_Box("prob_domain", hid_box_id, chandle%level_id, ia_box_global) 00516 00517 END IF 00518 00519 00520 00521 IF (HandleIsValid(chandle%level_id)) THEN 00522 00523 ! ** box data ** 00524 ! IF (i_box_data_size > 0) THEN 00525 00526 ! If the handle's offset is 0, then no data has been added to the dataset, 00527 ! meaning that it hasn't been initialized yet. This conditional will correct that. 00528 IF (chandle%box_offset == 0) THEN 00529 ! The dataset that holds the grid dimensions. 00530 CALL Initialize_HDF5_Dataset_Box("boxes", chandle%level_id, i_box_data_size) 00531 END IF 00532 00533 CALL IO_WriteBoxToChomboFile(chandle, "boxes", node%box%mGlobal) 00534 ! 00535 ! END IF 00536 00537 ! ** childbox data ** 00538 ! On levels below FinestLevel, store the child boxes of each node. These boxes 00539 ! are stored in breadth-first order relative to their parents. 00540 IF ((i_level < FinestLevel)) THEN 00541 00542 ! If the handle's offset is 0, then no data has been added to the dataset, 00543 ! meaning that it hasn't been initialized yet. This conditional will correct that. 00544 IF ((.NOT. lChildDatasetsInitialized) .AND. (chandle%childbox_offset == 0)) THEN 00545 ! The size of the childbox count dataset is always known--it is the number of boxes 00546 ! on the current level, since each element has a 1-1 correspondence with the current 00547 ! level. 00548 00549 CALL Initialize_HDF5_Dataset_Int("childbox_count", chandle%level_id, i_box_data_size) 00550 CALL Initialize_HDF5_Dataset_Box("child_boxes", chandle%level_id, i_childbox_data_size) 00551 lChildDatasetsInitialized = .TRUE. 00552 END IF 00553 00554 ! Write the number of children to the child_boxes array. 00555 CALL IO_WriteChildCountToChomboFile(chandle, child_count) 00556 00557 child_list => node%children 00558 00559 ! Loop over the child box list and add each child to the child boxes dataset. 00560 DO WHILE (ASSOCIATED(child_list)) 00561 CALL IO_WriteBoxToChomboFile(chandle, "child_boxes", child_list%self%box%mGlobal) 00562 child_list => child_list%next 00563 END DO 00564 00565 NULLIFY(child_list) 00566 00567 END IF 00568 00569 ! ** grid data ** 00570 ! Initialize the dataset that holds cell-centered data. Populating these 00571 ! datasets is more involved than the box datasets, so we'll do that later. 00572 IF ((i_grid_data_size > 0) .AND. (chandle%q_offset == 0)) THEN 00573 00574 CALL Initialize_HDF5_Dataset_Double("data:datatype=0", & 00575 chandle%level_id, & 00576 i_grid_data_size) 00577 00578 ! If this is not the finest level, then initialize a costmap dataset 00579 ! for this level. 00580 ! IF (i_level < FinestLevel) THEN 00581 ! CALL Initialize_HDF5_Dataset_Double("costmap", & 00582 ! chandle%level_id, & 00583 ! i_grid_data_size * 2 / NrVars) 00584 ! END IF 00585 00586 END IF 00587 00588 00589 ! Same as for the cell-centered data, except now we're creating a face-centered dataset. 00590 IF (MaintainAuxArrays .AND. (i_aux_data_size > 0) .AND. (chandle%aux_offset == 0)) THEN 00591 CALL Initialize_HDF5_Dataset_Double("MHD_data", chandle%level_id, i_aux_data_size) 00592 END IF 00593 00594 00595 ! Obtain the grid dimensions from the coordinate box. 00596 mx = node%box%mGlobal(1,2) - node%box%mGlobal(1,1) + 1 00597 my = node%box%mGlobal(2,2) - node%box%mGlobal(2,1) + 1 00598 mz = node%box%mGlobal(3,2) - node%box%mGlobal(3,1) + 1 00599 00600 ! Allocate memory for the current node's grid. 00601 ALLOCATE(dbla_chombo_qvar(mx * my * mz * (NrVars+NrDiagnosticVars)), stat=i_err) 00602 00603 ! Stops execution on a memory allocation error. 00604 IF (i_err /= 0) THEN 00605 PRINT *,"MakeChomboFile error: unable to allocate memory for q-var array." 00606 STOP 00607 END IF 00608 00609 ! Clear index; it will be used to reference chombo data array in the loop below. 00610 i_index = 0 00611 00612 ! Loop over each variable tracked by the program. 00613 DO i_variable = 1, NrVars 00614 00615 ! The nested loops below loop over each spatial dimension and reference the 00616 ! q value for each variable within a specific 3D coordinate; this value is 00617 ! then stored in the chombo data array. In effect, what this does is flatten 00618 ! the multi-dimensional data in q down to a 1D array for HDF5 storage. 00619 00620 DO k = 1, mz 00621 DO j = 1, my 00622 DO i = 1, mx 00623 00624 i_index = i_index + 1 00625 dbla_chombo_qvar(i_index) = Info%q(i, j, k, i_variable) 00626 00627 END DO ! End i 00628 END DO ! End j 00629 END DO ! End k 00630 00631 END DO ! End do i_variable = to Info%NrOutVars 00632 00633 00634 ! Loop over each variable tracked by the program. 00635 DO i_variable = 1, NrDiagnosticVars 00636 00637 ! The nested loops below loop over each spatial dimension and reference the 00638 ! q value for each variable within a specific 3D coordinate; this value is 00639 ! then stored in the chombo data array. In effect, what this does is flatten 00640 ! the multi-dimensional data in q down to a 1D array for HDF5 storage. 00641 00642 DO k = 1, mz 00643 DO j = 1, my 00644 DO i = 1, mx 00645 00646 i_index = i_index + 1 00647 dbla_chombo_qvar(i_index) = Info%diagnostics(i, j, k, i_variable) 00648 00649 END DO ! End i 00650 END DO ! End j 00651 END DO ! End k 00652 00653 END DO ! End do i_variable = to Info%NrOutVars 00654 00655 !Deallocate the diagnostic array always once it is outputted. 00656 IF (NrDiagnosticVars > 0) THEN 00657 IF (.NOT.(ASSOCIATED(Info%diagnostics))) THEN 00658 write(*,*) 'info diagnostics not associated' 00659 STOP 00660 END IF 00661 DEALLOCATE(Info%diagnostics) 00662 NULLIFY(Info%diagnostics) 00663 END IF 00664 00665 ! Write the data array to the Chombo file using the supplied offset. 00666 CALL Write_Slab_To_Dataset_Double("data:datatype=0", chandle%level_id, dbla_chombo_qvar, chandle%q_offset) 00667 ! Adjust the offset by the size of the grid just written. 00668 00669 chandle%q_offset = chandle%q_offset + i_index 00670 00671 ! IF( i_level < FinestLevel) THEN 00672 ! CALL Write_Slab_To_Dataset_Double("costmap", chandle%level_id, & 00673 ! RESHAPE(Info%costmap(1:mx,1:my,1:mz,1:2), (/ mx*my*mz*2 /)), chandle%costmap_offset) 00674 00675 ! Adjust the costmap offset. 00676 ! chandle%costmap_offset = chandle%costmap_offset + mx*my*mz*2 00677 ! END IF 00678 00679 ! Release array memory so that it can be used by the next grid. 00680 DEALLOCATE(dbla_chombo_qvar) 00681 00682 ! If MHD is running, then do the same thing for the Info%aux array. 00683 IF (MaintainAuxArrays) THEN 00684 00685 ! Allocate memory for the current node's aux grid. 00686 IF (nDim == 3) THEN 00687 ALLOCATE(dbla_chombo_aux(((mx + 1) * my * mz) + & 00688 (mx * (my + 1) * mz) + & 00689 (mx * my * (mz + 1))), stat=i_err) 00690 ELSE 00691 ALLOCATE(dbla_chombo_aux(((mx + 1) * my) + & 00692 (mx * (my + 1))), stat=i_err) 00693 END IF 00694 00695 ! Stops execution on a memory allocation error. 00696 IF (i_err /= 0) THEN 00697 PRINT *,"MakeChomboFile error: unable to allocate memory for aux array." 00698 STOP 00699 END IF 00700 00701 ! Clear index; it will be used to reference chombo data array in the loop below. 00702 i_mhd_index = 0 00703 00704 ! There will be one Bx variable for each dimension of the problem; we 00705 ! don't need to keep track of any other aux variables. 00706 DO i_variable = 1, nDim 00707 00708 ! Set the dimension of the current Bx variable to be extended by 1. 00709 ia_aux_extension = 0 00710 ia_aux_extension(i_variable) = 1 00711 00712 ! The nested loops below loop over each spatial dimension and reference the 00713 ! aux value for each variable within a specific 3D coordinate; this value is 00714 ! then stored in the chombo data array. In effect, what this does is flatten 00715 ! the multi-dimensional data in aux down to a 1D array for HDF5 storage. 00716 00717 DO k = 1, mz + ia_aux_extension(3) 00718 DO j = 1, my + ia_aux_extension(2) 00719 DO i = 1, mx + ia_aux_extension(1) 00720 00721 i_mhd_index = i_mhd_index + 1 00722 dbla_chombo_aux(i_mhd_index) = Info%aux(i, j, k, i_variable) 00723 00724 END DO ! End i 00725 END DO ! End j 00726 END DO ! End k 00727 00728 END DO ! End do i_variable = to Info%maux 00729 00730 ! Write the data array to the Chombo file using the supplied offset. 00731 CALL Write_Slab_To_Dataset_Double("MHD_data", chandle%level_id, & 00732 dbla_chombo_aux, chandle%aux_offset) 00733 00734 ! Adjust the offset by the size of the grid just written. 00735 chandle%aux_offset = chandle%aux_offset + i_mhd_index 00736 00737 ! Release array memory so that it can be used by the next grid. 00738 DEALLOCATE(dbla_chombo_aux) 00739 00740 END IF ! (if MaintainAuxArrays). 00741 00742 END IF 00743 END IF 00744 ! Destroy the structures created from remote data. 00745 IF (proc_id > Master) THEN 00746 00747 ! Deallocate the cell-centered data array. 00748 DEALLOCATE(Info%q) 00749 NULLIFY(Info%q) 00750 00751 ! Deallocate the aux array imported from the remote processor 00752 IF (MaintainAuxArrays) THEN 00753 DEALLOCATE(Info%aux) 00754 NULLIFY(Info%aux) 00755 END IF 00756 00757 DEALLOCATE(Info) 00758 NULLIFY(Info) 00759 00760 ! Deallocate the node's variable-length lists. 00761 ! DEALLOCATE(node%proclist, node%proctime) 00762 ! NULLIFY(node%proclist, node%proctime) 00763 00764 IF (ASSOCIATED(node%children)) CALL DestroyNodeList(node%children) 00765 00766 DEALLOCATE(node) 00767 NULLIFY(node) 00768 00769 ELSE 00770 ! [BDS] Advance to the next node on the level. 00771 NULLIFY(child_list) 00772 nodelist => nodelist%next 00773 00774 END IF 00775 00776 END DO ! Loop over nodes on i_level. 00777 00778 ! Destroy this processor's message to make room for a new one. 00779 IF (proc_id > Master) CALL DestroyPackedMessage(message) 00780 00781 END DO ! Loop over processors. 00782 00783 ! PRINT *, "Level ", i_level, " datasets finished." 00784 00785 !!!!!!!! END DATAZERO LEVEL TRAVERSAL 00786 00787 ! If there are boxes on this level, then add the remaining level data and close 00788 ! the level handle. Otherwise, set the FinestLevel to be the previous level and exit. 00789 IF (HandleIsValid(chandle%level_id)) THEN 00790 00791 ! Create the data attributes group. 00792 CALL h5gcreate_f(chandle%level_id, "data_attributes", hid_data_attributes_group_id, i_err) 00793 CALL CatchHDF5Error("h5gcreate_f('data_attributes')", i_err) 00794 00795 ! Adds the output_ghost attribute using the literal intvector [0, 0, 0]. 00796 CALL Add_HDF5_Attribute_IntVector("outputGhost", hid_intvect_id, & 00797 hid_data_attributes_group_id, (/0,0,0/)) 00798 00799 ! Adds the comps attribute (the same as num_components) 00800 CALL Add_HDF5_Attribute_Int("comps", hid_data_attributes_group_id, & 00801 NrVars+NrDiagnosticVars) 00802 00803 ! Adds the objectType attribute. 00804 CALL Add_HDF5_Attribute_String("objectType", hid_data_attributes_group_id, "FArrayBox") 00805 00806 ! Close data attributes group. 00807 CALL h5gclose_f(hid_data_attributes_group_id, i_err) 00808 CALL CatchHDF5Error("h5gclose_f('data_attributes')", i_err) 00809 00810 ! Close level_n group. 00811 CALL IO_CloseCurrentLevel(chandle) 00812 00813 ELSE 00814 ! If there were no boxes on this level, then some previous level was the finest level. 00815 ! 00816 IF (FinestLevel >= i_level) FinestLevel = i_level - 1 00817 END IF 00818 00819 00820 ELSE IF (i_level == CHOMBO_DOMAIN_LEVEL) THEN 00821 00822 IF (nDomains > 1) THEN 00823 PRINT*, 'not yet implemented for IO' 00824 STOP 00825 END IF 00826 ! Clear all level offsets. 00827 CALL ClearChomboHandleOffsets(chandle) 00828 chandle%next_level_cost = 0 00829 00830 ! Obtain level statistics for the local processor. 00831 CALL IO_GetDatasetSizes(i_level, FinestLevel, data_sizes) 00832 00833 !Stores the number of child grids on this level. 00834 i_childbox_data_size = data_sizes(IO_CHILDCOUNT) 00835 i_acc_childbox_data_size=i_acc_childbox_data_size+i_childbox_data_size 00836 00837 ! Open a chombo handle for this level. 00838 CALL IO_OpenCurrentLevel(chandle, i_level) 00839 00840 ! The -1 level stores its own boxes, child boxes (plus their count), and its costmap. 00841 ! It does not, however, have any cell- or face-centered data to store. 00842 ! CALL Initialize_HDF5_Dataset_Double("costmap", chandle%level_id, PRODUCT(GmX)*2, & 00843 ! L_NON_EXTENSIBLE) 00844 00845 CALL Initialize_HDF5_Dataset_Box("boxes", chandle%level_id, i_box_data_size, L_NON_EXTENSIBLE) 00846 CALL Initialize_HDF5_Dataset_Int("childbox_count", chandle%level_id, i_childbox_data_size) 00847 CALL Initialize_HDF5_Dataset_Box("child_boxes", chandle%level_id, i_childbox_data_size) 00848 00849 CALL IO_WriteDomainData(chandle, data_sizes(IO_CHILDCOUNT)) 00850 00851 ! Close level_n group. 00852 CALL IO_CloseCurrentLevel(chandle) 00853 00854 ELSE IF (i_level == CHOMBO_ROOT_LEVEL) THEN 00855 00856 ! Clear all level offsets. 00857 CALL ClearChomboHandleOffsets(chandle) 00858 chandle%next_level_cost = 0 00859 00860 ! Obtain level statistics for the local processor. 00861 CALL IO_GetDatasetSizes(i_level, FinestLevel, data_sizes) 00862 00863 !Stores the number of child grids on this level. 00864 i_childbox_data_size = data_sizes(IO_CHILDCOUNT) 00865 i_acc_childbox_data_size=i_acc_childbox_data_size+i_childbox_data_size 00866 00867 ! The -2 level only has one grid, and it's on the master. Since that grid is always generated 00868 ! by DomainInit(), the -2 level only needs a costmap, a single integer for its child count, and 00869 ! its child boxes. 00870 ! CALL Initialize_HDF5_Dataset_Double("root_costmap", chandle%root_group_id, & 00871 ! PRODUCT(GmX)*2, L_NON_EXTENSIBLE) 00872 00873 CALL Add_HDF5_Attribute_Int("root_child_count", chandle%root_group_id, i_childbox_data_size) 00874 CALL Initialize_HDF5_Dataset_Box("root_child_boxes", chandle%root_group_id, i_childbox_data_size) 00875 00876 END IF 00877 00878 PRINT *, 'Level ', i_level, ' data written.' 00879 ! PRINT * 00880 00881 END DO !i_level = -2 to FinestLevel. 00882 00883 00884 ! num_levels attribute (Chombo format requires that this be 1 greater the maximum 00885 ! level--essentially so that it acts as a level-index. 00886 CALL Add_HDF5_Attribute_Int("num_levels", chandle%root_group_id, FinestLevel + 1) 00887 00888 ! max_level attribute (use Info's current FinestLevel value). 00889 CALL Add_HDF5_Attribute_Int("max_level", chandle%root_group_id, FinestLevel) 00890 00891 !!! END OF LEVEL_N groups. 00892 00893 ! [BDS][20081126]: Add sink particles to chombo if required components are active. 00894 ! [BDS][20100712]: Sink particles are not currently implemented in scrambler, so this 00895 ! feature is deactivated. 00896 ! [BDS][20101130]: Sink particles are now part of scrambler. The routine below will stores them 00897 ! in the format described by the chombo DDF. 00898 CALL Particle_WriteData(chandle) 00899 00900 CALL WriteCoolingObjects(chandle) 00901 00902 CALL WritePointGravityObjects(chandle) 00903 00904 ! Close the chombo file. 00905 CALL CloseChomboHandle(chandle) 00906 00907 END SUBROUTINE MakeChomboFile 00908 00913 SUBROUTINE IO_WriteBoxToChomboFile(chandle, box_set_name, box) 00914 00915 TYPE(ChomboHandle), POINTER :: chandle 00916 CHARACTER(*) :: box_set_name 00917 INTEGER, DIMENSION(3,2) :: box 00918 00919 INTEGER, DIMENSION(6) :: box_data 00920 INTEGER(HSIZE_T) :: box_offset 00921 INTEGER(HID_T) :: source_group_id 00922 00923 00924 box_data(1:3) = box(1:3,1) - 1 00925 box_data(4:6) = box(1:3,2) - 1 00926 00927 ! Four possible options: "boxes", "child_boxes", "root_child_boxes" 00928 IF (box_set_name == "child_boxes") THEN 00929 box_offset = chandle%childbox_offset 00930 source_group_id = chandle%level_id 00931 00932 ELSE IF (box_set_name == "boxes") THEN 00933 box_offset = chandle%box_offset 00934 source_group_id = chandle%level_id 00935 00936 ELSE IF (box_set_name == "root_child_boxes") THEN 00937 box_offset = chandle%childbox_offset 00938 source_group_id = chandle%root_group_id 00939 00940 END IF 00941 00942 CALL Write_Slab_To_Dataset_Box(box_set_name, & 00943 source_group_id, & 00944 box_data, & 00945 box_offset) 00946 00947 IF (box_set_name == "child_boxes") THEN 00948 chandle%childbox_offset = chandle%childbox_offset + 1 00949 ELSE IF (box_set_name == "boxes") THEN 00950 chandle%box_offset = chandle%box_offset + 1 00951 ELSE 00952 chandle%domainbox_offset = chandle%domainbox_offset + 1 00953 END IF 00954 00955 END SUBROUTINE IO_WriteBoxToChomboFile 00956 00960 SUBROUTINE IO_WriteChildCountToChomboFile(chandle, int_data) 00961 00962 TYPE(ChomboHandle), POINTER :: chandle 00963 INTEGER :: int_data 00964 00965 00966 CALL Write_Slab_To_Dataset_Int("childbox_count", & 00967 chandle%level_id, & 00968 (/ int_data /), & 00969 chandle%childbox_count_offset) 00970 00971 chandle%childbox_count_offset = chandle%childbox_count_offset + 1 00972 00973 END SUBROUTINE IO_WriteChildCountToChomboFile 00974 00978 SUBROUTINE IO_WriteDomainData(chandle, master_childcount) 00979 00980 TYPE(ChomboHandle), POINTER :: chandle 00981 00982 TYPE(PackedMessage), POINTER :: message 00983 TYPE(NodedefList), POINTER :: nodelist, child_list 00984 REAL(KIND=qPrec), DIMENSION(:,:,:,:), POINTER :: cost_map 00985 INTEGER :: proc_id 00986 INTEGER :: master_childcount 00987 00988 INTEGER :: mx, my, mz 00989 INTEGER, DIMENSION(3,2) :: mGlobal 00990 TYPE(Nodedef), POINTER :: node 00991 INTEGER :: child_count 00992 INTEGER, DIMENSION(IO_LEVEL_STAT_COUNT) :: data_sizes 00993 INTEGER :: i_childbox_data_size 00994 INTEGER :: DummyInt, iErr 00995 00996 chandle%costmap_offset = 0 00997 child_count = 0 00998 00999 ! Points to the node list for the -1 (domain) level. 01000 nodelist => Nodes(CHOMBO_DOMAIN_LEVEL)%p 01001 01002 ! Loop over processors and retrieve any cost map data. 01003 DO proc_id = 0, MPI_np - 1 01004 01005 ! If proc-id is a worker, then retrieve its message. 01006 IF (proc_id > Master) THEN 01007 ! Create packed message object to recieve data. 01008 CALL CreatePackedMessage(CHOMBO_DOMAIN_LEVEL, & 01009 proc_id, & 01010 TRANSMIT_DOMAIN_DATA, & 01011 STAGE_RECV, & 01012 message) 01013 01014 ! Retrieve the level statistics from the remote processor. On the domain 01015 ! level, the q and aux data have no significance, but the child nodes do. 01016 CALL IO_UnparseLevelStatistics(message, data_sizes) 01017 01018 ! Store the cost of the next level's messages from this processor. 01019 chandle%next_level_cost(message%remote_proc) = TERMINATION_BOX_BYTES + & 01020 IO_LEVEL_STAT_BYTES + & 01021 data_sizes(IO_NEXTLEVELCOST) 01022 01023 ELSE 01024 ! Initialize the child count on the master processor. 01025 i_childbox_data_size = master_childcount 01026 END IF 01027 01028 DO ! Loop over domain nodes. 01029 01030 NULLIFY(cost_map) 01031 01032 IF (proc_id > Master) THEN 01033 01034 NULLIFY(node) 01035 01036 ! Extract a node that will go into the boxes array. 01037 CALL IO_UnparseRemoteNode(message, node) 01038 01039 IF (ASSOCIATED(node)) THEN 01040 ! Extract the child nodes from the message. 01041 CALL IO_UnparseNodeChildren(message, child_count, node) 01042 mGlobal=node%box%mGlobal 01043 ! Increment the size counter for the childbox dataset. 01044 i_childbox_data_size = i_childbox_data_size + child_count 01045 01046 ! Unparse a costmap from the packed message and break this loop if 01047 ! none can be found. 01048 ! CALL IO_UnparseLowLevelCostMap(message, mGlobal, cost_map) 01049 ELSE 01050 EXIT 01051 END IF 01052 01053 ! Obtain the grid dimensions from the box coordinates. 01054 mx = mGlobal(1,2) - mGlobal(1,1) + 1 01055 my = mGlobal(2,2) - mGlobal(2,1) + 1 01056 mz = mGlobal(3,2) - mGlobal(3,1) + 1 01057 01058 ELSE 01059 01060 ! Retrieve the costmap from the next InfoDef structure in the list. 01061 IF (ASSOCIATED(nodelist)) THEN 01062 ! Retrieve the InfoDef structure of node_ptr. 01063 node => nodelist%self 01064 ! cost_map => node%Info%costmap 01065 child_count = NodeCount(node%children) 01066 ELSE 01067 EXIT 01068 END IF 01069 01070 mx = node%box%mGlobal(1,2)-node%box%mGlobal(1,1) !Info%mX(1) 01071 my = node%box%mGlobal(2,2)-node%box%mGlobal(2,1) !Info%mX(2) 01072 mz = node%box%mGlobal(3,2)-node%box%mGlobal(3,1) !Info%mX(3) 01073 01074 mGlobal = node%box%mGlobal 01075 01076 END IF 01077 01078 ! Write box to domain box list. 01079 CALL IO_WriteBoxToChomboFile(chandle, "boxes", mGlobal) 01080 01081 ! Write box to root-level child box list (which we can do because there is no 01082 ! level group for level -2; we're just writing to a dataset in the root group, 01083 ! which stays open until the CloseChomboHandle() subroutine is called. 01084 CALL IO_WriteBoxToChomboFile(chandle, "root_child_boxes", mGlobal) 01085 01086 ! Write the data array to the Chombo file using the supplied offset. 01087 ! CALL Write_Slab_To_Dataset_Double("costmap", chandle%level_id, & 01088 ! RESHAPE(cost_map(1:mx,1:my,1:mz,1:2), (/ mx*my*mz*2 /)), chandle%costmap_offset) 01089 01090 ! Increment the offset by the size of the cost map. 01091 ! chandle%costmap_offset = chandle%costmap_offset + mx*my*mz*2 01092 01093 CALL IO_WriteChildCountToChomboFile(chandle, child_count) 01094 01095 child_list => node%children 01096 01097 ! The childbox attribute is being used twice on this level, so we clear it 01098 ! again here. It's klugey, but so's adding another offset that's only used once. 01099 chandle%childbox_offset = 0 01100 01101 ! 01102 DO WHILE (ASSOCIATED(child_list)) 01103 CALL IO_WriteBoxToChomboFile(chandle, "child_boxes", child_list%self%box%mGlobal) 01104 01105 child_list => child_list%next 01106 END DO 01107 01108 ! If the cost map came from a worker processor, then deallocated it. Otherwise, 01109 ! just advance to the next node. 01110 IF (proc_id > Master) THEN 01111 ! DEALLOCATE(cost_map) 01112 ! NULLIFY(cost_map) 01113 IF (ASSOCIATED(node%children)) CALL DestroyNodeList(node%children) 01114 ELSE 01115 nodelist => nodelist%next 01116 END IF 01117 01118 END DO ! node list loop 01119 01120 ! Destroy worker message so that a new one can be created. 01121 IF (proc_id > MASTER) CALL DestroyPackedMessage(message) 01122 01123 END DO ! processor loop 01124 01125 END SUBROUTINE IO_WriteDomainData 01126 01130 SUBROUTINE IO_ReadInLevelData(chandle, level) 01131 01132 TYPE(ChomboHandle), POINTER :: chandle 01133 INTEGER :: level 01134 01135 TYPE(NodedefList), POINTER :: nodelist, childlist 01136 TYPE(Nodedef), POINTER :: node, child 01137 TYPE(InfoDef), POINTER :: Info 01138 TYPE(NodeBox) :: box 01139 INTEGER :: box_count, child_count 01140 INTEGER :: i,j 01141 INTEGER, DIMENSION(3,2) :: mGlobal 01142 TYPE(NodeBoxList), POINTER :: masterbox_list, last_masterbox 01143 TYPE(NodeBoxList), POINTER :: new_child_list, last_new_child 01144 INTEGER :: effective_finest_level 01145 01146 01147 nodelist => Nodes(level)%p 01148 child_count = 0 01149 NULLIFY(new_child_list, last_new_child) 01150 01151 effective_finest_level = MIN(chandle%finest_level, MaxLevel) 01152 01153 IF (level >= -1) THEN 01154 01155 ! Obtain the number of boxes on this level that are stored in the chombo file. 01156 box_count = IO_GetDatasetElementCount(chandle%level_id, "boxes") 01157 01158 ! Temporarily cache the child boxes in a list stored in the chombo handle. 01159 DO i = 1, box_count 01160 ! Obtain the next box and store it in the chombo handle structure. 01161 CALL IO_GetBoxFromChomboFile(chandle, "boxes", mGlobal) 01162 CALL IO_CacheNodeBox(chandle, mGlobal, Master) 01163 01164 IF (level < effective_finest_level) THEN 01165 01166 ! Attempt to find the node on this level that matches this box. 01167 box%mGlobal = mGlobal 01168 box%MPI_id = 0 01169 CALL FindNode(level, box, node) 01170 01171 CALL IO_GetChildBoxCountFromChomboFile(chandle, child_count) 01172 01173 ! Retrieve each of this box's child boxes. 01174 DO j = 1, child_count 01175 ! Temporarily cache the child boxes in a list stored in the chombo handle. 01176 CALL IO_GetBoxFromChomboFile(chandle, "child_boxes", mGlobal) 01177 CALL IO_CacheChildNodeBox(chandle, mGlobal, Master) 01178 01179 ! If the current box is on the master processor, then cache these child boxes 01180 ! separately so that they can be redistributed. 01181 IF (ASSOCIATED(node)) THEN 01182 CALL AddNodeBoxToList(last_new_child, new_child_list) 01183 last_new_child%self%mGlobal = mGlobal 01184 last_new_child%self%MPI_id = Master 01185 END IF 01186 01187 END DO 01188 01189 ! If the current box is on this processor and has children, then 01190 ! redistribute the children. 01191 IF (ASSOCIATED(node) .AND. (child_count > 0)) THEN 01192 CALL ChomboReCreateChildren(node, level, child_count, new_child_list) 01193 01194 ! Clear the cached child list so that it can be used for the next 01195 ! round of local child boxes. 01196 CALL ClearNodeBoxList(new_child_list) 01197 NULLIFY(last_new_child) 01198 END IF 01199 01200 END IF 01201 01202 END DO 01203 01204 chandle%childbox_offset = 0 01205 chandle%childbox_count_offset = 0 01206 01207 ELSE 01208 ! There is only one node on level -2, and it's on the master. 01209 node => nodelist%self 01210 ! Info => node%Info 01211 ! CALL IO_GetCostMapFromChomboFile(chandle, node%box%mGlobal, -2, Info%costmap) 01212 01213 child_count = Read_HDF5_Attribute_Int("root_child_count", chandle%root_group_id) 01214 01215 ! Temporarily cache the child boxes in a list stored in the chombo handle. 01216 DO i = 1, child_count 01217 CALL IO_GetBoxFromChomboFile(chandle, "root_child_boxes", mGlobal) 01218 CALL IO_CacheChildNodeBox(chandle, mGlobal, Master) 01219 END DO 01220 01221 ! Redistribute children on level n. 01222 IF (child_count > 0) THEN 01223 CALL ChomboReCreateChildren(node, level, child_count, chandle%current_child_box) 01224 chandle%current_child_box => chandle%last_child_box 01225 END IF 01226 01227 END IF 01228 01229 CALL ClearChomboHandleOffsets(chandle) 01230 01231 END SUBROUTINE IO_ReadInLevelData 01232 01236 SUBROUTINE ChomboReloadLevel(level) 01237 01238 ! USE TreeLevelComms 01239 ! USE DataLevelOps 01240 01241 01242 TYPE(ChomboHandle), POINTER :: chandle 01243 INTEGER :: level 01244 01245 TYPE(StageMessageGroup), POINTER :: smg_recv_grids, smg_send_data 01246 TYPE(PackedMessage), POINTER :: pm_send_grids, pm_recv_data, message 01247 INTEGER :: n, i, message_size, status(MPI_STATUS_SIZE) 01248 INTEGER :: effective_finest_level 01249 INTEGER :: DummyInt, iErr 01250 01251 chandle=>curr_handle 01252 01253 ! IF (MPI_id == 0) PRINT *, "Starting level ", level, "..." 01254 01255 ! Determines the highest level to which Chombo refinements will be pursued. 01256 ! Essentially, this is the highest level of data that we will attempt to pull 01257 ! from the Chombo file. 01258 ! effective_finest_level = MIN(chandle%finest_level, MaxLevel) 01259 01260 NULLIFY(smg_recv_grids, smg_send_data, pm_send_grids, pm_recv_data) 01261 01262 IF (level >= -1) THEN 01263 CALL IO_OpenCurrentLevel(chandle, level) 01264 END IF 01265 01266 IF (MPI_ID == MASTER) THEN 01267 01268 ! Open level handles for level. 01269 ! IF (level >= -1) CALL IO_OpenCurrentLevel(chandle, level) 01270 01271 ! **** 01272 ! IF YOU ARE TRYING TO FOLLOW THE THREAD OF THE RESTART PROCESS, IT STARTS 01273 ! HERE, AT LEVEL -2. 01274 ! **** 01275 ! Obtain grid and child grid data for this level. This grid information is 01276 ! attached to the ChomboHandle structure for the time being. At this point, 01277 ! the master also distributes the child grids associated with its own nodes 01278 ! on this level. 01279 CALL IO_ReadInLevelData(chandle, level) 01280 01281 IF (level >= -1) THEN 01282 ! Each worker now sends its grids for this level to the master. This 01283 ! information is used to assign MPI_ids for each grid that the master 01284 ! has read in from the Chombo file. 01285 DO i=1,MPI_NP-1 01286 CALL MPI_RECV(message_size, 1, MPI_INTEGER, i, TRANSMIT_IO_WORKER_GRIDS, MPI_COMM_WORLD, status, iErr) 01287 CALL CreatePackedMessage(level, i, TRANSMIT_IO_WORKER_GRIDS, STAGE_RECV, message, message_size) 01288 CALL MPI_SEND(message_size, 1, MPI_INTEGER, i, TRANSMIT_IO_WORKER_GRIDS, MPI_COMM_WORLD, iErr) 01289 CALL ReceivePackedMessage(message) 01290 CALL IO_UnparseGridsFromWorkers(chandle, message) 01291 CALL DestroyPackedMessage(message) 01292 END DO 01293 END IF 01294 ! CALL IO_PostRecvGridsFromWorkers(smg_recv_grids, level) 01295 ! CALL IO_CompRecvGridsFromWorkers(chandle, smg_recv_grids) 01296 ! END DO 01297 ! END IF 01298 01299 ! Read in one box at a time from chombo, figure out where it lives, and send it. 01300 ! Now that they have the appropriate MPI_ids associated, the boxes read 01301 ! in from the Chombo file can be used to retrieve the data for this level. 01302 ! These routines pull q, aux, and costmap data from the file and send it to 01303 ! the appropriate processors. 01304 ! 01305 ! Child grids for workers are also packed and sent by these processors; the 01306 ! master processor has already distributed its child grids at this point, so 01307 ! it just copies over the InfoDef data from the Chombo files and ignores the 01308 ! child grids. 01309 CALL IO_MpiSendDataToWorkers(chandle, level) 01310 01311 ! Clear all the cached boxes from the ChomboHandle structure. 01312 CALL IO_ClearAllNodeBoxes(chandle) 01313 CALL IO_ClearAllChildBoxes(chandle) 01314 01315 ! Close level handles, since no further level data will be needed from the file. 01316 ! IF (level >= -1) CALL IO_CloseCurrentLevel(chandle) 01317 01318 ELSE 01319 ! Send grids to master to let it know what blocks of data to send. This information 01320 ! will be used on the master to assign MPI_ids to the boxes it reads in from the file. 01321 IF (level >= -1) THEN 01322 CALL IO_PostSendGridsToMaster(pm_send_grids, level) 01323 CALL IO_CompSendGridsToMaster(pm_send_grids) 01324 END IF 01325 01326 ! Receive data from master. This includes child boxes that will be transmitted 01327 ! to the children of this processor's nodes. 01328 ! CompRecvDataFromMaster() also copies the data to the InfoDef structures on this level. 01329 ! CALL IO_PostRecvDataFromMaster(pm_recv_data, level, chandle) 01330 CALL IO_CompRecvDataFromMaster(level, chandle) 01331 END IF 01332 01333 ! Close level handles, since no further level data will be needed from the file. 01334 IF (level >= -1) CALL IO_CloseCurrentLevel(chandle) 01335 01336 END SUBROUTINE ChomboReloadLevel 01337 01338 01341 SUBROUTINE ChomboRestartInit(nframe) 01342 01343 INTEGER :: nframe 01344 INTEGER :: n 01345 01346 CHARACTER(LEN=I_FILENAME_LENGTH) :: s_filename 01347 ! TYPE(ChomboHandle), POINTER :: chandle 01348 ! Retrieve the nframe-th chombo file name. 01349 WRITE(s_filename, '(A10,I5.5,A4)') 'out/chombo', nframe, '.hdf' 01350 01351 ! Create a chombo handle to manage the object. 01352 CALL CreateChomboHandle(s_filename, curr_handle, CHOMBO_HANDLE_READ) 01353 01354 current_time = Read_HDF5_Attribute_Double("time", curr_handle%root_group_id) 01355 olddt = Read_HDF5_Attribute_Double("olddt", curr_handle%root_group_id) 01356 overall_maxspeed = Read_HDF5_Attribute_Double("overall_maxspeed", curr_handle%root_group_id) 01357 maxcfl=Read_HDF5_Attribute_Double("maxcfl", curr_handle%root_group_id) 01358 levels(:)%tnow = current_time 01359 levels(:)%dt = 0.d0 01360 levels(:)%step = 1 01361 RestartLevel=min(MaxLevel,curr_handle%finest_level) 01362 NrInputVars = READ_HDF5_Attribute_Int("num_components", curr_handle%root_group_id) 01363 ! IF (MPI_ID == 0) write(*,*) 'found ', NrInputVars, ' variables', NrVars 01364 IF (NrInputVars < NrVars) THEN 01365 IF (MPI_ID == 0) PRINT*, 'Error - not enough variables in chombo file fro restart' 01366 STOP 01367 END IF 01368 01369 END SUBROUTINE ChomboRestartInit 01370 01371 SUBROUTINE ChomboRestartFinalize(nframe) 01372 INTEGER :: nframe 01373 ! close the chombo handle. 01374 CALL CloseChomboHandle(curr_handle) 01375 END SUBROUTINE ChomboRestartFinalize 01376 01377 01378 01381 SUBROUTINE WritePointGravityObjects(chandle) 01382 01383 USE PointGravitySrc, ONLY: FirstPointGravityObj, PointGravityDef, PointGravity_CountObjects, & 01384 PointGravity_InitChomboDatasets, PointGravity_WriteObjectToChombo 01385 01386 TYPE(ChomboHandle), POINTER :: chandle 01387 01388 TYPE(PointGravityDef), POINTER :: gravity_object 01389 INTEGER :: i_err 01390 INTEGER :: obj_count 01391 INTEGER :: iFixed 01392 01393 01394 ! Open a new particles group within the root level 01395 CALL h5gcreate_f(chandle%root_group_id, "point_gravity_objects", chandle%source_group_id, i_err) 01396 CALL CatchHDF5Error("WritePointGravityObjects::h5gcreate_f", i_err) 01397 01398 chandle%source_offset = 0 01399 01400 ! Retrieve the number of point gravity objects from the object list. 01401 obj_count = PointGravity_CountObjects() 01402 01403 ! Store the number of objects. 01404 CALL Add_HDF5_Attribute_Int("num_objects", chandle%source_group_id, obj_count) 01405 01406 ! If there are any gravity objects, then create datasets to store their data. Otherwise, 01407 ! don't create the datasets (since there's no need to take up file space). 01408 IF (obj_count > 0) THEN 01409 01410 CALL PointGravity_InitChomboDatasets(chandle, obj_count) 01411 01412 gravity_object => FirstPointGravityObj 01413 01414 ! Loop over the list of point gravity objects and write their data to the new component datasets. 01415 DO WHILE (ASSOCIATED(gravity_object)) 01416 01417 CALL PointGravity_WriteObjectToChombo(chandle, gravity_object) 01418 01419 gravity_object => gravity_object%next 01420 01421 END DO 01422 01423 END IF 01424 01425 ! Close the source term group. 01426 CALL h5gclose_f(chandle%source_group_id, i_err) 01427 CALL CatchHDF5Error("WritePointGravityObjects::h5gclose_f", i_err) 01428 01429 END SUBROUTINE WritePointGravityObjects 01430 01433 SUBROUTINE WriteCoolingObjects(chandle) 01434 01435 USE CoolingSrc, ONLY: CoolingDef, Cooling_CountObjects, firstcoolingobj, & 01436 Cooling_InitChomboDatasets, Cooling_WriteObjectToChombo 01437 01438 TYPE(ChomboHandle), POINTER :: chandle 01439 01440 TYPE(CoolingDef), POINTER :: cooling_object 01441 INTEGER :: i_err 01442 INTEGER :: obj_count 01443 INTEGER :: iFixed 01444 01445 01446 ! Open a new particles group within the root level 01447 CALL h5gcreate_f(chandle%root_group_id, "cooling_objects", chandle%source_group_id, i_err) 01448 CALL CatchHDF5Error("WriteCoolingObjects::h5gcreate_f", i_err) 01449 01450 chandle%source_offset = 0 01451 01452 ! Retrieve the number of cooling objects from the object list. 01453 obj_count = Cooling_CountObjects() 01454 01455 ! Store the number of objects. The INT() is required to recast obj_count from an HSIZE_T 01456 ! type for the function definition. 01457 CALL Add_HDF5_Attribute_Int("num_objects", chandle%source_group_id, obj_count) 01458 01459 ! If there are any cooling objects, then create datasets to store their data. Otherwise, 01460 ! don't create the datasets (since there's no need to take up file space). 01461 IF (obj_count > 0) THEN 01462 01463 ! If there are cooling objects to store, then create datasets for them. 01464 CALL Cooling_InitChomboDatasets(chandle, obj_count) 01465 01466 cooling_object => firstcoolingobj 01467 01468 ! Loop over the list of sink particles and write their data to the new component datasets. 01469 DO WHILE (ASSOCIATED(cooling_object)) 01470 01471 ! Writes the current object to the data file. Also increments the source offset. 01472 CALL Cooling_WriteObjectToChombo(chandle, cooling_object) 01473 01474 cooling_object => cooling_object%next 01475 END DO 01476 01477 END IF 01478 01479 ! Close the source term group. 01480 CALL h5gclose_f(chandle%source_group_id, i_err) 01481 CALL CatchHDF5Error("WriteCoolingObjects::h5gclose_f", i_err) 01482 01483 END SUBROUTINE WriteCoolingObjects 01484 01485 END MODULE IOChombo 01486