Scrambler  1
io_chombo.f90
Go to the documentation of this file.
00001 !#########################################################################
00002 !               
00003 !    Copyright (C) 2003-2012 Department of Physics and Astronomy,
00004 !                            University of Rochester,
00005 !                            Rochester, NY
00006 !
00007 !    io_chombo.f90 is part of AstroBEAR.
00008 !
00009 !    AstroBEAR is free software: you can redistribute it and/or modify    
00010 !    it under the terms of the GNU General Public License as published by 
00011 !    the Free Software Foundation, either version 3 of the License, or    
00012 !    (at your option) any later version.
00013 !
00014 !    AstroBEAR is distributed in the hope that it will be useful, 
00015 !    but WITHOUT ANY WARRANTY; without even the implied warranty of
00016 !    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00017 !    GNU General Public License for more details.
00018 !
00019 !    You should have received a copy of the GNU General Public License
00020 !    along with AstroBEAR.  If not, see <http://www.gnu.org/licenses/>.
00021 !
00022 !#########################################################################
00026 
00029 
00033 
00038 MODULE IOChombo
00039 
00040   USE HDF5
00041   USE HDF5Declarations
00042   USE ChomboDeclarations
00043   USE GlobalDeclarations
00044   USE PhysicsDeclarations
00045   USE ProcessingDeclarations
00046   USE TreeDeclarations
00047   USE DataDeclarations
00048   USE MessageDeclarations
00049   USE MpiTransmission
00050   USE EllipticDeclarations
00051   USE CommunicationDeclarations
00052   USE MpiPacking
00053   USE IOComms
00054   USE IOParsing
00055   USE DistributionControl
00056   USE TimeStep, ONLY : olddt, overall_maxspeed,maxcfl
00057   IMPLICIT NONE
00058   PRIVATE
00059 
00060   PUBLIC MakeChomboFile, ChomboRestartInit, ChomboRestartFinalize, ChomboReloadLevel !ChomboRestart
00061 
00062 CONTAINS
00063 
00064 
00068   SUBROUTINE MakeChomboFile(nframe)
00069 
00070     USE ParticleControl, ONLY: lParticles, Particle_WriteData
00071     USE CoolingSrc, ONLY: CoolingCheck
00072 
00073     ! input declarations
00074     INTEGER, INTENT(IN) :: nframe
00075     INTEGER :: FinestLevel
00076 
00077     ! Variable declarations
00078     CHARACTER(LEN=23) :: s_filename
00079 
00080     INTEGER(HID_T) :: hid_property_id
00081     INTEGER(HID_T) :: hid_file_id
00082 
00083     INTEGER(HID_T) :: hid_group_id
00084     INTEGER(HID_T) :: hid_chombo_global_group_id
00085     INTEGER(HID_T) :: hid_level_group_id
00086 
00087     INTEGER :: i_err
00088 
00089     INTEGER(HID_T) :: hid_boxes_property_id
00090     INTEGER(HID_T) :: hid_boxes_dataspace_id
00091     INTEGER(HID_T) :: hid_boxes_dataset_id
00092 
00093     INTEGER(HID_T) :: hid_data_attributes_group_id
00094 
00095     INTEGER(HSIZE_T), DIMENSION(1) :: ia_dataset_dims     ! Holds the size of the dataset.
00096     INTEGER, PARAMETER :: MaxNodesPerLevel=200
00097     INTEGER (HSIZE_T):: i_index        ! Used for any incidental loops along the way.
00098     INTEGER (HSIZE_T):: i_mhd_index    ! Used when looping through aux data.
00099 
00100     ! Stores the cumulative offset of the chombo dataset hyperslab that we are writing to.
00101     INTEGER(HSIZE_T) :: i_qvar_offset
00102 
00103     ! Stores the cumulative offset of the chombo dataset hyperslab that we are writing to.
00104     INTEGER(HSIZE_T) :: i_aux_offset    
00105 
00106     INTEGER :: i, j, k        ! indices to use when looping along spatial dimensions.
00107 
00108     REAL(KIND = qPrec), ALLOCATABLE, DIMENSION(:) :: dbla_chombo_qvar
00109 
00110     INTEGER :: i_level        ! Tracks the level in several loops.
00111     INTEGER :: i_dimension    ! Used in the loop that assembles the boxes data.
00112     INTEGER :: i_variable    ! Used in the loop that assembles chombo data.
00113 
00114 
00115     CHARACTER(LEN = I_MAX_CNAME_LENGTH), DIMENSION(0:I_MAX_COMPONENTS) :: sa_component_names
00116     CHARACTER(LEN = I_MAX_CNAME_LENGTH) :: s_tracer
00117     CHARACTER(LEN = I_MAX_CNAME_LENGTH) :: s_component_tag
00118 
00119     CHARACTER(LEN = I_MAX_CNAME_LENGTH) :: s_level_name
00120 
00121     INTEGER, DIMENSION(I_CHOMBO_BOX_SIZE) :: ia_box_global        ! array for holding box data.
00122 
00123     INTEGER :: i_level_node_index
00124 
00125     INTEGER :: NrCons
00126     INTEGER :: mx, my, mz
00127 
00128     REAL(KIND = qPrec) :: dbl_domain_offset
00129 
00130     INTEGER :: i_grid_offset
00131     INTEGER, DIMENSION(I_CHOMBO_BOX_SIZE) :: box_array
00132     INTEGER, ALLOCATABLE, DIMENSION(:,:) :: ia_box_data
00133 
00134     INTEGER(HSIZE_T) :: i_box_offset
00135     INTEGER :: i_grid_data_size
00136 
00137     ! Used for iterating over the nodes on a level.    
00138     TYPE(NodedefList), POINTER :: nodelist
00139     TYPE(InfoDef), POINTER :: Info
00140 
00141     ! MHD (auxiliary) variable handling.
00142     INTEGER :: i_aux_data_size
00143     INTEGER, DIMENSION(3) :: ia_aux_extension
00144     REAL(KIND = qPrec), ALLOCATABLE, DIMENSION(:) :: dbla_chombo_aux
00145 
00146     ! Additional communication-related variables.
00147     TYPE(PackedMessage), POINTER :: message
00148     TYPE(PackedMessage), POINTER :: next_message
00149     INTEGER, DIMENSION(4) :: data_sizes
00150 
00151     INTEGER :: i_box_data_size
00152     INTEGER :: i_counter
00153     INTEGER :: remote_grid_count
00154     INTEGER, DIMENSION(3) :: local_mX
00155     INTEGER :: proc_id
00156     TYPE(Nodedef), POINTER :: node
00157 
00158     TYPE(ChomboHandle), POINTER :: chandle
00159     TYPE(NodedefList), POINTER :: child_list
00160     INTEGER :: child_count
00161     INTEGER :: i_childbox_data_size, i_acc_childbox_data_size
00162     LOGICAL :: lChildDatasetsInitialized
00163     INTEGER :: DummyInt, iErr
00164     TYPE(DiagnosticListDef), POINTER :: Diagnostic
00165 
00166     NULLIFY(Info)
00167 
00168     WRITE(s_filename, '(A10,I5.5,A4)') 'out/chombo', nframe, '.hdf'
00169 
00170     CALL CreateChomboHandle(s_filename, chandle, CHOMBO_HANDLE_WRITE)
00171 
00172     ! Start out by assuming that the finest level is the maximum allowable level.
00173     ! If it turns out there is no data on the max level, then FinestLevel could change.
00174     FinestLevel = MaxLevel
00175 
00176 !!!!! Add required attributes for Chombo HDF specification.  New attributes are
00177 !!!!! added to the root group usingAdd_ChomboHDF_Attribute() subroutine, which
00178 !!!!! is described below.
00179 
00180     ! time attribute (uses Info's current time).
00181     CALL Add_HDF5_Attribute_Double("time", chandle%root_group_id, levels(0)%tnow)
00182     CALL Add_HDF5_Attribute_Double("olddt", chandle%root_group_id, olddt)
00183     CALL Add_HDF5_Attribute_Double("overall_maxspeed", chandle%root_group_id, overall_maxspeed)
00184     CALL Add_HDF5_Attribute_Double("maxcfl", chandle%root_group_id, maxcfl)
00185 
00186     ! iteration attribute (uses Info's current frame number).
00187     CALL Add_HDF5_Attribute_Int("iteration", chandle%root_group_id, nFrame)
00188 
00189     ! num_components attribute (use Info's number of output variables).
00190     CALL Add_HDF5_Attribute_Int("num_components", chandle%root_group_id, NrVars+NrDiagnosticVars)
00191 
00192     ! node centering (fixes contour maps in visit)
00193     CALL Add_HDF5_Attribute_Int("data_centering", chandle%root_group_id, 0) ! cell centered
00194 
00195 
00196  !!!This pro_lo attribute causes pseudocolor slices to not work with multiple root patches in visit
00197     !CALL Add_HDF5_Attribute_FloatVector("prob_lo", chandle%root_group_id, GxBounds(:,1))
00198 
00199 
00200 
00201 
00202 !!!!!    The scaling factors below are not part of the Chombo HDF specification, but they're
00203 !!!!!    useful enough that we're including them anyway.
00204 
00205     ! Number density.
00206     CALL Add_HDF5_Attribute_Double("n_density_scale", chandle%root_group_id, nScale)
00207 
00208     ! Mass density.
00209     CALL Add_HDF5_Attribute_Double("rho_scale", chandle%root_group_id, rScale)
00210 
00211     CALL Add_HDF5_Attribute_Double("length_scale", chandle%root_group_id, lScale)
00212 
00213     CALL Add_HDF5_Attribute_Double("velocity_scale", chandle%root_group_id, velScale)
00214 
00215     CALL Add_HDF5_Attribute_Double("pressure_scale", chandle%root_group_id, pScale)
00216 
00217     CALL Add_HDF5_Attribute_Double("temperature_scale", chandle%root_group_id, TempScale)
00218 
00219     !Uses var_index to keep track of what variables are in what slots
00220     NrCons = NrHydroVars - NrTracerVars
00221 
00222     ! Create tags for conservative variables using indices initialized in 
00223     ! PhysicsControl::SetVariableIndices().
00224     DO i=1, NrCons
00225        sa_component_names(i-1)= trim(FieldName(i))
00226     END DO
00227 
00228     ! Creates tags for tracer variables.
00229     DO i = 1, NrTracerVars !NrCons+1,NrHydroVars
00230        sa_component_names(nTracerLo-1+i-1) = trim(TracerName(i))
00231     END DO
00232 
00233     ! Creates tags for Elliptic variables.
00234     DO i = 1, NrEllipticVars !NrHydroVars+1,NrHydroVars+NrEllipticVars
00235        sa_component_names(nEllipticLo-1+i-1) = trim(EllipticName(i))       
00236 !       IF (i == iPhi) sa_component_names(i-1)="Phi"
00237 !       IF (i == iPhiSinks) sa_component_names(i-1)="PhiSinks"
00238 !       IF (i == iPhiGas) sa_component_names(i-1)="PhiGas"
00239 !       IF (i == iPhiDot) sa_component_names(i-1)="PhiDot"
00240     END DO
00241 
00242     Diagnostic=>FirstDiagnostic
00243     i=1
00244     DO WHILE (ASSOCIATED(Diagnostic))
00245        sa_component_names(NrHydroVars+NrEllipticVars+i-1) = trim(Diagnostic%Field%Name)
00246        i=i+1
00247        Diagnostic=>Diagnostic%next
00248     END DO
00249 
00250     ! If MaintainAuxArrays is true, then the current problem uses MHD.
00251     IF (MaintainAuxArrays) THEN    
00252 
00253        ! All MHD problems are three-dimensional, so you have to include all three
00254        ! directional momentum and magnetic field components.
00255        ! Add derived scaling factors.
00256        CALL Add_HDF5_Attribute_Double("px_scale", chandle%root_group_id, rScale * velScale)
00257        CALL Add_HDF5_Attribute_Double("py_scale", chandle%root_group_id, rScale * velScale)
00258        CALL Add_HDF5_Attribute_Double("pz_scale", chandle%root_group_id, rScale * velScale)
00259 
00260        IF (iE .ne. 0) CALL Add_HDF5_Attribute_Double("E_scale", chandle%root_group_id, pScale)
00261 
00262        CALL Add_HDF5_Attribute_Double("Bx_scale", chandle%root_group_id, pScale)
00263        CALL Add_HDF5_Attribute_Double("By_scale", chandle%root_group_id, pScale)
00264        CALL Add_HDF5_Attribute_Double("Bz_scale", chandle%root_group_id, pScale)
00265 
00266     ELSE
00267 
00268        ! If the project is three-dimensional, then the pz component must be added;
00269        ! otherwise, begin adding tracer components.
00270        IF (nDim == 2) THEN
00271 !          ! Add derived scaling factors.
00272           CALL Add_HDF5_Attribute_Double("px_scale", chandle%root_group_id, rScale * velScale)
00273           CALL Add_HDF5_Attribute_Double("py_scale", chandle%root_group_id, rScale * velScale)
00274           IF (iE .ne. 0) CALL Add_HDF5_Attribute_Double("E_scale", chandle%root_group_id, pScale)
00275        ELSE
00276           ! Add derived scaling factors.
00277           CALL Add_HDF5_Attribute_Double("px_scale", chandle%root_group_id, rScale * velScale)
00278           CALL Add_HDF5_Attribute_Double("py_scale", chandle%root_group_id, rScale * velScale)
00279           CALL Add_HDF5_Attribute_Double("pz_scale", chandle%root_group_id, rScale * velScale)    
00280           IF (iE .ne. 0) CALL Add_HDF5_Attribute_Double("E_scale", chandle%root_group_id, pScale)
00281        END IF
00282     END IF
00283 
00284 
00285     ! This part is separated out so that changing the assembly of the components
00286     ! is easy, but the addition of attributes may eventually have to be done
00287     ! inline in order to reduce overhead.
00288     !
00289     ! Construct new component_n attributes for each sa_component_name(n).
00290     DO i_index = 0, NrVars + NrDiagnosticVars - 1
00291 
00292        ! Create component tag (attribute name).
00293        IF (i_index <= 9) THEN
00294           WRITE(s_component_tag,'(A10,I1.1)') 'component_', i_index
00295        ELSE
00296           WRITE(s_component_tag,'(A10,I2.2)') 'component_', i_index
00297        END IF
00298 
00299        CALL Add_HDF5_Attribute_String(s_component_tag, &
00300             chandle%root_group_id, TRIM(sa_component_names(i_index)))
00301     END DO
00302 
00303 !!! End Assembling Component Array.
00304 
00305 !!! Create Group Chombo_global And Its Attributes
00306 
00307     ! Create Chombo_global group.
00308     CALL h5gcreate_f(chandle%file_handle, "Chombo_global", hid_chombo_global_group_id, i_err)
00309     CALL CatchHDF5Error("h5gcreate_f ('Chombo_global')", i_err)
00310 
00311     ! Add testReal attribute to Chombo_global group (value is a literal).
00312     CALL Add_HDF5_Attribute_Double("testReal", hid_chombo_global_group_id, &
00313          DBL_TEST_REAL_DAT) 
00314 
00315     ! Add SpaceDim attribute to Chombo_global group.  We can just use Info%nDim
00316     ! because we don't generally do 4D problems.
00317     IF (nDim == 1) THEN
00318           CALL Add_HDF5_Attribute_Int("SpaceDim", hid_chombo_global_group_id, 2)
00319        ELSE
00320           CALL Add_HDF5_Attribute_Int("SpaceDim", hid_chombo_global_group_id, nDim)
00321        END IF
00322     ! Close the Chombo_global group.
00323     CALL h5gclose_f(hid_chombo_global_group_id, i_err)
00324     CALL CatchHDF5Error("h5gclose_f ('Chombo_global')", i_err)
00325 
00326 !!! End Create Chombo_global group.
00327 
00328 !!! Storing the domain boundaries in the Chombo file.
00329     CALL Add_HDF5_Attribute_FloatVector("lower_bound", chandle%root_group_id, GxBounds(:,1))
00330     CALL Add_HDF5_Attribute_FloatVector("upper_bound", chandle%root_group_id, GxBounds(:,2))
00331 
00332 
00333     i_acc_childbox_data_size = 1 ! The initial value for i_childbox_data_size; after level -2
00334     ! it will have more meaningful values, but we want to make
00335     ! sure that the loop below functions properly.
00336 
00337     ! Clear the next_level_cost array before using it.
00338     chandle%next_level_cost = 0
00339 
00340 !!!    Assembling the group_level(n) groups.
00341 
00342     DO i_level = -2, FinestLevel
00343 
00344 !       PRINT "('Level ', i2, ' begun.')", i_level
00345 
00346        ! The size of the current level's boxes dataset is equal to the number of child boxes on the
00347        ! previous level.  Therefore, rather than recalculate it, we just move over the accumulated
00348        ! childbox count.
00349        i_box_data_size = i_acc_childbox_data_size
00350 
00351        ! Clear level data counters.
00352        i_grid_data_size = 0
00353        i_acc_childbox_data_size = 0
00354        IF (MaintainAuxArrays)  i_aux_data_size = 0
00355        lChildDatasetsInitialized = .FALSE.
00356 
00357        IF (i_level >= 0) THEN
00358 
00359           ! [BDS] Select the youngest node on level i_level.
00360           nodelist => Nodes(i_level)%p
00361 
00362           DO proc_id = 0, MPI_np - 1
00363 
00364              i_counter = 0
00365              
00366              IF (proc_id > Master) THEN
00367 
00368                 ! Tell worker to send data
00369                 CALL MPI_SEND(DummyInt, 1, MPI_INTEGER, proc_id, TRANSMIT_IO_WORKER_GRIDS, MPI_COMM_WORLD, iErr)
00370                 ! Create a message to receive data from this process.
00371                 CALL CreatePackedMessage(i_level, &
00372                                          proc_id, &
00373                                          TRANSMIT_FRAME_DATA, &
00374                                          STAGE_RECV, &
00375                                          message, &
00376                                          chandle%next_level_cost(proc_id))
00377 
00378                 ! Retrieve the level statistics from the remote processor.
00379                 CALL IO_UnparseLevelStatistics(message, data_sizes)
00380 
00381                 chandle%next_level_cost(message%remote_proc) = TERMINATION_BOX_BYTES + &
00382                                                                IO_LEVEL_STAT_BYTES + &
00383                                                                data_sizes(IO_NEXTLEVELCOST)
00384 
00385                 !Stores the number of child grids on this level.
00386                 i_childbox_data_size = data_sizes(IO_CHILDCOUNT)
00387 
00388                 IF (i_level >= 0) THEN
00389                    ! Initializes the size of the data:datatype=0 array.
00390                    i_grid_data_size = data_sizes(IO_Q_SIZE)
00391 
00392 
00393                    ! Initializes the size of the MHD_data array.  If there is no MHD in this problem
00394                    ! (i.e., if lMHD is false), then this will be 0.
00395                    i_aux_data_size = data_sizes(IO_AUX_SIZE)
00396                 END IF
00397 
00398              ELSE
00399                 ! Obtain level statistics for the local processor.
00400                 CALL IO_GetDatasetSizes(i_level, FinestLevel, data_sizes)
00401                 
00402 
00403                 !Stores the number of child grids on this level.
00404                 i_childbox_data_size = data_sizes(IO_CHILDCOUNT)
00405 
00406                 IF (i_level >= 0) THEN
00407 
00408                    ! Initializes the size of the data:datatype=0 array.
00409                    i_grid_data_size = data_sizes(IO_Q_SIZE)
00410 
00411                    ! Initializes the size of the MHD_data array.  If there is no MHD in this problem
00412                    ! (i.e., if lMHD is false), then this will be 0.
00413                    i_aux_data_size = data_sizes(IO_AUX_SIZE)
00414 
00415                    ! [BDS] Select the youngest node on level i_level.
00416                    nodelist => Nodes(i_level)%p
00417 
00418                 END IF
00419 
00420              END IF
00421 
00422              i_acc_childbox_data_size = i_acc_childbox_data_size + i_childbox_data_size
00423 
00424              DO   ! Loop over nodes on i_level.
00425 
00426                 IF (proc_id > Master) THEN
00427 
00428                    NULLIFY(node, Info)
00429 
00430                    ! Extract the next node from the message.  This will return null on
00431                    ! a termination box.
00432                    CALL IO_UnparseRemoteNode(message, node)
00433                    
00434                    IF (ASSOCIATED(node)) THEN
00435                       ! If the level is less than the finest level, then unparse any
00436                       ! child nodes that have been sent along with the current node.
00437                       IF (i_level < FinestLevel)  CALL IO_UnparseNodeChildren(message, child_count, node)
00438 
00439                       ! Unparse the grid data associated with the current node.
00440                       IF (i_level > -1) CALL IO_UnparseRemoteGrid(message, Info, FinestLevel)
00441                    ELSE
00442                       EXIT
00443                    END IF
00444 
00445                 ELSE
00446 
00447                    ! If node_ptr is now null, then there are no nodes
00448                    ! left on level i_level and we can exit the loop.
00449                    IF (.NOT. ASSOCIATED(nodelist)) EXIT
00450                    IF (.NOT. ASSOCIATED(nodelist%self))  EXIT
00451 
00452                    ! Retrieve the InfoDef structure of node_ptr.
00453                    node => nodelist%self
00454                    child_count = NodeCount(node%children)
00455                    Info => nodelist%self%Info
00456 
00457                    ! Die if unable to obtain node_ptr's Info structure.
00458                    IF (.NOT. ASSOCIATED(Info)) THEN
00459                       PRINT *, "MakeChomboFile error ", i_err, &
00460                            ": unable to obtain node info structure."
00461                       STOP
00462                    END IF
00463                    
00464                 END IF
00465 
00466                 ! If there is cell-centered data to be stored in the dataset and the offset
00467                 ! is empty, then assume the level has not been created and create it.  If 
00468                 ! there is no grid data, then do nothing.  If there is data to store and the 
00469                 ! offset is not 0, then the dataset extension code will fix the problem when
00470                 ! this routine goes to write the data to the dataset.
00471 
00472                 ! q data is never sent without costmap data, so an empty q offset will be accompanied
00473                 ! by an empty costmap offset as well.
00474                 
00475                 IF (i_grid_data_size > 0) THEN
00476                    IF (.NOT. HandleIsValid(chandle%level_id)) THEN
00477 
00478                       ! Open a chombo handle for this level.
00479                       CALL IO_OpenCurrentLevel(chandle, i_level)
00480 
00481                       ! Create the dx attribute
00482                       CALL Add_HDF5_Attribute_Double("dx", chandle%level_id, levels(i_level)%dx)
00483                       ! Create the anisotropic attribute (optional chomobo attrib for anisotropic spacing)
00484                       ! [BDS][20100712]:  The design does not currently support anisotropic systems.
00485                       CALL Add_HDF5_Attribute_FloatVector("anisotropic", chandle%level_id, (/1d0,1d0,1d0/))!Info%dX/Info%dX(1))         
00486 
00487                       ! Create the ref_ratio attribute.
00488                       CALL Add_HDF5_Attribute_Int("ref_ratio", chandle%level_id, levels(i_level)%CoarsenRatio)
00489 
00490                       ! loop over the three spatial dimensions to construct the box object..
00491                       DO i_index = 1,3
00492 
00493                          ! Set the lower-bounds of the box object.  This will be the 
00494                          ! lower-bound value in mGlobal, minus one as an offset for 
00495                          ! Chombo's field-space.
00496 
00497                          ia_box_global(i_index) = 0
00498 
00499                          ! ia_box_global = #cells along each dimension, adjusted by the 
00500                          !           refinement level.  This can be found by taking
00501                          !          the difference between the max and min values
00502                          !          for each dimension and dividing by dX(dim).
00503 
00504                          IF (levels(i_level)%dX /= zero) THEN
00505                             ia_box_global(i_index+3)=Gmx(i_index) * product(levels(0:i_level-1)%coarsenratio)-1 !(GxBounds(i_index,2) - GxBounds(i_index,1)) / levels(i_level)%dX
00506                          END IF
00507 
00508                          ! If there is a fractional part to the domain offset, then 
00509                          ! add another cell to the global box to accomodate it.
00510 !                         ia_box_global(i_index + 3) = MAX(dbl_domain_offset - 1, 0)
00511 
00512                       END DO
00513 
00514                       ! Create prob_domain attribute using a box object.
00515                       CALL Add_HDF5_Attribute_Box("prob_domain", hid_box_id, chandle%level_id, ia_box_global)
00516 
00517                    END IF
00518 
00519 
00520 
00521                    IF (HandleIsValid(chandle%level_id)) THEN
00522 
00523                       ! ** box data **
00524 !                      IF (i_box_data_size > 0) THEN
00525 
00526                           ! If the handle's offset is 0, then no data has been added to the dataset,
00527                           ! meaning that it hasn't been initialized yet.  This conditional will correct that.
00528                           IF (chandle%box_offset == 0) THEN
00529                               ! The dataset that holds the grid dimensions.
00530                               CALL Initialize_HDF5_Dataset_Box("boxes", chandle%level_id, i_box_data_size)
00531                           END IF
00532 
00533                           CALL IO_WriteBoxToChomboFile(chandle, "boxes", node%box%mGlobal)
00534 !
00535  !                     END IF
00536 
00537                       ! ** childbox data **
00538                       ! On levels below FinestLevel, store the child boxes of each node.  These boxes
00539                       ! are stored in breadth-first order relative to their parents.
00540                       IF ((i_level < FinestLevel)) THEN
00541 
00542                          ! If the handle's offset is 0, then no data has been added to the dataset,
00543                          ! meaning that it hasn't been initialized yet.  This conditional will correct that.
00544                          IF ((.NOT. lChildDatasetsInitialized) .AND. (chandle%childbox_offset == 0)) THEN
00545                              ! The size of the childbox count dataset is always known--it is the number of boxes
00546                              ! on the current level, since each element has a 1-1 correspondence with the current
00547                              ! level.
00548 
00549                              CALL Initialize_HDF5_Dataset_Int("childbox_count", chandle%level_id, i_box_data_size)
00550                              CALL Initialize_HDF5_Dataset_Box("child_boxes", chandle%level_id, i_childbox_data_size)
00551                              lChildDatasetsInitialized = .TRUE.
00552                          END IF
00553 
00554                          ! Write the number of children to the child_boxes array.
00555                          CALL IO_WriteChildCountToChomboFile(chandle, child_count)
00556 
00557                          child_list => node%children
00558 
00559                          ! Loop over the child box list and add each child to the child boxes dataset.
00560                          DO WHILE (ASSOCIATED(child_list))
00561                             CALL IO_WriteBoxToChomboFile(chandle, "child_boxes", child_list%self%box%mGlobal)
00562                             child_list => child_list%next
00563                          END DO
00564 
00565                          NULLIFY(child_list)
00566 
00567                       END IF
00568 
00569                       ! ** grid data **
00570                       ! Initialize the dataset that holds cell-centered data.  Populating these
00571                       ! datasets is more involved than the box datasets, so we'll do that later.
00572                       IF ((i_grid_data_size > 0) .AND. (chandle%q_offset == 0)) THEN
00573 
00574                          CALL Initialize_HDF5_Dataset_Double("data:datatype=0", &
00575                                                              chandle%level_id, &
00576                                                              i_grid_data_size)
00577 
00578                          ! If this is not the finest level, then initialize a costmap dataset
00579                          ! for this level.
00580 !                         IF (i_level < FinestLevel) THEN
00581 !                               CALL Initialize_HDF5_Dataset_Double("costmap", &
00582 !                                                                   chandle%level_id, &
00583 !                                                                   i_grid_data_size * 2 / NrVars)
00584 !                         END IF
00585 
00586                       END IF
00587 
00588 
00589                       ! Same as for the cell-centered data, except now we're creating a face-centered dataset.
00590                       IF (MaintainAuxArrays .AND. (i_aux_data_size > 0) .AND. (chandle%aux_offset == 0)) THEN
00591                          CALL Initialize_HDF5_Dataset_Double("MHD_data", chandle%level_id, i_aux_data_size)
00592                       END IF
00593 
00594  
00595                       ! Obtain the grid dimensions from the coordinate box.
00596                       mx = node%box%mGlobal(1,2) - node%box%mGlobal(1,1) + 1
00597                       my = node%box%mGlobal(2,2) - node%box%mGlobal(2,1) + 1
00598                       mz = node%box%mGlobal(3,2) - node%box%mGlobal(3,1) + 1
00599 
00600                       ! Allocate memory for the current node's grid.
00601                       ALLOCATE(dbla_chombo_qvar(mx * my * mz * (NrVars+NrDiagnosticVars)), stat=i_err)
00602 
00603                       ! Stops execution on a memory allocation error.
00604                       IF (i_err /= 0) THEN
00605                          PRINT *,"MakeChomboFile error: unable to allocate memory for q-var array."
00606                          STOP
00607                       END IF
00608 
00609                       ! Clear index; it will be used to reference chombo data array in the loop below.
00610                       i_index = 0
00611 
00612                       ! Loop over each variable tracked by the program.    
00613                       DO i_variable = 1, NrVars
00614 
00615                          ! The nested loops below loop over each spatial dimension and reference the
00616                          ! q value for each variable within a specific 3D coordinate; this value is
00617                          ! then stored in the chombo data array.  In effect, what this does is flatten
00618                          ! the multi-dimensional data in q down to a 1D array for HDF5 storage.
00619 
00620                          DO k = 1, mz
00621                             DO j = 1, my
00622                                DO i = 1, mx
00623 
00624                                   i_index = i_index + 1
00625                                   dbla_chombo_qvar(i_index) = Info%q(i, j, k, i_variable)
00626 
00627                                END DO ! End i
00628                             END DO ! End j
00629                          END DO ! End k
00630 
00631                       END DO ! End do i_variable =  to Info%NrOutVars
00632 
00633 
00634                       ! Loop over each variable tracked by the program.    
00635                       DO i_variable = 1, NrDiagnosticVars
00636 
00637                          ! The nested loops below loop over each spatial dimension and reference the
00638                          ! q value for each variable within a specific 3D coordinate; this value is
00639                          ! then stored in the chombo data array.  In effect, what this does is flatten
00640                          ! the multi-dimensional data in q down to a 1D array for HDF5 storage.
00641 
00642                          DO k = 1, mz
00643                             DO j = 1, my
00644                                DO i = 1, mx
00645 
00646                                   i_index = i_index + 1
00647                                   dbla_chombo_qvar(i_index) = Info%diagnostics(i, j, k, i_variable)
00648 
00649                                END DO ! End i
00650                             END DO ! End j
00651                          END DO ! End k
00652 
00653                       END DO ! End do i_variable =  to Info%NrOutVars
00654                       
00655                       !Deallocate the diagnostic array always once it is outputted.
00656                       IF (NrDiagnosticVars > 0) THEN
00657                          IF (.NOT.(ASSOCIATED(Info%diagnostics))) THEN
00658                             write(*,*) 'info diagnostics not associated'
00659                             STOP
00660                          END IF
00661                          DEALLOCATE(Info%diagnostics)
00662                          NULLIFY(Info%diagnostics)
00663                       END IF
00664                       
00665                       ! Write the data array to the Chombo file using the supplied offset.
00666                       CALL Write_Slab_To_Dataset_Double("data:datatype=0", chandle%level_id, dbla_chombo_qvar, chandle%q_offset)
00667                       ! Adjust the offset by the size of the grid just written.
00668 
00669                       chandle%q_offset = chandle%q_offset + i_index
00670 
00671 !                      IF( i_level < FinestLevel) THEN
00672 !                         CALL Write_Slab_To_Dataset_Double("costmap", chandle%level_id, &
00673 !                              RESHAPE(Info%costmap(1:mx,1:my,1:mz,1:2), (/ mx*my*mz*2 /)), chandle%costmap_offset)
00674 
00675                          ! Adjust the costmap offset.
00676 !                         chandle%costmap_offset = chandle%costmap_offset + mx*my*mz*2
00677 !                      END IF
00678 
00679                       ! Release array memory so that it can be used by the next grid.
00680                       DEALLOCATE(dbla_chombo_qvar)
00681 
00682                       ! If MHD is running, then do the same thing for the Info%aux array.
00683                       IF (MaintainAuxArrays) THEN 
00684 
00685                          ! Allocate memory for the current node's aux grid.
00686                          IF (nDim == 3) THEN
00687                             ALLOCATE(dbla_chombo_aux(((mx + 1) * my * mz) + &
00688                                  (mx * (my + 1) * mz) + &
00689                                  (mx * my * (mz + 1))), stat=i_err)
00690                          ELSE
00691                             ALLOCATE(dbla_chombo_aux(((mx + 1) * my) + &
00692                                  (mx * (my + 1))), stat=i_err)
00693                          END IF
00694 
00695                          ! Stops execution on a memory allocation error.
00696                          IF (i_err /= 0) THEN
00697                             PRINT *,"MakeChomboFile error: unable to allocate memory for aux array."
00698                             STOP
00699                          END IF
00700 
00701                          ! Clear index; it will be used to reference chombo data array in the loop below.
00702                          i_mhd_index = 0
00703 
00704                          ! There will be one Bx variable for each dimension of the problem; we
00705                          ! don't need to keep track of any other aux variables.
00706                          DO i_variable = 1, nDim
00707 
00708                             ! Set the dimension of the current Bx variable to be extended by 1.
00709                             ia_aux_extension = 0
00710                             ia_aux_extension(i_variable) = 1
00711 
00712                             ! The nested loops below loop over each spatial dimension and reference the
00713                             ! aux value for each variable within a specific 3D coordinate; this value is
00714                             ! then stored in the chombo data array.  In effect, what this does is flatten
00715                             ! the multi-dimensional data in aux down to a 1D array for HDF5 storage.
00716 
00717                             DO k = 1, mz + ia_aux_extension(3)
00718                                DO j = 1, my + ia_aux_extension(2)
00719                                   DO i = 1, mx + ia_aux_extension(1)
00720 
00721                                      i_mhd_index = i_mhd_index + 1
00722                                      dbla_chombo_aux(i_mhd_index) = Info%aux(i, j, k, i_variable)
00723 
00724                                   END DO    ! End i
00725                                END DO        ! End j
00726                             END DO            ! End k
00727 
00728                          END DO        ! End do i_variable =  to Info%maux
00729 
00730                          ! Write the data array to the Chombo file using the supplied offset.
00731                          CALL Write_Slab_To_Dataset_Double("MHD_data", chandle%level_id, &
00732                               dbla_chombo_aux, chandle%aux_offset)
00733 
00734                          ! Adjust the offset by the size of the grid just written.
00735                          chandle%aux_offset = chandle%aux_offset + i_mhd_index
00736 
00737                          ! Release array memory so that it can be used by the next grid.
00738                          DEALLOCATE(dbla_chombo_aux)
00739 
00740                       END IF         ! (if MaintainAuxArrays).
00741 
00742                    END IF
00743                 END IF
00744                 ! Destroy the structures created from remote data.
00745                 IF (proc_id > Master) THEN
00746 
00747                    ! Deallocate the cell-centered data array.
00748                    DEALLOCATE(Info%q)
00749                    NULLIFY(Info%q)
00750 
00751                    ! Deallocate the aux array imported from the remote processor
00752                    IF (MaintainAuxArrays) THEN
00753                       DEALLOCATE(Info%aux)
00754                       NULLIFY(Info%aux)
00755                    END IF
00756 
00757                    DEALLOCATE(Info)
00758                    NULLIFY(Info)
00759 
00760                    ! Deallocate the node's variable-length lists.
00761 !                   DEALLOCATE(node%proclist, node%proctime)
00762 !                   NULLIFY(node%proclist, node%proctime)
00763 
00764                    IF (ASSOCIATED(node%children))  CALL DestroyNodeList(node%children)
00765 
00766                    DEALLOCATE(node)
00767                    NULLIFY(node)
00768 
00769                 ELSE
00770                    ! [BDS] Advance to the next node on the level.
00771                    NULLIFY(child_list)
00772                    nodelist => nodelist%next
00773 
00774                 END IF
00775 
00776              END DO  ! Loop over nodes on i_level.
00777 
00778              ! Destroy this processor's message to make room for a new one.
00779              IF (proc_id > Master)  CALL DestroyPackedMessage(message)
00780 
00781           END DO  ! Loop over processors.
00782 
00783 !          PRINT *, "Level ", i_level, " datasets finished."
00784 
00785 !!!!!!!! END DATAZERO LEVEL TRAVERSAL
00786 
00787           ! If there are boxes on this level, then add the remaining level data and close
00788           ! the level handle.  Otherwise, set the FinestLevel to be the previous level and exit.
00789           IF (HandleIsValid(chandle%level_id)) THEN
00790 
00791              ! Create the data attributes group.
00792              CALL h5gcreate_f(chandle%level_id, "data_attributes", hid_data_attributes_group_id, i_err)
00793              CALL CatchHDF5Error("h5gcreate_f('data_attributes')", i_err)
00794 
00795              ! Adds the output_ghost attribute using the literal intvector [0, 0, 0].
00796              CALL Add_HDF5_Attribute_IntVector("outputGhost", hid_intvect_id, &
00797                   hid_data_attributes_group_id, (/0,0,0/))
00798 
00799              ! Adds the comps attribute (the same as num_components)
00800              CALL Add_HDF5_Attribute_Int("comps", hid_data_attributes_group_id, &
00801                   NrVars+NrDiagnosticVars)
00802 
00803              ! Adds the objectType attribute.
00804              CALL Add_HDF5_Attribute_String("objectType", hid_data_attributes_group_id, "FArrayBox")
00805 
00806              ! Close data attributes group.
00807              CALL h5gclose_f(hid_data_attributes_group_id, i_err)
00808              CALL CatchHDF5Error("h5gclose_f('data_attributes')", i_err)
00809 
00810              ! Close level_n group.
00811              CALL IO_CloseCurrentLevel(chandle)
00812 
00813           ELSE
00814              ! If there were no boxes on this level, then some previous level was the finest level.
00815              ! 
00816              IF (FinestLevel >= i_level)  FinestLevel = i_level - 1
00817           END IF
00818 
00819 
00820        ELSE IF (i_level == CHOMBO_DOMAIN_LEVEL) THEN
00821 
00822           IF (nDomains > 1) THEN 
00823              PRINT*, 'not yet implemented for IO'
00824              STOP
00825           END IF
00826           ! Clear all level offsets.
00827           CALL ClearChomboHandleOffsets(chandle)
00828           chandle%next_level_cost = 0
00829 
00830           ! Obtain level statistics for the local processor.
00831           CALL IO_GetDatasetSizes(i_level, FinestLevel, data_sizes)
00832 
00833           !Stores the number of child grids on this level.
00834           i_childbox_data_size = data_sizes(IO_CHILDCOUNT)
00835           i_acc_childbox_data_size=i_acc_childbox_data_size+i_childbox_data_size
00836 
00837           ! Open a chombo handle for this level.
00838           CALL IO_OpenCurrentLevel(chandle, i_level)
00839 
00840           ! The -1 level stores its own boxes, child boxes (plus their count), and its costmap.
00841           ! It does not, however, have any cell- or face-centered data to store. 
00842 !          CALL Initialize_HDF5_Dataset_Double("costmap", chandle%level_id, PRODUCT(GmX)*2, &
00843 !               L_NON_EXTENSIBLE)
00844 
00845           CALL Initialize_HDF5_Dataset_Box("boxes", chandle%level_id, i_box_data_size, L_NON_EXTENSIBLE)
00846           CALL Initialize_HDF5_Dataset_Int("childbox_count", chandle%level_id, i_childbox_data_size)
00847           CALL Initialize_HDF5_Dataset_Box("child_boxes", chandle%level_id, i_childbox_data_size)
00848 
00849           CALL IO_WriteDomainData(chandle, data_sizes(IO_CHILDCOUNT))
00850 
00851           ! Close level_n group.
00852           CALL IO_CloseCurrentLevel(chandle)
00853 
00854        ELSE IF (i_level == CHOMBO_ROOT_LEVEL) THEN
00855 
00856           ! Clear all level offsets.
00857           CALL ClearChomboHandleOffsets(chandle)
00858           chandle%next_level_cost = 0
00859 
00860           ! Obtain level statistics for the local processor.
00861           CALL IO_GetDatasetSizes(i_level, FinestLevel, data_sizes)
00862 
00863           !Stores the number of child grids on this level.
00864           i_childbox_data_size = data_sizes(IO_CHILDCOUNT)
00865           i_acc_childbox_data_size=i_acc_childbox_data_size+i_childbox_data_size
00866 
00867           ! The -2 level only has one grid, and it's on the master.  Since that grid is always generated
00868           ! by DomainInit(), the -2 level only needs a costmap, a single integer for its child count, and
00869           ! its child boxes.
00870 !          CALL Initialize_HDF5_Dataset_Double("root_costmap", chandle%root_group_id, &
00871 !               PRODUCT(GmX)*2, L_NON_EXTENSIBLE)
00872 
00873           CALL Add_HDF5_Attribute_Int("root_child_count", chandle%root_group_id, i_childbox_data_size)
00874           CALL Initialize_HDF5_Dataset_Box("root_child_boxes", chandle%root_group_id, i_childbox_data_size)
00875 
00876        END IF
00877 
00878        PRINT *, 'Level ', i_level, ' data written.'
00879 !       PRINT *
00880 
00881     END DO         !i_level = -2 to FinestLevel.
00882 
00883 
00884     ! num_levels attribute (Chombo format requires that this be 1 greater the maximum
00885     ! level--essentially so that it acts as a level-index.
00886     CALL Add_HDF5_Attribute_Int("num_levels", chandle%root_group_id, FinestLevel + 1)
00887 
00888     ! max_level attribute (use Info's current FinestLevel value).
00889     CALL Add_HDF5_Attribute_Int("max_level", chandle%root_group_id, FinestLevel)
00890 
00891 !!! END OF LEVEL_N groups.
00892 
00893     ! [BDS][20081126]:  Add sink particles to chombo if required components are active.
00894     ! [BDS][20100712]:  Sink particles are not currently implemented in scrambler, so this
00895     !                   feature is deactivated.
00896     ! [BDS][20101130]:  Sink particles are now part of scrambler.  The routine below will stores them
00897         !                   in the format described by the chombo DDF.
00898     CALL Particle_WriteData(chandle)
00899 
00900     CALL WriteCoolingObjects(chandle)
00901 
00902     CALL WritePointGravityObjects(chandle)
00903 
00904     ! Close the chombo file.
00905     CALL CloseChomboHandle(chandle)
00906 
00907   END SUBROUTINE MakeChomboFile
00908 
00913   SUBROUTINE IO_WriteBoxToChomboFile(chandle, box_set_name, box)
00914 
00915     TYPE(ChomboHandle), POINTER :: chandle
00916     CHARACTER(*) :: box_set_name
00917     INTEGER, DIMENSION(3,2) :: box
00918 
00919     INTEGER, DIMENSION(6) :: box_data
00920     INTEGER(HSIZE_T) :: box_offset
00921     INTEGER(HID_T) :: source_group_id
00922 
00923 
00924     box_data(1:3) = box(1:3,1) - 1
00925     box_data(4:6) = box(1:3,2) - 1
00926 
00927     ! Four possible options:  "boxes", "child_boxes", "root_child_boxes"
00928     IF (box_set_name == "child_boxes") THEN
00929        box_offset = chandle%childbox_offset
00930        source_group_id = chandle%level_id
00931 
00932     ELSE IF (box_set_name == "boxes") THEN
00933        box_offset = chandle%box_offset
00934        source_group_id = chandle%level_id
00935 
00936     ELSE IF (box_set_name == "root_child_boxes") THEN
00937        box_offset = chandle%childbox_offset
00938        source_group_id = chandle%root_group_id
00939 
00940     END IF
00941 
00942     CALL Write_Slab_To_Dataset_Box(box_set_name, &
00943          source_group_id, &
00944          box_data, &
00945          box_offset)
00946 
00947     IF (box_set_name == "child_boxes") THEN
00948        chandle%childbox_offset = chandle%childbox_offset + 1
00949     ELSE IF (box_set_name == "boxes") THEN
00950        chandle%box_offset = chandle%box_offset + 1
00951     ELSE
00952        chandle%domainbox_offset = chandle%domainbox_offset + 1
00953     END IF
00954 
00955   END SUBROUTINE IO_WriteBoxToChomboFile
00956 
00960   SUBROUTINE IO_WriteChildCountToChomboFile(chandle, int_data)
00961 
00962     TYPE(ChomboHandle), POINTER :: chandle
00963     INTEGER :: int_data
00964 
00965 
00966     CALL Write_Slab_To_Dataset_Int("childbox_count", &
00967          chandle%level_id, &
00968          (/ int_data /), &
00969          chandle%childbox_count_offset)
00970 
00971     chandle%childbox_count_offset = chandle%childbox_count_offset + 1
00972 
00973   END SUBROUTINE IO_WriteChildCountToChomboFile
00974 
00978   SUBROUTINE IO_WriteDomainData(chandle, master_childcount)
00979 
00980     TYPE(ChomboHandle), POINTER :: chandle
00981 
00982     TYPE(PackedMessage), POINTER :: message
00983     TYPE(NodedefList), POINTER :: nodelist, child_list
00984     REAL(KIND=qPrec), DIMENSION(:,:,:,:), POINTER :: cost_map
00985     INTEGER :: proc_id
00986     INTEGER :: master_childcount
00987 
00988     INTEGER :: mx, my, mz
00989     INTEGER, DIMENSION(3,2) :: mGlobal
00990     TYPE(Nodedef), POINTER :: node
00991     INTEGER :: child_count
00992     INTEGER, DIMENSION(IO_LEVEL_STAT_COUNT) :: data_sizes
00993     INTEGER :: i_childbox_data_size
00994     INTEGER :: DummyInt, iErr
00995 
00996     chandle%costmap_offset = 0
00997     child_count = 0
00998 
00999     ! Points to the node list for the -1 (domain) level.
01000     nodelist => Nodes(CHOMBO_DOMAIN_LEVEL)%p
01001     
01002     ! Loop over processors and retrieve any cost map data.
01003     DO proc_id = 0, MPI_np - 1
01004 
01005        ! If proc-id is a worker, then retrieve its message.
01006        IF (proc_id > Master) THEN
01007           ! Create packed message object to recieve data.
01008           CALL CreatePackedMessage(CHOMBO_DOMAIN_LEVEL, &
01009                                    proc_id, &
01010                                    TRANSMIT_DOMAIN_DATA, &
01011                                    STAGE_RECV, &
01012                                    message)
01013 
01014           ! Retrieve the level statistics from the remote processor.  On the domain
01015           ! level, the q and aux data have no significance, but the child nodes do.
01016           CALL IO_UnparseLevelStatistics(message, data_sizes)
01017           
01018           ! Store the cost of the next level's messages from this processor.
01019           chandle%next_level_cost(message%remote_proc) = TERMINATION_BOX_BYTES + &
01020                                                          IO_LEVEL_STAT_BYTES + &
01021                                                          data_sizes(IO_NEXTLEVELCOST)
01022 
01023        ELSE
01024           ! Initialize the child count on the master processor.
01025           i_childbox_data_size = master_childcount
01026        END IF
01027 
01028        DO   ! Loop over domain nodes.
01029 
01030           NULLIFY(cost_map)
01031 
01032           IF (proc_id > Master) THEN
01033 
01034              NULLIFY(node)
01035 
01036              ! Extract a node that will go into the boxes array.
01037              CALL IO_UnparseRemoteNode(message, node)
01038 
01039              IF (ASSOCIATED(node)) THEN
01040                 ! Extract the child nodes from the message.
01041                 CALL IO_UnparseNodeChildren(message, child_count, node)
01042                 mGlobal=node%box%mGlobal
01043                 ! Increment the size counter for the childbox dataset.
01044                 i_childbox_data_size = i_childbox_data_size + child_count
01045 
01046                 ! Unparse a costmap from the packed message and break this loop if
01047                 ! none can be found.
01048 !                CALL IO_UnparseLowLevelCostMap(message, mGlobal, cost_map)
01049              ELSE
01050                 EXIT
01051              END IF
01052 
01053              ! Obtain the grid dimensions from the box coordinates.
01054              mx = mGlobal(1,2) - mGlobal(1,1) + 1
01055              my = mGlobal(2,2) - mGlobal(2,1) + 1
01056              mz = mGlobal(3,2) - mGlobal(3,1) + 1
01057 
01058           ELSE
01059 
01060              ! Retrieve the costmap from the next InfoDef structure in the list.
01061              IF (ASSOCIATED(nodelist)) THEN
01062                 ! Retrieve the InfoDef structure of node_ptr.
01063                 node => nodelist%self
01064 !                cost_map => node%Info%costmap
01065                 child_count = NodeCount(node%children)
01066              ELSE
01067                 EXIT
01068              END IF
01069 
01070              mx = node%box%mGlobal(1,2)-node%box%mGlobal(1,1) !Info%mX(1)
01071              my = node%box%mGlobal(2,2)-node%box%mGlobal(2,1) !Info%mX(2)
01072              mz = node%box%mGlobal(3,2)-node%box%mGlobal(3,1) !Info%mX(3)
01073 
01074              mGlobal = node%box%mGlobal
01075 
01076           END IF
01077 
01078           ! Write box to domain box list.
01079           CALL IO_WriteBoxToChomboFile(chandle, "boxes", mGlobal)
01080 
01081           ! Write box to root-level child box list (which we can do because there is no
01082           ! level group for level -2; we're just writing to a dataset in the root group,
01083           ! which stays open until the CloseChomboHandle() subroutine is called.
01084           CALL IO_WriteBoxToChomboFile(chandle, "root_child_boxes", mGlobal)
01085 
01086           ! Write the data array to the Chombo file using the supplied offset.
01087 !          CALL Write_Slab_To_Dataset_Double("costmap", chandle%level_id, &
01088 !               RESHAPE(cost_map(1:mx,1:my,1:mz,1:2), (/ mx*my*mz*2 /)), chandle%costmap_offset)
01089 
01090           ! Increment the offset by the size of the cost map.
01091 !          chandle%costmap_offset = chandle%costmap_offset + mx*my*mz*2
01092           
01093           CALL IO_WriteChildCountToChomboFile(chandle, child_count)
01094 
01095           child_list => node%children
01096 
01097           ! The childbox attribute is being used twice on this level, so we clear it
01098           ! again here.  It's klugey, but so's adding another offset that's only used once.
01099           chandle%childbox_offset = 0
01100 
01101           ! 
01102           DO WHILE (ASSOCIATED(child_list))
01103              CALL IO_WriteBoxToChomboFile(chandle, "child_boxes", child_list%self%box%mGlobal)
01104 
01105              child_list => child_list%next
01106           END DO
01107 
01108           ! If the cost map came from a worker processor, then deallocated it.  Otherwise,
01109           ! just advance to the next node.
01110           IF (proc_id > Master) THEN
01111 !             DEALLOCATE(cost_map)
01112 !             NULLIFY(cost_map)
01113              IF (ASSOCIATED(node%children))  CALL DestroyNodeList(node%children)
01114           ELSE
01115              nodelist => nodelist%next
01116           END IF
01117 
01118        END DO ! node list loop
01119 
01120        ! Destroy worker message so that a new one can be created.
01121        IF (proc_id > MASTER)  CALL DestroyPackedMessage(message)
01122 
01123     END DO ! processor loop
01124 
01125   END SUBROUTINE IO_WriteDomainData
01126 
01130   SUBROUTINE IO_ReadInLevelData(chandle, level)
01131 
01132     TYPE(ChomboHandle), POINTER :: chandle
01133     INTEGER :: level
01134 
01135     TYPE(NodedefList), POINTER :: nodelist, childlist
01136     TYPE(Nodedef), POINTER :: node, child
01137     TYPE(InfoDef), POINTER :: Info
01138     TYPE(NodeBox) :: box
01139     INTEGER :: box_count, child_count
01140     INTEGER :: i,j
01141     INTEGER, DIMENSION(3,2) :: mGlobal
01142     TYPE(NodeBoxList), POINTER :: masterbox_list, last_masterbox
01143     TYPE(NodeBoxList), POINTER :: new_child_list, last_new_child
01144     INTEGER :: effective_finest_level
01145 
01146 
01147     nodelist => Nodes(level)%p
01148     child_count = 0
01149     NULLIFY(new_child_list, last_new_child)
01150 
01151     effective_finest_level = MIN(chandle%finest_level, MaxLevel)
01152 
01153     IF (level >= -1) THEN
01154 
01155        ! Obtain the number of boxes on this level that are stored in the chombo file.
01156        box_count = IO_GetDatasetElementCount(chandle%level_id, "boxes")
01157 
01158        ! Temporarily cache the child boxes in a list stored in the chombo handle.
01159        DO i = 1, box_count
01160           ! Obtain the next box and store it in the chombo handle structure.
01161           CALL IO_GetBoxFromChomboFile(chandle, "boxes", mGlobal)
01162           CALL IO_CacheNodeBox(chandle, mGlobal, Master)
01163 
01164           IF (level < effective_finest_level) THEN
01165 
01166              ! Attempt to find the node on this level that matches this box.
01167              box%mGlobal = mGlobal
01168              box%MPI_id = 0
01169              CALL FindNode(level, box, node)
01170 
01171              CALL IO_GetChildBoxCountFromChomboFile(chandle, child_count)
01172 
01173              ! Retrieve each of this box's child boxes.
01174              DO j = 1, child_count 
01175                 ! Temporarily cache the child boxes in a list stored in the chombo handle.
01176                 CALL IO_GetBoxFromChomboFile(chandle, "child_boxes", mGlobal)
01177                 CALL IO_CacheChildNodeBox(chandle, mGlobal, Master)
01178 
01179                 ! If the current box is on the master processor, then cache these child boxes
01180                 ! separately so that they can be redistributed.
01181                 IF (ASSOCIATED(node)) THEN
01182                    CALL AddNodeBoxToList(last_new_child, new_child_list)
01183                    last_new_child%self%mGlobal = mGlobal
01184                    last_new_child%self%MPI_id = Master
01185                 END IF
01186 
01187              END DO
01188 
01189              ! If the current box is on this processor and has children, then
01190              ! redistribute the children.
01191              IF (ASSOCIATED(node) .AND. (child_count > 0)) THEN
01192                 CALL ChomboReCreateChildren(node, level, child_count, new_child_list)
01193 
01194                 ! Clear the cached child list so that it can be used for the next
01195                 ! round of local child boxes.
01196                 CALL ClearNodeBoxList(new_child_list)
01197                 NULLIFY(last_new_child)
01198              END IF
01199 
01200           END IF
01201 
01202        END DO
01203 
01204        chandle%childbox_offset = 0
01205        chandle%childbox_count_offset = 0
01206 
01207     ELSE
01208        ! There is only one node on level -2, and it's on the master.
01209        node => nodelist%self
01210 !       Info => node%Info
01211 !       CALL IO_GetCostMapFromChomboFile(chandle, node%box%mGlobal, -2, Info%costmap)
01212 
01213        child_count = Read_HDF5_Attribute_Int("root_child_count", chandle%root_group_id)
01214 
01215        ! Temporarily cache the child boxes in a list stored in the chombo handle.
01216        DO i = 1, child_count 
01217           CALL IO_GetBoxFromChomboFile(chandle, "root_child_boxes", mGlobal)
01218           CALL IO_CacheChildNodeBox(chandle, mGlobal, Master)
01219        END DO
01220 
01221        ! Redistribute children on level n.
01222        IF (child_count > 0) THEN
01223           CALL ChomboReCreateChildren(node, level, child_count, chandle%current_child_box)
01224           chandle%current_child_box => chandle%last_child_box
01225        END IF
01226 
01227     END IF
01228 
01229     CALL ClearChomboHandleOffsets(chandle)
01230 
01231   END SUBROUTINE IO_ReadInLevelData
01232 
01236   SUBROUTINE ChomboReloadLevel(level)
01237 
01238 !    USE TreeLevelComms
01239 !    USE DataLevelOps
01240 
01241 
01242     TYPE(ChomboHandle), POINTER :: chandle
01243     INTEGER :: level
01244 
01245     TYPE(StageMessageGroup), POINTER :: smg_recv_grids, smg_send_data
01246     TYPE(PackedMessage), POINTER :: pm_send_grids, pm_recv_data, message
01247     INTEGER :: n, i, message_size, status(MPI_STATUS_SIZE)
01248     INTEGER :: effective_finest_level
01249     INTEGER :: DummyInt, iErr
01250     
01251     chandle=>curr_handle
01252 
01253 !    IF (MPI_id == 0)  PRINT *, "Starting level ", level, "..."
01254 
01255     ! Determines the highest level to which Chombo refinements will be pursued.
01256     ! Essentially, this is the highest level of data that we will attempt to pull 
01257     ! from the Chombo file.
01258 !    effective_finest_level = MIN(chandle%finest_level, MaxLevel)
01259 
01260     NULLIFY(smg_recv_grids, smg_send_data, pm_send_grids, pm_recv_data)
01261 
01262     IF (level >= -1) THEN
01263        CALL IO_OpenCurrentLevel(chandle, level)
01264     END IF
01265 
01266     IF (MPI_ID == MASTER) THEN
01267 
01268        ! Open level handles for level.
01269 !       IF (level >= -1)  CALL IO_OpenCurrentLevel(chandle, level)
01270 
01271        ! **** 
01272        !      IF YOU ARE TRYING TO FOLLOW THE THREAD OF THE RESTART PROCESS, IT STARTS
01273        !      HERE, AT LEVEL -2.
01274        ! ****
01275        ! Obtain grid and child grid data for this level.  This grid information is
01276        ! attached to the ChomboHandle structure for the time being.  At this point, 
01277        ! the master also distributes the child grids associated with its own nodes
01278        ! on this level.
01279        CALL IO_ReadInLevelData(chandle, level)
01280 
01281        IF (level >= -1) THEN
01282           ! Each worker now sends its grids for this level to the master.  This
01283           ! information is used to assign MPI_ids for each grid that the master
01284           ! has read in from the Chombo file.
01285           DO i=1,MPI_NP-1
01286              CALL MPI_RECV(message_size, 1, MPI_INTEGER, i, TRANSMIT_IO_WORKER_GRIDS, MPI_COMM_WORLD, status, iErr)
01287              CALL CreatePackedMessage(level, i, TRANSMIT_IO_WORKER_GRIDS, STAGE_RECV, message, message_size)
01288              CALL MPI_SEND(message_size, 1, MPI_INTEGER, i, TRANSMIT_IO_WORKER_GRIDS, MPI_COMM_WORLD, iErr)          
01289              CALL ReceivePackedMessage(message)
01290              CALL IO_UnparseGridsFromWorkers(chandle, message)
01291              CALL DestroyPackedMessage(message)
01292           END DO
01293        END IF
01294 !             CALL IO_PostRecvGridsFromWorkers(smg_recv_grids, level)
01295  !            CALL IO_CompRecvGridsFromWorkers(chandle, smg_recv_grids)
01296 !          END DO
01297 !       END IF
01298 
01299        ! Read in one box at a time from chombo, figure out where it lives, and send it.
01300        ! Now that they have the appropriate MPI_ids associated, the boxes read
01301        ! in from the Chombo file can be used to retrieve the data for this level.
01302        ! These routines pull q, aux, and costmap data from the file and send it to
01303        ! the appropriate processors.
01304        !
01305        ! Child grids for workers are also packed and sent by these processors; the
01306        ! master processor has already distributed its child grids at this point, so
01307        ! it just copies over the InfoDef data from the Chombo files and ignores the
01308        ! child grids.
01309        CALL IO_MpiSendDataToWorkers(chandle, level)
01310 
01311        ! Clear all the cached boxes from the ChomboHandle structure.
01312        CALL IO_ClearAllNodeBoxes(chandle)
01313        CALL IO_ClearAllChildBoxes(chandle)
01314 
01315        ! Close level handles, since no further level data will be needed from the file.
01316 !       IF (level >= -1)  CALL IO_CloseCurrentLevel(chandle)
01317 
01318     ELSE
01319        ! Send grids to master to let it know what blocks of data to send.  This information
01320        ! will be used on the master to assign MPI_ids to the boxes it reads in from the file.
01321        IF (level >= -1) THEN
01322           CALL IO_PostSendGridsToMaster(pm_send_grids, level)
01323           CALL IO_CompSendGridsToMaster(pm_send_grids)
01324        END IF
01325 
01326        ! Receive data from master.  This includes child boxes that will be transmitted
01327        ! to the children of this processor's nodes.
01328        ! CompRecvDataFromMaster() also copies the data to the InfoDef structures on this level.
01329 !       CALL IO_PostRecvDataFromMaster(pm_recv_data, level, chandle)
01330        CALL IO_CompRecvDataFromMaster(level, chandle)
01331     END IF
01332 
01333     ! Close level handles, since no further level data will be needed from the file.
01334     IF (level >= -1)  CALL IO_CloseCurrentLevel(chandle)
01335 
01336  END SUBROUTINE ChomboReloadLevel
01337 
01338 
01341   SUBROUTINE ChomboRestartInit(nframe)
01342 
01343     INTEGER :: nframe
01344     INTEGER :: n
01345 
01346     CHARACTER(LEN=I_FILENAME_LENGTH) :: s_filename
01347 !    TYPE(ChomboHandle), POINTER :: chandle
01348     ! Retrieve the nframe-th chombo file name.
01349     WRITE(s_filename, '(A10,I5.5,A4)') 'out/chombo', nframe, '.hdf'
01350 
01351     ! Create a chombo handle to manage the object.
01352     CALL CreateChomboHandle(s_filename, curr_handle, CHOMBO_HANDLE_READ)
01353 
01354     current_time = Read_HDF5_Attribute_Double("time", curr_handle%root_group_id)
01355     olddt = Read_HDF5_Attribute_Double("olddt", curr_handle%root_group_id)
01356     overall_maxspeed = Read_HDF5_Attribute_Double("overall_maxspeed", curr_handle%root_group_id)
01357     maxcfl=Read_HDF5_Attribute_Double("maxcfl", curr_handle%root_group_id)
01358     levels(:)%tnow = current_time
01359     levels(:)%dt = 0.d0
01360     levels(:)%step = 1
01361     RestartLevel=min(MaxLevel,curr_handle%finest_level)
01362     NrInputVars = READ_HDF5_Attribute_Int("num_components", curr_handle%root_group_id)
01363 !    IF (MPI_ID == 0) write(*,*) 'found ', NrInputVars, ' variables', NrVars
01364     IF (NrInputVars < NrVars) THEN
01365        IF (MPI_ID == 0) PRINT*, 'Error - not enough variables in chombo file fro restart'
01366        STOP
01367     END IF
01368           
01369  END SUBROUTINE ChomboRestartInit
01370 
01371  SUBROUTINE ChomboRestartFinalize(nframe)
01372     INTEGER :: nframe
01373     ! close the chombo handle.
01374     CALL CloseChomboHandle(curr_handle)
01375  END SUBROUTINE ChomboRestartFinalize
01376 
01377 
01378 
01381 SUBROUTINE WritePointGravityObjects(chandle)
01382 
01383     USE PointGravitySrc, ONLY: FirstPointGravityObj, PointGravityDef, PointGravity_CountObjects, &
01384                                PointGravity_InitChomboDatasets, PointGravity_WriteObjectToChombo
01385 
01386     TYPE(ChomboHandle), POINTER :: chandle
01387 
01388         TYPE(PointGravityDef), POINTER :: gravity_object
01389         INTEGER :: i_err
01390     INTEGER :: obj_count
01391     INTEGER :: iFixed
01392 
01393 
01394     ! Open a new particles group within the root level
01395     CALL h5gcreate_f(chandle%root_group_id, "point_gravity_objects", chandle%source_group_id, i_err)
01396     CALL CatchHDF5Error("WritePointGravityObjects::h5gcreate_f", i_err)
01397 
01398     chandle%source_offset = 0
01399 
01400     ! Retrieve the number of point gravity objects from the object list.
01401     obj_count = PointGravity_CountObjects()
01402 
01403     ! Store the number of objects.
01404     CALL Add_HDF5_Attribute_Int("num_objects", chandle%source_group_id, obj_count)
01405 
01406     ! If there are any gravity objects, then create datasets to store their data.  Otherwise,
01407          ! don't create the datasets (since there's no need to take up file space).
01408     IF (obj_count > 0) THEN
01409 
01410         CALL PointGravity_InitChomboDatasets(chandle, obj_count)
01411 
01412         gravity_object => FirstPointGravityObj
01413          
01414         ! Loop over the list of point gravity objects and write their data to the new component datasets.
01415         DO WHILE (ASSOCIATED(gravity_object))
01416 
01417             CALL PointGravity_WriteObjectToChombo(chandle, gravity_object)
01418 
01419             gravity_object => gravity_object%next
01420 
01421         END DO
01422 
01423     END IF
01424 
01425     ! Close the source term group.
01426     CALL h5gclose_f(chandle%source_group_id, i_err)
01427     CALL CatchHDF5Error("WritePointGravityObjects::h5gclose_f", i_err)
01428 
01429 END SUBROUTINE WritePointGravityObjects
01430 
01433 SUBROUTINE WriteCoolingObjects(chandle)
01434 
01435     USE CoolingSrc, ONLY: CoolingDef, Cooling_CountObjects, firstcoolingobj, &
01436                           Cooling_InitChomboDatasets, Cooling_WriteObjectToChombo
01437 
01438     TYPE(ChomboHandle), POINTER :: chandle
01439 
01440         TYPE(CoolingDef), POINTER :: cooling_object
01441         INTEGER :: i_err
01442     INTEGER :: obj_count
01443     INTEGER :: iFixed
01444 
01445 
01446     ! Open a new particles group within the root level
01447     CALL h5gcreate_f(chandle%root_group_id, "cooling_objects", chandle%source_group_id, i_err)
01448     CALL CatchHDF5Error("WriteCoolingObjects::h5gcreate_f", i_err)
01449 
01450     chandle%source_offset = 0
01451 
01452     ! Retrieve the number of cooling objects from the object list.
01453     obj_count = Cooling_CountObjects()
01454 
01455     ! Store the number of objects.  The INT() is required to recast obj_count from an HSIZE_T
01456     ! type for the function definition.
01457     CALL Add_HDF5_Attribute_Int("num_objects", chandle%source_group_id, obj_count)
01458 
01459     ! If there are any cooling objects, then create datasets to store their data.  Otherwise,
01460          ! don't create the datasets (since there's no need to take up file space).
01461     IF (obj_count > 0) THEN
01462 
01463         ! If there are cooling objects to store, then create datasets for them.
01464         CALL Cooling_InitChomboDatasets(chandle, obj_count)
01465 
01466         cooling_object => firstcoolingobj
01467          
01468         ! Loop over the list of sink particles and write their data to the new component datasets.
01469         DO WHILE (ASSOCIATED(cooling_object))
01470 
01471             ! Writes the current object to the data file.  Also increments the source offset.
01472             CALL Cooling_WriteObjectToChombo(chandle, cooling_object)
01473 
01474             cooling_object => cooling_object%next
01475         END DO
01476 
01477     END IF
01478 
01479     ! Close the source term group.
01480     CALL h5gclose_f(chandle%source_group_id, i_err)
01481     CALL CatchHDF5Error("WriteCoolingObjects::h5gclose_f", i_err)
01482 
01483 END SUBROUTINE WriteCoolingObjects
01484 
01485 END MODULE IOChombo
01486 
 All Classes Files Functions Variables