diff --git a/README.md b/README.md index ffca403..13bf26e 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,8 @@ UMT can run different test problems, depending on how it was configured and buil Blueprint test problem ------------------------ UMT can dynamically generate an unstructured 2D or 3D mesh when built against Conduit -version 0.8.9 or later. This mode is selected using the -B command line argument. +version 0.8.9 or later, or their develop branch as of 1/1/2024 or later. This mode is +selected using the -B command line argument. Each MPI rank in UMT will construct a mesh domain by repeating a Blueprint tile topology into a larger, repeated pattern arranged as a square tiling of the tile topology. By default, UMT will use the tile pattern that is built into Conduit, although other diff --git a/build_and_run_umt.sh b/build_and_run_umt.sh index 6f0747d..c5b3266 100755 --- a/build_and_run_umt.sh +++ b/build_and_run_umt.sh @@ -8,16 +8,16 @@ # FC = # Default to GNU -CC=gcc -CXX=g++ -FC=gfortran +#CC=gcc +#CXX=g++ +#FC=gfortran -FFLAGS=-fallow-argument-mismatch +#FFLAGS=-fallow-argument-mismatch # Intel example -# CC=icx -# CXX=icpx -# FC=ifx +CC=icx +CXX=icpx +FC=ifx # This script will compile a basic Release build of UMT. Additional CMake options can be added to the command line args of this script, and they will be picked up and added to the UMT CMake command at the bottom of this script. @@ -52,7 +52,7 @@ cmake ${UMT_REPO_PATH}/src -DCMAKE_Fortran_FLAGS=${FFLAGS} -DCMAKE_BUILD_TYPE=Re gmake -j install cd .. -srun -n 8 ${INSTALL_PATH}/bin/test_driver -c 10 -B -d 8,8,0 --benchmark_problem 2 -srun -n 8 ${INSTALL_PATH}/bin/test_driver -c 10 -B -d 4,4,4 --benchmark_problem 2 +srun -n 8 ${INSTALL_PATH}/bin/test_driver -c 10 -B local -d 8,8,0 --benchmark_problem 2 +srun -n 8 ${INSTALL_PATH}/bin/test_driver -c 10 -B local -d 4,4,4 --benchmark_problem 2 # Test UMT on SSP1 unstructured 3d mesh problem on two mpi ranks. Refine the mesh via -r and -R arguments. diff --git a/host-configs/example.gnu.cmake b/host-configs/example.gnu.cmake index cc81adf..5bec829 100644 --- a/host-configs/example.gnu.cmake +++ b/host-configs/example.gnu.cmake @@ -13,7 +13,4 @@ set(CMAKE_Fortran_FLAGS "-ffree-line-length-none" CACHE PATH "") set(ENABLE_OPENMP ON CACHE BOOL "") set(ENABLE_OPENMP_OFFLOAD OFF CACHE BOOL "") -set(CONDUIT_ROOT ${TPL_ROOT}/conduit/0.8.2 CACHE PATH "") -set(MFEM_ROOT ${TPL_ROOT}/mfem/4.4 CACHE PATH "") -set(HYPRE_ROOT ${TPL_ROOT}/hypre/2.24.0 CACHE PATH "") -set(METIS_ROOT ${TPL_ROOT}/metis/5.1.0 CACHE PATH "") +set(CONDUIT_ROOT ${TPL_ROOT}/conduit/develop CACHE PATH "") diff --git a/src/cmake/InitBuildTypeCompilerFlags.cmake b/src/cmake/InitBuildTypeCompilerFlags.cmake index 651771e..36a54ce 100644 --- a/src/cmake/InitBuildTypeCompilerFlags.cmake +++ b/src/cmake/InitBuildTypeCompilerFlags.cmake @@ -74,8 +74,8 @@ if (NOT "${CMAKE_BUILD_TYPE}" STREQUAL "") elseif("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "IntelLLVM") set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -DNDEBUG") - set(CMAKE_Fortran_FLAGS_RELWITHDEBINFO "-O2 -g -warn all,noexternal,nointerfaces -diag-enable=remark -fpe-all=0 -fpe0 -traceback") - set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -g -warn all,noexternal,nointerfaces -diag-enable=remark -check all -fpe-all=0 -fpe0 -traceback") + set(CMAKE_Fortran_FLAGS_RELWITHDEBINFO "-O2 -g -warn all,noexternal,nointerfaces -diag-enable=remark -fpen=0-traceback") + set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -g -warn all,noexternal,nointerfaces -diag-enable=remark -check all -fpen=0 -traceback") elseif("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "PGI") diff --git a/src/teton/aux/DestructMeshData.F90 b/src/teton/aux/DestructMeshData.F90 index 6e68f9b..de8df7f 100644 --- a/src/teton/aux/DestructMeshData.F90 +++ b/src/teton/aux/DestructMeshData.F90 @@ -129,15 +129,11 @@ subroutine DestructMeshData(nonLTE) BIND(C,NAME="teton_destructmeshdata") ! Communication Sets - if (Size% ndim >= 2) then + do setID=1,nCommSets+nGTASets + CSet => getCommSetData(Quad, setID) - do setID=1,nCommSets+nGTASets - CSet => getCommSetData(Quad, setID) - - call destruct(CSet) - enddo - - endif + call destruct(CSet) + enddo ! Deallocate Quadrature data (One for Sn, One for GTA) diff --git a/src/teton/control/finalizeSets.F90 b/src/teton/control/finalizeSets.F90 index 9991895..5b1bfad 100644 --- a/src/teton/control/finalizeSets.F90 +++ b/src/teton/control/finalizeSets.F90 @@ -91,55 +91,137 @@ subroutine finalizeSets ! Unmap zone sets - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% nCornerSet) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% nCornerBatch) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% offset) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% cornerList) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% cornerMap) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% zoneList) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% cornerConverged) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% Te) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% TeOld) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% delta) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% sumT) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% netRate) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% dTCompton) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% B) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% dBdT) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% Snu0) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% dSnu0dT) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% AD) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% z) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% fk2) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% nI) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% nS) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% ex) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% expPH) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% comptonDeltaEr) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% dComptonDT) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% comptonSe) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% AU) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(ZSet% AL) + UMPIRE_DEVICE_POOL_FREE(ZSet% nCornerSet) + TOMP(target exit data map(always,release:ZSet% nCornerSet)) + + UMPIRE_DEVICE_POOL_FREE(ZSet% nCornerBatch) + TOMP(target exit data map(always,release:ZSet% nCornerBatch)) + + UMPIRE_DEVICE_POOL_FREE(ZSet% offset) + TOMP(target exit data map(always,release:ZSet% offset)) + + UMPIRE_DEVICE_POOL_FREE(ZSet% cornerList) + TOMP(target exit data map(always,release:ZSet% cornerList)) + + UMPIRE_DEVICE_POOL_FREE(ZSet% cornerMap) + TOMP(target exit data map(always,release:ZSet% cornerMap)) + + UMPIRE_DEVICE_POOL_FREE(ZSet% zoneList) + TOMP(target exit data map(always,release:ZSet% zoneList)) + + UMPIRE_DEVICE_POOL_FREE(ZSet% cornerConverged) + TOMP(target exit data map(always,release:ZSet% cornerConverged)) + + UMPIRE_DEVICE_POOL_FREE(ZSet% Te) + TOMP(target exit data map(always,release:ZSet% Te)) + + UMPIRE_DEVICE_POOL_FREE(ZSet% TeOld) + TOMP(target exit data map(always,release:ZSet% TeOld)) + + UMPIRE_DEVICE_POOL_FREE(ZSet% delta) + TOMP(target exit data map(always,release:ZSet% delta)) + + UMPIRE_DEVICE_POOL_FREE(ZSet% sumT) + TOMP(target exit data map(always,release:ZSet% sumT)) + + UMPIRE_DEVICE_POOL_FREE(ZSet% netRate) + TOMP(target exit data map(always,release:ZSet% netRate)) + + UMPIRE_DEVICE_POOL_FREE(ZSet% dTCompton) + TOMP(target exit data map(always,release:ZSet% dTCompton)) + + UMPIRE_DEVICE_POOL_FREE(ZSet% B) + TOMP(target exit data map(always,release:ZSet% B)) + + UMPIRE_DEVICE_POOL_FREE(ZSet% dBdT) + TOMP(target exit data map(always,release:ZSet% dBdT)) + + UMPIRE_DEVICE_POOL_FREE(ZSet% Snu0) + TOMP(target exit data map(always,release:ZSet% Snu0)) + + UMPIRE_DEVICE_POOL_FREE(ZSet% dSnu0dT) + TOMP(target exit data map(always,release:ZSet% dSnu0dT)) + + UMPIRE_DEVICE_POOL_FREE(ZSet% AD) + TOMP(target exit data map(always,release:ZSet% AD)) + + UMPIRE_DEVICE_POOL_FREE(ZSet% z) + TOMP(target exit data map(always,release:ZSet% z)) + + UMPIRE_DEVICE_POOL_FREE(ZSet% fk2) + TOMP(target exit data map(always,release:ZSet% fk2)) + + UMPIRE_DEVICE_POOL_FREE(ZSet% nI) + TOMP(target exit data map(always,release:ZSet% nI)) + + UMPIRE_DEVICE_POOL_FREE(ZSet% nS) + TOMP(target exit data map(always,release:ZSet% nS)) + + UMPIRE_DEVICE_POOL_FREE(ZSet% ex) + TOMP(target exit data map(always,release:ZSet% ex)) + + UMPIRE_DEVICE_POOL_FREE(ZSet% expPH) + TOMP(target exit data map(always,release:ZSet% expPH)) + + UMPIRE_DEVICE_POOL_FREE(ZSet% comptonDeltaEr) + TOMP(target exit data map(always,release:ZSet% comptonDeltaEr)) + + UMPIRE_DEVICE_POOL_FREE(ZSet% dComptonDT) + TOMP(target exit data map(always,release:ZSet% dComptonDT)) + + UMPIRE_DEVICE_POOL_FREE(ZSet% comptonSe) + TOMP(target exit data map(always,release:ZSet% comptonSe)) + + UMPIRE_DEVICE_POOL_FREE(ZSet% AU) + TOMP(target exit data map(always,release:ZSet% AU)) + + UMPIRE_DEVICE_POOL_FREE(ZSet% AL) + TOMP(target exit data map(always,release:ZSet% AL)) + TOMP(target exit data map(release: ZSet)) ! Unmap group sets do gSetID=1,nGroupSets - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% GrpSetPtr(gSetID)% STotal) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% GrpSetPtr(gSetID)% Sigt) + UMPIRE_DEVICE_POOL_FREE(Quad% GrpSetPtr(gSetID)% STotal) + TOMP(target exit data map(always,release:Quad% GrpSetPtr(gSetID)% STotal)) + + UMPIRE_DEVICE_POOL_FREE(Quad% GrpSetPtr(gSetID)% Sigt) + TOMP(target exit data map(always,release:Quad% GrpSetPtr(gSetID)% Sigt)) + enddo do aSetID=1,nAngleSets+nGTASets - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% AngSetPtr(aSetID)% nextZ) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% AngSetPtr(aSetID)% nextC) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% AngSetPtr(aSetID)% StartingDirection) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% AngSetPtr(aSetID)% FinishingDirection) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% AngSetPtr(aSetID)% Omega) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% AngSetPtr(aSetID)% Weight) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% AngSetPtr(aSetID)% numCycles) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% AngSetPtr(aSetID)% cycleOffSet) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% AngSetPtr(aSetID)% cycleList) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% AngSetPtr(aSetID)% nHyperPlanes) + UMPIRE_DEVICE_POOL_FREE(Quad% AngSetPtr(aSetID)% nextZ) + TOMP(target exit data map(always,release:Quad% AngSetPtr(aSetID)% nextZ)) + + UMPIRE_DEVICE_POOL_FREE(Quad% AngSetPtr(aSetID)% nextC) + TOMP(target exit data map(always,release:Quad% AngSetPtr(aSetID)% nextC)) + + UMPIRE_DEVICE_POOL_FREE(Quad% AngSetPtr(aSetID)% StartingDirection) + TOMP(target exit data map(always,release:Quad% AngSetPtr(aSetID)% StartingDirection)) + + UMPIRE_DEVICE_POOL_FREE(Quad% AngSetPtr(aSetID)% FinishingDirection) + TOMP(target exit data map(always,release:Quad% AngSetPtr(aSetID)% FinishingDirection)) + + UMPIRE_DEVICE_POOL_FREE(Quad% AngSetPtr(aSetID)% Omega) + TOMP(target exit data map(always,release:Quad% AngSetPtr(aSetID)% Omega)) + + UMPIRE_DEVICE_POOL_FREE(Quad% AngSetPtr(aSetID)% Weight) + TOMP(target exit data map(always,release:Quad% AngSetPtr(aSetID)% Weight)) + + UMPIRE_DEVICE_POOL_FREE(Quad% AngSetPtr(aSetID)% numCycles) + TOMP(target exit data map(always,release:Quad% AngSetPtr(aSetID)% numCycles)) + + UMPIRE_DEVICE_POOL_FREE(Quad% AngSetPtr(aSetID)% cycleOffSet) + TOMP(target exit data map(always,release:Quad% AngSetPtr(aSetID)% cycleOffSet)) + + UMPIRE_DEVICE_POOL_FREE(Quad% AngSetPtr(aSetID)% cycleList) + TOMP(target exit data map(always,release:Quad% AngSetPtr(aSetID)% cycleList)) + + UMPIRE_DEVICE_POOL_FREE(Quad% AngSetPtr(aSetID)% nHyperPlanes) + TOMP(target exit data map(always,release:Quad% AngSetPtr(aSetID)% nHyperPlanes)) + ! This loop unmaps internal components of HypPlanePtr and BdyExitPtr. ! Delay unmapping these until this loop is done. @@ -156,21 +238,36 @@ subroutine finalizeSets endif enddo - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% AngSetPtr(aSetID)% HypPlanePtr) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% AngSetPtr(aSetID)% BdyExitPtr) + UMPIRE_DEVICE_POOL_FREE(Quad% AngSetPtr(aSetID)% HypPlanePtr) + TOMP(target exit data map(always,release:Quad% AngSetPtr(aSetID)% HypPlanePtr)) + + UMPIRE_DEVICE_POOL_FREE(Quad% AngSetPtr(aSetID)% BdyExitPtr) + TOMP(target exit data map(always,release:Quad% AngSetPtr(aSetID)% BdyExitPtr)) + if ( aSetID <= nAngleSets ) then - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% AngSetPtr(aSetID)% AfpNorm) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% AngSetPtr(aSetID)% AezNorm) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% AngSetPtr(aSetID)% ANormSum) + UMPIRE_DEVICE_POOL_FREE(Quad% AngSetPtr(aSetID)% AfpNorm) + TOMP(target exit data map(always,release:Quad% AngSetPtr(aSetID)% AfpNorm)) + + UMPIRE_DEVICE_POOL_FREE(Quad% AngSetPtr(aSetID)% AezNorm) + TOMP(target exit data map(always,release:Quad% AngSetPtr(aSetID)% AezNorm)) + + UMPIRE_DEVICE_POOL_FREE(Quad% AngSetPtr(aSetID)% ANormSum) + TOMP(target exit data map(always,release:Quad% AngSetPtr(aSetID)% ANormSum)) + endif if (Size% ndim == 2) then - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% AngSetPtr(aSetID)% angDerivFac) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% AngSetPtr(aSetID)% quadTauW1) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% AngSetPtr(aSetID)% quadTauW2) + UMPIRE_DEVICE_POOL_FREE(Quad% AngSetPtr(aSetID)% angDerivFac) + TOMP(target exit data map(always,release:Quad% AngSetPtr(aSetID)% angDerivFac)) + + UMPIRE_DEVICE_POOL_FREE(Quad% AngSetPtr(aSetID)% quadTauW1) + TOMP(target exit data map(always,release:Quad% AngSetPtr(aSetID)% quadTauW1)) + + UMPIRE_DEVICE_POOL_FREE(Quad% AngSetPtr(aSetID)% quadTauW2) + TOMP(target exit data map(always,release:Quad% AngSetPtr(aSetID)% quadTauW2)) endif @@ -178,101 +275,223 @@ subroutine finalizeSets ! Geometry - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Geom% Volume) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Geom% VolumeOld) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Geom% VolumeZone) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Geom% cOffSet) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Geom% numCorner) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Geom% CToZone) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Geom% corner1) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Geom% corner2) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Geom% zone1) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Geom% zone2) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Geom% cEZ) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Geom% cFP) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Geom% A_ez) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Geom% A_fp) + UMPIRE_DEVICE_POOL_FREE(Geom% Volume) + TOMP(target exit data map(always,release:Geom% Volume)) + + UMPIRE_DEVICE_POOL_FREE(Geom% VolumeOld) + TOMP(target exit data map(always,release:Geom% VolumeOld)) + + UMPIRE_DEVICE_POOL_FREE(Geom% VolumeZone) + TOMP(target exit data map(always,release:Geom% VolumeZone)) + + UMPIRE_DEVICE_POOL_FREE(Geom% cOffSet) + TOMP(target exit data map(always,release:Geom% cOffSet)) + + UMPIRE_DEVICE_POOL_FREE(Geom% numCorner) + TOMP(target exit data map(always,release:Geom% numCorner)) + + UMPIRE_DEVICE_POOL_FREE(Geom% CToZone) + TOMP(target exit data map(always,release:Geom% CToZone)) + + UMPIRE_DEVICE_POOL_FREE(Geom% corner1) + TOMP(target exit data map(always,release:Geom% corner1)) + + UMPIRE_DEVICE_POOL_FREE(Geom% corner2) + TOMP(target exit data map(always,release:Geom% corner2)) + + UMPIRE_DEVICE_POOL_FREE(Geom% zone1) + TOMP(target exit data map(always,release:Geom% zone1)) + + UMPIRE_DEVICE_POOL_FREE(Geom% zone2) + TOMP(target exit data map(always,release:Geom% zone2)) + + UMPIRE_DEVICE_POOL_FREE(Geom% cEZ) + TOMP(target exit data map(always,release:Geom% cEZ)) + + UMPIRE_DEVICE_POOL_FREE(Geom% cFP) + TOMP(target exit data map(always,release:Geom% cFP)) + + UMPIRE_DEVICE_POOL_FREE(Geom% A_ez) + TOMP(target exit data map(always,release:Geom% A_ez)) + + UMPIRE_DEVICE_POOL_FREE(Geom% A_fp) + TOMP(target exit data map(always,release:Geom% A_fp)) + if (Size% ndim == 2) then - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Geom% Area) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Geom% RadiusEZ) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Geom% RadiusFP) + UMPIRE_DEVICE_POOL_FREE(Geom% Area) + TOMP(target exit data map(always,release:Geom% Area)) + + UMPIRE_DEVICE_POOL_FREE(Geom% RadiusEZ) + TOMP(target exit data map(always,release:Geom% RadiusEZ)) + + UMPIRE_DEVICE_POOL_FREE(Geom% RadiusFP) + TOMP(target exit data map(always,release:Geom% RadiusFP)) + elseif (Size% ndim == 3) then - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Geom% nCFacesArray) + UMPIRE_DEVICE_POOL_FREE(Geom% nCFacesArray) + TOMP(target exit data map(always,release:Geom% nCFacesArray)) + endif TOMP(target exit data map(release:Geom)) ! Radiation Intensity - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Rad% PhiTotal) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Rad% radEnergy) + UMPIRE_DEVICE_POOL_FREE(Rad% PhiTotal) + TOMP(target exit data map(always,release:Rad% PhiTotal)) + + UMPIRE_DEVICE_POOL_FREE(Rad% radEnergy) + TOMP(target exit data map(always,release:Rad% radEnergy)) + TOMP(target exit data map(release:Rad)) ! GTA if (Size%useNewGTASolver) then - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(GTA% TT) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(GTA% Pvv) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(GTA% GreySigTotal) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(GTA% GreySigScat) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(GTA% GreySigScatVol) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(GTA% GreySigtInv) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(GTA% PhiInc) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(GTA% Q) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(GTA% TsaSource) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(GTA% AfpNorm) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(GTA% AezNorm) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(GTA% ANormSum) + UMPIRE_DEVICE_POOL_FREE(GTA% TT) + TOMP(target exit data map(always,release:GTA% TT)) + + UMPIRE_DEVICE_POOL_FREE(GTA% Pvv) + TOMP(target exit data map(always,release:GTA% Pvv)) + + UMPIRE_DEVICE_POOL_FREE(GTA% GreySigTotal) + TOMP(target exit data map(always,release:GTA% GreySigTotal)) + + UMPIRE_DEVICE_POOL_FREE(GTA% GreySigScat) + TOMP(target exit data map(always,release:GTA% GreySigScat)) + + UMPIRE_DEVICE_POOL_FREE(GTA% GreySigScatVol) + TOMP(target exit data map(always,release:GTA% GreySigScatVol)) + + UMPIRE_DEVICE_POOL_FREE(GTA% GreySigtInv) + TOMP(target exit data map(always,release:GTA% GreySigtInv)) + + UMPIRE_DEVICE_POOL_FREE(GTA% PhiInc) + TOMP(target exit data map(always,release:GTA% PhiInc)) + + UMPIRE_DEVICE_POOL_FREE(GTA% Q) + TOMP(target exit data map(always,release:GTA% Q)) + + UMPIRE_DEVICE_POOL_FREE(GTA% TsaSource) + TOMP(target exit data map(always,release:GTA% TsaSource)) + + UMPIRE_DEVICE_POOL_FREE(GTA% AfpNorm) + TOMP(target exit data map(always,release:GTA% AfpNorm)) + + UMPIRE_DEVICE_POOL_FREE(GTA% AezNorm) + TOMP(target exit data map(always,release:GTA% AezNorm)) + + UMPIRE_DEVICE_POOL_FREE(GTA% ANormSum) + TOMP(target exit data map(always,release:GTA% ANormSum)) + if (Size% ndim == 2) then - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(GTA% Tvv) + UMPIRE_DEVICE_POOL_FREE(GTA% Tvv) + TOMP(target exit data map(always,release:GTA% Tvv)) + endif endif - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(GTA% GreySource) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(GTA% GreyCorrection) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(GTA% Chi) + UMPIRE_DEVICE_POOL_FREE(GTA% GreySource) + TOMP(target exit data map(always,release:GTA% GreySource)) + + UMPIRE_DEVICE_POOL_FREE(GTA% GreyCorrection) + TOMP(target exit data map(always,release:GTA% GreyCorrection)) + + UMPIRE_DEVICE_POOL_FREE(GTA% Chi) + TOMP(target exit data map(always,release:GTA% Chi)) + TOMP(target exit data map(release:GTA)) do setID=nSets+1,nSets+nGTASets - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% SetDataPtr(setID)% AngleOrder) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% SetDataPtr(setID)% tPsi) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% SetDataPtr(setID)% pInc) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% SetDataPtr(setID)% src) + UMPIRE_DEVICE_POOL_FREE(Quad% SetDataPtr(setID)% AngleOrder) + TOMP(target exit data map(always,release:Quad% SetDataPtr(setID)% AngleOrder)) + + UMPIRE_DEVICE_POOL_FREE(Quad% SetDataPtr(setID)% tPsi) + TOMP(target exit data map(always,release:Quad% SetDataPtr(setID)% tPsi)) + + UMPIRE_DEVICE_POOL_FREE(Quad% SetDataPtr(setID)% pInc) + TOMP(target exit data map(always,release:Quad% SetDataPtr(setID)% pInc)) + + UMPIRE_DEVICE_POOL_FREE(Quad% SetDataPtr(setID)% src) + TOMP(target exit data map(always,release:Quad% SetDataPtr(setID)% src)) + if (Size% ndim == 2) then - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% SetDataPtr(setID)% tPsiM) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% SetDataPtr(setID)% tInc) + UMPIRE_DEVICE_POOL_FREE(Quad% SetDataPtr(setID)% tPsiM) + TOMP(target exit data map(always,release:Quad% SetDataPtr(setID)% tPsiM)) + + UMPIRE_DEVICE_POOL_FREE(Quad% SetDataPtr(setID)% tInc) + TOMP(target exit data map(always,release:Quad% SetDataPtr(setID)% tInc)) + endif enddo ! Material - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Mat% Tec) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Mat% Tecn) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Mat% denec) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Mat% cve) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Mat% rho) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Mat% nez) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Mat% stimComptonMult) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Mat% Siga) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Mat% Sigs) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Mat% Eta) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Mat% EmissionRate) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Mat% SMatEff) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Mat% PowerEmitted) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Mat% PowerCompton) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Mat% nonLinearIterations) + UMPIRE_DEVICE_POOL_FREE(Mat% Tec) + TOMP(target exit data map(always,release:Mat% Tec)) + + UMPIRE_DEVICE_POOL_FREE(Mat% Tecn) + TOMP(target exit data map(always,release:Mat% Tecn)) + + UMPIRE_DEVICE_POOL_FREE(Mat% denec) + TOMP(target exit data map(always,release:Mat% denec)) + + UMPIRE_DEVICE_POOL_FREE(Mat% cve) + TOMP(target exit data map(always,release:Mat% cve)) + + UMPIRE_DEVICE_POOL_FREE(Mat% rho) + TOMP(target exit data map(always,release:Mat% rho)) + + UMPIRE_DEVICE_POOL_FREE(Mat% nez) + TOMP(target exit data map(always,release:Mat% nez)) + + UMPIRE_DEVICE_POOL_FREE(Mat% stimComptonMult) + TOMP(target exit data map(always,release:Mat% stimComptonMult)) + + UMPIRE_DEVICE_POOL_FREE(Mat% Siga) + TOMP(target exit data map(always,release:Mat% Siga)) + + UMPIRE_DEVICE_POOL_FREE(Mat% Sigs) + TOMP(target exit data map(always,release:Mat% Sigs)) + + UMPIRE_DEVICE_POOL_FREE(Mat% Eta) + TOMP(target exit data map(always,release:Mat% Eta)) + + UMPIRE_DEVICE_POOL_FREE(Mat% EmissionRate) + TOMP(target exit data map(always,release:Mat% EmissionRate)) + + UMPIRE_DEVICE_POOL_FREE(Mat% SMatEff) + TOMP(target exit data map(always,release:Mat% SMatEff)) + + UMPIRE_DEVICE_POOL_FREE(Mat% PowerEmitted) + TOMP(target exit data map(always,release:Mat% PowerEmitted)) + + UMPIRE_DEVICE_POOL_FREE(Mat% PowerCompton) + TOMP(target exit data map(always,release:Mat% PowerCompton)) + + UMPIRE_DEVICE_POOL_FREE(Mat% nonLinearIterations) + TOMP(target exit data map(always,release:Mat% nonLinearIterations)) + TOMP(target exit data map(release:Mat)) ! IF THESE ARE UNALLOCATED WILL CRASH? #if !defined(TETON_ENABLE_MINIAPP_BUILD) if (getComptonFlag(Compton) /= comptonType_None) then - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Compton% gamMean) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Compton% gamSqdDGam) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Compton% gamCubedDGam) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Compton% gamD) + UMPIRE_DEVICE_POOL_FREE(Compton% gamMean) + TOMP(target exit data map(always,release:Compton% gamMean)) + + UMPIRE_DEVICE_POOL_FREE(Compton% gamSqdDGam) + TOMP(target exit data map(always,release:Compton% gamSqdDGam)) + + UMPIRE_DEVICE_POOL_FREE(Compton% gamCubedDGam) + TOMP(target exit data map(always,release:Compton% gamCubedDGam)) + + UMPIRE_DEVICE_POOL_FREE(Compton% gamD) + TOMP(target exit data map(always,release:Compton% gamD)) + endif TOMP(target exit data map(release:Compton)) @@ -295,7 +514,9 @@ subroutine finalizeSets if ( Size% useGPU ) then #if defined(TETON_ENABLE_OPENMP_OFFLOAD) call finalizeGPUMemory(setID) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% SetDataPtr(setID)% AngleOrder) + UMPIRE_DEVICE_POOL_FREE(Quad% SetDataPtr(setID)% AngleOrder) + TOMP(target exit data map(always,release:Quad% SetDataPtr(setID)% AngleOrder)) + #endif endif @@ -325,9 +546,16 @@ subroutine finalizeSets if ( Size% useGPU ) then - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad%AngSetPtr) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad%GrpSetPtr) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad%SetDataPtr) + UMPIRE_DEVICE_POOL_FREE(Quad%AngSetPtr) + + TOMP(target exit data map(always,release:Quad%AngSetPtr)) + + UMPIRE_DEVICE_POOL_FREE(Quad%GrpSetPtr) + TOMP(target exit data map(always,release:Quad%GrpSetPtr)) + + UMPIRE_DEVICE_POOL_FREE(Quad%SetDataPtr) + TOMP(target exit data map(always,release:Quad%SetDataPtr)) + TOMP(target exit data map(release:Quad)) endif diff --git a/src/teton/control/initializeSets.F90 b/src/teton/control/initializeSets.F90 index 6d4333f..71b29c7 100644 --- a/src/teton/control/initializeSets.F90 +++ b/src/teton/control/initializeSets.F90 @@ -127,70 +127,172 @@ subroutine initializeSets ! Map Quadrature List TOMP(target enter data map(to:Quad)) - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad%SetDataPtr) - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad%GrpSetPtr) - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad%AngSetPtr) + + UMPIRE_DEVICE_POOL_ALLOC(Quad%SetDataPtr) + TOMP(target enter data map(always,to:Quad%SetDataPtr)) + + UMPIRE_DEVICE_POOL_ALLOC(Quad%GrpSetPtr) + TOMP(target enter data map(always,to:Quad%GrpSetPtr)) + + UMPIRE_DEVICE_POOL_ALLOC(Quad%AngSetPtr) + TOMP(target enter data map(always,to:Quad%AngSetPtr)) + ! Map Group Sets do gSetID=1,nGroupSets - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% GrpSetPtr(gSetID)% STotal) - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% GrpSetPtr(gSetID)% Sigt) + UMPIRE_DEVICE_POOL_ALLOC(Quad% GrpSetPtr(gSetID)% STotal) + TOMP(target enter data map(always,to:Quad% GrpSetPtr(gSetID)% STotal)) + + UMPIRE_DEVICE_POOL_ALLOC(Quad% GrpSetPtr(gSetID)% Sigt) + TOMP(target enter data map(always,to:Quad% GrpSetPtr(gSetID)% Sigt)) + enddo ! Map ZoneSets TOMP(target enter data map(to:ZSet)) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% AL) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% AU) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% nCornerSet) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% nCornerBatch) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% offset) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% cornerList) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% cornerMap) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% zoneList) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% cornerConverged) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% Te) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% TeOld) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% delta) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% sumT) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% netRate) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% dTCompton) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% B) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% dBdT) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% Snu0) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% dSnu0dT) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% AD) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% z) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% fk2) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% nI) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% nS) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% ex) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% expPH) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% comptonDeltaEr) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% dComptonDT) - TOMP_TARGET_ENTER_DATA_MAP_TO(ZSet% comptonSe) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% AL) + TOMP(target enter data map(always,to:ZSet% AL)) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% AU) + TOMP(target enter data map(always,to:ZSet% AU)) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% nCornerSet) + TOMP(target enter data map(always,to:ZSet% nCornerSet)) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% nCornerBatch) + TOMP(target enter data map(always,to:ZSet% nCornerBatch)) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% offset) + TOMP(target enter data map(always,to:ZSet% offset)) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% cornerList) + TOMP(target enter data map(always,to:ZSet% cornerList)) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% cornerMap) + TOMP(target enter data map(always,to:ZSet% cornerMap)) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% zoneList) + TOMP(target enter data map(always,to:ZSet% zoneList)) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% cornerConverged) + TOMP(target enter data map(always,to:ZSet% cornerConverged)) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% Te) + TOMP(target enter data map(always,to:ZSet% Te)) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% TeOld) + TOMP(target enter data map(always,to:ZSet% TeOld)) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% delta) + TOMP(target enter data map(always,to:ZSet% delta)) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% sumT) + TOMP(target enter data map(always,to:ZSet% sumT)) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% netRate) + TOMP(target enter data map(always,to:ZSet% netRate)) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% dTCompton) + TOMP(target enter data map(always,to:ZSet% dTCompton)) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% B) + TOMP(target enter data map(always,to:ZSet% B)) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% dBdT) + TOMP(target enter data map(always,to:ZSet% dBdT)) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% Snu0) + TOMP(target enter data map(always,to:ZSet% Snu0)) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% dSnu0dT) + TOMP(target enter data map(always,to:ZSet% dSnu0dT)) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% AD) + TOMP(target enter data map(always,to:ZSet% AD)) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% z) + TOMP(target enter data map(always,to:ZSet% z)) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% fk2) + TOMP(target enter data map(always,to:ZSet% fk2)) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% nI) + TOMP(target enter data map(always,to:ZSet% nI)) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% nS) + TOMP(target enter data map(always,to:ZSet% nS)) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% ex) + TOMP(target enter data map(always,to:ZSet% ex)) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% expPH) + TOMP(target enter data map(always,to:ZSet% expPH)) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% comptonDeltaEr) + TOMP(target enter data map(always,to:ZSet% comptonDeltaEr)) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% dComptonDT) + TOMP(target enter data map(always,to:ZSet% dComptonDT)) + + UMPIRE_DEVICE_POOL_ALLOC(ZSet% comptonSe) + TOMP(target enter data map(always,to:ZSet% comptonSe)) + ! Map Angle Sets do aSetID=1,nAngleSets+nGTASets - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% AngSetPtr(aSetID)% nextZ) - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% AngSetPtr(aSetID)% nextC) - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% AngSetPtr(aSetID)% StartingDirection) - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% AngSetPtr(aSetID)% FinishingDirection) - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% AngSetPtr(aSetID)% Omega) - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% AngSetPtr(aSetID)% Weight) - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% AngSetPtr(aSetID)% numCycles) - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% AngSetPtr(aSetID)% cycleOffSet) - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% AngSetPtr(aSetID)% cycleList) - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% AngSetPtr(aSetID)% nHyperPlanes) + UMPIRE_DEVICE_POOL_ALLOC(Quad% AngSetPtr(aSetID)% nextZ) + + TOMP(target enter data map(always,to:Quad% AngSetPtr(aSetID)% nextZ)) + + UMPIRE_DEVICE_POOL_ALLOC(Quad% AngSetPtr(aSetID)% nextC) + TOMP(target enter data map(always,to:Quad% AngSetPtr(aSetID)% nextC)) + + UMPIRE_DEVICE_POOL_ALLOC(Quad% AngSetPtr(aSetID)% StartingDirection) + TOMP(target enter data map(always,to:Quad% AngSetPtr(aSetID)% StartingDirection)) + + UMPIRE_DEVICE_POOL_ALLOC(Quad% AngSetPtr(aSetID)% FinishingDirection) + TOMP(target enter data map(always,to:Quad% AngSetPtr(aSetID)% FinishingDirection)) + + UMPIRE_DEVICE_POOL_ALLOC(Quad% AngSetPtr(aSetID)% Omega) + TOMP(target enter data map(always,to:Quad% AngSetPtr(aSetID)% Omega)) + + UMPIRE_DEVICE_POOL_ALLOC(Quad% AngSetPtr(aSetID)% Weight) + TOMP(target enter data map(always,to:Quad% AngSetPtr(aSetID)% Weight)) + + UMPIRE_DEVICE_POOL_ALLOC(Quad% AngSetPtr(aSetID)% numCycles) + TOMP(target enter data map(always,to:Quad% AngSetPtr(aSetID)% numCycles)) + + UMPIRE_DEVICE_POOL_ALLOC(Quad% AngSetPtr(aSetID)% cycleOffSet) + TOMP(target enter data map(always,to:Quad% AngSetPtr(aSetID)% cycleOffSet)) + + UMPIRE_DEVICE_POOL_ALLOC(Quad% AngSetPtr(aSetID)% cycleList) + TOMP(target enter data map(always,to:Quad% AngSetPtr(aSetID)% cycleList)) + + UMPIRE_DEVICE_POOL_ALLOC(Quad% AngSetPtr(aSetID)% nHyperPlanes) + TOMP(target enter data map(always,to:Quad% AngSetPtr(aSetID)% nHyperPlanes)) + + + UMPIRE_DEVICE_POOL_ALLOC(Quad% AngSetPtr(aSetID)% HypPlanePtr) + + TOMP(target enter data map(always,to:Quad% AngSetPtr(aSetID)% HypPlanePtr)) + + UMPIRE_DEVICE_POOL_ALLOC(Quad% AngSetPtr(aSetID)% BdyExitPtr) + TOMP(target enter data map(always,to:Quad% AngSetPtr(aSetID)% BdyExitPtr)) - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% AngSetPtr(aSetID)% HypPlanePtr) - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% AngSetPtr(aSetID)% BdyExitPtr) if ( aSetID <= nAngleSets ) then - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% AngSetPtr(aSetID)% AfpNorm) - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% AngSetPtr(aSetID)% AezNorm) - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% AngSetPtr(aSetID)% ANormSum) + UMPIRE_DEVICE_POOL_ALLOC(Quad% AngSetPtr(aSetID)% AfpNorm) + TOMP(target enter data map(always,to:Quad% AngSetPtr(aSetID)% AfpNorm)) + + UMPIRE_DEVICE_POOL_ALLOC(Quad% AngSetPtr(aSetID)% AezNorm) + TOMP(target enter data map(always,to:Quad% AngSetPtr(aSetID)% AezNorm)) + + UMPIRE_DEVICE_POOL_ALLOC(Quad% AngSetPtr(aSetID)% ANormSum) + TOMP(target enter data map(always,to:Quad% AngSetPtr(aSetID)% ANormSum)) + endif do angle=1,Quad% AngSetPtr(aSetID)% numAngles @@ -209,9 +311,15 @@ subroutine initializeSets enddo if (Size% ndim == 2) then - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% AngSetPtr(aSetID)% angDerivFac) - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% AngSetPtr(aSetID)% quadTauW1) - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% AngSetPtr(aSetID)% quadTauW2) + UMPIRE_DEVICE_POOL_ALLOC(Quad% AngSetPtr(aSetID)% angDerivFac) + TOMP(target enter data map(always,to:Quad% AngSetPtr(aSetID)% angDerivFac)) + + UMPIRE_DEVICE_POOL_ALLOC(Quad% AngSetPtr(aSetID)% quadTauW1) + TOMP(target enter data map(always,to:Quad% AngSetPtr(aSetID)% quadTauW1)) + + UMPIRE_DEVICE_POOL_ALLOC(Quad% AngSetPtr(aSetID)% quadTauW2) + TOMP(target enter data map(always,to:Quad% AngSetPtr(aSetID)% quadTauW2)) + endif enddo @@ -219,69 +327,149 @@ subroutine initializeSets ! Geometry TOMP(target enter data map(to:Geom)) - TOMP_TARGET_ENTER_DATA_MAP_TO(Geom% Volume) - TOMP_TARGET_ENTER_DATA_MAP_TO(Geom% VolumeOld) - TOMP_TARGET_ENTER_DATA_MAP_TO(Geom% VolumeZone) - TOMP_TARGET_ENTER_DATA_MAP_TO(Geom% cOffSet) - TOMP_TARGET_ENTER_DATA_MAP_TO(Geom% numCorner) - TOMP_TARGET_ENTER_DATA_MAP_TO(Geom% CToZone) - TOMP_TARGET_ENTER_DATA_MAP_TO(Geom% corner1) - TOMP_TARGET_ENTER_DATA_MAP_TO(Geom% corner2) - TOMP_TARGET_ENTER_DATA_MAP_TO(Geom% zone1) - TOMP_TARGET_ENTER_DATA_MAP_TO(Geom% zone2) - TOMP_TARGET_ENTER_DATA_MAP_TO(Geom% cEZ) - TOMP_TARGET_ENTER_DATA_MAP_TO(Geom% cFP) - TOMP_TARGET_ENTER_DATA_MAP_TO(Geom% A_ez) - TOMP_TARGET_ENTER_DATA_MAP_TO(Geom% A_fp) + UMPIRE_DEVICE_POOL_ALLOC(Geom% Volume) + TOMP(target enter data map(always,to:Geom% Volume)) + + UMPIRE_DEVICE_POOL_ALLOC(Geom% VolumeOld) + TOMP(target enter data map(always,to:Geom% VolumeOld)) + + UMPIRE_DEVICE_POOL_ALLOC(Geom% VolumeZone) + TOMP(target enter data map(always,to:Geom% VolumeZone)) + + UMPIRE_DEVICE_POOL_ALLOC(Geom% cOffSet) + TOMP(target enter data map(always,to:Geom% cOffSet)) + + UMPIRE_DEVICE_POOL_ALLOC(Geom% numCorner) + TOMP(target enter data map(always,to:Geom% numCorner)) + + UMPIRE_DEVICE_POOL_ALLOC(Geom% CToZone) + TOMP(target enter data map(always,to:Geom% CToZone)) + + UMPIRE_DEVICE_POOL_ALLOC(Geom% corner1) + TOMP(target enter data map(always,to:Geom% corner1)) + + UMPIRE_DEVICE_POOL_ALLOC(Geom% corner2) + TOMP(target enter data map(always,to:Geom% corner2)) + + UMPIRE_DEVICE_POOL_ALLOC(Geom% zone1) + TOMP(target enter data map(always,to:Geom% zone1)) + + UMPIRE_DEVICE_POOL_ALLOC(Geom% zone2) + TOMP(target enter data map(always,to:Geom% zone2)) + + UMPIRE_DEVICE_POOL_ALLOC(Geom% cEZ) + TOMP(target enter data map(always,to:Geom% cEZ)) + + UMPIRE_DEVICE_POOL_ALLOC(Geom% cFP) + TOMP(target enter data map(always,to:Geom% cFP)) + + UMPIRE_DEVICE_POOL_ALLOC(Geom% A_ez) + TOMP(target enter data map(always,to:Geom% A_ez)) + + UMPIRE_DEVICE_POOL_ALLOC(Geom% A_fp) + TOMP(target enter data map(always,to:Geom% A_fp)) + if (Size% ndim == 2) then - TOMP_TARGET_ENTER_DATA_MAP_TO(Geom% Area) - TOMP_TARGET_ENTER_DATA_MAP_TO(Geom% RadiusEZ) - TOMP_TARGET_ENTER_DATA_MAP_TO(Geom% RadiusFP) + UMPIRE_DEVICE_POOL_ALLOC(Geom% Area) + TOMP(target enter data map(always,to:Geom% Area)) + + UMPIRE_DEVICE_POOL_ALLOC(Geom% RadiusEZ) + TOMP(target enter data map(always,to:Geom% RadiusEZ)) + + UMPIRE_DEVICE_POOL_ALLOC(Geom% RadiusFP) + TOMP(target enter data map(always,to:Geom% RadiusFP)) + elseif (Size% ndim == 3) then - TOMP_TARGET_ENTER_DATA_MAP_TO(Geom% nCFacesArray) + UMPIRE_DEVICE_POOL_ALLOC(Geom% nCFacesArray) + TOMP(target enter data map(always,to:Geom% nCFacesArray)) + endif ! Radiation Intensity TOMP(target enter data map(to:Rad)) - TOMP_TARGET_ENTER_DATA_MAP_TO(Rad% PhiTotal) - TOMP_TARGET_ENTER_DATA_MAP_TO(Rad% radEnergy) + UMPIRE_DEVICE_POOL_ALLOC(Rad% PhiTotal) + TOMP(target enter data map(always,to:Rad% PhiTotal)) + + UMPIRE_DEVICE_POOL_ALLOC(Rad% radEnergy) + TOMP(target enter data map(always,to:Rad% radEnergy)) + #if !defined(TETON_ENABLE_MINIAPP_BUILD) TOMP(target enter data map(to:Compton)) if (getComptonFlag(Compton) /= comptonType_None) then - TOMP_TARGET_ENTER_DATA_MAP_TO(Compton% gamMean) - TOMP_TARGET_ENTER_DATA_MAP_TO(Compton% gamSqdDGam) - TOMP_TARGET_ENTER_DATA_MAP_TO(Compton% gamCubedDGam) - TOMP_TARGET_ENTER_DATA_MAP_TO(Compton% gamD) + UMPIRE_DEVICE_POOL_ALLOC(Compton% gamMean) + TOMP(target enter data map(always,to:Compton% gamMean)) + + UMPIRE_DEVICE_POOL_ALLOC(Compton% gamSqdDGam) + TOMP(target enter data map(always,to:Compton% gamSqdDGam)) + + UMPIRE_DEVICE_POOL_ALLOC(Compton% gamCubedDGam) + TOMP(target enter data map(always,to:Compton% gamCubedDGam)) + + UMPIRE_DEVICE_POOL_ALLOC(Compton% gamD) + TOMP(target enter data map(always,to:Compton% gamD)) + endif #endif ! GTA TOMP(target enter data map(to:GTA)) - TOMP_TARGET_ENTER_DATA_MAP_TO(GTA% GreySource) - TOMP_TARGET_ENTER_DATA_MAP_TO(GTA% GreyCorrection) - TOMP_TARGET_ENTER_DATA_MAP_TO(GTA% Chi) + UMPIRE_DEVICE_POOL_ALLOC(GTA% GreySource) + TOMP(target enter data map(always,to:GTA% GreySource)) + + UMPIRE_DEVICE_POOL_ALLOC(GTA% GreyCorrection) + TOMP(target enter data map(always,to:GTA% GreyCorrection)) + + UMPIRE_DEVICE_POOL_ALLOC(GTA% Chi) + TOMP(target enter data map(always,to:GTA% Chi)) + if (Size%useNewGTASolver) then - TOMP_TARGET_ENTER_DATA_MAP_TO(GTA% TT) - TOMP_TARGET_ENTER_DATA_MAP_TO(GTA% Pvv) - TOMP_TARGET_ENTER_DATA_MAP_TO(GTA% GreySigTotal) - TOMP_TARGET_ENTER_DATA_MAP_TO(GTA% GreySigScat) - TOMP_TARGET_ENTER_DATA_MAP_TO(GTA% GreySigScatVol) - TOMP_TARGET_ENTER_DATA_MAP_TO(GTA% GreySigtInv) - TOMP_TARGET_ENTER_DATA_MAP_TO(GTA% PhiInc) - TOMP_TARGET_ENTER_DATA_MAP_TO(GTA% Q) - TOMP_TARGET_ENTER_DATA_MAP_TO(GTA% TsaSource) - TOMP_TARGET_ENTER_DATA_MAP_TO(GTA% AfpNorm) - TOMP_TARGET_ENTER_DATA_MAP_TO(GTA% AezNorm) - TOMP_TARGET_ENTER_DATA_MAP_TO(GTA% ANormSum) + UMPIRE_DEVICE_POOL_ALLOC(GTA% TT) + TOMP(target enter data map(always,to:GTA% TT)) + + UMPIRE_DEVICE_POOL_ALLOC(GTA% Pvv) + TOMP(target enter data map(always,to:GTA% Pvv)) + + UMPIRE_DEVICE_POOL_ALLOC(GTA% GreySigTotal) + TOMP(target enter data map(always,to:GTA% GreySigTotal)) + + UMPIRE_DEVICE_POOL_ALLOC(GTA% GreySigScat) + TOMP(target enter data map(always,to:GTA% GreySigScat)) + + UMPIRE_DEVICE_POOL_ALLOC(GTA% GreySigScatVol) + TOMP(target enter data map(always,to:GTA% GreySigScatVol)) + + UMPIRE_DEVICE_POOL_ALLOC(GTA% GreySigtInv) + TOMP(target enter data map(always,to:GTA% GreySigtInv)) + + UMPIRE_DEVICE_POOL_ALLOC(GTA% PhiInc) + TOMP(target enter data map(always,to:GTA% PhiInc)) + + UMPIRE_DEVICE_POOL_ALLOC(GTA% Q) + TOMP(target enter data map(always,to:GTA% Q)) + + UMPIRE_DEVICE_POOL_ALLOC(GTA% TsaSource) + TOMP(target enter data map(always,to:GTA% TsaSource)) + + UMPIRE_DEVICE_POOL_ALLOC(GTA% AfpNorm) + TOMP(target enter data map(always,to:GTA% AfpNorm)) + + UMPIRE_DEVICE_POOL_ALLOC(GTA% AezNorm) + TOMP(target enter data map(always,to:GTA% AezNorm)) + + UMPIRE_DEVICE_POOL_ALLOC(GTA% ANormSum) + TOMP(target enter data map(always,to:GTA% ANormSum)) + if (Size% ndim == 2) then - TOMP_TARGET_ENTER_DATA_MAP_TO(GTA% Tvv) + UMPIRE_DEVICE_POOL_ALLOC(GTA% Tvv) + TOMP(target enter data map(always,to:GTA% Tvv)) + endif endif endif ! Size%useGPU @@ -337,7 +525,9 @@ subroutine initializeSets if (Size%useGPU) then do setID=1,nSets - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% SetDataPtr(setID)% AngleOrder) + UMPIRE_DEVICE_POOL_ALLOC(Quad% SetDataPtr(setID)% AngleOrder) + TOMP(target enter data map(always,to:Quad% SetDataPtr(setID)% AngleOrder)) + enddo endif @@ -358,21 +548,51 @@ subroutine initializeSets if ( Size% useGPU ) then TOMP(target enter data map(to:Mat)) - TOMP_TARGET_ENTER_DATA_MAP_TO(Mat% Tec) - TOMP_TARGET_ENTER_DATA_MAP_TO(Mat% Tecn) - TOMP_TARGET_ENTER_DATA_MAP_TO(Mat% denec) - TOMP_TARGET_ENTER_DATA_MAP_TO(Mat% cve) - TOMP_TARGET_ENTER_DATA_MAP_TO(Mat% rho) - TOMP_TARGET_ENTER_DATA_MAP_TO(Mat% nez) - TOMP_TARGET_ENTER_DATA_MAP_TO(Mat% stimComptonMult) - TOMP_TARGET_ENTER_DATA_MAP_TO(Mat% Siga) - TOMP_TARGET_ENTER_DATA_MAP_TO(Mat% Sigs) - TOMP_TARGET_ENTER_DATA_MAP_TO(Mat% Eta) - TOMP_TARGET_ENTER_DATA_MAP_TO(Mat% EmissionRate) - TOMP_TARGET_ENTER_DATA_MAP_TO(Mat% SMatEff) - TOMP_TARGET_ENTER_DATA_MAP_TO(Mat% PowerEmitted) - TOMP_TARGET_ENTER_DATA_MAP_TO(Mat% PowerCompton) - TOMP_TARGET_ENTER_DATA_MAP_TO(Mat% nonLinearIterations) + UMPIRE_DEVICE_POOL_ALLOC(Mat% Tec) + TOMP(target enter data map(always,to:Mat% Tec)) + + UMPIRE_DEVICE_POOL_ALLOC(Mat% Tecn) + TOMP(target enter data map(always,to:Mat% Tecn)) + + UMPIRE_DEVICE_POOL_ALLOC(Mat% denec) + TOMP(target enter data map(always,to:Mat% denec)) + + UMPIRE_DEVICE_POOL_ALLOC(Mat% cve) + TOMP(target enter data map(always,to:Mat% cve)) + + UMPIRE_DEVICE_POOL_ALLOC(Mat% rho) + TOMP(target enter data map(always,to:Mat% rho)) + + UMPIRE_DEVICE_POOL_ALLOC(Mat% nez) + TOMP(target enter data map(always,to:Mat% nez)) + + UMPIRE_DEVICE_POOL_ALLOC(Mat% stimComptonMult) + TOMP(target enter data map(always,to:Mat% stimComptonMult)) + + UMPIRE_DEVICE_POOL_ALLOC(Mat% Siga) + TOMP(target enter data map(always,to:Mat% Siga)) + + UMPIRE_DEVICE_POOL_ALLOC(Mat% Sigs) + TOMP(target enter data map(always,to:Mat% Sigs)) + + UMPIRE_DEVICE_POOL_ALLOC(Mat% Eta) + TOMP(target enter data map(always,to:Mat% Eta)) + + UMPIRE_DEVICE_POOL_ALLOC(Mat% EmissionRate) + TOMP(target enter data map(always,to:Mat% EmissionRate)) + + UMPIRE_DEVICE_POOL_ALLOC(Mat% SMatEff) + TOMP(target enter data map(always,to:Mat% SMatEff)) + + UMPIRE_DEVICE_POOL_ALLOC(Mat% PowerEmitted) + TOMP(target enter data map(always,to:Mat% PowerEmitted)) + + UMPIRE_DEVICE_POOL_ALLOC(Mat% PowerCompton) + TOMP(target enter data map(always,to:Mat% PowerCompton)) + + UMPIRE_DEVICE_POOL_ALLOC(Mat% nonLinearIterations) + TOMP(target enter data map(always,to:Mat% nonLinearIterations)) + endif ! Map GTA set variables @@ -380,14 +600,26 @@ subroutine initializeSets if ( Size% useGPU ) then do setID=nSets+1,nSets+nGTASets - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% SetDataPtr(setID)% AngleOrder) - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% SetDataPtr(setID)% tPsi) - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% SetDataPtr(setID)% pInc) - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% SetDataPtr(setID)% src) + UMPIRE_DEVICE_POOL_ALLOC(Quad% SetDataPtr(setID)% AngleOrder) + TOMP(target enter data map(always,to:Quad% SetDataPtr(setID)% AngleOrder)) + + UMPIRE_DEVICE_POOL_ALLOC(Quad% SetDataPtr(setID)% tPsi) + TOMP(target enter data map(always,to:Quad% SetDataPtr(setID)% tPsi)) + + UMPIRE_DEVICE_POOL_ALLOC(Quad% SetDataPtr(setID)% pInc) + TOMP(target enter data map(always,to:Quad% SetDataPtr(setID)% pInc)) + + UMPIRE_DEVICE_POOL_ALLOC(Quad% SetDataPtr(setID)% src) + TOMP(target enter data map(always,to:Quad% SetDataPtr(setID)% src)) + if (Size% ndim == 2) then - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% SetDataPtr(setID)% tPsiM) - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% SetDataPtr(setID)% tInc) + UMPIRE_DEVICE_POOL_ALLOC(Quad% SetDataPtr(setID)% tPsiM) + TOMP(target enter data map(always,to:Quad% SetDataPtr(setID)% tPsiM)) + + UMPIRE_DEVICE_POOL_ALLOC(Quad% SetDataPtr(setID)% tInc) + TOMP(target enter data map(always,to:Quad% SetDataPtr(setID)% tInc)) + endif enddo diff --git a/src/teton/driver/test_driver.cc b/src/teton/driver/test_driver.cc index 06430a1..4fb6f48 100644 --- a/src/teton/driver/test_driver.cc +++ b/src/teton/driver/test_driver.cc @@ -1,5 +1,6 @@ #include "mpi.h" #include +#include #include #include #include @@ -34,13 +35,19 @@ extern "C" void xl__trce(int, siginfo_t *, void *); #include "cuda_runtime_api.h" #endif +#if defined(TETON_ENABLE_HIP) +#include "hip/hip_runtime_api.h" +#endif + #include "TetonConduitInterface.hh" #include "TetonInterface.hh" #include "conduit/conduit.hpp" #include "conduit/conduit_blueprint.hpp" #include "conduit/conduit_blueprint_mesh_utils.hpp" +#include "conduit/conduit_blueprint_mpi_mesh_utils.hpp" #include "conduit/conduit_relay.hpp" +#include "conduit/conduit_relay_mpi_io_blueprint.hpp" #if defined(TETON_ENABLE_MFEM) #include "mfem.hpp" @@ -241,7 +248,7 @@ class TetonDriver #if defined(TETON_ENABLE_CALIPER) cali::ConfigManager mgr{}; #endif - bool blueprintMesh{false}; + int blueprintMesh{0}; int dims[3]{10, 10, 10}; //!< Number of cells in blueprint mesh. }; @@ -342,7 +349,7 @@ int TetonDriver::processArguments(int argc, char *argv[]) static struct option long_options[] = { {"apply_label", no_argument, 0, 'l'}, {"benchmark_problem", required_argument, 0, 'b'}, - {"blueprint", no_argument, 0, 'B'}, + {"blueprint", required_argument, 0, 'B'}, {"dims", required_argument, 0, 'd'}, {"caliper", required_argument, 0, 'p'}, {"input_sanitizer_level", required_argument, 0, 'y'}, @@ -381,10 +388,10 @@ int TetonDriver::processArguments(int argc, char *argv[]) int option_index = 0; #if defined(TETON_ENABLE_MFEM) - auto optString = "A:Bb:c:D:d:eG:gHhi:k:l:M:mn:o:P:p:s:S:t:u:Vv:y:" // Base options - "C:R:r:z:Z:"; // MFEM-only options + auto optString = "A:B:b:c:D:d:eG:gHhi:k:l:M:mn:o:P:p:s:S:t:u:Vv:y:" // Base options + "C:R:r:z:Z:"; // MFEM-only options #else - auto optString = "A:Bb:c:D:d:eG:gHhi:k:l:M:mn:o:P:p:s:S:t:u:Vv:y:"; // Base options + auto optString = "A:B:b:c:D:d:eG:gHhi:k:l:M:mn:o:P:p:s:S:t:u:Vv:y:"; // Base options #endif int opt = getopt_long(argc, argv, optString, long_options, &option_index); @@ -398,7 +405,12 @@ int TetonDriver::processArguments(int argc, char *argv[]) switch (opt) { case 'B': - blueprintMesh = true; + if (strcmp(optarg, "local") == 0) + blueprintMesh = 1; + else if (strcmp(optarg, "global") == 0) + blueprintMesh = 2; + else + blueprintMesh = 0; break; case 'b': benchmarkProblem = atoi(optarg); @@ -477,7 +489,7 @@ int TetonDriver::processArguments(int argc, char *argv[]) break; case 'M': meshOrdering = std::string(optarg); - if (meshOrdering != "normal" && meshOrdering != "kdtree") + if (meshOrdering != "normal" && meshOrdering != "kdtree" && meshOrdering != "hilbert") { throw std::runtime_error("Unsupported mesh ordering " + meshOrdering); } @@ -710,7 +722,10 @@ void TetonDriver::printUsage(const std::string &argv0) const std::cout << " -G, --num_Groups Number energy groups" << std::endl; #endif #if defined(TETON_CONDUIT_HAS_TILED_FUNCTION) - std::cout << " -B, --blueprint Generate Blueprint tiled mesh in memory." << std::endl; + std::cout << " -B, --blueprint local|global Generate Blueprint tiled mesh in memory using the specified scheme." + << " The \"local\" scheme creates the same sized mesh on each MPI rank, allowing for weak scaling. The " + << "\"global\" scheme creates the specified mesh size globally and decomposes that size over the available" + << "MPI ranks, allowing for strong scaling." << std::endl; std::cout << " -d, --dims i,j,k The size of the Blueprint mesh in tiles in i,j,k. k=0 builds a 2D mesh." << std::endl; @@ -724,6 +739,7 @@ int TetonDriver::execute() { conduit::Node &options = myTetonObject.getOptions(); conduit::Node &meshBlueprint = myTetonObject.getMeshBlueprint(); + //========================================================== // If benchmark problem specified, set the parameters. // UMT SP #1 @@ -733,37 +749,36 @@ int TetonDriver::execute() // UMT SP #2 // 2 x 2 product quadrature // 16 groups + // + // ! Iteration control defaults. //========================================================== if (benchmarkProblem > 0) { + options["iteration/relativeTolerance"] = energy_check_tolerance / 10.0; + +// If running UMT, only the sweep kernel is active. Increase the number of +// allowed inner flux iterations to enable it to converge on its own. +#if defined(TETON_ENABLE_MINIAPP_BUILD) + options["iteration/incidentFluxMaxIt"] = 99; +#endif + fixedDT = 1e-3; + if (cycles == 0) + { + cycles = 5; + } + if (benchmarkProblem == 1) { - double relTol = 1.0e-10; - options["iteration/relativeTolerance"] = relTol; - fixedDT = 1e-3; - if (cycles == 0) - { - cycles = 2; - } numPolarUser = 3; numAzimuthalUser = 3; numGroupsUser = 128; - energy_check_tolerance = 10 * relTol; label = "UMTSPP1"; } else if (benchmarkProblem == 2) { - double relTol = 1.0e-10; - options["iteration/relativeTolerance"] = relTol; - fixedDT = 1e-3; - if (cycles == 0) - { - cycles = 2; - } numPolarUser = 2; numAzimuthalUser = 2; numGroupsUser = 16; - energy_check_tolerance = 10 * relTol; label = "UMTSPP2"; } else @@ -799,7 +814,7 @@ int TetonDriver::execute() //========================================================== //========================================================== - if (blueprintMesh) + if (blueprintMesh > 0) { buildBlueprintTiledMesh(); } @@ -1747,34 +1762,13 @@ void TetonDriver::buildBlueprintTiledMesh() conduit::Node &options = myTetonObject.getOptions(); conduit::Node &meshBlueprint = myTetonObject.getMeshBlueprint(); - // Figure out a suitable domain decomposition for the number of ranks and - // where the current rank exists within it. - int ndims = (dims[2] > 0) ? 3 : 2; - int domain[] = {0, 0, 0}; - int domains[] = {1, 1, myRank}; - decompose(myRank, mySize, ndims, domain, domains); - // Entire problem domain is 1x1x1 const double extents[] = {0., 1., 0., 1., 0., 1.}; - // Figure out this domain's extents. - double sideX = (extents[1] - extents[0]) / static_cast(domains[0]); - double sideY = (extents[3] - extents[2]) / static_cast(domains[1]); - double sideZ = (extents[5] - extents[4]) / static_cast(domains[2]); - double domainExt[] = {extents[0] + domain[0] * sideX, - extents[0] + (domain[0] + 1) * sideX, - extents[2] + domain[1] * sideY, - extents[2] + (domain[1] + 1) * sideY, - extents[4] + domain[2] * sideZ, - extents[4] + (domain[2] + 1) * sideZ}; - - // Make options. + + // Common options. conduit::Node bopts; bopts["meshname"] = "main"; - bopts["domain"].set(domain, 3); - bopts["domains"].set(domains, 3); - bopts["extents"].set(domainExt, 6); bopts["datatype"] = "int32"; - bopts["reorder"] = meshOrdering; // If the inputPath was set to a file path and the file exists, try reading it // as a new tile definition. if ((endsWith(inputPath, ".yaml") || endsWith(inputPath, ".json")) && access(inputPath.c_str(), F_OK) == 0) @@ -1782,8 +1776,46 @@ void TetonDriver::buildBlueprintTiledMesh() conduit::relay::io::load(inputPath, bopts["tile"]); } - // Build the mesh - conduit::blueprint::mesh::examples::tiled(dims[0], dims[1], dims[2], meshBlueprint, bopts); + if (mySize > 1 && blueprintMesh == 2) // global - strong scaling mesh + { + // Make options + bopts["numDomains"] = mySize; + bopts["extents"].set(extents, 6); + bopts["selectedDomains"].set(std::vector{myRank}); // Just build the local domain + bopts["curveSplitting"] = 0; // Disabled for now. + + // Build mesh + conduit::blueprint::mesh::examples::tiled(dims[0], dims[1], dims[2], meshBlueprint, bopts); + } + else // local - weak scaling mesh + { + // Figure out a suitable domain decomposition for the number of ranks and + // where the current rank exists within it. + int ndims = (dims[2] > 0) ? 3 : 2; + int domain[] = {0, 0, 0}; + int domains[] = {1, 1, myRank}; + decompose(myRank, mySize, ndims, domain, domains); + + // Figure out this domain's extents. + double sideX = (extents[1] - extents[0]) / static_cast(domains[0]); + double sideY = (extents[3] - extents[2]) / static_cast(domains[1]); + double sideZ = (extents[5] - extents[4]) / static_cast(domains[2]); + double domainExt[] = {extents[0] + domain[0] * sideX, + extents[0] + (domain[0] + 1) * sideX, + extents[2] + domain[1] * sideY, + extents[2] + (domain[1] + 1) * sideY, + extents[4] + domain[2] * sideZ, + extents[4] + (domain[2] + 1) * sideZ}; + + // Make options. + bopts["domain"].set(domain, 3); + bopts["domains"].set(domains, 3); + bopts["extents"].set(domainExt, 6); + bopts["reorder"] = meshOrdering; + + // Build the mesh + conduit::blueprint::mesh::examples::tiled(dims[0], dims[1], dims[2], meshBlueprint, bopts); + } const conduit::Node &main_topo = meshBlueprint.fetch_existing("topologies/main"); auto nelem = conduit::blueprint::mesh::topology::length(main_topo); @@ -1950,19 +1982,21 @@ void TetonDriver::writeEndSummary(double end_time, outfile.close(); - double relative_energy_check = std::abs(energy_check / (energy_radiation + 1.0e-50)); - std::cerr << "VERIFICATION STEP: Unaccounted for residual energy, relative to total radiation energy, is " << relative_energy_check << "." << std::endl; - if (relative_energy_check <= energy_check_tolerance) + double rel_energy_check_result = std::abs(energy_check / (energy_radiation + 1.0e-50)); + if (rel_energy_check_result <= energy_check_tolerance) { - std::cerr << "VERIFICATION PASSED: Residual energy is within tolerance of +/-" << energy_check_tolerance << "." << std::endl; + std::cout << "RESULT CHECK PASSED: Energy check (this is relative to total energy) " + << rel_energy_check_result << " within tolerance of +/- " << energy_check_tolerance << "; check '" + << filePath << "' for tally details\n" + << std::endl; } else { - std::cerr << "VERIFICATION FAILED: Residual energy exceeds tolerance of +/-" << energy_check_tolerance << "." << std::endl; + std::cerr << "RESULT CHECK FAILED: Energy check (this is relative to total energy) " + << rel_energy_check_result << " exceeded tolerance of +/- " << energy_check_tolerance + << "; check '" << filePath << "' for tally details\n" + << std::endl; - std::cerr << "Residual energy ( absolute ): " << energy_check << std::endl; - std::cerr << "Residual energy ( relative ): " << relative_energy_check << std::endl; - std::cerr << "Residual energy tolerance ( relative ): " << energy_check_tolerance << std::endl; std::cerr << "Energy radiation: " << energy_radiation << std::endl; std::cerr << "Power incident: " << power_incident << std::endl; std::cerr << "Power escaped: " << power_escape << std::endl; @@ -2101,6 +2135,7 @@ int main(int argc, char *argv[]) std::string w(e.what()); if (!w.empty()) std::cerr << "Teton driver: " << w << std::endl; + retval = 1; } // NOTE: The teton object must be destroyed first, before MPI is finalized. Teton uses MPI during its destructor. diff --git a/src/teton/gpu/OMPWrappers.F90.templates b/src/teton/gpu/OMPWrappers.F90.templates index 230277a..b1806da 100644 --- a/src/teton/gpu/OMPWrappers.F90.templates +++ b/src/teton/gpu/OMPWrappers.F90.templates @@ -65,23 +65,16 @@ ! Allocate device memory via umpire allocator !$omp critical -#if defined(TETON_ENABLE_UMPIRE) d_ptr = Allocator%umpire_device_allocator%allocate_pointer(num_bytes) -#endif !$omp end critical ! Verify that pointer now points to something. TETON_VERIFY( C_ASSOCIATED(d_ptr), "Failed to allocate " // label // " on device.") -#if defined(TETON_ENABLE_OPENMP_OFFLOAD) ! Associate host and device c pointers. err_code = omp_target_associate_ptr( C_LOC(h_ptr), d_ptr, num_bytes, offset, omp_get_default_device() ) -#endif TETON_VERIFY( err_code == 0, "Failed to associate host variable " // label // " with device pointer.") - -#if defined(TETON_ENABLE_OPENMP_OFFLOAD) TETON_VERIFY( omp_target_is_present(C_LOC(h_ptr), omp_get_default_device()) /= 0, "Runtime failed to add " // label // " in host<->device table.") -#endif endif end subroutine FTM_CAT4(target_alloc_and_pair_ptrs,FTM_TYPE,FTM_KIND,FTM_RANK) @@ -114,24 +107,18 @@ TETON_VERIFY( c_associated(d_ptr), "Failed to retrieve device pointer for " // label ) ! Disassociate the pointer -#if defined(TETON_ENABLE_OPENMP_OFFLOAD) ! Workaround for El Capitan EA platform. Otherwise, the runtime complains about an invalid reference count in the pointer table. ! (Kostas has put in a bug ticket to compiler team already) TOMP(target exit data map(release:h_ptr)) err_code = omp_target_disassociate_ptr( C_LOC(h_ptr), omp_get_default_device() ) -#endif TETON_VERIFY( err_code == 0, "Failed to disassociate host variable " // label ) -#if defined(TETON_ENABLE_OPENMP_OFFLOAD) TETON_VERIFY( omp_target_is_present(C_LOC(h_ptr), omp_get_default_device()) == 0, "Runtime failed to remove variable " // label // " from host<->device table.") -#endif !$omp critical -#if defined(TETON_ENABLE_UMPIRE) call Allocator%umpire_device_allocator%deallocate_pointer(d_ptr) -#endif !$omp end critical endif diff --git a/src/teton/gpu/OMPWrappers_mod.F90 b/src/teton/gpu/OMPWrappers_mod.F90 index 5adbe9d..93667b2 100644 --- a/src/teton/gpu/OMPWrappers_mod.F90 +++ b/src/teton/gpu/OMPWrappers_mod.F90 @@ -4,12 +4,6 @@ ! OMPWrappers - A set of subroutines that wrap OpenMP operations for * ! mapping data structures to/from a target device, or * ! updating data between the host and device. * -! * -! Providing this API layer over the OpenMP pragmas provides two main features * -! - the ability to provide custom OpenMP map and update implementations, * -! allowing for the use of an external memory allocator such as UMPIRE. (WIP)* -! - Specific to SIERRA XLF - the ability to minimize compiler OpenMP offload * -! flags on source files. * !****************************************************************************** module OMPWrappers_mod @@ -17,6 +11,14 @@ module OMPWrappers_mod #if defined(TETON_ENABLE_OPENMP) use omp_lib #endif + +!****************************************************************************** +! If using the UMPIRE library with OpenMP offload kernels, a family of +! procedures is provided below to allow the code to make use of UMPIRE for +! device allocations. +!****************************************************************************** +#if defined(TETON_ENABLE_OPENMP_OFFLOAD) && defined(TETON_ENABLE_UMPIRE) + use iso_c_binding, only : c_double, c_int, c_bool, c_size_t, c_ptr, c_null_ptr, c_loc, c_associated use AngleSet_mod, only : AngleSet, HypPlane, BdyExit use Geometry_mod, only : Geometry @@ -58,8 +60,7 @@ module OMPWrappers_mod target_free_and_unpair_ptrs_type_SetData_1 end interface -#if defined(TETON_ENABLE_OPENMP_OFFLOAD) -#if !defined(TETON_OPENMP_HAS_FORTRAN_INTERFACE) +#if !defined(TETON_OPENMP_HAS_FORTRAN_INTERFACE) public :: omp_target_alloc, omp_target_free, omp_target_associate_ptr, omp_target_disassociate_ptr, omp_target_is_present @@ -109,7 +110,6 @@ end function omp_target_is_present end interface # endif -#endif contains @@ -183,4 +183,6 @@ end function omp_target_is_present #define FTM_RANK 1 #include "OMPWrappers.F90.templates" +#endif + end module OMPWrappers_mod diff --git a/src/teton/gpu/finalizeGPUMemory.F90 b/src/teton/gpu/finalizeGPUMemory.F90 index c31569d..ca21972 100644 --- a/src/teton/gpu/finalizeGPUMemory.F90 +++ b/src/teton/gpu/finalizeGPUMemory.F90 @@ -32,16 +32,28 @@ subroutine finalizeGPUMemory(setID) if (Size% ndim == 2) then - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% SetDataPtr(setID)% PsiM) + UMPIRE_DEVICE_POOL_FREE(Quad% SetDataPtr(setID)% PsiM) + TOMP(target exit data map(always,release:Quad% SetDataPtr(setID)% PsiM)) endif - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% SetDataPtr(setID)% Psi) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% SetDataPtr(setID)% Psi1) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% SetDataPtr(setID)% PsiB) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% SetDataPtr(setID)% Q) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% SetDataPtr(setID)% S) - TOMP_TARGET_EXIT_DATA_MAP_RELEASE(Quad% SetDataPtr(setID)% cyclePsi) + UMPIRE_DEVICE_POOL_FREE(Quad% SetDataPtr(setID)% Psi) + TOMP(target exit data map(always,release:Quad% SetDataPtr(setID)% Psi)) + + UMPIRE_DEVICE_POOL_FREE(Quad% SetDataPtr(setID)% Psi1) + TOMP(target exit data map(always,release:Quad% SetDataPtr(setID)% Psi1)) + + UMPIRE_DEVICE_POOL_FREE(Quad% SetDataPtr(setID)% PsiB) + TOMP(target exit data map(always,release:Quad% SetDataPtr(setID)% PsiB)) + + UMPIRE_DEVICE_POOL_FREE(Quad% SetDataPtr(setID)% Q) + TOMP(target exit data map(always,release:Quad% SetDataPtr(setID)% Q)) + + UMPIRE_DEVICE_POOL_FREE(Quad% SetDataPtr(setID)% S) + TOMP(target exit data map(always,release:Quad% SetDataPtr(setID)% S)) + + UMPIRE_DEVICE_POOL_FREE(Quad% SetDataPtr(setID)% cyclePsi) + TOMP(target exit data map(always,release:Quad% SetDataPtr(setID)% cyclePsi)) return end subroutine finalizeGPUMemory diff --git a/src/teton/gpu/initializeGPUMemory.F90 b/src/teton/gpu/initializeGPUMemory.F90 index 368b8c4..da985d2 100644 --- a/src/teton/gpu/initializeGPUMemory.F90 +++ b/src/teton/gpu/initializeGPUMemory.F90 @@ -32,15 +32,28 @@ subroutine initializeGPUMemory do setID=1,nSets if (Size% ndim == 2) then - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% SetDataPtr(setID) % PsiM) + UMPIRE_DEVICE_POOL_ALLOC(Quad% SetDataPtr(setID) % PsiM) + TOMP(target enter data map(always,to:Quad% SetDataPtr(setID) % PsiM)) + endif - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% SetDataPtr(setID)% Psi) - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% SetDataPtr(setID)% Psi1) - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% SetDataPtr(setID)% PsiB) - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% SetDataPtr(setID)% Q) - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% SetDataPtr(setID)% S) - TOMP_TARGET_ENTER_DATA_MAP_TO(Quad% SetDataPtr(setID)% cyclePsi) + UMPIRE_DEVICE_POOL_ALLOC(Quad% SetDataPtr(setID)% Psi) + TOMP(target enter data map(always,to:Quad% SetDataPtr(setID)% Psi)) + + UMPIRE_DEVICE_POOL_ALLOC(Quad% SetDataPtr(setID)% Psi1) + TOMP(target enter data map(always,to:Quad% SetDataPtr(setID)% Psi1)) + + UMPIRE_DEVICE_POOL_ALLOC(Quad% SetDataPtr(setID)% PsiB) + TOMP(target enter data map(always,to:Quad% SetDataPtr(setID)% PsiB)) + + UMPIRE_DEVICE_POOL_ALLOC(Quad% SetDataPtr(setID)% Q) + TOMP(target enter data map(always,to:Quad% SetDataPtr(setID)% Q)) + + UMPIRE_DEVICE_POOL_ALLOC(Quad% SetDataPtr(setID)% S) + TOMP(target enter data map(always,to:Quad% SetDataPtr(setID)% S)) + + UMPIRE_DEVICE_POOL_ALLOC(Quad% SetDataPtr(setID)% cyclePsi) + TOMP(target enter data map(always,to:Quad% SetDataPtr(setID)% cyclePsi)) enddo diff --git a/src/teton/include/TetonBlueprint.hh b/src/teton/include/TetonBlueprint.hh index 3599fe4..1ca247c 100644 --- a/src/teton/include/TetonBlueprint.hh +++ b/src/teton/include/TetonBlueprint.hh @@ -67,8 +67,7 @@ class TetonBlueprint std::vector &face_to_bcid, int rank); - -/* ------------------------------------------------------------------------------------------------------------------ + /* ------------------------------------------------------------------------------------------------------------------ This builds several arrays containing the boundary condition information for each boundary element in 1D. The 'shared' boundary info is pulled from the blueprint face topology adjacency sets. TODO: need to fix for MPI-decomposed meshes @@ -78,7 +77,6 @@ class TetonBlueprint std::vector &boundaries_types, int rank); -/* ------------------------------------------------------------------------------------------------------------------ /* ------------------------------------------------------------------------------------------------------------------ Set of methods used to create the teton mesh connectivity array. This array consists of, for each zone,: zoneID, - id of zone @@ -180,8 +178,14 @@ class TetonBlueprint std::vector m_face_to_bcid; // For handling Teton's boundary conditions - enum BC_Type { bc_reflecting = 32, bc_vaccuum = 35, - bc_source_temp = 34, bc_source_fd = 36, bc_shared = 33 }; + enum BC_Type + { + bc_reflecting = 32, + bc_vaccuum = 35, + bc_source_temp = 34, + bc_source_fd = 36, + bc_shared = 33 + }; public: TetonBlueprint(conduit::Node &blueprintNode, conduit::Node ¶metersNode) diff --git a/src/teton/include/omp_wrappers.h b/src/teton/include/omp_wrappers.h index 27f765c..748034f 100644 --- a/src/teton/include/omp_wrappers.h +++ b/src/teton/include/omp_wrappers.h @@ -14,23 +14,17 @@ #if defined(TETON_ENABLE_OPENMP_OFFLOAD) # define TOMP(source) !$omp source # define TOMPC(source) !$omp& source +# if defined(TETON_ENABLE_UMPIRE) +# define UMPIRE_DEVICE_POOL_ALLOC(source) call target_alloc_and_pair_ptrs(source) +# define UMPIRE_DEVICE_POOL_FREE(source) call target_free_and_unpair_ptrs(source) +# else +# define UMPIRE_DEVICE_POOL_ALLOC(source) +# define UMPIRE_DEVICE_POOL_FREE(source) +# endif -#define TOMP_TARGET_ENTER_DATA_MAP_TO(source) \ -call target_alloc_and_pair_ptrs(source, "source");\ -!$omp target enter data map(always,alloc:source);\ -!$omp target update to(source) - -#define TOMP_TARGET_EXIT_DATA_MAP_RELEASE(source) \ -call target_free_and_unpair_ptrs(source, "source"); \ -!$omp target exit data map(always, release:source) - -# define TOMP_TARGET_ENTER_DATA_MAP_ALLOC(source) !$omp target enter data map(alloc:source) -# define TOMP_TARGET_ENTER_DATA_MAP_FROM(source) !$omp target exit data map(from:source) #else # define TOMP(source) # define TOMPC(source) -# define TOMP_TARGET_ENTER_DATA_MAP_TO(source) -# define TOMP_TARGET_ENTER_DATA_MAP_ALLOC(source) -# define TOMP_TARGET_ENTER_DATA_MAP_FROM(source) -# define TOMP_TARGET_EXIT_DATA_MAP_RELEASE(source) +# define UMPIRE_DEVICE_POOL_ALLOC(source) +# define UMPIRE_DEVICE_POOL_FREE(source) #endif diff --git a/src/teton/interface/TetonBlueprint.cc b/src/teton/interface/TetonBlueprint.cc index 57506ff..be3b04e 100644 --- a/src/teton/interface/TetonBlueprint.cc +++ b/src/teton/interface/TetonBlueprint.cc @@ -1229,6 +1229,21 @@ void TetonBlueprint::CreateConnectivityArrays(conduit::Node &meshNode, MPI_Comm meshNode["arrays/zone_to_corners"].set(zone_to_corners, ncorners); } +/** + @brief This method creates a face_attribute field for the main_face topology + with values that come from boundary_attribute where a face matches a + boundary face and 0 for other faces. + + The boundary topology is iterated and for each face, we compute a hash + of the ids and map them to the id of the boundary face. The face topology + is then iterated and a hash is made for each of its faces. We look up that + hash to try and map a face in the boundary topology. If we find a matching + hash then its id is used to copy from the boundary_attributes to the + face_attribute field. + + @param meshNode The Conduit node that contaisn the meshes. + @param rank The MPI rank. + */ void TetonBlueprint::CreateConduitFaceAttributes(conduit::Node &meshNode, int rank) { CALI_CXX_MARK_FUNCTION; @@ -1453,6 +1468,23 @@ void TetonBlueprint::ProcessSurfaceEdits(int rank) } } +/** + @brief Compute data that lives on the face and associates a face with boundaries, + if applicable. If the face is not associated with a boundary then it is a + face that is shared with another domain, figure out which using the face + adjset. The face adjset is an element-associated adjset that contains a list + of face ids that touch other domains. + + @param boundaries A map of boundaries to faces where the keys in the map are values + present in the fields/boundary_attribute/values field and the + values are vectors of face ids in the main_faces topology that + use those boundary numbers. + @param[out] nbelem_corner_faces The number of corner faces that touch boundary faces. + @param[out] boundaries_types Counts of the various boundary types (how many faces of each) + @param[out] boundary_conditions An encoding of boundary condition ids, number of corner faces. + @param[out] face_to_bcid A "field" for the face mesh that indicates for each face + the boundary condition id for that face. + */ void TetonBlueprint::ComputeFaceIDs(std::map> &boundaries, int &nbelem_corner_faces, std::vector &boundaries_types, @@ -1733,24 +1765,24 @@ void TetonBlueprint::ComputeFaceIDs(std::map> &boundaries, // BCTypeInt = {type1, type2}, where 31 <= type1 <= 37 is Teton's boundary ID type mapping, // where type1 corresponds to the first boundary and type2 corresponds to the second boundary. // Here BCCornerFaces = {1,1} since always 1 corner per boundary condition (is this ever not true?). -// BCNeighborID = {rank1, rank2}, where rank1 corresponds to the corresponding rank of a shared boundary. -// If not a shared boundary, then rank1 = -1 (same with rank2). +// BCNeighborID = {rank1, rank2}, where rank1 corresponds to the corresponding rank of a shared boundary. +// If not a shared boundary, then rank1 = -1 (same with rank2). // teton_addboundary(&numBCTotal, &BCTypeInt[0], &BCCornerFaces[0], &BCNeighborID[0]); // Also need // teton_setzone1d(&zoneID, &numBCTotal, &BCZoneID[0]); -// Here numBCTotal = 2 (is this ever not true)? and +// Here numBCTotal = 2 (is this ever not true)? and // Teton expects two boundary conditions, one for zone1D == 1 and one for zoneID == nzones // Here BCZoneID[0] == 1, BCZoneID[1] == nzones or BCZoneID[0] == nzones and BCZoneID[1] == 1 -void TetonBlueprint::ComputeFaceIDs1D(int *boundary_connectivity, // boundary to vertex mapping - int *boundary_attributes, // boundary to mesh attribute mapping +void TetonBlueprint::ComputeFaceIDs1D(int *boundary_connectivity, // boundary to vertex mapping + int *boundary_attributes, // boundary to mesh attribute mapping std::vector &boundaries_types, // boundary to Teton boundary type mapping int rank) { // need to fill these out and store them in mMeshNode std::vector bc_type_int(2); std::vector bc_corner_faces(2); - std::vector bc_nghbr_id(2); + std::vector bc_nghbr_id(2); std::vector bc_zone_id(2); // TODO: check if there is ever a case where this doesn't hold @@ -1761,11 +1793,14 @@ void TetonBlueprint::ComputeFaceIDs1D(int *boundary_connectivity, // boundary to // map boundary ID to corresponding Teton boundary type (c.g. 34 for a vaccuum boundary condition) if (mParametersNode.has_path("boundary_conditions/id_to_type_map")) { + conduit::int_accessor boundary_ids = mParametersNode.fetch_existing("boundary_conditions/id_to_type_map/ids") + .value(); + conduit::int_accessor teton_bc_types = mParametersNode.fetch_existing("boundary_conditions/id_to_type_map/types") + .value(); - conduit::int_accessor boundary_ids = mParametersNode.fetch_existing("boundary_conditions/id_to_type_map/ids").value(); - conduit::int_accessor teton_bc_types = mParametersNode.fetch_existing("boundary_conditions/id_to_type_map/types").value(); - - TETON_VERIFY_C(rank, boundary_ids.number_of_elements() == teton_bc_types.number_of_elements(), "boundary ids list length != boundary types list length"); + TETON_VERIFY_C(rank, + boundary_ids.number_of_elements() == teton_bc_types.number_of_elements(), + "boundary ids list length != boundary types list length"); for (int i = 0; i < boundary_ids.number_of_elements(); ++i) { @@ -1784,9 +1819,9 @@ void TetonBlueprint::ComputeFaceIDs1D(int *boundary_connectivity, // boundary to //for (int face = 0; face < nfaces; ++face) // elem_to_bcid[face] = 0; - for (int j = 0; j < 2; ++j) + for (int j = 0; j < 2; ++j) { - int bc_id = boundary_attributes[j]; + int bc_id = boundary_attributes[j]; { int bc_type = boundary_id_to_type.at(bc_id); number_boundary_types[bc_type] += 1; @@ -1813,41 +1848,41 @@ void TetonBlueprint::ComputeFaceIDs1D(int *boundary_connectivity, // boundary to int nreflec, nvacuum; //, nsource; nreflec = boundaries_types[0]; nvacuum = boundaries_types[1]; -// nsource = boundaries_types[2]; -// We don't appear to use nsource for anything -- black27 + // nsource = boundaries_types[2]; + // We don't appear to use nsource for anything -- black27 std::vector bc_ids_ordered(num_bndrs); int jreflect = 0, jvaccuum = 0, jsource = 0; - for (int j = 0; j < 2; ++j) + for (int j = 0; j < 2; ++j) { int bc_id = boundary_attributes[j]; int bc_type = boundary_id_to_type.at(bc_id); if (bc_type == BC_Type::bc_reflecting) // reflecting { bc_ids_ordered[jreflect] = bc_id; - bc_type_int[jreflect] = bc_type; + bc_type_int[jreflect] = bc_type; int bc_vertex_id = boundary_connectivity[j]; - if (bc_vertex_id == 0) + if (bc_vertex_id == 0) { - bc_zone_id[jreflect] = 1; // vertexID 0 corresponds to zoneID = 1 in Teton + bc_zone_id[jreflect] = 1; // vertexID 0 corresponds to zoneID = 1 in Teton } else { - bc_zone_id[jreflect] = bc_vertex_id; // vertexID nzones corresponds to zoneID = nzones in Teton + bc_zone_id[jreflect] = bc_vertex_id; // vertexID nzones corresponds to zoneID = nzones in Teton } } else if (bc_type == BC_Type::bc_vaccuum) // vaccuum { int index = jvaccuum + nreflec; bc_ids_ordered[index] = bc_id; - bc_type_int[index] = bc_type; + bc_type_int[index] = bc_type; int bc_vertex_id = boundary_connectivity[j]; - if (bc_vertex_id == 0) + if (bc_vertex_id == 0) { - bc_zone_id[index] = 1; // vertexID 0 corresponds to zoneID = 1 in Teton + bc_zone_id[index] = 1; // vertexID 0 corresponds to zoneID = 1 in Teton } else { - bc_zone_id[index] = bc_vertex_id; // vertexID nzones corresponds to zoneID = nzones in Teton + bc_zone_id[index] = bc_vertex_id; // vertexID nzones corresponds to zoneID = nzones in Teton } jvaccuum += 1; } @@ -1855,15 +1890,15 @@ void TetonBlueprint::ComputeFaceIDs1D(int *boundary_connectivity, // boundary to { int index = jsource + nreflec + nvacuum; bc_ids_ordered[index] = bc_id; - bc_type_int[index] = bc_type; + bc_type_int[index] = bc_type; int bc_vertex_id = boundary_connectivity[j]; - if (bc_vertex_id == 0) + if (bc_vertex_id == 0) { - bc_zone_id[index] = 1; // vertexID 0 corresponds to zoneID = 1 in Teton + bc_zone_id[index] = 1; // vertexID 0 corresponds to zoneID = 1 in Teton } else { - bc_zone_id[index] = bc_vertex_id; // vertexID nzones corresponds to zoneID = nzones in Teton + bc_zone_id[index] = bc_vertex_id; // vertexID nzones corresponds to zoneID = nzones in Teton } jsource += 1; } @@ -1887,8 +1922,8 @@ void TetonBlueprint::ComputeFaceIDs1D(int *boundary_connectivity, // boundary to //int *szones_ptr = group["values"].value(); const conduit::int32 *szones_ptr = group["values"].as_int32_ptr(); - bc_type_int[index] = BC_Type::bc_shared; // shared boundary - bc_zone_id[index] = szones_ptr[0]; // only one shared zone per face-neighbor in 1d + bc_type_int[index] = BC_Type::bc_shared; // shared boundary + bc_zone_id[index] = szones_ptr[0]; // only one shared zone per face-neighbor in 1d bc_nghbr_id[index] = nbr_rank; bc_corner_faces[index] = 1; } @@ -1905,34 +1940,32 @@ void TetonBlueprint::ComputeFaceIDs1D(int *boundary_connectivity, // boundary to mParametersNode["boundary_conditions/num_total"] = num_bndrs; std::vector elem_ids(2); - int nelem = mMeshNode["topologies/mesh/elements/dims/i"].value(); + int nelem = mMeshNode["topologies/mesh/elements/dims/i"].value(); mParametersNode["boundary_conditions/type"].set(bc_type_int.data(), 2); mParametersNode["boundary_conditions/zone_ids"].set(bc_zone_id.data(), 2); mParametersNode["boundary_conditions/neighbor_ids"].set(bc_nghbr_id.data(), 2); mParametersNode["boundary_conditions/bc_ncorner_faces"].set(bc_corner_faces.data(), 2); - } - void TetonBlueprint::OutputTetonMesh(int rank, MPI_Comm comm) { const conduit::Node &coords = mMeshNode["coordsets/coords/values"]; if (!coords.has_path("y") && !coords.has_path("z")) { - int nelem = mMeshNode["topologies/mesh/elements/dims/i"].value(); + int nelem = mMeshNode["topologies/mesh/elements/dims/i"].value(); // TODO Move to mesh conduit Node // TODO: fix rank - int rank = 0; + int rank = 0; mParametersNode["size/ndim"] = 1; mParametersNode["size/rank"] = rank; mParametersNode["size/nzones"] = nelem; - mParametersNode["size/nverts"] = 2*nelem; - mParametersNode["size/ncornr"] = 2*nelem; - mParametersNode["size/nsides"] = 2*nelem; + mParametersNode["size/nverts"] = 2 * nelem; + mParametersNode["size/ncornr"] = 2 * nelem; + mParametersNode["size/nsides"] = 2 * nelem; mParametersNode["size/nbelem"] = 2; mParametersNode["size/maxcf"] = 2; mParametersNode["size/maxCorner"] = 2; - // TODO: enable slab geometry option as well + // TODO: enable slab geometry option as well mParametersNode["size/geomType"] = 42; //spherical //mParametersNode["size/geomType"] = 41; //slab int num_face_nbrs_teton = mParametersNode["boundary_conditions/num_comm"].value(); @@ -1974,6 +2007,7 @@ void TetonBlueprint::OutputTetonMesh(int rank, MPI_Comm comm) // for each face CreateConduitFaceAttributes(mMeshNode, rank); + // Populate boundaries, a list of face ids for each boundary condition id. const conduit::int32 *face_to_attr = mMeshNode["fields/face_attribute/values"].as_int32_ptr(); try @@ -1988,7 +2022,6 @@ void TetonBlueprint::OutputTetonMesh(int rank, MPI_Comm comm) int bdnry_cond_id = face_to_attr[face]; if (boundaries.find(bdnry_cond_id) == boundaries.end()) { - //int newface = face_to_newface_mapping[face]; boundaries[bdnry_cond_id] = {face}; } else @@ -2025,7 +2058,6 @@ void TetonBlueprint::OutputTetonMesh(int rank, MPI_Comm comm) if (mMeshNode.has_path("adjsets/main_face")) { const conduit::Node &face_adjset = mMeshNode["adjsets/main_face"]; - //const conduit::Node &corner_adjset = mMeshNode["adjsets/main_corner"]; conduit::NodeConstIterator groups_it = face_adjset["groups"].children(); while (groups_it.has_next()) { @@ -2171,5 +2203,3 @@ void TetonBlueprint::OutputTetonMesh(int rank, MPI_Comm comm) verifyInput(mMeshNode, comm); } - - diff --git a/src/teton/interface/TetonConduitInterface.cc b/src/teton/interface/TetonConduitInterface.cc index 137e58d..8c2f9c3 100644 --- a/src/teton/interface/TetonConduitInterface.cc +++ b/src/teton/interface/TetonConduitInterface.cc @@ -131,8 +131,11 @@ void Teton::initialize(MPI_Comm communicator, bool fromRestart) MPI_Barrier(communicator); int ndim = options.fetch_existing("size/ndim").value(); - if (ndim > 1) teton_setoppositeface(); //Prerequisite for calling setMeshSizeAndPositions() in 2D/3D - setCommunication(); //Prerequisite for calling setMeshSizeAndPositions() + if (ndim > 1) + { + teton_setoppositeface(); //Prerequisite for calling setMeshSizeAndPositions() in 2D/3D + setCommunication(); //Prerequisite for calling setMeshSizeAndPositions() + } setMeshSizeAndPositions(); // This is an awful hack to make sure Z%VolumeOld is initialized @@ -318,9 +321,8 @@ void Teton::constructBoundaries() int *BCNeighborID = options.fetch_existing("boundary_conditions/neighbor_ids").as_int_ptr(); int *BCCornerFaces = options.fetch_existing("boundary_conditions/bc_ncorner_faces").as_int_ptr(); int numBCTotal = 2; - - teton_addboundary(&numBCTotal, &BCTypeInt[0], &BCCornerFaces[0], &BCNeighborID[0]); + teton_addboundary(&numBCTotal, &BCTypeInt[0], &BCCornerFaces[0], &BCNeighborID[0]); } } @@ -1171,13 +1173,13 @@ void Teton::setMeshSizeAndPositions() { int nvertices = blueprint["coordsets/coords/values/x"].dtype().number_of_elements(); double *vertex_coords = blueprint["coordsets/coords/values/x"].value(); - int nzones = nvertices-1; + int nzones = nvertices - 1; std::vector zoneCoordinates(2); for (int zone = 0; zone < nzones; ++zone) { int zoneID = zone + 1; zoneCoordinates[0] = vertex_coords[zone]; - zoneCoordinates[1] = vertex_coords[zone+1]; + zoneCoordinates[1] = vertex_coords[zone + 1]; // TODO: add asset that zoneCoordinates[0] < zoneCoordinates[1] teton_setnodeposition(&zoneID, &zoneCoordinates[0]); } @@ -1290,7 +1292,7 @@ void Teton::setMeshConnectivity() int *BCZoneID = options.fetch_existing("boundary_conditions/zone_ids").value(); for (int zone = 0; zone < nzones; ++zone) { - int zoneID = zone+1; + int zoneID = zone + 1; teton_setzone1d(&zoneID, &numBCTotal, &BCZoneID[0]); } } @@ -1781,7 +1783,7 @@ void Teton::getRadiationForceDensity1D(double *RadiationForceDensityX) CornerVolumeSumsAtVertex[v2] += CornerVolumes[1]; } - for (int v = 0; v < nzones+1; ++v) + for (int v = 0; v < nzones + 1; ++v) { RadiationForceDensityX[v] /= CornerVolumeSumsAtVertex[v]; } @@ -1795,7 +1797,7 @@ void Teton::getRadiationForceDensity(double *RadiationForceDensityX, { conduit::Node &options = getOptions(); int ndim = options.fetch_existing("size/ndim").value(); - if (ndim == 1) + if (ndim == 1) { getRadiationForceDensity1D(RadiationForceDensityX); return; @@ -1837,7 +1839,7 @@ void Teton::getRadiationForceDensity(double *RadiationForceDensityX, int vertexID = mCornerToVertex[cornerID]; corner_counter += 1; RadiationForceDensityX[vertexID] += RadiationForce[c * ndim + 0]; - if (ndim > 1) + if (ndim > 1) RadiationForceDensityY[vertexID] += RadiationForce[c * ndim + 1]; if (ndim == 3) RadiationForceDensityZ[vertexID] += RadiationForce[c * ndim + 2]; @@ -1848,7 +1850,7 @@ void Teton::getRadiationForceDensity(double *RadiationForceDensityX, for (int v = 0; v < nverts; ++v) { RadiationForceDensityX[v] /= CornerVolumeSumsAtVertex[v]; - if (ndim > 1) + if (ndim > 1) RadiationForceDensityY[v] /= CornerVolumeSumsAtVertex[v]; if (ndim == 3) RadiationForceDensityZ[v] /= CornerVolumeSumsAtVertex[v];