From 96c7a6837258bb8170d1852b3219774e0c968a92 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Tue, 16 Aug 2022 16:24:24 -0600 Subject: [PATCH 1/8] update to newest parthenon --- CMakeLists.txt | 8 -- external/parthenon | 2 +- src/compile_constants.hpp.in | 2 - src/fluid/fluid.cpp | 34 ------ src/fluid/fluid.hpp | 42 ++++++- src/monopole_gr/interp_3d_to_1d.hpp | 6 +- src/pgen/homogeneous_sphere.cpp | 1 - src/pgen/radiation_advection.cpp | 4 - src/pgen/radiation_equilibration.cpp | 4 - src/pgen/rotor.cpp | 4 +- src/phoebus_driver.cpp | 174 ++++++++++++++++----------- 11 files changed, 154 insertions(+), 127 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 52452d46..c6698709 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -173,14 +173,6 @@ include(cmake/Geometry.cmake) # Fluid include(cmake/Fluid.cmake) -# Print RHS -option(PHOEBUS_PRINT_RHS "Print right hand sides" OFF) -if (PHOEBUS_PRINT_RHS) - set(PRINT_RHS 1) -else() - set(PRINT_RHS 0) -endif() - # Phoebus src message("\nConfiguring src") add_subdirectory(src) diff --git a/external/parthenon b/external/parthenon index 22fd4442..e0e2c487 160000 --- a/external/parthenon +++ b/external/parthenon @@ -1 +1 @@ -Subproject commit 22fd4442c4486b9da70557112e2630e46c187da2 +Subproject commit e0e2c48716cb1520308711dd78a4c2b17c520cb7 diff --git a/src/compile_constants.hpp.in b/src/compile_constants.hpp.in index 0eb3537b..57ebf83c 100644 --- a/src/compile_constants.hpp.in +++ b/src/compile_constants.hpp.in @@ -28,6 +28,4 @@ #define CON2PRIM_STATISTICS @CON2PRIM_STATISTICS@ -#define PRINT_RHS @PRINT_RHS@ - #endif //COMPILE_CONSTANTS_HPP_ diff --git a/src/fluid/fluid.cpp b/src/fluid/fluid.cpp index 1c65e30e..c0a911de 100644 --- a/src/fluid/fluid.cpp +++ b/src/fluid/fluid.cpp @@ -493,40 +493,6 @@ TaskStatus ConservedToPrimitiveClassic(T *rc, const IndexRange &ib, const IndexR return TaskStatus::complete; } -#if SET_FLUX_SRC_DIAGS -TaskStatus CopyFluxDivergence(MeshBlockData *rc) { - auto *pmb = rc->GetParentPointer().get(); - auto &fluid = pmb->packages.Get("fluid"); - if (!fluid->Param("active")) return TaskStatus::complete; - - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); - - std::vector vars( - {fluid_cons::density, fluid_cons::momentum, fluid_cons::energy}); - PackIndexMap imap; - auto divf = rc->PackVariables(vars, imap); - const int crho = imap[fluid_cons::density].first; - const int cmom_lo = imap[fluid_cons::momentum].first; - const int cmom_hi = imap[fluid_cons::momentum].second; - const int ceng = imap[fluid_cons::energy].first; - std::vector diag_vars({diagnostic_variables::divf}); - auto diag = rc->PackVariables(diag_vars); - - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "CopyDivF", DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, - ib.e, KOKKOS_LAMBDA(const int k, const int j, const int i) { - diag(0, k, j, i) = divf(crho, k, j, i); - diag(1, k, j, i) = divf(cmom_lo, k, j, i); - diag(2, k, j, i) = divf(cmom_lo + 1, k, j, i); - diag(3, k, j, i) = divf(cmom_lo + 2, k, j, i); - diag(4, k, j, i) = divf(ceng, k, j, i); - }); - return TaskStatus::complete; -} -#endif - TaskStatus CalculateFluidSourceTerms(MeshBlockData *rc, MeshBlockData *rc_src) { constexpr int ND = Geometry::NDFULL; diff --git a/src/fluid/fluid.hpp b/src/fluid/fluid.hpp index 8db37fe4..ed50f3f1 100644 --- a/src/fluid/fluid.hpp +++ b/src/fluid/fluid.hpp @@ -20,6 +20,8 @@ #include using namespace parthenon::package::prelude; +#include "phoebus_utils/variables.hpp" + namespace fluid { std::shared_ptr Initialize(ParameterInput *pin); @@ -40,18 +42,56 @@ TaskStatus ConservedToPrimitiveVanDerHolst(T *rc, const IndexRange &ib, const IndexRange &jb, const IndexRange &kb); TaskStatus CalculateFluidSourceTerms(MeshBlockData *rc, MeshBlockData *rc_src); -TaskStatus CopyFluxDivergence(MeshBlockData *rc); TaskStatus CalculateFluxes(MeshBlockData *rc); TaskStatus FluxCT(MeshBlockData *rc); TaskStatus CalculateDivB(MeshBlockData *rc); Real EstimateTimestepBlock(MeshBlockData *rc); +template +TaskStatus CopyFluxDivergence(T *rc); + template using c2p_type = TaskStatus (*)(T *, const IndexRange &, const IndexRange &, const IndexRange &); using c2p_meshblock_type = c2p_type>; using c2p_mesh_type = c2p_type>; +#if SET_FLUX_SRC_DIAGS +template +TaskStatus CopyFluxDivergence(T *rc) { + auto pmb = rc->GetParentPointer(); + auto &fluid = pmb->packages.Get("fluid"); + const Params ¶ms = fluid->AllParams(); + if (!params.Get("active")) return TaskStatus::complete; + + std::vector vars( + {fluid_cons::density, fluid_cons::momentum, fluid_cons::energy}); + PackIndexMap imap; + auto divf = rc->PackVariables(vars, imap); + const int crho = imap[fluid_cons::density].first; + const int cmom_lo = imap[fluid_cons::momentum].first; + const int cmom_hi = imap[fluid_cons::momentum].second; + const int ceng = imap[fluid_cons::energy].first; + std::vector diag_vars({diagnostic_variables::divf}); + auto diag = rc->PackVariables(diag_vars); + + // TODO(JMM): If we expose a way to get cellbounds from the mesh or + // meshdata object, that would be better. + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "CopyDivF", DevExecSpace(), 0, divf.GetDim(5) - 1, 0, + divf.GetDim(3) - 1, 0, divf.GetDim(2) - 1, 0, divf.GetDim(1) - 1, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + diag(b, 0, k, j, i) = divf(b, crho, k, j, i); + diag(b, 1, k, j, i) = divf(b, cmom_lo, k, j, i); + diag(b, 2, k, j, i) = divf(b, cmom_lo + 1, k, j, i); + diag(b, 3, k, j, i) = divf(b, cmom_lo + 2, k, j, i); + diag(b, 4, k, j, i) = divf(b, ceng, k, j, i); + }); + return TaskStatus::complete; +} + +#endif + } // namespace fluid #endif // FLUID_HPP_ diff --git a/src/monopole_gr/interp_3d_to_1d.hpp b/src/monopole_gr/interp_3d_to_1d.hpp index 3a121bbe..15af5982 100644 --- a/src/monopole_gr/interp_3d_to_1d.hpp +++ b/src/monopole_gr/interp_3d_to_1d.hpp @@ -225,9 +225,9 @@ GetMonopoleVarsHelper(const EnergyMomentum &tmunu, const Geometry_t &geom, const Real X2 = coords.x2v(j); const Real X3 = coords.x3v(k); transform(X1, X2, X3, C, c2s, s2c); - const Real r = C[0]; - const Real th = C[1]; - const Real ph = C[2]; + // const Real r = C[0]; + // const Real th = C[1]; + // const Real ph = C[2]; // J jr = 0; diff --git a/src/pgen/homogeneous_sphere.cpp b/src/pgen/homogeneous_sphere.cpp index eead59cd..2fd24813 100644 --- a/src/pgen/homogeneous_sphere.cpp +++ b/src/pgen/homogeneous_sphere.cpp @@ -52,7 +52,6 @@ void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { const Real vx = pin->GetOrAddReal("homogeneous_sphere", "vx", 0.0); auto &coords = pmb->coords; - auto pmesh = pmb->pmy_mesh; IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::entire); IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::entire); diff --git a/src/pgen/radiation_advection.cpp b/src/pgen/radiation_advection.cpp index 6219939c..d7eb11e7 100644 --- a/src/pgen/radiation_advection.cpp +++ b/src/pgen/radiation_advection.cpp @@ -91,10 +91,6 @@ void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { auto rad = pmb->packages.Get("radiation").get(); auto species = rad->Param>("species"); auto num_species = rad->Param("num_species"); - singularity::RadiationType species_d[3] = {}; - for (int s = 0; s < num_species; s++) { - species_d[s] = species[s]; - } const Real W = 1 / sqrt(1 - vx * vx); const Real t0p = 1.5 * kappa * width * width; diff --git a/src/pgen/radiation_equilibration.cpp b/src/pgen/radiation_equilibration.cpp index 328b0742..595afc55 100644 --- a/src/pgen/radiation_equilibration.cpp +++ b/src/pgen/radiation_equilibration.cpp @@ -52,10 +52,6 @@ void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { const auto specB = idJ.GetBounds(1); const Real J = pin->GetOrAddReal("radiation_equilibration", "J", 0.0); - auto &coords = pmb->coords; - auto pmesh = pmb->pmy_mesh; - int ndim = pmesh->ndim; - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::entire); IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::entire); IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::entire); diff --git a/src/pgen/rotor.cpp b/src/pgen/rotor.cpp index 7c6a0ac0..0368e220 100644 --- a/src/pgen/rotor.cpp +++ b/src/pgen/rotor.cpp @@ -64,9 +64,9 @@ void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { auto emax = pmb->packages.Get("eos")->Param("sie_max"); auto gpkg = pmb->packages.Get("geometry"); auto geom = Geometry::GetCoordinateSystem(rc.get()); - Real a_snake, k_snake, alpha, betax, betay, betaz; + Real a_snake, k_snake, alpha, betay; alpha = 1; - a_snake = k_snake = betax = betay = betaz = 0; + a_snake = k_snake = betay = 0; Real tf = pin->GetReal("parthenon/time", "tlim"); if (is_snake) { a_snake = gpkg->Param("a"); diff --git a/src/phoebus_driver.cpp b/src/phoebus_driver.cpp index 9dfaf07c..7eac33b7 100644 --- a/src/phoebus_driver.cpp +++ b/src/phoebus_driver.cpp @@ -17,7 +17,9 @@ #include // TODO(JCD): this should be exported by parthenon +#include #include +#include #include #include #include @@ -99,8 +101,6 @@ void PhoebusDriver::PostInitializationCommunication() { TaskID none(0); BlockList_t &blocks = pmesh->block_list; - const auto &stage_name = integrator->stage_name; - auto rad = pmesh->packages.Get("radiation"); auto fluid = pmesh->packages.Get("fluid"); const bool rad_active = rad->Param("active"); @@ -112,27 +112,46 @@ void PhoebusDriver::PostInitializationCommunication() { rad_mocmc_active = (rad->Param("method") == "mocmc"); } + const int num_partitions = pmesh->DefaultNumPartitions(); + TaskRegion &sync_region = tc.AddRegion(num_partitions); + for (int ib = 0; ib < num_partitions; ib++) { + auto &md = pmesh->mesh_data.GetOrAdd("base", ib); + auto &tl = sync_region[ib]; + + const auto any = parthenon::BoundaryType::any; + const auto local = parthenon::BoundaryType::local; + const auto nonlocal = parthenon::BoundaryType::nonlocal; + + auto start_recv = + tl.AddTask(none, parthenon::cell_centered_bvars::StartReceiveBoundBufs, md); + + auto send = tl.AddTask(start_recv, + parthenon::cell_centered_bvars::SendBoundBufs, md); + + auto send_local = + tl.AddTask(start_recv, parthenon::cell_centered_bvars::SendBoundBufs, md); + auto recv_local = tl.AddTask( + start_recv, parthenon::cell_centered_bvars::ReceiveBoundBufs, md); + auto set_local = + tl.AddTask(recv_local, parthenon::cell_centered_bvars::SetBounds, md); + + auto recv = tl.AddTask( + start_recv, parthenon::cell_centered_bvars::ReceiveBoundBufs, md); + auto set = tl.AddTask(recv, parthenon::cell_centered_bvars::SetBounds, md); + + if (pmesh->multilevel) { + tl.AddTask(set | set_local, + parthenon::cell_centered_refinement::RestrictPhysicalBounds, md.get()); + } + } + TaskRegion &async_region = tc.AddRegion(blocks.size()); for (int ib = 0; ib < blocks.size(); ib++) { auto pmb = blocks[ib].get(); auto &tl = async_region[ib]; auto &sc = pmb->meshblock_data.Get(); - auto &md = - pmesh->mesh_data.GetOrAdd(stage_name[0], ib); // TODO(BRR) This gives an empty md - - auto start_recv = tl.AddTask(none, &MeshBlockData::StartReceiving, sc.get(), - BoundaryCommSubset::all); - auto send = - tl.AddTask(start_recv, parthenon::cell_centered_bvars::SendBoundaryBuffers, md); - auto recv = - tl.AddTask(send, parthenon::cell_centered_bvars::ReceiveBoundaryBuffers, md); - auto fill_from_bufs = - tl.AddTask(recv, parthenon::cell_centered_bvars::SetBoundaries, md); - auto clear_comm_flags = - tl.AddTask(fill_from_bufs, &MeshBlockData::ClearBoundary, sc.get(), - BoundaryCommSubset::all); - auto prolongBound = tl.AddTask(clear_comm_flags, parthenon::ProlongateBoundaries, sc); + auto prolongBound = tl.AddTask(none, parthenon::ProlongateBoundaries, sc); auto set_bc = tl.AddTask(prolongBound, parthenon::ApplyBoundaryConditions, sc); @@ -223,21 +242,40 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { } bool time_dependent_geom = geom_params.Get("time_dependent", false); - auto num_independent_task_lists = blocks.size(); - TaskRegion &async_region_1 = tc.AddRegion(num_independent_task_lists); - for (int ib = 0; ib < num_independent_task_lists; ib++) { - auto pmb = blocks[ib].get(); - auto &tl = async_region_1[ib]; - - // first make other useful containers - auto &base = pmb->meshblock_data.Get(); - if (stage == 1) { + // First we need to make all the meshblock-level containers + if (stage == 1) { + for (int i = 0; i < blocks.size(); ++i) { + auto &pmb = blocks[i]; + auto &base = pmb->meshblock_data.Get(); pmb->meshblock_data.Add("dUdt", base); for (int i = 1; i < integrator->nstages; i++) { pmb->meshblock_data.Add(stage_name[i], base); + pmb->meshblock_data.Add("geometric source terms", base, src_w_diag); } - pmb->meshblock_data.Add("geometric source terms", base, src_w_diag); } + } + + // be ready for flux corrections and boundary comms + const int num_partitions = pmesh->DefaultNumPartitions(); + TaskRegion &sync_region_0 = tc.AddRegion(num_partitions); + for (int ib = 0; ib < num_partitions; ++ib) { + auto &base = pmesh->mesh_data.GetOrAdd("base", ib); + auto &sc0 = pmesh->mesh_data.GetOrAdd(stage_name[stage - 1], ib); + auto &sc1 = pmesh->mesh_data.GetOrAdd(stage_name[stage], ib); + auto &tl = sync_region_0[ib]; + + const auto any = parthenon::BoundaryType::any; + tl.AddTask(none, parthenon::cell_centered_bvars::StartReceiveBoundBufs, sc1); + if (pmesh->multilevel) { + tl.AddTask(none, parthenon::cell_centered_bvars::StartReceiveFluxCorrections, sc0); + } + } + + auto num_independent_task_lists = blocks.size(); + TaskRegion &async_region_1 = tc.AddRegion(num_independent_task_lists); + for (int ib = 0; ib < num_independent_task_lists; ib++) { + auto pmb = blocks[ib].get(); + auto &tl = async_region_1[ib]; // pull out the container we'll use to get fluxes and/or compute RHSs auto &sc0 = pmb->meshblock_data.Get(stage_name[stage - 1]); @@ -250,8 +288,6 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { // pull out a container for the geometric source terms auto &gsrc = pmb->meshblock_data.Get("geometric source terms"); - auto start_recv = tl.AddTask(none, &MeshBlockData::StartReceiving, sc1.get(), - BoundaryCommSubset::all); TaskID geom_src(0); TaskID sndrcv_flux_depend(0); @@ -299,34 +335,6 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { geom_src = geom_src | eddington; } - - auto send_flux = tl.AddTask(sndrcv_flux_depend, - &MeshBlockData::SendFluxCorrection, sc0.get()); - - auto recv_flux = tl.AddTask(sndrcv_flux_depend, - &MeshBlockData::ReceiveFluxCorrection, sc0.get()); - - // compute the divergence of fluxes of conserved variables - auto flux_div = - tl.AddTask(recv_flux, parthenon::Update::FluxDivergence>, - sc0.get(), dudt.get()); - -#if SET_FLUX_SRC_DIAGS - auto copy_flux_div = - tl.AddTask(flux_div | geom_src, fluid::CopyFluxDivergence, dudt.get()); -#endif - - auto add_rhs = - tl.AddTask(flux_div | geom_src, SumData>, - src_names, dudt.get(), gsrc.get(), dudt.get()); - -#if PRINT_RHS - auto print_rhs = - tl.AddTask(add_rhs, Debug::PrintRHS>, dudt.get()); - auto next = print_rhs; -#else - auto next = add_rhs; -#endif } if (rad_mocmc_active && stage == integrator->nstages) { @@ -370,13 +378,13 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { } } - const int num_partitions = pmesh->DefaultNumPartitions(); TaskRegion &sync_region = tc.AddRegion(num_partitions); for (int ib = 0; ib < num_partitions; ib++) { auto &base = pmesh->mesh_data.GetOrAdd("base", ib); auto &sc0 = pmesh->mesh_data.GetOrAdd(stage_name[stage - 1], ib); auto &sc1 = pmesh->mesh_data.GetOrAdd(stage_name[stage], ib); auto &dudt = pmesh->mesh_data.GetOrAdd("dUdt", ib); + auto &gsrc = pmesh->mesh_data.GetOrAdd("geometric source terms", ib); auto &tl = sync_region[ib]; int reg_dep_id = 0; @@ -393,8 +401,29 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { sync_region.AddRegionalDependencies(reg_dep_id++, ib, interp_to_monopole); } + // fluxes now sent/received in mesh-data. + auto send_flux = + tl.AddTask(none, parthenon::cell_centered_bvars::LoadAndSendFluxCorrections, sc0); + auto recv_flux = + tl.AddTask(none, parthenon::cell_centered_bvars::ReceiveFluxCorrections, sc0); + auto set_flux = + tl.AddTask(recv_flux, parthenon::cell_centered_bvars::SetFluxCorrections, sc0); + + // compute the divergence of fluxes of conserved variables + auto flux_div = + tl.AddTask(set_flux, parthenon::Update::FluxDivergence>, sc0.get(), + dudt.get()); + +#if SET_FLUX_SRC_DIAGS + auto copy_flux_div = + tl.AddTask(flux_div, fluid::CopyFluxDivergence>, dudt.get()); +#endif + + auto add_rhs = tl.AddTask(flux_div, SumData>, src_names, + dudt.get(), gsrc.get(), dudt.get()); + // update step - auto avg_data = tl.AddTask(none, AverageIndependentData>, sc0.get(), + auto avg_data = tl.AddTask(add_rhs, AverageIndependentData>, sc0.get(), base.get(), beta); auto update = tl.AddTask(avg_data, UpdateIndependentData>, sc0.get(), dudt.get(), beta * dt, sc1.get()); @@ -456,12 +485,26 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { } // update ghost cells + const auto local = parthenon::BoundaryType::local; + const auto nonlocal = parthenon::BoundaryType::nonlocal; auto send = - tl.AddTask(update, parthenon::cell_centered_bvars::SendBoundaryBuffers, sc1); - auto recv = - tl.AddTask(send, parthenon::cell_centered_bvars::ReceiveBoundaryBuffers, sc1); - auto fill_from_bufs = - tl.AddTask(recv, parthenon::cell_centered_bvars::SetBoundaries, sc1); + tl.AddTask(update, parthenon::cell_centered_bvars::SendBoundBufs, sc1); + + auto send_local = + tl.AddTask(update, parthenon::cell_centered_bvars::SendBoundBufs, sc1); + auto recv_local = + tl.AddTask(update, parthenon::cell_centered_bvars::ReceiveBoundBufs, sc1); + auto set_local = + tl.AddTask(recv_local, parthenon::cell_centered_bvars::SetBounds, sc1); + + auto recv = tl.AddTask( + update, parthenon::cell_centered_bvars::ReceiveBoundBufs, sc1); + auto set = tl.AddTask(recv, parthenon::cell_centered_bvars::SetBounds, sc1); + + if (pmesh->multilevel) { + tl.AddTask(set | set_local, + parthenon::cell_centered_refinement::RestrictPhysicalBounds, sc1.get()); + } } TaskRegion &async_region_2 = tc.AddRegion(num_independent_task_lists); @@ -470,9 +513,6 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { auto &tl = async_region_2[ib]; auto &sc1 = pmb->meshblock_data.Get(stage_name[stage]); - auto clear_comm_flags = tl.AddTask(none, &MeshBlockData::ClearBoundary, - sc1.get(), BoundaryCommSubset::all); - auto prolongBound = tl.AddTask(none, parthenon::ProlongateBoundaries, sc1); // set physical boundaries From 16f29bb8ecbbaf7c4e197f38aedee46f90888433 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Tue, 16 Aug 2022 17:53:17 -0600 Subject: [PATCH 2/8] update singularity-eos --- external/singularity-eos | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/singularity-eos b/external/singularity-eos index 89480875..332bf8e2 160000 --- a/external/singularity-eos +++ b/external/singularity-eos @@ -1 +1 @@ -Subproject commit 89480875001f8d1c3fa0998a4f9a22887c0330b1 +Subproject commit 332bf8e2e3cc9142473c4546302d20d6bd82dd32 From d7a792b2d185e847e6eb9f89b13cf97a6cb815b0 Mon Sep 17 00:00:00 2001 From: Jonah Maxwell Miller Date: Wed, 17 Aug 2022 12:41:41 -0600 Subject: [PATCH 3/8] update to parthenon with fixed debug --- external/parthenon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/parthenon b/external/parthenon index e0e2c487..e45adca8 160000 --- a/external/parthenon +++ b/external/parthenon @@ -1 +1 @@ -Subproject commit e0e2c48716cb1520308711dd78a4c2b17c520cb7 +Subproject commit e45adca88d79360b4e0d8a16b2d919d289ddfd0e From db47fe57b276d417822ab90c9713a4cf9be1bd93 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Wed, 17 Aug 2022 17:09:53 -0600 Subject: [PATCH 4/8] Add all the relevant cmake changes for singularity-eos --- CMakeLists.txt | 4 ++++ src/CMakeLists.txt | 4 +--- tst/CMakeLists.txt | 4 +--- tst/unit/geometry/CMakeLists.txt | 3 +-- tst/unit/monopole_gr/CMakeLists.txt | 4 +--- tst/unit/phoebus_utils/CMakeLists.txt | 4 +--- tst/unit/radiation/CMakeLists.txt | 3 +-- 7 files changed, 10 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c6698709..e89500c8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -159,6 +159,10 @@ set(SINGULARITY_SUBMODULE_MODE ON CACHE BOOL "" FORCE) set(SINGULARITY_USE_KOKKOSKERNELS OFF CACHE BOOL "" FORCE) set(SINGULARITY_BUILD_CLOSURE OFF CACHE BOOL "" FORCE) +# Kill cmake's package registry because it can interfere +set(CMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY ON CACHE BOOL "" FORCE) +set(CMAKE_FIND_PACKAGE_NO_SYSTEM_PACKAGE_REGISTRY ON CACHE BOOL "" FORCE) + # singularity eos message("Configuring singularity-eos") add_subdirectory(external/singularity-eos singularity-eos) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2845fdbd..bc6fdc59 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -156,8 +156,6 @@ endif() # target_compile_options(phoebus PUBLIC -G -lineinfo) target_link_libraries(phoebus PRIVATE parthenon - singularity-eos::flags - singularity-eos + singularity-eos::singularity-eos singularity-opac::singularity-opac - $<$:singularity-eos::hdf5> ) diff --git a/tst/CMakeLists.txt b/tst/CMakeLists.txt index 63f563ba..b4094ac3 100644 --- a/tst/CMakeLists.txt +++ b/tst/CMakeLists.txt @@ -18,9 +18,7 @@ target_link_libraries(phoebus_unit_tests parthenon Catch2::Catch2 Kokkos::kokkos - singularity-eos::flags - singularity-eos - $<$:singularity-eos::hdf5> + singularity-eos::singularity-eos ) if (PHOEBUS_ENABLE_MPI) target_compile_definitions(phoebus_unit_tests diff --git a/tst/unit/geometry/CMakeLists.txt b/tst/unit/geometry/CMakeLists.txt index 36261b8a..89bb2f86 100644 --- a/tst/unit/geometry/CMakeLists.txt +++ b/tst/unit/geometry/CMakeLists.txt @@ -42,8 +42,7 @@ add_library(geometry_unit_tests OBJECT target_link_libraries(geometry_unit_tests PRIVATE parthenon - singularity-eos::flags - singularity-eos + singularity-eos::singularity-eos Catch2::Catch2 Kokkos::kokkos) diff --git a/tst/unit/monopole_gr/CMakeLists.txt b/tst/unit/monopole_gr/CMakeLists.txt index ecdd5f17..bca55647 100644 --- a/tst/unit/monopole_gr/CMakeLists.txt +++ b/tst/unit/monopole_gr/CMakeLists.txt @@ -33,9 +33,7 @@ target_link_libraries(monopole_gr_unit_tests parthenon Catch2::Catch2 Kokkos::kokkos - singularity-eos::flags - singularity-eos - $<$:singularity-eos::hdf5> + singularity-eos::singularity-eos ) target_include_directories(monopole_gr_unit_tests PRIVATE diff --git a/tst/unit/phoebus_utils/CMakeLists.txt b/tst/unit/phoebus_utils/CMakeLists.txt index faf5d378..565ac35e 100644 --- a/tst/unit/phoebus_utils/CMakeLists.txt +++ b/tst/unit/phoebus_utils/CMakeLists.txt @@ -21,9 +21,7 @@ target_link_libraries(phoebus_utils_unit_tests parthenon Catch2::Catch2 Kokkos::kokkos - singularity-eos::flags - singularity-eos - $<$:singularity-eos::hdf5> + singularity-eos::singularity-eos ) target_include_directories(phoebus_utils_unit_tests PRIVATE diff --git a/tst/unit/radiation/CMakeLists.txt b/tst/unit/radiation/CMakeLists.txt index 95f0d26a..589357cd 100644 --- a/tst/unit/radiation/CMakeLists.txt +++ b/tst/unit/radiation/CMakeLists.txt @@ -20,8 +20,7 @@ add_library(radiation_unit_tests OBJECT target_link_libraries(radiation_unit_tests PRIVATE parthenon - singularity-eos::flags - singularity-eos + singularity-eos::singularity-eos Catch2::Catch2 Kokkos::kokkos) From 4958d00e9570f98f1046191835e3c9622da0824d Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Wed, 17 Aug 2022 17:39:33 -0600 Subject: [PATCH 5/8] remove namespace conflict with Robust --- src/radiation/mocmc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/radiation/mocmc.cpp b/src/radiation/mocmc.cpp index 51d00e1f..80652540 100644 --- a/src/radiation/mocmc.cpp +++ b/src/radiation/mocmc.cpp @@ -37,8 +37,8 @@ namespace cr = radmoment_cons; namespace pr = radmoment_prim; namespace ir = radmoment_internal; namespace im = mocmc_internal; -using namespace singularity; using namespace singularity::neutrinos; +using singularity::EOS; using singularity::RadiationType; using vpack_types::FlatIdx; From 46bc6542803e914394f78c7e6e72709aa57bb0d8 Mon Sep 17 00:00:00 2001 From: Jonah Maxwell Miller Date: Fri, 19 Aug 2022 19:06:53 -0600 Subject: [PATCH 6/8] one fewer warning --- src/geometry/boyer_lindquist.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/geometry/boyer_lindquist.hpp b/src/geometry/boyer_lindquist.hpp index fef6d815..e646dda1 100644 --- a/src/geometry/boyer_lindquist.hpp +++ b/src/geometry/boyer_lindquist.hpp @@ -77,7 +77,6 @@ class BoyerLindquist { const Real th = X2; Real r2, r3, sth, cth, DD, mu; ComputeDeltaMu_(r, th, r2, r3, sth, cth, DD, mu); - const Real r3dm = r3 * DD * mu; const Real sth2 = sth * sth; LinearAlgebra::SetZero(g, NDFULL, NDFULL); g[0][0] = -(1.0 - ratio(2.0, r * mu)); From a49a8dd35e86b8b7c9a61f55654bd2374a7af149 Mon Sep 17 00:00:00 2001 From: Jonah Maxwell Miller Date: Mon, 22 Aug 2022 16:40:52 -0600 Subject: [PATCH 7/8] get around stupid cray issue --- CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index e89500c8..c42a14d5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -99,6 +99,9 @@ if(PHOEBUS_ENABLE_CUDA) set(Kokkos_ENABLE_CUDA_LAMBDA ON CACHE BOOL "" FORCE) set(Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE ON CACHE BOOL "" FORCE) set(Kokkos_ENABLE_CUDA_LAMBDA ON CACHE BOOL "" FORCE) + # Get around a bug with atomics on Cray + # This option will eventually be deprecated, so care required. + set(Kokkos_ENABLE_IML_DESUL_ATOMICS OFF CACHE BOOL "" FORCE) endif() # HDF5 From 26be574c4761f0e3ab1e94c5626582445544aae5 Mon Sep 17 00:00:00 2001 From: Jonah Maxwell Miller Date: Mon, 22 Aug 2022 17:00:40 -0600 Subject: [PATCH 8/8] oops --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c42a14d5..d48317d0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -101,7 +101,7 @@ if(PHOEBUS_ENABLE_CUDA) set(Kokkos_ENABLE_CUDA_LAMBDA ON CACHE BOOL "" FORCE) # Get around a bug with atomics on Cray # This option will eventually be deprecated, so care required. - set(Kokkos_ENABLE_IML_DESUL_ATOMICS OFF CACHE BOOL "" FORCE) + set(Kokkos_ENABLE_IMPL_DESUL_ATOMICS OFF CACHE BOOL "" FORCE) endif() # HDF5