diff --git a/src/fluid/fluid.cpp b/src/fluid/fluid.cpp index f3cd8058..f40025f0 100644 --- a/src/fluid/fluid.cpp +++ b/src/fluid/fluid.cpp @@ -318,6 +318,11 @@ std::shared_ptr Initialize(ParameterInput *pin) { HstSum, ReduceMom, "total X" + std::to_string(d + 1) + " momentum")); } + AllReduce> net_field_totals; + AllReduce> net_field_totals_2; + physics->AddParam<>("net_field_totals", net_field_totals, true); + physics->AddParam<>("net_field_totals_2", net_field_totals_2, true); + params.Add(parthenon::hist_param_key, hst_vars); // Fill Derived and Estimate Timestep diff --git a/src/main.cpp b/src/main.cpp index e21e65e8..692c1543 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -57,27 +57,30 @@ int main(int argc, char *argv[]) { Boundaries::ProcessBoundaryConditions(pman); // call ParthenonInit to set up the mesh + /* scope the driver so that its destructors are called before MPI/Kokkos Finalize */ + /* Not necessary,as is, but safer. */ pman.ParthenonInitPackagesAndMesh(); + { - // call post-initialization - if (!pman.IsRestart()) { - phoebus::PostInitializationModifier(pman.pinput.get(), pman.pmesh.get()); - } + // call post-initialization + if (!pman.IsRestart()) { + phoebus::PostInitializationModifier(pman.pinput.get(), pman.pmesh.get()); + } - // Initialize the driver - phoebus::PhoebusDriver driver(pman.pinput.get(), pman.app_input.get(), pman.pmesh.get(), - pman.IsRestart()); + // Initialize the driver + phoebus::PhoebusDriver driver(pman.pinput.get(), pman.app_input.get(), + pman.pmesh.get(), pman.IsRestart()); - // Communicate ghost buffers before executing - driver.PostInitializationCommunication(); + // Communicate ghost buffers before executing + driver.PostInitializationCommunication(); - // This line actually runs the simulation - auto driver_status = driver.Execute(); + // This line actually runs the simulation + auto driver_status = driver.Execute(); #if CON2PRIM_STATISTICS - con2prim_statistics::Stats::report(); + con2prim_statistics::Stats::report(); #endif - + } // call MPI_Finalize and Kokkos::finalize if necessary pman.ParthenonFinalize(); diff --git a/src/phoebus_driver.cpp b/src/phoebus_driver.cpp index 2e406fcf..82125e2e 100644 --- a/src/phoebus_driver.cpp +++ b/src/phoebus_driver.cpp @@ -335,11 +335,17 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { // Extra per-step user work TaskRegion &sync_region_5 = tc.AddRegion(num_partitions); int sync_region_5_dep_id; - net_field_totals.val.resize(2); // Mdot, Phi - net_field_totals_2.val.resize(2); + + // Mdot, Phi + AllReduce> *net_field_totals = + fluid->MutableParam>>("net_field_totals"); + AllReduce> *net_field_totals_2 = + fluid->MutableParam>>("net_field_totals_2"); + net_field_totals->val.resize(2); + net_field_totals_2->val.resize(2); for (int i = 0; i < 2; i++) { - net_field_totals.val[i] = 0.; - net_field_totals_2.val[i] = 0.; + net_field_totals->val[i] = 0.; + net_field_totals_2->val[i] = 0.; } for (int i = 0; i < num_partitions; i++) { sync_region_5_dep_id = 0; @@ -352,17 +358,17 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { // Evaluate current Mdot, Phi auto sum_mdot_1 = tl.AddTask(none, fixup::SumMdotPhiForNetFieldScaling, md.get(), t, - stage, &(net_field_totals.val)); + stage, &(net_field_totals->val)); sync_region_5.AddRegionalDependencies(sync_region_5_dep_id, i, sum_mdot_1); sync_region_5_dep_id++; TaskID start_reduce_1 = (i == 0 ? tl.AddTask(sum_mdot_1, fixup::NetFieldStartReduce, - md.get(), t, stage, &net_field_totals) + md.get(), t, stage, net_field_totals) : none); // Test the reduction until it completes TaskID finish_reduce_1 = tl.AddTask(start_reduce_1, fixup::NetFieldCheckReduce, - md.get(), t, stage, &net_field_totals); + md.get(), t, stage, net_field_totals); sync_region_5.AddRegionalDependencies(sync_region_5_dep_id, i, finish_reduce_1); sync_region_5_dep_id++; @@ -371,16 +377,16 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { // Evaluate Mdot, Phi (only Phi changes) after modifying B field auto sum_mdot_2 = tl.AddTask(mod_net, fixup::SumMdotPhiForNetFieldScaling, md.get(), - t, stage, &(net_field_totals_2.val)); + t, stage, &(net_field_totals_2->val)); sync_region_5.AddRegionalDependencies(sync_region_5_dep_id, i, sum_mdot_2); sync_region_5_dep_id++; TaskID start_reduce_2 = (i == 0 ? tl.AddTask(sum_mdot_2, fixup::NetFieldStartReduce, - md.get(), t, stage, &net_field_totals_2) + md.get(), t, stage, net_field_totals_2) : none); // Test the reduction until it completes TaskID finish_reduce_2 = tl.AddTask(start_reduce_2, fixup::NetFieldCheckReduce, - md.get(), t, stage, &net_field_totals_2); + md.get(), t, stage, net_field_totals_2); sync_region_5.AddRegionalDependencies(sync_region_5_dep_id, i, finish_reduce_2); sync_region_5_dep_id++; @@ -390,7 +396,7 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { auto set_scale = tl.AddTask(mod_net_2, fixup::UpdateNetFieldScaleControls, md.get(), t, dt, stage, - &(net_field_totals.val), &(net_field_totals_2.val)); + &(net_field_totals->val), &(net_field_totals_2->val)); // End tuning region that only evaluates occasionally and on first stage @@ -596,26 +602,28 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { auto floors = tl.AddTask(radfixup, fixup::ApplyFloors>, sc.get()); if (rad_mocmc_active && stage == integrator->nstages) { - particle_resolution.val.resize(1); // total + AllReduce> *particle_resolution = + rad->MutableParam>>("particle_resolution"); + particle_resolution->val.resize(1); // total // for (int i = 0; i < particle_resolution.val.size(); i++) { - particle_resolution.val[i] = 0.; + particle_resolution->val[i] = 0.; //} int reg_dep_id = 0; // auto &tl = async_region_3[0]; auto update_count = tl.AddTask(none, radiation::MOCMCUpdateParticleCount, pmesh, - &particle_resolution.val); + &particle_resolution->val); async_region_3.AddRegionalDependencies(reg_dep_id, 0, update_count); reg_dep_id++; auto start_count_reduce = tl.AddTask(update_count, &AllReduce>::StartReduce, - &particle_resolution, MPI_SUM); + particle_resolution, MPI_SUM); auto finish_count_reduce = tl.AddTask(start_count_reduce, &AllReduce>::CheckReduce, - &particle_resolution); + particle_resolution); async_region_3.AddRegionalDependencies(reg_dep_id, 0, finish_count_reduce); reg_dep_id++; @@ -629,7 +637,7 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { << std::endl; return TaskStatus::complete; }, - &particle_resolution.val) + &particle_resolution->val) : none); } @@ -914,27 +922,30 @@ TaskListStatus PhoebusDriver::MonteCarloStep() { **/ TaskRegion &tuning_region = tc.AddRegion(1); { - particle_resolution.val.resize(4); // made, absorbed, scattered, total + auto rad = pmesh->packages.Get("radiation"); + AllReduce> *particle_resolution = + rad->MutableParam>>("particle_resolution"); + particle_resolution->val.resize(4); // made, absorbed, scattered, total for (int i = 0; i < 4; i++) { - particle_resolution.val[i] = 0.; + particle_resolution->val[i] = 0.; } int reg_dep_id = 0; auto &tl = tuning_region[0]; auto update_resolution = tl.AddTask(none, radiation::MonteCarloUpdateParticleResolution, pmesh, - &particle_resolution.val); + &particle_resolution->val); tuning_region.AddRegionalDependencies(reg_dep_id, 0, update_resolution); reg_dep_id++; auto start_resolution_reduce = tl.AddTask(update_resolution, &AllReduce>::StartReduce, - &particle_resolution, MPI_SUM); + particle_resolution, MPI_SUM); auto finish_resolution_reduce = tl.AddTask(start_resolution_reduce, &AllReduce>::CheckReduce, - &particle_resolution); + particle_resolution); tuning_region.AddRegionalDependencies(reg_dep_id, 0, finish_resolution_reduce); reg_dep_id++; @@ -949,12 +960,12 @@ TaskListStatus PhoebusDriver::MonteCarloStep() { << " total = " << (*res)[3] << std::endl; return TaskStatus::complete; }, - &particle_resolution.val) + &particle_resolution->val) : none); auto update_tuning = tl.AddTask(finish_resolution_reduce, radiation::MonteCarloUpdateTuning, pmesh, - &particle_resolution.val, t0, dt); + &particle_resolution->val, t0, dt); } status = tc.Execute(); diff --git a/src/phoebus_driver.hpp b/src/phoebus_driver.hpp index 9caf2486..13495ab7 100644 --- a/src/phoebus_driver.hpp +++ b/src/phoebus_driver.hpp @@ -42,12 +42,6 @@ class PhoebusDriver : public EvolutionDriver { const bool is_restart_; Real dt_init, dt_init_fact; - AllReduce dNtot; - AllReduce> particle_resolution; - AllReduce particles_outstanding; - - AllReduce> net_field_totals; - AllReduce> net_field_totals_2; }; parthenon::Packages_t ProcessPackages(std::unique_ptr &pin); diff --git a/src/radiation/radiation.cpp b/src/radiation/radiation.cpp index 2e89efeb..ea7203aa 100644 --- a/src/radiation/radiation.cpp +++ b/src/radiation/radiation.cpp @@ -319,6 +319,14 @@ std::shared_ptr Initialize(ParameterInput *pin) { } if (method == "monte_carlo" || method == "mocmc") { + // reducers + AllReduce dNtot; + AllReduce> particle_resolution; + AllReduce particles_outstanding; + physics->AddParam<>("particle_resolution", particle_resolution, true); + physics->AddParam<>("particles_outstanding", particles_outstanding, true); + physics->AddParam<>("dNtot", dNtot, true); + // Initialize random number generator pool int rng_seed = pin->GetOrAddInteger("radiation", "rng_seed", 238947); physics->AddParam<>("rng_seed", rng_seed);