Passing all tests on cpu and gpu

src/compute/boundaryConditions/periodics.cpp

@@ -7,8 +7,8 @@
 
 void periodicRotHigh(block_ b,
                      face_ face,
-                     const std::function<void(block_, thtrdat_, int, std::string)> &eos,
-                     const thtrdat_ th,
+                     const std::function<void(block_, thtrdat_, int, std::string)> &/*eos*/,
+                     const thtrdat_ /*th*/,
                      const std::string terms,
                      const double /*tme*/) {
   //-------------------------------------------------------------------------------------------|
@@ -87,8 +87,8 @@ void periodicRotHigh(block_ b,
 
 void periodicRotLow(block_ b,
                      face_ face,
-                     const std::function<void(block_, thtrdat_, int, std::string)> &eos,
-                     const thtrdat_ th,
+                     const std::function<void(block_, thtrdat_, int, std::string)> &/*eos*/,
+                     const thtrdat_ /*th*/,
                      const std::string terms,
                      const double /*tme*/) {
   //-------------------------------------------------------------------------------------------|

src/peregrinepy/consistify.py

@@ -27,7 +27,6 @@ def consistify(mb):
         # Update spatial derivatives
         mb.dqdxyz(blk)
 
-        # TODO: can we get rid of this if check?
         if mb.config["RHS"]["diffusion"]:
             # Apply viscous boundary conditions
             for face in blk.faces:

src/peregrinepy/grid/unifySolverGrid.py

@@ -11,6 +11,10 @@ def unifySolverGrid(mb):
     for _ in range(3):
         mpiComm.communicate(mb, ["x", "y", "z"])
 
+    # Device is up to date after communicate, so pull back down
+    for blk in mb:
+        blk.updateHostView(["x", "y", "z"])
+
     for blk in mb:
         for face in blk.faces:
             bc = face.bcType
@@ -42,6 +46,6 @@ def unifySolverGrid(mb):
                     y[:] = tempy[:]
                     z[:] = tempz[:]
 
+    # Push back up the device
     for blk in mb:
-        # Push back up the device
         blk.updateDeviceView(["x", "y", "z"])

tests/boundaryConditions/test_periodics.py

@@ -23,12 +23,10 @@
 class TestPeriodics:
     @classmethod
     def setup_class(self):
-        # MPI.Init()
         pass
 
     @classmethod
     def teardown_class(self):
-        # MPI.Finalize()
         pass
 
     def test_rotationalPeriodics(self, my_setup, adv, spdata):
@@ -91,6 +89,8 @@ def test_rotationalPeriodics(self, my_setup, adv, spdata):
         mb.eos(blk, mb.thtrdat, 0, "prims")
         pg.consistify(mb)
 
+        blk.updateHostView(["q", "dqdx", "dqdy", "dqdz"])
+
         u = blk.array["q"][:, :, :, 1]
         v = blk.array["q"][:, :, :, 2]
         w = blk.array["q"][:, :, :, 3]
@@ -119,8 +119,6 @@ def test_rotationalPeriodics(self, my_setup, adv, spdata):
             )
             velo6 = np.column_stack((u[s6c].ravel(), v[s6c].ravel(), w[s6c].ravel()))
 
-            # dot the velocities with the face normals kills two birds with one stone
-            # check that velo and gradients are rotated correctly and also halo
             assert np.allclose(
                 np.sum(normals5 * velo5, axis=1), np.sum(normals6 * velo6, axis=1)
             )
@@ -145,3 +143,42 @@ def test_rotationalPeriodics(self, my_setup, adv, spdata):
                 assert np.allclose(
                     np.sum(normals5 * dqdx5, axis=1), np.sum(normals6 * dqdx6, axis=1)
                 )
+
+            # Now check the other way
+            # face6 halo compares to interior on side 5
+            s6 = np.s_[:, :, -(g + 1)]
+            s5c = np.s_[:, :, 2 * ng - g - 1]
+            s5f = np.s_[:, :, 2 * ng - g]
+
+            normals6 = np.column_stack((nx[s6].ravel(), ny[s6].ravel(), nz[s6].ravel()))
+            velo6 = np.column_stack((u[s6].ravel(), v[s6].ravel(), w[s6].ravel()))
+
+            normals5 = np.column_stack(
+                (nx[s5f].ravel(), ny[s5f].ravel(), nz[s5f].ravel())
+            )
+            velo5 = np.column_stack((u[s5c].ravel(), v[s5c].ravel(), w[s5c].ravel()))
+
+            assert np.allclose(
+                np.sum(normals6 * velo6, axis=1), np.sum(normals5 * velo5, axis=1)
+            )
+
+            # check the gradients
+            for i in range(blk.ne):
+                dqdx6 = np.column_stack(
+                    (
+                        dqdx[s6][:, :, i].ravel(),
+                        dqdy[s6][:, :, i].ravel(),
+                        dqdz[s6][:, :, i].ravel(),
+                    )
+                )
+
+                dqdx5 = np.column_stack(
+                    (
+                        dqdx[s5c][:, :, i].ravel(),
+                        dqdy[s5c][:, :, i].ravel(),
+                        dqdz[s5c][:, :, i].ravel(),
+                    )
+                )
+                assert np.allclose(
+                    np.sum(normals6 * dqdx6, axis=1), np.sum(normals5 * dqdx5, axis=1)
+                )

tests/conftest.py

@@ -9,8 +9,8 @@
 
 @pytest.fixture(scope="session")
 def my_setup(request):
-    kokkos.initialize()
     MPI.Init()
+    kokkos.initialize()
 
     def fin():
         kokkos.finalize()