Merge branch 'brhillman/mmf/integrate-rrtmgpxx-new' (PR E3SM-Project#…

…4585)

Enable using the C++ back end of RRTMGP for both E3SM and E3SM-MMF.

The C++ code uses YAKL to enable running on CPU or GPU with the same
codebase. Maintains the ability to run the F90 back end as well
(remains the default). Building with the C++ version is enabled by
adding -rrtmgpxx to CAM_CONFIG_OPTS in env_build.xml. Note that this
is in addition to using the -rad rrtmgp option. The two flags are
required because -rad rrtmgp sets much of the logic that determines
input data and settings in configure and build-namelist, which we do
not want to duplicate just because we are running the C++ back end
instead of the F90 back end. A testmod (eam-rrtmgpxx) is also added
so that RRTMGPXX can be built via create_test without any user
interaction. Also updates both the YAKL and RRTMGP submodules.
Believe it or not, this is BFB.

[BFB]

* brhillman/mmf/integrate-rrtmgpxx-new: (71 commits)
  Explicitly use pool allocator for ESMT test
  Make sure rrtmgp_interface gets netcdf includes
  Only copy to/from device as needed
  Upgrading to new YAKL with a new pool allocator that uses the avilable space more efficiently and turns off the pool for targets without a separate memory address space.
  Add RRTMGPXX test to E3SM integration
  Make RRTMGPXX default for all MMF compsets
  Removed unused commented out code
  Revert to RRTMGP I/O
  Update to latest RRTMGP master after rrtmgp_const fix
  Add radiation_final for CRM rrtmg
  Remove netcdf-cxx4 from submodules
  Add radiation_final to rrtmg interfaces
  Removing unused declarations, and changing rrtmgp interface build system to use the new YAKL target-based approach
  Changing const.h to rrtmgp_const.h for rrtmgpxx
  Getting rid of fast math in DEBUG CUDA_FLAGS, and adding -g to DEBUG CUDA_FLAGS
  Update to latest RRTMGP master with bugfix
  Adding debug flags to CXXFLAGS for summit+gnu
  Wrapping data in host Arrays, then copying to device, then copying back at the end; for SW and LW
  Moving to latest YAKL, which no longer fails upon multiple calls to yakl::initialize and yakl::finalize
  Fix CUDA_FLAGS for DEBUG=TRUE
  ...

cime_config/machines/config_compilers.xml

@@ -1492,6 +1492,9 @@ flags should be captured within MPAS CMake files.
   <FFLAGS>
     <append DEBUG="FALSE"> -O2 </append>
   </FFLAGS>
+  <CXXFLAGS>
+    <append DEBUG="TRUE" > -O0 -g </append>
+  </CXXFLAGS>
   <CPPDEFS>
     <append COMP_NAME="gptl"> -DHAVE_SLASHPROC </append>
   </CPPDEFS>
@@ -1547,7 +1550,8 @@ flags should be captured within MPAS CMake files.
   <PNETCDF_PATH>$ENV{PNETCDF_PATH}</PNETCDF_PATH>
   <SUPPORTS_CXX>TRUE</SUPPORTS_CXX>
   <CUDA_FLAGS>
-    <append> -O3 -arch sm_70 --use_fast_math </append>
+    <append DEBUG="FALSE"> -O3 -arch sm_70 --use_fast_math </append>
+    <append DEBUG="TRUE" > -O0 -g -arch sm_70 </append>
   </CUDA_FLAGS>
   <USE_CUDA>TRUE</USE_CUDA>
 </compiler>
@@ -1770,7 +1774,8 @@ flags should be captured within MPAS CMake files.
   <PNETCDF_PATH>$ENV{PNETCDF_PATH}</PNETCDF_PATH>
   <SUPPORTS_CXX>TRUE</SUPPORTS_CXX>
   <CUDA_FLAGS>
-    <append> -O3 -arch sm_70 --use_fast_math </append>
+    <append DEBUG="FALSE"> -O3 -arch sm_70 --use_fast_math </append>
+    <append DEBUG="TRUE" > -O0 -g -arch sm_70 </append>
   </CUDA_FLAGS>
   <USE_CUDA>TRUE</USE_CUDA>
 </compiler>

cime_config/tests.py

@@ -70,6 +70,7 @@
             "SMS_Ln1.ne4_oQU240.F2010.eam-chem_pp",
             "SMS_D_Ln5.ne4_oQU240.F2010.eam-clubb_sp",
             "ERS_Ld5.ne4_oQU240.F2010.eam-rrtmgp",
+            "ERS_Ld5.ne4_oQU240.F2010.eam-rrtmgpxx",
             "REP_Ln5.ne4_oQU240.F2010",
             "SMS_Ld9.ne4pg2_oQU480.F2010.eam-thetahy_sl_pg2_mass",
             )

components/cmake/build_model.cmake

@@ -105,6 +105,31 @@ function(build_model COMP_CLASS COMP_NAME)
                              cmake/atm/../../eam/src/physics/crm/crm_ecpp_output_module.F90 )
     endif()
 
+    # Add rrtmgp++ source code if asked for
+    if (USE_RRTMGPXX)
+      message(STATUS "Building RRTMGPXX")
+      # Build the static rrtmgpxx library
+      set(RRTMGPXX_BIN ${CMAKE_CURRENT_BINARY_DIR}/rrtmgp)
+      add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../../eam/src/physics/rrtmgp/external/cpp ${RRTMGPXX_BIN})
+      # Build the interface code
+      set(RRTMGPXX_INTERFACE_BIN ${CMAKE_CURRENT_BINARY_DIR}/rrtmgp_interface)
+      add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../../eam/src/physics/rrtmgp/cpp ${RRTMGPXX_INTERFACE_BIN})
+      # Interface code needs some additional headers
+      target_include_directories(rrtmgp_interface PRIVATE
+          ${CMAKE_CURRENT_SOURCE_DIR}/../../eam/src/physics/rrtmgp/external/cpp/extensions/fluxes_byband
+          ${CMAKE_CURRENT_SOURCE_DIR}/../../eam/src/physics/rrtmgp/external/cpp/extensions/cloud_optics
+          ${CMAKE_CURRENT_SOURCE_DIR}/../../eam/src/physics/rrtmgp/cpp
+      )
+      # The interface code needs to know about the NETCDF includes defined
+      # above. The easiest way I know of to do this is to pass all of the
+      # accumulated includes to the target.
+      # TODO: this can go away if the above NETCDF section is refactored to
+      # use find_library instead of appending to INCLDIR.
+      target_include_directories(rrtmgp_interface PRIVATE ${INCLDIR})
+      # Add the source files for the interface code to the main E3SM build
+      set(RRTMGPXX_F90 cmake/atm/../../eam/src/physics/rrtmgp/cpp/rrtmgp_interface.F90)
+      set(SOURCES ${SOURCES} ${RRTMGPXX_F90})
+    endif()
   endif()
 
   #-------------------------------------------------------------------------------
@@ -248,6 +273,9 @@ function(build_model COMP_CLASS COMP_NAME)
       if (USE_SAMXX)
         target_link_libraries(${TARGET_NAME} PRIVATE samxx)
       endif()
+      if (USE_RRTMGPXX)
+          target_link_libraries(${TARGET_NAME} PRIVATE rrtmgp rrtmgp_interface)
+      endif()
     endif()
     if (USE_KOKKOS)
       target_link_libraries (${TARGET_NAME} PRIVATE Kokkos::kokkos)

components/cmake/common_setup.cmake

@@ -35,8 +35,18 @@ if (NOT HAS_SAMXX EQUAL -1)
   set(USE_SAMXX TRUE)
 endif()
 
-# If samxx is being used, then YAKL must be used as well
-set(USE_YAKL ${USE_SAMXX})
+string(FIND "${CAM_CONFIG_OPTS}" "-rrtmgpxx" HAS_RRTMGPXX)
+if (NOT HAS_RRTMGPXX EQUAL -1)
+  # The following is for the RRTMGPXX code:
+  set(USE_RRTMGPXX TRUE)
+endif()
+
+# If samxx or rrtmgpxx is being used, then YAKL must be used as well
+if (USE_SAMXX OR USE_RRTMGPXX)
+    set(USE_YAKL TRUE)
+else()
+    set(USE_YAKL FALSE)
+endif()
 
 # If YAKL is being used, then we need to enable USE_CXX
 if (${USE_YAKL})

components/eam/bld/configure

@@ -279,6 +279,7 @@ OPTIONS
      -use_ECPP          use CRM clouds for vertical transport, aqueous chemistry and wet removable of aerosols
      -crm_adv           CRM advection scheme [MPDATA | UM5]
      -crm <model>       CRM model [sam | samomp | samxx]
+     -rrtmgpxx          Use RRTMGP++ code
 EOF
 }
 
@@ -351,6 +352,7 @@ GetOptions(
     "MMF_microphysics_scheme=s"     => \$opts{'MMF_microphysics_scheme'},
     "crm_adv=s"                 => \$opts{'crm_adv'},
     "crm=s"                     => \$opts{'crm'},
+    "rrtmgpxx"                  => \$opts{'rrtmgpxx'},
     "debug"                     => \$opts{'debug'},
     "rain_evap_to_coarse_aero"  => \$opts{'rain_evap_to_coarse_aero'},
     "bc_dep_to_snow_updates"    => \$opts{'bc_dep_to_snow_updates'},
@@ -2608,13 +2610,16 @@ sub write_filepath
         print $fh "$camsrcdir/eam/src/physics/rrtmg/ext/rrtmg_sw\n";
     } elsif ($rad eq 'rrtmgp') {
         print $fh "$camsrcdir/eam/src/physics/rrtmgp\n";
-        print $fh "$camsrcdir/eam/src/physics/rrtmgp/external/rte\n";
-        print $fh "$camsrcdir/eam/src/physics/rrtmgp/external/rte/kernels\n";
-        print $fh "$camsrcdir/eam/src/physics/rrtmgp/external/rrtmgp\n";
-        print $fh "$camsrcdir/eam/src/physics/rrtmgp/external/rrtmgp/kernels\n";
-        print $fh "$camsrcdir/eam/src/physics/rrtmgp/external/extensions\n";
-        print $fh "$camsrcdir/eam/src/physics/rrtmgp/external/extensions/rng\n";
-        print $fh "$camsrcdir/eam/src/physics/rrtmgp/external/examples\n";
+        if (not defined $opts{'rrtmgpxx'} ) {
+            print $fh "$camsrcdir/eam/src/physics/rrtmgp/external/rte\n";
+            print $fh "$camsrcdir/eam/src/physics/rrtmgp/external/rte/kernels\n";
+            print $fh "$camsrcdir/eam/src/physics/rrtmgp/external/rrtmgp\n";
+            print $fh "$camsrcdir/eam/src/physics/rrtmgp/external/rrtmgp/kernels\n";
+            print $fh "$camsrcdir/eam/src/physics/rrtmgp/external/extensions\n";
+            print $fh "$camsrcdir/eam/src/physics/rrtmgp/external/extensions/rng\n";
+            print $fh "$camsrcdir/eam/src/physics/rrtmgp/external/examples\n";
+            print $fh "$camsrcdir/eam/src/physics/rrtmgp/f90\n";
+        }
     }
 
     print $fh "$camsrcdir/eam/src/physics/cam\n";

components/eam/cime_config/config_component.xml

@@ -57,7 +57,7 @@
       <value compset="_EAM%MMF"        >-use_MMF -crm_adv MPDATA -nlev 60 -crm_nz 50 </value>
       <value compset="_EAM%MMF"        >-crm_dx 2000 -crm_dt 10 </value>
       <value compset="_EAM%MMF"        >-crm_nx 64 -crm_nx_rad 4 -crm_ny 1 -crm_ny_rad 1 </value>
-      <value compset="_EAM%MMF"        >-rad rrtmgp </value>
+      <value compset="_EAM%MMF"        >-rad rrtmgp -rrtmgpxx</value>
       <value compset="_EAM%MMF[12]"    >-crm sam</value>
       <value compset="_EAM%MMFXX"      >-crm samxx</value>
       <value compset="_EAM%MMFOMP"     >-crm samomp</value>

components/eam/cime_config/testdefs/testmods_dirs/eam/mmf_use_ESMT/shell_commands

@@ -1 +1 @@
-./xmlchange --append -id CAM_CONFIG_OPTS -val " -cppdefs ' -DMMF_ESMT -DMMF_USE_ESMT ' "
+./xmlchange --append -id CAM_CONFIG_OPTS -val " -cppdefs ' -DMMF_ESMT -DMMF_USE_ESMT -DYAKL_SEPARATE_MEMORY_SPACE ' "

components/eam/cime_config/testdefs/testmods_dirs/eam/rrtmgpxx/shell_commands

@@ -0,0 +1,2 @@
+#!/bin/bash
+./xmlchange --append CAM_CONFIG_OPTS='-rad rrtmgp -rrtmgpxx'

components/eam/src/physics/cam/flux_avg.F90

@@ -76,11 +76,11 @@ subroutine flux_avg_init(cam_in,  pbuf2d)
       ncol = get_ncols_p(lchnk)
       pbuf2d_chunk => pbuf_get_chunk(pbuf2d, lchnk)
 
-      call pbuf_set_field(pbuf2d_chunk, lhflx_idx,  cam_in(lchnk)%lhf(:ncol))
-      call pbuf_set_field(pbuf2d_chunk, shflx_idx,  cam_in(lchnk)%shf(:ncol))
-      call pbuf_set_field(pbuf2d_chunk, qflx_idx,   cam_in(lchnk)%cflx(:ncol,1))
-      call pbuf_set_field(pbuf2d_chunk, taux_idx,   cam_in(lchnk)%wsx(:ncol))
-      call pbuf_set_field(pbuf2d_chunk, tauy_idx,   cam_in(lchnk)%wsy(:ncol))
+      call pbuf_set_field(pbuf2d_chunk, lhflx_idx,  cam_in(lchnk)%lhf(:))
+      call pbuf_set_field(pbuf2d_chunk, shflx_idx,  cam_in(lchnk)%shf(:))
+      call pbuf_set_field(pbuf2d_chunk, qflx_idx,   cam_in(lchnk)%cflx(:,1))
+      call pbuf_set_field(pbuf2d_chunk, taux_idx,   cam_in(lchnk)%wsx(:))
+      call pbuf_set_field(pbuf2d_chunk, tauy_idx,   cam_in(lchnk)%wsy(:))
 
       call pbuf_set_field(pbuf2d_chunk, shflx_res_idx, 0.0_r8)
       call pbuf_set_field(pbuf2d_chunk, lhflx_res_idx, 0.0_r8)

components/eam/src/physics/cam/physpkg.F90

@@ -1357,6 +1357,7 @@ subroutine phys_final( phys_state, phys_tend, pbuf2d )
     use physics_buffer, only : physics_buffer_desc, pbuf_deallocate
     use chemistry, only : chem_final
     use wv_saturation, only : wv_sat_final
+    use radiation, only: radiation_final
     !----------------------------------------------------------------------- 
     ! 
     ! Purpose: 
@@ -1383,6 +1384,10 @@ subroutine phys_final( phys_state, phys_tend, pbuf2d )
     call wv_sat_final
     call t_stopf ('wv_sat_final')
 
+    call t_startf ('radiation_final')
+    call radiation_final()
+    call t_stopf ('radiation_final')
+
     call t_startf ('print_cost_p')
     call print_cost_p
     call t_stopf ('print_cost_p')

components/eam/src/physics/crm/physpkg.F90

@@ -985,7 +985,8 @@ subroutine phys_final( phys_state, phys_tend, pbuf2d )
   use physics_buffer, only : physics_buffer_desc, pbuf_deallocate
   use chemistry,      only : chem_final
   use wv_saturation,  only : wv_sat_final
-  use crm_physics, only: crm_physics_final
+  use crm_physics,    only : crm_physics_final
+  use radiation,      only : radiation_final
   !----------------------------------------------------------------------- 
   ! Purpose: Finalization of physics package
   !-----------------------------------------------------------------------
@@ -1011,6 +1012,10 @@ subroutine phys_final( phys_state, phys_tend, pbuf2d )
   call wv_sat_final
   call t_stopf ('wv_sat_final')
 
+  call t_startf ('radiation_final')
+  call radiation_final()
+  call t_stopf ('radiation_final')
+
   call t_startf ('crm_physics_final')
   call crm_physics_final()
   call t_stopf ('crm_physics_final')

components/eam/src/physics/crm/rrtmg/radiation.F90

@@ -50,6 +50,7 @@ module radiation
    radiation_nextsw_cday, &! calendar day of next radiation calculation
    radiation_do,          &! query which radiation calcs are done this timestep
    radiation_init,        &! calls radini
+   radiation_final,       &! deallocate
    radiation_readnl,      &! read radiation namelist
    radiation_tend          ! moved from radctl.F90
 
@@ -839,6 +840,13 @@ subroutine radiation_init(phys_state)
 
   end subroutine radiation_init
 
+!===============================================================================
+
+  subroutine radiation_final()
+    ! Do any needed clean-up and deallocation before model exit. Empty for now
+    ! but required for consistency with RRTMGPXX interface.
+  end subroutine radiation_final
+
 !===============================================================================
 
   subroutine radiation_tend( state, ptend,pbuf, &