Switch global ocean in init mode to E3SM shared constants

cime_config/machines/cmake_macros/nvidiagpu_muller-gpu.cmake

@@ -6,8 +6,9 @@ if (COMP_NAME STREQUAL gptl)
 endif()
 string(APPEND CPPDEFS " -DTHRUST_IGNORE_CUB_VERSION_CHECK")
 string(APPEND CMAKE_CUDA_FLAGS " -ccbin CC -O2 -arch sm_80 --use_fast_math")
-string(APPEND CMAKE_EXE_LINKER_FLAGS " -acc -gpu=cc70,cc60 -Minfo=accel")
+set(CMAKE_CUDA_ARCHITECTURES "80")
+string(APPEND CMAKE_Fortran_FLAGS " -acc -gpu=cc80 -Minfo=accel")
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -acc -gpu=cc80 -Minfo=accel")
 set(SCC "cc")
 set(SCXX "CC")
 set(SFC "ftn")
-string(APPEND CMAKE_Fortran_FLAGS " -acc -gpu=cc70,cc60 -Minfo=accel")

cime_config/machines/cmake_macros/nvidiagpu_pm-gpu.cmake

@@ -6,8 +6,9 @@ if (COMP_NAME STREQUAL gptl)
 endif()
 string(APPEND CPPDEFS " -DTHRUST_IGNORE_CUB_VERSION_CHECK")
 string(APPEND CMAKE_CUDA_FLAGS " -ccbin CC -O2 -arch sm_80 --use_fast_math")
-string(APPEND CMAKE_EXE_LINKER_FLAGS " -acc -gpu=cc70,cc60 -Minfo=accel")
+set(CMAKE_CUDA_ARCHITECTURES "80")
+string(APPEND CMAKE_Fortran_FLAGS " -acc -gpu=cc80 -Minfo=accel")
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -acc -gpu=cc80 -Minfo=accel")
 set(SCC "cc")
 set(SCXX "CC")
 set(SFC "ftn")
-string(APPEND CMAKE_Fortran_FLAGS " -acc -gpu=cc70,cc60 -Minfo=accel")

cime_config/machines/config_machines.xml

@@ -2859,34 +2859,45 @@
         <command name="load">anaconda3/5.2.0</command>
       </modules>
       <modules compiler="intel">
-        <command name="load">intel/18.0.4-443hhug</command>
-        <command name="load">intel-mkl/2018.4.274-jwaeshj</command>
-        <command name="load">hdf5/1.10.5-3mk3uik</command>
-        <command name="load">netcdf/4.7.0-krelxcz</command>
-        <command name="load">netcdf-fortran/4.4.5-74lj75q</command>
+        <command name="load">gcc/7.4.0</command>
+        <command name="load">intel/20.0.4-lednsve</command>
+        <command name="load">intel-mkl/2020.4.304-voqlapk</command>
       </modules>
       <modules compiler="intel" mpilib="impi">
-        <command name="load">intel-mpi/2018.4.274-4hmwfl6</command>
-        <command name="load">parallel-netcdf/1.11.0-acswzws</command>
+        <command name="load">intel-mpi/2019.9.304-i42whlw</command>
+        <command name="load">hdf5/1.10.7-ugvomvt</command>
+        <command name="load">netcdf-c/4.4.1-blyisdg</command>
+        <command name="load">netcdf-cxx/4.2-gkqc6fq</command>
+        <command name="load">netcdf-fortran/4.4.4-eanrh5t</command>
+        <command name="load">parallel-netcdf/1.11.0-y3nmmej</command>
       </modules>
       <modules compiler="intel" mpilib="mvapich">
-        <command name="load">mvapich2/2.3.1-verbs-omjz3ck</command>
-        <command name="load">parallel-netcdf/1.11.2-7fy6qz3</command>
+        <command name="load">mvapich2/2.3.6-verbs-x4iz7lq</command>
+        <command name="load">hdf5/1.10.7-igh6foh</command>
+        <command name="load">netcdf-c/4.4.1-gei7x7w</command>
+        <command name="load">netcdf-cxx/4.2-db2f5or</command>
+        <command name="load">netcdf-fortran/4.4.4-b4ldb3a</command>
+        <command name="load">parallel-netcdf/1.11.0-kj4jsvt</command>
       </modules>
       <modules compiler="gnu">
-        <command name="load">gcc/8.2.0-g7hppkz</command>
-        <command name="load">intel-mkl/2018.4.274-2amycpi</command>
-        <command name="load">hdf5/1.8.16-mz7lmxh</command>
-        <command name="load">netcdf/4.4.1-xkjcghm</command>
-        <command name="load">netcdf-fortran/4.4.4-mpstomu</command>
+        <command name="load">gcc/8.2.0-xhxgy33</command>
+        <command name="load">intel-mkl/2020.4.304-d6zw4xa</command>
       </modules>
       <modules compiler="gnu" mpilib="impi">
-        <command name="load">intel-mpi/2018.4.274-ozfo327</command>
-        <command name="load">parallel-netcdf/1.11.0-filvnis</command>
+        <command name="load">intel-mpi/2019.9.304-rxpzd6p</command>
+        <command name="load">hdf5/1.10.7-oy6d2nm</command>
+        <command name="load">netcdf-c/4.4.1-fysjgfx</command>
+        <command name="load">netcdf-cxx/4.2-oaiw2v6</command>
+        <command name="load">netcdf-fortran/4.4.4-kxgkaop</command>
+        <command name="load">parallel-netcdf/1.11.0-fce7akl</command>
       </modules>
       <modules compiler="gnu" mpilib="mvapich">
-        <command name="load">mvapich2/2.3-bebop-3xi4hiu</command>
-        <command name="load">parallel-netcdf/1.11.2-hfn33fd</command>
+        <command name="load">mvapich2/2.3-bebop-a66r4jf</command>
+        <command name="load">hdf5/1.10.5-ejeshwh</command>
+        <command name="load">netcdf/4.4.1-ve2zfkw</command>
+        <command name="load">netcdf-cxx/4.2-2rkopdl</command>
+        <command name="load">netcdf-fortran/4.4.4-thtylny</command>
+        <command name="load">parallel-netcdf/1.11.0-kozyofv</command>
       </modules>
     </module_system>
     <RUNDIR>$CIME_OUTPUT_ROOT/$CASE/run</RUNDIR>
@@ -2901,7 +2912,7 @@
     <environment_variables mpilib="!mpi-serial">
       <env name="PNETCDF_PATH">$SHELL{dirname $(dirname $(which pnetcdf_version))}</env>
     </environment_variables>
-    <environment_variables compiler="gnu">
+    <environment_variables>
       <env name="HDF5_ROOT">$SHELL{which h5dump | xargs dirname | xargs dirname}</env>
     </environment_variables>
     <environment_variables BUILD_THREADED="TRUE">

components/data_comps/drof/src/drof_comp_mod.F90

@@ -25,7 +25,11 @@ module drof_comp_mod
   use drof_shr_mod   , only: rest_file      ! namelist input
   use drof_shr_mod   , only: rest_file_strm ! namelist input
   use drof_shr_mod   , only: nullstr
-
+#ifdef HAVE_MOAB
+  use seq_comm_mct,     only : mrofid ! id of moab rof app
+  use seq_comm_mct,     only : mbrof_data ! turn on if the data rof
+  use iso_c_binding
+#endif
   !
   ! !PUBLIC TYPES:
   implicit none
@@ -67,6 +71,12 @@ subroutine drof_comp_init(Eclock, x2r, r2x, &
        SDROF, gsmap, ggrid, mpicom, compid, my_task, master_task, &
        inst_suffix, inst_name, logunit, read_restart)
 
+#ifdef HAVE_MOAB
+    use iMOAB, only: iMOAB_DefineTagStorage, iMOAB_GetDoubleTagStorage, &
+                     iMOAB_SetIntTagStorage, iMOAB_SetDoubleTagStorage, &
+                     iMOAB_ResolveSharedEntities, iMOAB_CreateVertices, &
+                     iMOAB_GetMeshInfo, iMOAB_UpdateMeshInfo, iMOAB_WriteMesh
+#endif
     ! !DESCRIPTION: initialize drof model
     implicit none
 
@@ -92,7 +102,19 @@ subroutine drof_comp_init(Eclock, x2r, r2x, &
     logical       :: exists      ! file existance logical
     integer(IN)   :: nu          ! unit number
     character(CL) :: calendar ! model calendar
-
+#ifdef HAVE_MOAB
+    character*400  tagname
+    real(R8) latv, lonv
+    integer iv, tagindex, ilat, ilon, ierr  !, arrsize, nfields
+    real(R8), allocatable, target :: data(:)
+    integer(IN), pointer :: idata(:)   ! temporary
+    real(r8), dimension(:), allocatable :: moab_vert_coords  ! temporary
+    !real(R8), allocatable, target :: vtags_zero(:, :)
+
+#ifdef MOABDEBUG
+    character*100 outfile, wopts
+#endif
+#endif
     !--- formats ---
     character(*), parameter :: F00   = "('(drof_comp_init) ',8a)"
     character(*), parameter :: F0L   = "('(drof_comp_init) ',a, l2)"
@@ -164,6 +186,121 @@ subroutine drof_comp_init(Eclock, x2r, r2x, &
 
     call t_stopf('drof_initmctdom')
 
+
+#ifdef HAVE_MOAB
+   ilat = mct_aVect_indexRA(ggrid%data,'lat')
+   ilon = mct_aVect_indexRA(ggrid%data,'lon')
+   allocate(moab_vert_coords(lsize*3))
+   do iv = 1, lsize
+      lonv = ggrid%data%rAttr(ilon, iv) * SHR_CONST_PI/180.
+      latv = ggrid%data%rAttr(ilat, iv) * SHR_CONST_PI/180.
+      moab_vert_coords(3*iv-2)=COS(latv)*COS(lonv)
+      moab_vert_coords(3*iv-1)=COS(latv)*SIN(lonv)
+      moab_vert_coords(3*iv  )=SIN(latv)
+   enddo
+
+   ! create the vertices with coordinates from MCT domain
+   ierr = iMOAB_CreateVertices(mrofid, lsize*3, 3, moab_vert_coords)
+   if (ierr .ne. 0)  &
+      call shr_sys_abort('Error: fail to create MOAB vertices in land model')
+
+   tagname='GLOBAL_ID'//C_NULL_CHAR
+   ierr = iMOAB_DefineTagStorage(mrofid, tagname, &
+                                 0, & ! dense, integer
+                                 1, & ! number of components
+                                 tagindex )
+   if (ierr .ne. 0)  &
+      call shr_sys_abort('Error: fail to retrieve GLOBAL_ID tag ')
+
+   ! get list of global IDs for Dofs
+   call mct_gsMap_orderedPoints(gsMap, my_task, idata)
+
+   ierr = iMOAB_SetIntTagStorage ( mrofid, tagname, lsize, &
+                                    0, & ! vertex type
+                                    idata)
+   if (ierr .ne. 0)  &
+      call shr_sys_abort('Error: fail to set GLOBAL_ID tag ')
+
+   ierr = iMOAB_ResolveSharedEntities( mrofid, lsize, idata );
+   if (ierr .ne. 0)  &
+      call shr_sys_abort('Error: fail to resolve shared entities')
+
+   deallocate(moab_vert_coords)
+   deallocate(idata)
+
+   ierr = iMOAB_UpdateMeshInfo( mrofid )
+   if (ierr .ne. 0)  &
+      call shr_sys_abort('Error: fail to update mesh info ')
+
+   allocate(data(lsize))
+   ierr = iMOAB_DefineTagStorage( mrofid, "area:aream:frac:mask"//C_NULL_CHAR, &
+                                    1, & ! dense, double
+                                    1, & ! number of components
+                                    tagindex )
+   if (ierr > 0 )  &
+      call shr_sys_abort('Error: fail to create tag: area:aream:frac:mask' )
+
+   data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'area'),:)
+   tagname='area'//C_NULL_CHAR
+   ierr = iMOAB_SetDoubleTagStorage ( mrofid, tagname, lsize, &
+                                    0, & ! set data on vertices
+                                    data)
+   if (ierr > 0 )  &
+      call shr_sys_abort('Error: fail to get area tag ')
+
+   ! set the same data for aream (model area) as area
+   ! data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'aream'),:)
+   tagname='aream'//C_NULL_CHAR
+   ierr = iMOAB_SetDoubleTagStorage ( mrofid, tagname, lsize, &
+                                    0, & ! set data on vertices
+                                    data)
+   if (ierr > 0 )  &
+      call shr_sys_abort('Error: fail to set aream tag ')
+
+   data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'mask'),:)
+   tagname='mask'//C_NULL_CHAR
+   ierr = iMOAB_SetDoubleTagStorage ( mrofid, tagname, lsize, &
+                                    0, & ! set data on vertices
+                                    data)
+   if (ierr > 0 )  &
+      call shr_sys_abort('Error: fail to set mask tag ')
+
+   data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'frac'),:)
+   tagname='frac'//C_NULL_CHAR
+   ierr = iMOAB_SetDoubleTagStorage ( mrofid, tagname, lsize, &
+                                    0, & ! set data on vertices
+                                    data)
+   if (ierr > 0 )  &
+      call shr_sys_abort('Error: fail to set frac tag ')
+
+   deallocate(data)
+
+   ! define tags
+   ierr = iMOAB_DefineTagStorage( mrofid, trim(seq_flds_x2r_fields)//C_NULL_CHAR, &
+                                    1, & ! dense, double
+                                    1, & ! number of components
+                                    tagindex )
+   if (ierr > 0 )  &
+      call shr_sys_abort('Error: fail to create seq_flds_x2r_fields tags ')
+
+   ierr = iMOAB_DefineTagStorage( mrofid, trim(seq_flds_r2x_fields)//C_NULL_CHAR, &
+                                    1, & ! dense, double
+                                    1, & ! number of components
+                                    tagindex )
+   if (ierr > 0 )  &
+      call shr_sys_abort('Error: fail to create seq_flds_r2x_fields tags ')
+   mbrof_data = .true. ! will have effects 
+#ifdef MOABDEBUG
+      !      debug test
+   outfile = 'RofDataMesh.h5m'//C_NULL_CHAR
+   wopts   = ';PARALLEL=WRITE_PART'//C_NULL_CHAR !
+      !      write out the mesh file to disk
+   ierr = iMOAB_WriteMesh(mrofid, trim(outfile), trim(wopts))
+   if (ierr .ne. 0) then
+      call shr_sys_abort(subname//' ERROR in writing data mesh rof ')
+   endif
+#endif
+#endif
     !----------------------------------------------------------------------------
     ! Initialize MCT attribute vectors
     !----------------------------------------------------------------------------
@@ -256,6 +393,13 @@ subroutine drof_comp_run(EClock, x2r, r2x, &
        SDROF, gsmap, ggrid, mpicom, compid, my_task, master_task, &
        inst_suffix, logunit, case_name)
 
+#ifdef MOABDEBUG
+    use iMOAB, only: iMOAB_WriteMesh
+#endif
+#ifdef HAVE_MOAB
+    use seq_flds_mod    , only: seq_flds_r2x_fields 
+    use seq_flds_mod    , only: moab_set_tag_from_av
+#endif
     ! !DESCRIPTION:  run method for drof model
     implicit none
 
@@ -285,7 +429,18 @@ subroutine drof_comp_run(EClock, x2r, r2x, &
     integer(IN)   :: nu                ! unit number
     integer(IN)   :: nflds_r2x
     character(len=18) :: date_str
+#ifdef HAVE_MOAB
+    real(R8), allocatable, target :: datam(:)
+    type(mct_list) :: temp_list
+    integer :: size_list, index_list
+    type(mct_string)    :: mctOStr  !
+    character*400  tagname, mct_field
+#ifdef MOABDEBUG
+    integer  :: cur_drof_stepno, ierr
+    character*100 outfile, wopts, lnum
+#endif
 
+#endif
     character(*), parameter :: F00   = "('(drof_comp_run) ',8a)"
     character(*), parameter :: F04   = "('(drof_comp_run) ',2a,2i8,'s')"
     character(*), parameter :: subName = "(drof_comp_run) "
@@ -384,6 +539,32 @@ subroutine drof_comp_run(EClock, x2r, r2x, &
     !----------------------------------------------------------------------------
     ! Log output for model date
     !----------------------------------------------------------------------------
+#ifdef HAVE_MOAB
+    lsize = mct_avect_lsize(r2x) ! is it the same as mct_avect_lsize(avstrm) ?
+    allocate(datam(lsize)) ! 
+    call mct_list_init(temp_list ,seq_flds_r2x_fields)
+    size_list=mct_list_nitem (temp_list)
+    do index_list = 1, size_list
+      call mct_list_get(mctOStr,index_list,temp_list)
+      mct_field = mct_string_toChar(mctOStr)
+      tagname= trim(mct_field)//C_NULL_CHAR
+      call moab_set_tag_from_av(tagname, r2x, index_list, mrofid, datam, lsize) ! loop over all a2x fields, not just a few
+    enddo
+    call mct_list_clean(temp_list)
+    deallocate(datam) ! maybe we should keep it around, deallocate at the final only?
+
+#ifdef MOABDEBUG
+    call seq_timemgr_EClockGetData( EClock, stepno=cur_drof_stepno )
+    write(lnum,"(I0.2)")cur_drof_stepno
+    outfile = 'drof_comp_run_'//trim(lnum)//'.h5m'//C_NULL_CHAR
+    wopts   = 'PARALLEL=WRITE_PART'//C_NULL_CHAR
+    ierr = iMOAB_WriteMesh(mrofid, outfile, wopts)
+    if (ierr > 0 )  then
+       write(logunit,*) 'Failed to write data rof component state '
+    endif
+#endif
+
+#endif
 
     call t_startf('drof_run2')
     if (my_task == master_task) then