Add oneAPI config for Narwhal, add GNU config for Atlantis, remove Sandy platform; make python variant for esmf configurable

configs/common/packages.yaml

@@ -54,17 +54,19 @@ packages:
   # Also, check the acorn and derecho site configs which have esmf modifications.
   esmf:
     require:
-      - '~xerces ~pnetcdf +shared +external-parallelio +python'
-      - any_of: ['@=8.6.1 snapshot=none', '@=8.7.0b11 snapshot=b11']
-      - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"']
-        when: "%intel"
-        message: "Extra ESMF compile options for Intel"
-      #- any_of: ['']
-      #  when: "%gcc"
-      #  message: "Extra ESMF compile options for GCC"
-      #- any_of: ['']
-      #  when: "%apple-clang"
-      #  message: "Extra ESMF compile options for GCC"
+    - '~xerces ~pnetcdf +shared +external-parallelio'
+    - any_of: ['@=8.6.1 snapshot=none', '@=8.7.0b11 snapshot=b11']
+    - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"']
+      when: "%intel"
+      message: "Extra ESMF compile options for Intel"
+    #- any_of: ['']
+    #  when: "%gcc"
+    #  message: "Extra ESMF compile options for GCC"
+    #- any_of: ['']
+    #  when: "%apple-clang"
+    #  message: "Extra ESMF compile options for GCC"
+    prefer:
+    - '+python'
   fckit:
     require: '@0.11.0 +eckit'
   fftw:

configs/sites/tier1/sandy/mirrors.yaml → configs/sites/tier1/atlantis/mirrors.yaml

@@ -1,15 +1,17 @@
 mirrors:
   local-source:
     fetch:
-      url: file:///gpfs/fs1/neptune/spack-stack/source-cache
+      # TODO UPDATE?
+      url: file:///neptune_diagnostics/spack-stack/source-cache/
       access_pair:
       - null
       - null
       access_token: null
       profile: null
       endpoint_url: null
     push:
-      url: file:///gpfs/fs1/neptune/spack-stack/source-cache
+      # TODO UPDATE?
+      url: file:///neptune_diagnostics/spack-stack/source-cache/
       access_pair:
       - null
       - null

configs/sites/tier1/atlantis/packages_gcc.yaml

@@ -0,0 +1,13 @@
+packages:
+  all:
+    compiler:: [gcc@11.2.0]
+    providers:
+      mpi:: [openmpi@4.1.5]
+  mpi:
+    buildable: False
+  openmpi:
+    buildable: False
+    externals:
+    - spec: openmpi@4.1.5%gcc@=11.2.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath fabrics=ucx schedulers=slurm
+      modules:
+      - openmpi/mlnx/gcc/64/4.1.5a1

configs/sites/tier1/atlantis/packages_intel.yaml

@@ -15,6 +15,6 @@ packages:
   intel-oneapi-mkl:
     externals:
     - spec: intel-oneapi-mkl@2022.1.0
+      prefix: /cm/shared/apps/intel/oneapi
       modules:
       -  mkl/2022.1.0
-      prefix: /cm/shared/apps/intel/oneapi

configs/sites/tier1/narwhal/compilers.yaml

@@ -21,6 +21,30 @@ compilers::
         set:
           CRAYPE_LINK_TYPE: 'dynamic'
       extra_rpaths: []
+  - compiler:
+      spec: oneapi@2024.2.0
+      paths:
+        cc: cc
+        cxx: CC
+        f77: ftn
+        fc: ftn
+      flags: {}
+      operating_system: sles15
+      modules:
+      - PrgEnv-intel/8.4.0
+      - intel/2024.2
+      - cray-libsci/23.05.1.4
+      - libfabric/1.12.1.2.2.1
+      environment:
+        prepend_path:
+          PATH: '/opt/cray/pe/gcc/10.3.0/snos/bin'
+          CPATH: '/opt/cray/pe/gcc/10.3.0/snos/include'
+          LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/23.05.1.4/INTEL/2022.2/x86_64/lib:/opt/cray/pe/gcc/10.3.0/snos/lib:/opt/cray/pe/gcc/10.3.0/snos/lib64'
+        append_path:
+          CPATH: '/opt/intel/oneapi_2024.2.0.634/compiler/2024.2/opt/compiler/include/intel64'
+        set:
+          CRAYPE_LINK_TYPE: 'dynamic'
+      extra_rpaths: []
   - compiler:
       spec: gcc@10.3.0
       paths:

configs/sites/tier1/narwhal/packages_oneapi.yaml

@@ -0,0 +1,27 @@
+packages:
+  all:
+    compiler:: [oneapi@2024.2.0,gcc@10.3.0]
+    providers:
+      mpi:: [cray-mpich@8.1.21]
+  mpi:
+    buildable: False
+  cray-mpich:
+    externals:
+    - spec: cray-mpich@8.1.21%oneapi@2024.2.0 ~wrappers
+      modules:
+      - cray-mpich-ucx/8.1.21
+      - craype-network-ucx
+  intel-oneapi-mkl:
+    externals:
+    - spec: intel-oneapi-mkl@2024.2%oneapi@2024.2.0
+      prefix: /opt/intel/oneapi_2024.2.0.634
+  intel-oneapi-tbb:
+    externals:
+    - spec: intel-oneapi-tbb@2021.13%oneapi@2024.2.0
+      prefix: /opt/intel/oneapi_2024.2.0.634
+      modules:
+      - tbb/2021.13
+  intel-oneapi-runtime:
+    externals:
+    - spec: intel-oneapi-runtime@2024.2.0%oneapi@2024.2.0
+      prefix: /opt/intel/oneapi_2024.2.0.634

doc/source/PreConfiguredSites.rst

@@ -65,7 +65,7 @@ Pre-configured sites (tier 1)
 +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+
 |                     | Narwhal               | GCC, Intel, oneAPI | ``/p/app/projects/NEPTUNE/spack-stack/``               | NRL             |
 | U.S. Navy (HPCMP)   +-----------------------+--------------------+--------------------------------------------------------+-----------------+
-|                     | Nautilus              | Intel              | ``/p/app/projects/NEPTUNE/spack-stack/``               | NRL             |
+|                     | Nautilus              | GCC, Intel, oneAPI | ``/p/app/projects/NEPTUNE/spack-stack/``               | NRL             |
 +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+
 | Univ. of Wisconsin  | S4                    | Intel              | ``/data/prod/jedi/spack-stack/``                       | JCSDA           |
 +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+
@@ -164,6 +164,30 @@ The following is required for building new spack environments with Intel on this
    module unload cray-libsci
    module load cray-libsci/23.05.1.4
 
+The following is required for building new spack environments with Intel oneAPI on this platform.. Don't use ``module purge`` on Narwhal!
+
+.. code-block:: console
+
+   umask 0022
+   module unload PrgEnv-cray
+   module load PrgEnv-intel/8.4.0
+   module unload intel
+   module load intel/2024.2
+   module unload cray-mpich
+   module unload craype-network-ofi
+   # Warning. Do not load craype-network-ucx
+   # or cray-mpich-ucx/8.1.21!
+   # There is a bug in the modulefile that prevents
+   # spack from setting the environment for its
+   # build steps when the module is already
+   # loaded. Instead, let spack load it when the
+   # package requires it.
+   #module load craype-network-ucx
+   #module load cray-mpich-ucx/8.1.21
+   module load libfabric/1.12.1.2.2.1
+   module unload cray-libsci
+   module load cray-libsci/23.05.1.4
+
 The following is required for building new spack environments with GNU on this platform.. Don't use ``module purge`` on Narwhal!
 
 .. code-block:: console

spack-ext/repos/spack-stack/packages/neptune-env/package.py

@@ -47,5 +47,6 @@ class NeptuneEnv(BundlePackage):
     # Basic Python dependencies that are always needed
     depends_on("py-f90nml", type="run")
     depends_on("py-python-dateutil", type="run")
+    depends_on("py-pyyaml", type="run")
 
     # There is no need for install() since there is no code.

spack-ext/repos/spack-stack/packages/neptune-python-env/package.py

@@ -22,6 +22,8 @@ class NeptunePythonEnv(BundlePackage):
     variant("xnrl", default=False, description="Build non-pulic XNRL")
 
     depends_on("neptune-env", type="run")
+    # Enable the Python variant for ESMF
+    depends_on("esmf +python", type="run")
 
     depends_on("py-h5py", type="run")
     depends_on("py-netcdf4", type="run")