Add ai-env virtual package; remove python variant from jedi-base-env; bug fixes for Discover SCU16/17; doc updates for ParallelWorks; etc

.gitmodules

@@ -1,9 +1,11 @@
 [submodule "spack"]
   path = spack
-  #url = https://github.com/spack/spack
-  #branch = develop
-  url = https://github.com/jcsda/spack
-  branch = jcsda_emc_spack_stack
+  ##url = https://github.com/spack/spack
+  ##branch = develop
+  #url = https://github.com/jcsda/spack
+  #branch = jcsda_emc_spack_stack
+  url = https://github.com/climbfuji/spack
+  branch = feature/pytorch_internal_protobuf
 [submodule "doc/CMakeModules"]
   path = doc/CMakeModules
   url = https://github.com/noaa-emc/cmakemodules

configs/common/modules_lmod.yaml

@@ -71,6 +71,8 @@ modules:
       - openssl
       - perl
       - pkgconf
+      - protobuf
+      - py-protobuf
       - qt
       - randrproto
       - readline

configs/common/modules_tcl.yaml

@@ -73,6 +73,8 @@ modules:
       - openssl
       - perl
       - pkgconf
+      - protobuf
+      - py-protobuf
       - qt
       - randrproto
       - readline

configs/common/packages.yaml

@@ -242,6 +242,8 @@
       variants: +rust_bootstrap
     py-shapely:
       require: ['@1.8.0']
+    py-torch:
+      require: "+internal-protobuf ~mkldnn"
     qt:
       version: ['5.15.3']
     scotch:

configs/sites/derecho/compilers.yaml

@@ -1,42 +1,47 @@
 compilers::
-  - compiler:
-      spec: intel@2021.10.0
-      paths:
-        cc: /glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2023.2.1/compiler/2023.2.1/linux/bin/intel64/icc
-        cxx: /glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2023.2.1/compiler/2023.2.1/linux/bin/intel64/icpc
-        f77: /glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2023.2.1/compiler/2023.2.1/linux/bin/intel64/ifort
-        fc: /glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2023.2.1/compiler/2023.2.1/linux/bin/intel64/ifort
-      flags: {}
-      operating_system: sles15
-      target: x86_64
-      modules:
-      - ncarenv/23.09
-      - intel-classic/2023.2.1
-      environment:
-        prepend_path:
-          PATH: '/opt/cray/pe/gcc/12.2.0/bin'
-          CPATH: '/opt/cray/pe/gcc/12.2.0/snos/include'
-          LD_LIBRARY_PATH: '/glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2023.2.1/compiler/2023.2.1/linux/compiler/lib/intel64_lin:/opt/cray/pe/gcc/12.2.0/snos/lib:/opt/cray/pe/gcc/12.2.0/lib64'
-        set:
-          # https://github.com/JCSDA/spack-stack/issues/957
-          FI_CXI_RX_MATCH_MODE: 'hybrid'
-      extra_rpaths: []
-  - compiler:
-      spec: gcc@12.2.0
-      paths:
-        cc: /opt/cray/pe/gcc/12.2.0/bin/gcc
-        cxx: /opt/cray/pe/gcc/12.2.0/bin/g++
-        f77: /opt/cray/pe/gcc/12.2.0/bin/gfortran
-        fc: /opt/cray/pe/gcc/12.2.0/bin/gfortran
-      flags: {}
-      operating_system: sles15
-      target: x86_64
-      modules:
-      - ncarenv/23.09
-      - gcc/12.2.0
-      environment:
-        set:
-          # https://github.com/JCSDA/spack-stack/issues/957
-          FI_CXI_RX_MATCH_MODE: 'hybrid'
+- compiler:
+    spec: intel@2021.10.0
+    paths:
+      cc: /glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2023.2.1/compiler/2023.2.1/linux/bin/intel64/icc
+      cxx: /glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2023.2.1/compiler/2023.2.1/linux/bin/intel64/icpc
+      f77: /glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2023.2.1/compiler/2023.2.1/linux/bin/intel64/ifort
+      fc: /glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2023.2.1/compiler/2023.2.1/linux/bin/intel64/ifort
+    flags:
+      cflags: -diag-disable=10441
+      cxxflags: -diag-disable=10441
+      fflags: -diag-disable=10448
+    operating_system: sles15
+    target: x86_64
+    modules:
+    - ncarenv/23.09
+    - intel-classic/2023.2.1
+    environment:
+      prepend_path:
+        PATH: '/opt/cray/pe/gcc/12.2.0/bin'
+        CPATH: '/opt/cray/pe/gcc/12.2.0/snos/include'
+        LD_LIBRARY_PATH: '/glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2023.2.1/compiler/2023.2.1/linux/compiler/lib/intel64_lin:/opt/cray/pe/gcc/12.2.0/snos/lib:/opt/cray/pe/gcc/12.2.0/lib64'
+      set:
+        # https://github.com/JCSDA/spack-stack/issues/957
+        FI_CXI_RX_MATCH_MODE: 'hybrid'
+        # https://github.com/JCSDA/spack-stack/issues/1012
+        I_MPI_EXTRA_FILESYSTEM: 'ON'
       extra_rpaths: []
+- compiler:
+    spec: gcc@12.2.0
+    paths:
+      cc: /opt/cray/pe/gcc/12.2.0/bin/gcc
+      cxx: /opt/cray/pe/gcc/12.2.0/bin/g++
+      f77: /opt/cray/pe/gcc/12.2.0/bin/gfortran
+      fc: /opt/cray/pe/gcc/12.2.0/bin/gfortran
+    flags: {}
+    operating_system: sles15
+    target: x86_64
+    modules:
+    - ncarenv/23.09
+    - gcc/12.2.0
+    environment:
+      set:
+        # https://github.com/JCSDA/spack-stack/issues/957
+        FI_CXI_RX_MATCH_MODE: 'hybrid'
+    extra_rpaths: []
 

configs/sites/discover-scu16/compilers.yaml

@@ -6,7 +6,8 @@ compilers:
       cxx: /usr/local/intel/oneapi/2021/compiler/2022.0.1/linux/bin/intel64/icpc
       f77: /usr/local/intel/oneapi/2021/compiler/2022.0.1/linux/bin/intel64/ifort
       fc: /usr/local/intel/oneapi/2021/compiler/2022.0.1/linux/bin/intel64/ifort
-    flags: {}
+    flags:
+      ldflags: '-L/usr/local/other/gcc/11.2.0/lib64'
     operating_system: sles12
     target: x86_64
     modules:

configs/sites/discover-scu17/compilers.yaml

@@ -19,8 +19,9 @@ compilers:
         PATH: '/usr/local/other/gcc/11.4.0/bin'
         CPATH: '/usr/local/other/gcc/11.4.0/include'
         LD_LIBRARY_PATH: '/usr/local/intel/oneapi/2021/compiler/2023.2.1/linux/compiler/lib/intel64_lin:/usr/local/other/gcc/11.4.0/lib64'
-      #set:
-      #  I_MPI_ROOT: '/usr/local/intel/oneapi/2021/mpi/2021.5.0'
+      set:
+        # https://github.com/JCSDA/spack-stack/issues/1012
+        I_MPI_EXTRA_FILESYSTEM: 'ON'
     extra_rpaths: []
 - compiler:
     spec: gcc@=12.3.0

configs/sites/hercules/compilers.yaml

@@ -6,7 +6,10 @@ compilers:
       cxx: /apps/spack-managed/gcc-11.3.1/intel-oneapi-compilers-2023.1.0-sb753366rvywq75zeg4ml5k5c72xgj72/compiler/2023.1.0/linux/bin/intel64/icpc
       f77: /apps/spack-managed/gcc-11.3.1/intel-oneapi-compilers-2023.1.0-sb753366rvywq75zeg4ml5k5c72xgj72/compiler/2023.1.0/linux/bin/intel64/ifort
       fc: /apps/spack-managed/gcc-11.3.1/intel-oneapi-compilers-2023.1.0-sb753366rvywq75zeg4ml5k5c72xgj72/compiler/2023.1.0/linux/bin/intel64/ifort
-    flags: {}
+    flags:
+      cflags: -diag-disable=10441
+      cxxflags: -diag-disable=10441
+      fflags: -diag-disable=10448
     operating_system: rocky9
     target: x86_64
     modules:

configs/sites/noaa-aws/README.md

@@ -6,6 +6,11 @@ that the site configuration files in `configs/sites/noaa-aws` work out of the bo
 
 sudo su
 chmod 777 /contrib
+# The following three commands were necessary to fix failures accessing these repos
+yum-config-manager --disable intel-clck-2019-repo
+yum-config-manager --disable intel-hpc-platform
+yum-config-manager --disable intelpython
+#
 yum install -y qt5-qtbase-devel
 yum install -y qt5-qtsvg-devel
 yum install -y xorg-x11-xauth
@@ -126,6 +131,7 @@ mkdir ecflow
 # Create the modulefile from the template in doc/modulefile_templates
 
 ############## Steps to perform when starting a new cluster ##############
+
 source /contrib/admin/basic_setup.sh  # sudo privileges requred to install packages
 module unuse /opt/cray/craype/default/modulefiles
 module unuse /opt/cray/modulefiles
@@ -135,3 +141,12 @@ module load ecflow/5.8.4
 module load mysql/8.0.31
 module load git-lfs/2.4.1
 
+git lfs install
+git config --global credential.helper cache # or store
+git config --global user.email "your.name@domain.com"
+git config --global user.name "Your Name"
+
+mkdir ~/.aws
+# Create ~/.aws/config
+# Create ~/.aws/credentials
+chmod 400 ~/.aws/credentials

configs/sites/noaa-azure/README.md

@@ -6,7 +6,11 @@ that the site configuration files in `configs/sites/noaa-azure` work out of the
 
 sudo su
 chmod 777 /contrib
-yum install -y m4
+# The following three commands were necessary to fix failures accessing these repos
+yum-config-manager --disable intel-clck-2019-repo
+yum-config-manager --disable intel-hpc-platform
+yum-config-manager --disable intelpython
+#
 yum install -y qt5-qtbase-devel
 yum install -y qt5-qtsvg-devel
 yum install -y xorg-x11-xauth
@@ -130,6 +134,7 @@ mkdir ecflow
 # Create the modulefile from the template in doc/modulefile_templates
 
 ############## Steps to perform when starting a new cluster ##############
+
 source /contrib/admin/basic_setup.sh  # sudo privileges requred to install packages
 module unuse /opt/cray/craype/default/modulefiles
 module unuse /opt/cray/modulefiles
@@ -140,4 +145,12 @@ module load mysql/8.0.31
 module load git-lfs/2.4.1
 #
 
+git lfs install
+git config --global credential.helper cache # or store
+git config --global user.email "your.name@domain.com"
+git config --global user.name "Your Name"
 
+mkdir ~/.aws
+# Create ~/.aws/config
+# Create ~/.aws/credentials
+chmod 400 ~/.aws/credentials

configs/sites/noaa-gcloud/README.md

@@ -6,6 +6,11 @@ that the site configuration files in `configs/sites/noaa-gcloud` work out of the
 
 sudo su
 chmod 777 /contrib
+# The following three commands were necessary to fix failures accessing these repos
+yum-config-manager --disable intel-clck-2019-repo
+yum-config-manager --disable intel-hpc-platform
+yum-config-manager --disable intelpython
+#
 yum install -y qt5-qtbase-devel
 yum install -y qt5-qtsvg-devel
 yum install -y xorg-x11-xauth
@@ -126,6 +131,7 @@ mkdir ecflow
 # Create the modulefile from the template in doc/modulefile_templates
 
 ############## Steps to perform when starting a new cluster ##############
+
 source /contrib/admin/basic_setup.sh  # sudo privileges requred to install packages
 module unuse /opt/cray/craype/default/modulefiles
 module unuse /opt/cray/modulefiles
@@ -135,3 +141,12 @@ module load ecflow/5.8.4
 module load mysql/8.0.31
 module load git-lfs/2.4.1
 
+git lfs install
+git config --global credential.helper cache # or store
+git config --global user.email "your.name@domain.com"
+git config --global user.name "Your Name"
+
+mkdir ~/.aws
+# Create ~/.aws/config
+# Create ~/.aws/credentials
+chmod 400 ~/.aws/credentials

configs/templates/unified-dev/spack.yaml

@@ -9,6 +9,7 @@ spack:
   - compilers: ['%aocc', '%apple-clang', '%gcc', '%intel']
   - packages:
       - ewok-env +ecflow +cylc
+      - ai-env
       - geos-gcm-env
       - global-workflow-env
       - gmao-swell-env
@@ -44,5 +45,7 @@ spack:
       - [$packages]
       - [$compilers]
       exclude:
+        # py-torch in ai-env doesn't build with Intel
+        - ai-env%intel
         # jedi-tools doesn't build with Intel
         - jedi-tools-env%intel

doc/source/MaintainersSection.rst

@@ -345,14 +345,24 @@ NASA Discover SCU17
 On Discover SCU17 ``ecflow`` needs to be installed as a one-off before spack can be used.
 
 ecflow
-  ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library. After loading the following modules, follow the instructions in :numref:`Section %s <MaintainersSection_ecFlow>`.
+  ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library. After loading the following modules, follow the instructions in :numref:`Section %s <MaintainersSection_ecFlow>` (cont'd below).
 
 .. code-block:: console
 
    module purge
-   module load comp/gcc/12.3.0
    module load cmake/3.28.2
 
+The following workaround is required after installing ``ecflow`` and creating the modulefile: edit ``path/to/ecflow/bin/ecflow_ui`` and change the last few lines to (i.e. prepend the ``LD_PRELOAD`` command):
+
+.. code-block:: console
+
+   if [ $ECFLOWUI_BT != "no" ]
+   then
+       LD_PRELOAD=/usr/lib64/libstdc++.so.6 catchsegv ${ECFLOWUI_USER_START_CMD} "$exe"
+   else
+       LD_PRELOAD=/usr/lib64/libstdc++.so.6 ${ECFLOWUI_USER_START_CMD} "$exe"
+   fi
+
 .. _MaintainersSection_Narwhal:
 
 ------------------------------

doc/source/PreConfiguredSites.rst

@@ -26,9 +26,9 @@ On selected systems, developmental versions / release candidates are installed t
 | MSU                 +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+
 |                     | Orion                            | GCC, Intel      | ``/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env``                      | Cam Book / Dom Heinzeller     |
 +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+
-|                     | Discover SCU16                   | GCC, Intel      | ``/gpfsm/dswdev/jcsda/spack-stack/scu16/spack-stack-20240207/envs/unified-env-*``                       | Dom Heinzeller / ???          |
+|                     | Discover SCU16                   | GCC, Intel      | ``/gpfsm/dswdev/jcsda/spack-stack/scu16/spack-stack-20240228/envs/unified-env-*``                       | Dom Heinzeller / ???          |
 | NASA                +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+
-|                     | Discover SCU17                   | GCC, Intel      | ``/gpfsm/dnb55/projects/p01/s2127/dheinzel/spstmil/envs/unified-env-*`` **TEMP FOR ACCEPTANCE TESTING** | Dom Heinzeller / ???          |
+|                     | Discover SCU17                   | GCC, Intel      | ``/gpfsm/dswdev/jcsda/spack-stack/scu17/spack-stack-20240228/envs/unified-env-*``                       | Dom Heinzeller / ???          |
 +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+
 |                     | Casper                           | GCC             | ``/glade/work/epicufsrt/contrib/spack-stack/casper/spack-stack-1.6.0/envs/unified-env``                 | Dom Heinzeller / ???          |
 | NCAR-Wyoming        +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+
@@ -177,29 +177,26 @@ The following is required for building new spack environments and for using spac
    module load miniconda/3.9.7
    module load ecflow/5.8.4
 
-For ``spack-stack-20240207`` with Intel, proceed with loading the following modules:
+For ``spack-stack-20240228`` with Intel, proceed with loading the following modules:
 
 .. code-block:: console
 
-   module use /gpfsm/dswdev/jcsda/spack-stack/scu16/spack-stack-20240207/envs/unified-env-intel-2021.5.0/install/modulefiles/Core
+   module use /gpfsm/dswdev/jcsda/spack-stack/scu16/spack-stack-20240228/envs/unified-env-intel-2021.5.0/install/modulefiles/Core
    module load stack-intel/2021.5.0
    module load stack-intel-oneapi-mpi/2021.5.0
    module load stack-python/3.10.13
    module available
 
-For ``spack-stack-20240207`` with GNU, proceed with loading the following modules:
+For ``spack-stack-20240228`` with GNU, proceed with loading the following modules:
 
 .. code-block:: console
 
-   module use /gpfsm/dswdev/jcsda/spack-stack/scu16/spack-stack-20240207/envs/unified-env-gcc-12.1.0/install/modulefiles/Core
+   module use /gpfsm/dswdev/jcsda/spack-stack/scu16/spack-stack-20240228/envs/unified-env-gcc-12.1.0/install/modulefiles/Core
    module load stack-gcc/12.1.0
    module load stack-openmpi/4.1.3
    module load stack-python/3.10.13
    module available
 
-.. note::
-   When using Intel, it may be required to set the environment variable ``LDFLAGS="-L/usr/local/other/gcc/11.2.0/lib64"`` for building applications like JEDI.
-
 ------------------------------
 NASA Discover SCU17
 ------------------------------
@@ -213,23 +210,21 @@ The following is required for building new spack environments and for using spac
    module use /discover/swdev/jcsda/spack-stack/scu17/modulefiles
    module load ecflow/5.11.4
 
-For ``spack-stack-20240207`` with Intel, load the following modules after loading ecflow:
-
-**TEMPORARY LOCATIONS FOR ACCEPTANCE TESTING**!
+For ``spack-stack-20240228`` with Intel, load the following modules after loading ecflow:
 
 .. code-block:: console
 
-   module use /gpfsm/dnb55/projects/p01/s2127/dheinzel/spstmil/envs/unified-env-intel-2021.10.0/install/modulefiles/Core
+   module use /gpfsm/dswdev/jcsda/spack-stack/scu17/spack-stack-20240228/envs/unified-env-intel-2021.10.0/install/modulefiles/Core
    module load stack-intel/2021.10.0
    module load stack-intel-oneapi-mpi/2021.10.0
    module load stack-python/3.10.13
    module available
 
-For ``spack-stack-20240207`` with GNU, load the following modules after loading ecflow:
+For ``spack-stack-20240228`` with GNU, load the following modules after loading ecflow:
 
 .. code-block:: console
 
-   module use /gpfsm/dnb55/projects/p01/s2127/dheinzel/spstmil/envs/unified-env-gcc-12.3.0/install/modulefiles/Core
+   module use /gpfsm/dswdev/jcsda/spack-stack/scu17/spack-stack-20240228/envs/unified-env-gcc-12.3.0/install/modulefiles/Core
    module load stack-gcc/12.3.0
    module load stack-openmpi/4.1.6
    module load stack-python/3.10.13