From aa5c4a08ac5ecfbb8712e0ea46ff0a543663d0ff Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 24 Jan 2024 19:44:27 -0700 Subject: [PATCH 01/11] Update aws-pcluste site config for pcluster-3.8.0 --- configs/sites/aws-pcluster/compilers.yaml | 43 +++++++------------- configs/sites/aws-pcluster/packages.yaml | 48 ++++++++++------------- 2 files changed, 34 insertions(+), 57 deletions(-) diff --git a/configs/sites/aws-pcluster/compilers.yaml b/configs/sites/aws-pcluster/compilers.yaml index e1f780a1b..6363de90a 100644 --- a/configs/sites/aws-pcluster/compilers.yaml +++ b/configs/sites/aws-pcluster/compilers.yaml @@ -1,48 +1,33 @@ compilers: - compiler: - spec: intel@2022.1.0 + spec: oneapi@2024.0.2 paths: - cc: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/icc - cxx: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/icpc - f77: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/ifort - fc: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/ifort + cc: /opt/intel/oneapi/compiler/2024.0/bin/icx + cxx: /opt/intel/oneapi/compiler/2024.0/bin/icpx + f77: /opt/intel/oneapi/compiler/2024.0/bin/ifort + fc: /opt/intel/oneapi/compiler/2024.0/bin/ifort flags: {} - operating_system: ubuntu20.04 + operating_system: ubuntu22.04 target: x86_64 - modules: [] + modules: + - tbb/2021.11 + - compiler-rt/2024.0.2 + - oclfpga/2024.0.0 + - compiler/2024.0.2 + - ifort/2024.0.2 environment: - prepend_path: - LD_LIBRARY_PATH: '/opt/intel/oneapi/compiler/2022.1.0/linux/compiler/lib/intel64_lin' set: I_MPI_PMI_LIBRARY: '/opt/slurm/lib/libpmi.so' extra_rpaths: [] -# Spack gets confused if there is an Intel and a OneAPI compiler ... -#- compiler: -# spec: oneapi@2022.1.0 -# paths: -# cc: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/icx -# cxx: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/icpx -# f77: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/ifx -# fc: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/ifx -# flags: {} -# operating_system: ubuntu20.04 -# target: x86_64 -# modules: [] -# environment: -# #prepend_path: -# # LD_LIBRARY_PATH: '/opt/intel/oneapi/compiler/2022.1.0/linux/compiler/lib/intel64_lin' -# set: -# I_MPI_PMI_LIBRARY: '/opt/slurm/lib/libpmi.so' -# extra_rpaths: [] - compiler: - spec: gcc@9.4.0 + spec: gcc@11.4.0 paths: cc: /usr/bin/gcc cxx: /usr/bin/g++ f77: /usr/bin/gfortran fc: /usr/bin/gfortran flags: {} - operating_system: ubuntu20.04 + operating_system: ubuntu22.04 target: x86_64 modules: [] environment: {} diff --git a/configs/sites/aws-pcluster/packages.yaml b/configs/sites/aws-pcluster/packages.yaml index 50830f9a6..2a8f95ebc 100644 --- a/configs/sites/aws-pcluster/packages.yaml +++ b/configs/sites/aws-pcluster/packages.yaml @@ -1,34 +1,26 @@ packages: all: - compiler: [intel@2022.1.0, gcc@9.4.0] - #compiler: [oneapi@2022.1.0] + compiler: [oneapi@2024.0.2, gcc@11.4.0] providers: - mpi: [intel-oneapi-mpi@2021.6.0, openmpi@4.1.4] + mpi: [intel-oneapi-mpi@2021.11, openmpi@4.1.6] ### MPI, Python, MKL mpi: buildable: false intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.6.0%intel@2022.1.0 + - spec: intel-oneapi-mpi@2021.11%oneapi@2024.0.2 prefix: /opt/intel modules: - - libfabric-aws/1.16.0~amzn4.0 - - intelmpi - #externals: - #- spec: intel-oneapi-mpi@2021.6.0%oneapi@2022.1.0 - # prefix: /opt/intel - # modules: - # - libfabric-aws/1.16.0~amzn4.0 - # - intelmpi + - libfabric-aws/1.19.0amzn4.0 + - mpi/2021.11 openmpi: externals: - - spec: openmpi@4.1.4%gcc@9.4.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath + - spec: openmpi@4.1.6%gcc@11.4.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath fabrics=ofi schedulers=slurm - prefix: /opt/amazon/openmpi modules: - - libfabric-aws/1.16.0~amzn4.0 - - openmpi/4.1.4 + - libfabric-aws/1.19.0amzn4.0 + - openmpi/4.1.6 ### Modification of common packages @@ -49,11 +41,11 @@ packages: externals: - spec: binutils@2.34 prefix: /usr - boost: - buildable: False - externals: - - spec: boost@1.71.0 +atomic +chrono +date_time +exception +filesystem +graph +iostreams +locale +log +math +mpi +numpy +pic +program_options +python +random +regex +serialization +signals +system +test +thread +timer +wave cxxstd=17 visibility=hidden - prefix: /usr + ### boost: + ### buildable: False + ### externals: + ### - spec: boost@1.71.0 +atomic +chrono +date_time +exception +filesystem +graph +iostreams +locale +log +math +mpi +numpy +pic +program_options +python +random +regex +serialization +signals +system +test +thread +timer +wave cxxstd=17 visibility=hidden + ### prefix: /usr coreutils: externals: - spec: coreutils@8.30 @@ -62,11 +54,11 @@ packages: externals: - spec: diffutils@3.7 prefix: /usr - ecflow: - buildable: False - externals: - - spec: ecflow@5.8.4 +ui - prefix: /home/ubuntu/jedi/ecflow-5.8.4 + ### ecflow: + ### buildable: False + ### externals: + ### - spec: ecflow@5.8.4 +ui + ### prefix: /home/ubuntu/jedi/ecflow-5.8.4 findutils: externals: - spec: findutils@4.7.0 @@ -106,7 +98,7 @@ packages: mysql: buildable: False externals: - - spec: mysql@8.0.32 + - spec: mysql@8.0.35 prefix: /usr perl: externals: @@ -119,7 +111,7 @@ packages: qt: buildable: false externals: - - spec: qt@5.12.8 + - spec: qt@5.15.3 prefix: /usr sed: externals: From c65ac19fcea46ad8b2c0b59bdc51a57c01addded Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 25 Jan 2024 13:38:50 -0700 Subject: [PATCH 02/11] Update .gitmodules and submodule pointer for spack --- .gitmodules | 10 ++++++---- spack | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index b1f475b64..a196480f8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,11 @@ [submodule "spack"] path = spack - #url = https://github.com/spack/spack - #branch = develop - url = https://github.com/jcsda/spack - branch = jcsda_emc_spack_stack + ##url = https://github.com/spack/spack + ##branch = develop + #url = https://github.com/jcsda/spack + #branch = jcsda_emc_spack_stack + url = https://github.com/climbfuji/spack + branch = bugfix/wgrib2_oneapi [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index e7ef79104..980062fdb 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit e7ef791045012b6ef989be97c6eee24ec571c4b6 +Subproject commit 980062fdb689e9463b90422102dbb2e689eb9005 From 76e398698a9506edcf9648b4e68b6fc73fc5e914 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 25 Jan 2024 13:40:30 -0700 Subject: [PATCH 03/11] Update configs/sites/aws-pcluster/packages.yaml --- configs/sites/aws-pcluster/packages.yaml | 46 ++++++++++-------------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/configs/sites/aws-pcluster/packages.yaml b/configs/sites/aws-pcluster/packages.yaml index 2a8f95ebc..604553658 100644 --- a/configs/sites/aws-pcluster/packages.yaml +++ b/configs/sites/aws-pcluster/packages.yaml @@ -10,7 +10,7 @@ packages: intel-oneapi-mpi: externals: - spec: intel-oneapi-mpi@2021.11%oneapi@2024.0.2 - prefix: /opt/intel + prefix: /opt/intel/oneapi modules: - libfabric-aws/1.19.0amzn4.0 - mpi/2021.11 @@ -31,37 +31,27 @@ packages: ### All other external packages listed alphabetically autoconf: externals: - - spec: autoconf@2.69 + - spec: autoconf@2.71 prefix: /usr automake: externals: - - spec: automake@1.16.1 + - spec: automake@1.16.5 prefix: /usr binutils: externals: - - spec: binutils@2.34 + - spec: binutils@2.38 prefix: /usr - ### boost: - ### buildable: False - ### externals: - ### - spec: boost@1.71.0 +atomic +chrono +date_time +exception +filesystem +graph +iostreams +locale +log +math +mpi +numpy +pic +program_options +python +random +regex +serialization +signals +system +test +thread +timer +wave cxxstd=17 visibility=hidden - ### prefix: /usr coreutils: externals: - - spec: coreutils@8.30 + - spec: coreutils@8.32 prefix: /usr diffutils: externals: - - spec: diffutils@3.7 + - spec: diffutils@3.8 prefix: /usr - ### ecflow: - ### buildable: False - ### externals: - ### - spec: ecflow@5.8.4 +ui - ### prefix: /home/ubuntu/jedi/ecflow-5.8.4 findutils: externals: - - spec: findutils@4.7.0 + - spec: findutils@4.8.0 prefix: /usr flex: externals: @@ -69,19 +59,19 @@ packages: prefix: /usr gawk: externals: - - spec: gawk@5.0.1 + - spec: gawk@5.1.0 prefix: /usr git: externals: - - spec: git@2.25.1~tcltk + - spec: git@2.34.1~tcltk prefix: /usr git-lfs: externals: - - spec: git-lfs@2.9.2 + - spec: git-lfs@3.0.2 prefix: /usr gmake: externals: - - spec: gmake@4.2.1 + - spec: gmake@4.3 prefix: /usr groff: externals: @@ -102,11 +92,11 @@ packages: prefix: /usr perl: externals: - - spec: perl@5.30.0~cpanm+shared+threads + - spec: perl@5.34.0~cpanm+shared+threads prefix: /usr pkg-config: externals: - - spec: pkg-config@0.29.1 + - spec: pkg-config@0.29.2 prefix: /usr qt: buildable: false @@ -115,21 +105,21 @@ packages: prefix: /usr sed: externals: - - spec: sed@4.7 + - spec: sed@4.8 prefix: /usr tar: externals: - - spec: tar@1.30 + - spec: tar@1.34 prefix: /usr texinfo: externals: - - spec: texinfo@6.7 + - spec: texinfo@6.8 prefix: /usr texlive: externals: - - spec: texlive@20190410 + - spec: texlive@20220204 prefix: /usr wget: externals: - - spec: wget@1.20.3 + - spec: wget@1.21.1 prefix: /usr From ae9cc735a525f24424203545e137637e40552617 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 25 Jan 2024 13:40:43 -0700 Subject: [PATCH 04/11] Update compiler compatibility matrix in doc/source/NewSiteConfigs.rst --- doc/source/NewSiteConfigs.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/NewSiteConfigs.rst b/doc/source/NewSiteConfigs.rst index 783005b9a..e717299d8 100644 --- a/doc/source/NewSiteConfigs.rst +++ b/doc/source/NewSiteConfigs.rst @@ -15,9 +15,9 @@ It is also instructive to peruse the GitHub actions scripts in ``.github/workflo +===========================================+======================================================================+===========================+ | Intel classic (icc, icpc, ifort) | 2021.3.0 to the latest available version in oneAPI 2023.1.0 | ``intel@`` | +-------------------------------------------+----------------------------------------------------------------------+---------------------------+ -| Intel mixed (icx, icpx, ifort) | all versions up to latest available version in oneAPI 2023.1.0 | ``intel@`` | +| Intel mixed (icx, icpx, ifort) | 2024.0 to the latest available version in oneAPI 2024.0.2 | ``oneapi@`` | +-------------------------------------------+----------------------------------------------------------------------+---------------------------+ -| GNU (gcc, g++, gfortran) | 9.2.0 to 12.2.0 (note: 13.x.y is **not** yet supported) | ``gcc@`` | +| GNU (gcc, g++, gfortran) | 9.2.0 to 12.3.0 (note: 13.x.y is **not** yet supported) | ``gcc@`` | +-------------------------------------------+----------------------------------------------------------------------+---------------------------+ | Apple clang (clang, clang++, w/ gfortran) | 13.1.6 to 15.0.0 | ``apple-clang@`` | +-------------------------------------------+----------------------------------------------------------------------+---------------------------+ From 2ca1af4b986a37e63ba0343859f23f190c9462b2 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 25 Jan 2024 15:24:42 -0700 Subject: [PATCH 05/11] Set Fortran compiler flags in configs/sites/aws-pcluster/compilers.yaml for oneAPI compilers to suppress deprecation warning messages --- configs/sites/aws-pcluster/compilers.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configs/sites/aws-pcluster/compilers.yaml b/configs/sites/aws-pcluster/compilers.yaml index 6363de90a..30486b296 100644 --- a/configs/sites/aws-pcluster/compilers.yaml +++ b/configs/sites/aws-pcluster/compilers.yaml @@ -6,7 +6,8 @@ compilers: cxx: /opt/intel/oneapi/compiler/2024.0/bin/icpx f77: /opt/intel/oneapi/compiler/2024.0/bin/ifort fc: /opt/intel/oneapi/compiler/2024.0/bin/ifort - flags: {} + flags: + fflags: -diag-disable=10448 operating_system: ubuntu22.04 target: x86_64 modules: From 388744d54891b6e8824a5bc9604d3a4d3861f3bf Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 26 Jan 2024 09:53:51 -0700 Subject: [PATCH 06/11] oneapi support for shumlib --- spack-ext/repos/jcsda-emc/packages/shumlib/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack-ext/repos/jcsda-emc/packages/shumlib/package.py b/spack-ext/repos/jcsda-emc/packages/shumlib/package.py index df89f717b..4e830afcf 100644 --- a/spack-ext/repos/jcsda-emc/packages/shumlib/package.py +++ b/spack-ext/repos/jcsda-emc/packages/shumlib/package.py @@ -39,7 +39,7 @@ def build(self, spec, prefix): os.system("make -f make/vm-x86-gfortran-clang.mk") elif spec.satisfies("%gcc"): os.system("make -f make/vm-x86-gfortran-gcc.mk") - elif spec.satisfies("%intel"): + elif spec.satisfies("%intel") or spec.satisfies("%oneapi"): os.system("make -f make/vm-x86-ifort-icc.mk") else: raise InstallError("No shumlib make config for this compiler") From 8c48ce3120895720e3df55ea4d6d1c8a3e143da8 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 14 Mar 2024 17:52:15 +0000 Subject: [PATCH 07/11] Update .gitmodules and submodule pointer for spack for code review and testing --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index a196480f8..e8b5c0aba 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,7 +5,7 @@ #url = https://github.com/jcsda/spack #branch = jcsda_emc_spack_stack url = https://github.com/climbfuji/spack - branch = bugfix/wgrib2_oneapi + branch = feature/oneapi [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules From 9dbfea4909e9847f433566dfc032face2c3957ae Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 15 Mar 2024 20:01:12 -0600 Subject: [PATCH 08/11] wgrib2 doesn't build with oneAPI --- configs/templates/unified-dev/spack.yaml | 3 +++ spack | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml index 8f9b0dfde..3866c7f99 100644 --- a/configs/templates/unified-dev/spack.yaml +++ b/configs/templates/unified-dev/spack.yaml @@ -45,3 +45,6 @@ spack: - ai-env%intel # jedi-tools doesn't build with Intel - jedi-tools-env%intel + # wgrib2 in global-workflow-env/ufs-srw-app-env doesn't build with Intel oneAPI + - global-workflow-env%oneapi + - ufs-srw-app-env%oneapi diff --git a/spack b/spack index 57466339a..abfee55a0 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 57466339a69b5cbcb4b346fbe18c2778ee248bbc +Subproject commit abfee55a08b857ef839f4f43b229ad1ed4e0cd74 From aa5fbc2e269c06690a646791def65093db6f2c04 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 15 Mar 2024 20:01:27 -0600 Subject: [PATCH 09/11] Unpin py-netcdf4 --- configs/common/packages.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 31a4dc967..4ac1bff59 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -227,9 +227,6 @@ # README.md #py-mysql-connector-python: # version: ['8.0.32'] - py-netcdf4: - version: ['1.5.8'] - variants: ~mpi py-numpy: require: ['@1.22.3'] py-pandas: From ec567acfe9ea5fa8ed01f0709f2e1b947bb28e08 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 18 Mar 2024 12:03:44 -0600 Subject: [PATCH 10/11] Update submodule pointer for spack --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index abfee55a0..d2dd591de 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit abfee55a08b857ef839f4f43b229ad1ed4e0cd74 +Subproject commit d2dd591de1e295f3c835f69ebe218abd35424b42 From c7d4314841bd9dc8d106ca55e2123616d0355463 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 18 Mar 2024 13:33:52 -0600 Subject: [PATCH 11/11] Update aws-pcluster site config --- configs/sites/aws-pcluster/compilers.yaml | 9 ++++++--- configs/sites/aws-pcluster/packages.yaml | 8 -------- spack | 2 +- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/configs/sites/aws-pcluster/compilers.yaml b/configs/sites/aws-pcluster/compilers.yaml index 150a39055..0a7b7377a 100644 --- a/configs/sites/aws-pcluster/compilers.yaml +++ b/configs/sites/aws-pcluster/compilers.yaml @@ -4,10 +4,13 @@ compilers: paths: cc: /opt/intel/oneapi/compiler/2024.0/bin/icx cxx: /opt/intel/oneapi/compiler/2024.0/bin/icpx - f77: /opt/intel/oneapi/compiler/2024.0/bin/ifort - fc: /opt/intel/oneapi/compiler/2024.0/bin/ifort + f77: /opt/intel/oneapi/compiler/2024.0/bin/ifx + fc: /opt/intel/oneapi/compiler/2024.0/bin/ifx flags: - fflags: -diag-disable=10448 + cflags: --gcc-install-dir=/usr/lib/gcc/x86_64-linux-gnu/11 + cxxflags: --gcc-install-dir=/usr/lib/gcc/x86_64-linux-gnu/11 + # not sure this works/is needed fflags: --gcc-install-dir=/usr/lib/gcc/x86_64-linux-gnu/11 + # with ifort, need this for fflags: fflags: -diag-disable=10448 operating_system: ubuntu22.04 target: x86_64 modules: diff --git a/configs/sites/aws-pcluster/packages.yaml b/configs/sites/aws-pcluster/packages.yaml index ae4e62216..3f92b8ae5 100644 --- a/configs/sites/aws-pcluster/packages.yaml +++ b/configs/sites/aws-pcluster/packages.yaml @@ -69,10 +69,6 @@ packages: externals: - spec: git-lfs@3.0.2 prefix: /usr - gmake: - externals: - - spec: gmake@4.3 - prefix: /usr groff: externals: - spec: groff@1.22.4 @@ -90,10 +86,6 @@ packages: externals: - spec: mysql@8.0.35 prefix: /usr - perl: - externals: - - spec: perl@5.34.0~cpanm+shared+threads - prefix: /usr pkg-config: externals: - spec: pkg-config@0.29.2 diff --git a/spack b/spack index d2dd591de..3eb84a57c 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit d2dd591de1e295f3c835f69ebe218abd35424b42 +Subproject commit 3eb84a57ce43592b72d6876cefc0072196751c37