Merge 'ABHI/abagusetty/homme/aurora' (PR E3SM-Project#6690)

Update Cmake machine config for Aurora Cleanup certain warnings related to headers and Intel compilers Linking Fortran/C++/SYCL with CXX linker (-fortlib) instead of Fortran language for SYCL builds. Since Fortran linking with SYCL is no longer supported (link is in the PR).
E3SM-Ocean-Discussion · Oct 29, 2024 · d8a408c · d8a408c
2 parents d4a5531 + aab68c6
commit d8a408c
Show file tree

Hide file tree

Showing 6 changed files with 28 additions and 34 deletions.
diff --git a/components/homme/cmake/HommeMacros.cmake b/components/homme/cmake/HommeMacros.cmake
@@ -112,7 +112,13 @@ macro(createTestExec execName execType macroNP macroNC
   ADD_DEFINITIONS(-DHAVE_CONFIG_H)
 
   ADD_EXECUTABLE(${execName} ${EXEC_SOURCES})
-  SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE Fortran)
+  # For SYCL builds it is suggested to use CXX linker with `-fortlib`
+  # for mixed-language setups
+  IF(SYCL_BUILD)
+    SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE CXX)
+  ELSE()
+    SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE Fortran)
+  ENDIF()
   IF(BUILD_HOMME_WITHOUT_PIOLIBRARY)
     TARGET_COMPILE_DEFINITIONS(${execName} PUBLIC HOMME_WITHOUT_PIOLIBRARY)
   ENDIF()

diff --git a/components/homme/cmake/machineFiles/aurora-aot.cmake b/components/homme/cmake/machineFiles/aurora-aot.cmake
@@ -1,11 +1,9 @@
 #module restore
-#module load oneapi/eng-compiler/2022.12.30.005
-#module load intel_compute_runtime/release/agama-devel-627
-#module load spack cmake
+#module load spack-pe-base cmake
 #module list
 
 
-SET (SUNSPOT_MACHINE TRUE CACHE BOOL "")
+SET (AURORA_MACHINE TRUE CACHE BOOL "")
 
 SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "")
 SET(HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "")
@@ -18,12 +16,12 @@ SET(USE_QUEUING FALSE CACHE BOOL "")
 
 #temp hack
 SET(HOMME_USE_KOKKOS TRUE CACHE BOOL "")
+SET(HOMME_USE_MKL TRUE CACHE BOOL "")
 
 SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "")
 SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "")
 
-#set(KOKKOS_HOME "/home/onguba/kokkos-build/mar05-aot/install" CACHE STRING "")
-#set(E3SM_KOKKOS_PATH ${KOKKOS_HOME} CACHE STRING "")
+set(Kokkos_ROOT $ENV{KOKKOS_HOME} CACHE STRING "")
 
 SET(USE_TRILINOS OFF CACHE BOOL "")
 
@@ -36,19 +34,14 @@ SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "")
 SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "")
 SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "")
 
-# -fsycl-link-huge-device-code for theta to get build
-#JIT flags
-#SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda")
-#SET(SYCL_LINK_FLAGS "-fsycl -fsycl-link-huge-device-code -fsycl-device-code-split=per_kernel -fsycl-targets=spir64")
-
 #AOT flags
 SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda")
-SET(SYCL_LINK_FLAGS "-fsycl-max-parallel-link-jobs=32 -fsycl-link-huge-device-code -fsycl -fsycl-device-code-split=per_kernel -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device 12.60.7\"")
+SET(SYCL_LINK_FLAGS "-fsycl-max-parallel-link-jobs=32 -fsycl -fsycl-device-code-split=per_kernel -fsycl-targets=intel_gpu_pvc")
 
 SET(ADD_Fortran_FLAGS "-fc=ifx -fpscomp logicals -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "")
 SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "")
 
-SET(ADD_CXX_FLAGS "-std=c++17 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "")
+SET(ADD_CXX_FLAGS "-std=c++17 -fp-model=precise -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "")
 SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "")
 
 set (ENABLE_OPENMP OFF CACHE BOOL "")
@@ -57,8 +50,6 @@ set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "")
 
 set (HOMME_TESTING_PROFILE "dev" CACHE STRING "")
 
-set (USE_NUM_PROCS 4 CACHE STRING "")
-
-SET (USE_MPI_OPTIONS "--bind-to core" CACHE FILEPATH "")
-
+set (USE_NUM_PROCS 12 CACHE STRING "")
 
+SET (USE_MPI_OPTIONS "--pmi=pmix --cpu-bind list:0-7,104-111:8-15,112-119:16-23,120-127:24-31,128-135:32-39,136-143:40-47,144-151:52-59,156-163:60-67,164-171:68-75,172-179:76-83,180-187:84-91,188-195:92-99,196-203 gpu_tile_compact.sh" CACHE FILEPATH "")
diff --git a/components/homme/cmake/machineFiles/aurora-jit.cmake b/components/homme/cmake/machineFiles/aurora-jit.cmake
@@ -1,10 +1,9 @@
 #module restore
-#module load oneapi/eng-compiler/2022.12.30.005
-#module load intel_compute_runtime/release/agama-devel-627
-#module load spack cmake
+#module load spack-pe-base cmake
 #module list
 
 
+SET (AURORA_MACHINE TRUE CACHE BOOL "")
 
 SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "")
 SET(HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "")
@@ -17,12 +16,12 @@ SET(USE_QUEUING FALSE CACHE BOOL "")
 
 #temp hack
 SET(HOMME_USE_KOKKOS TRUE CACHE BOOL "")
+SET(HOMME_USE_MKL TRUE CACHE BOOL "")
 
 SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "")
 SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "")
 
-#set(KOKKOS_HOME "/home/onguba/kokkos-build/jan03-2024/install" CACHE STRING "")
-#set(E3SM_KOKKOS_PATH ${KOKKOS_HOME} CACHE STRING "")
+set(Kokkos_ROOT $ENV{KOKKOS_HOME} CACHE STRING "")
 
 SET(USE_TRILINOS OFF CACHE BOOL "")
 
@@ -35,14 +34,14 @@ SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "")
 SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "")
 SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "")
 
-# -fsycl-link-huge-device-code for theta to get build
+#AOT flags
 SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda")
-SET(SYCL_LINK_FLAGS "-fsycl -fsycl-link-huge-device-code -fsycl-device-code-split=per_kernel -fsycl-targets=spir64")
+SET(SYCL_LINK_FLAGS "-fsycl-max-parallel-link-jobs=32 -fsycl")
 
-SET(ADD_Fortran_FLAGS "-fc=ifx -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "")
+SET(ADD_Fortran_FLAGS "-fc=ifx -fpscomp logicals -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "")
 SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "")
 
-SET(ADD_CXX_FLAGS "-std=c++17 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "")
+SET(ADD_CXX_FLAGS "-std=c++17 -fp-model=precise -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "")
 SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "")
 
 set (ENABLE_OPENMP OFF CACHE BOOL "")
@@ -51,8 +50,6 @@ set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "")
 
 set (HOMME_TESTING_PROFILE "dev" CACHE STRING "")
 
-set (USE_NUM_PROCS 4 CACHE STRING "")
-
-SET (USE_MPI_OPTIONS "--bind-to core" CACHE FILEPATH "")
-
+set (USE_NUM_PROCS 12 CACHE STRING "")
 
+SET (USE_MPI_OPTIONS "--pmi=pmix --cpu-bind list:0-7,104-111:8-15,112-119:16-23,120-127:24-31,128-135:32-39,136-143:40-47,144-151:52-59,156-163:60-67,164-171:68-75,172-179:76-83,180-187:84-91,188-195:92-99,196-203 gpu_tile_compact.sh" CACHE FILEPATH "")
diff --git a/components/homme/src/share/cxx/ExecSpaceDefs.cpp b/components/homme/src/share/cxx/ExecSpaceDefs.cpp
@@ -22,7 +22,7 @@
 #endif
 
 #ifdef KOKKOS_ENABLE_SYCL
-#include <CL/sycl.hpp>
+#include <sycl/sycl.hpp>
 #endif
 
 namespace Homme {

diff --git a/components/homme/src/share/cxx/utilities/scream_tridiag.hpp b/components/homme/src/share/cxx/utilities/scream_tridiag.hpp
@@ -120,7 +120,7 @@ int get_team_nthr (const TeamMember& team) {
   return team.team_size();
 }
 
-// Impl details for Nvidia and AMD GPUs.
+// Impl details for Nvidia, AMD and Intel GPUs.
 
 template <typename TeamMember> KOKKOS_FORCEINLINE_FUNCTION
 int get_thread_id_within_team_gpu (const TeamMember& team) {

diff --git a/components/homme/src/share/cxx/vector/vector_pragmas.hpp b/components/homme/src/share/cxx/vector/vector_pragmas.hpp
@@ -7,7 +7,7 @@
 #ifndef HOMMEXX_VECTOR_PRAGMAS_HPP
 #define HOMMEXX_VECTOR_PRAGMAS_HPP
 
-#if defined(__INTEL_COMPILER)
+#if defined(__INTEL_COMPILER) || defined(__INTEL_CLANG_COMPILER) || defined(__INTEL_LLVM_COMPILER)
 
 #define VECTOR_IVDEP_LOOP _Pragma("ivdep")
 #define ALWAYS_VECTORIZE_LOOP _Pragma("vector always")