Draft: QMCkl

CMakeLists.txt

@@ -137,6 +137,8 @@ option(EXT_SPEEDTEST  "Enable speed tests targets"           OFF)
 option(EXT_STATICPACK "Produce static packaging if possible" OFF)
 option(EXT_DETAIL     "Produce more detailed config output"  OFF)
 option(EXT_NVTX       "Turn on Nvidia NVTX ranges"           OFF)
+option(EXT_OMP        "Use OpenMP"                            ON)
+option(EXT_PROFILED   "Use profiled compile option"          OFF)
 
 set(EXT_BLAS_LIB   ""    CACHE STRING    "Blas libraries")
 set(EXT_LAPACK_LIB ""    CACHE STRING    "Lapack libraries")
@@ -297,6 +299,27 @@ foreach(OTHER_LIB ${EXT_OTHER_LIB})
   link_libraries(${OTHER_LIB})
 endforeach()
 
+find_package(TREXIO)
+if (TREXIO_FOUND)
+    list( APPEND PPDIRECTIVES _TREXIO )
+    link_libraries(${TREXIO_LIBRARIES})
+    include_directories( ${TREXIO_INCLUDE_DIRS} )
+endif()
+
+find_package(QMCKL)
+if (QMCKL_FOUND)
+    list( APPEND PPDIRECTIVES _QMCKL )
+    link_libraries(${QMCKL_LIBRARIES})
+    include_directories( ${QMCKL_INCLUDE_DIRS} )
+endif()
+
+find_package(QMCKL_GPU)
+if (QMCKL_GPU_FOUND)
+    list( APPEND PPDIRECTIVES _QMCKL_GPU )
+    link_libraries(${QMCKL_GPU_LIBRARIES})
+    include_directories( ${QMCKL_GPU_INCLUDE_DIRS} )
+endif()
+
 ######################################################################
 #
 # Setting native OMP to off
@@ -307,7 +330,9 @@ set( NATIVE_OMP_B OFF )
 #
 ######################################################################
 
-find_package(OpenMP)
+if(${EXT_OMP})
+  find_package(OpenMP)
+endif()
 find_package(Threads)
 
 if(EXT_GPU)
@@ -588,6 +613,7 @@ if(EXT_MODTEST)
   add_target_exe_serial_wrapper(test_openmp_reduction_complex test_tools)
   add_target_exe_serial_wrapper(test_upwinvp test_tools)
   add_target_exe_serial_wrapper(test_upwinvp_complex test_tools)
+  add_target_exe_serial_wrapper(test_makefun test_tools)
   #add_target_exe_serial_wrapper(test_upwinvp_pfaff test_tools)
   #add_target_exe_serial_wrapper(test_upwinvp_pfaff_complex test_tools)
 

Makefile

@@ -28,12 +28,15 @@ ifeq ($(make_inc_c) ,c)
     $(info )
     $(info 1 GNU Fortran Compiler )
     $(info 2 GNU Fortran Compiler with MPI) 
+    $(info 3 Intel Fortran Compiler with MPI) 
     $(info )
-    $(shell read -p "Select an example [1-2]: " ans; \
+    $(shell read -p "Select an example [1-3]: " ans; \
         if [ "$$ans" = "1" ]; then \
             cp devel_tools/make.inc.examples/make.inc.example.gcc make.inc; \
         elif [ "$$ans" = "2" ]; then \
             cp devel_tools/configs/make.inc.example.gccmpi make.inc; \
+        elif [ "$$ans" = "3" ]; then \
+            cp devel_tools/configs/make.inc.example.mpiifort make.inc; \
         fi)
 
 endif

cmake/FindQMCKL_GPU.cmake

@@ -0,0 +1,79 @@
+#===========================================
+
+# Try to find the QMCkl GPU library;
+# If found, it will define the following variables (note the plural form):
+#  QMCKL_GPU_FOUND - System has libqmckl;
+#  QMCKL_GPU_INCLUDE_DIRS - The QMCKL GPU include directories;
+#  QMCKL_GPU_LIBRARIES - The libraries needed to use QMCKL GPU;
+
+# If QMCKL GPU has been installed in a non-standard location, one can set an
+# environment variable $QMCKL_GPU_DIR in the current shell:
+# $ export QMCKL_GPU_DIR=<custom_path>
+# to indicate the prefix used during the QMCKL installation
+# (typically `./configure prefix=<custom_path> ..` or `cmake -DCMAKE_INSTALL_DIR=<custom_path> ..`)
+
+# This file should be located WITHIN your project source tree.
+# (e.g. in cmake/FindQMCKL_GPU.cmake)
+# How to use it in your project CMakeLists.txt:
+
+# This is needed to locate FindQMCKL_GPU.cmake file, modify it according to your source tree.
+# list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/")
+
+# find_package(QMCKL_GPU)
+# if (QMCKL_GPU_FOUND)
+#   include_directories(${QMCKL_GPU_INCLUDE_DIRS})
+#   target_link_libraries(your_target ${QMCKL_GPU_LIBRARIES})
+# endif()
+
+#===========================================
+
+# This file is distirbuted under the BSD 3-Clause License.
+# Copyright (c) 2021, TREX Center of Excellence
+
+#===========================================
+
+message(" ")
+message("Looking for the QMCKL_GPU library:")
+
+set(QMCKL_GPU_SEARCH_PATHS
+        ~/Library/Frameworks
+        /Library/Frameworks
+        /usr/local
+        /usr
+        /sw # Fink
+        /opt/local # DarwinPorts
+        /opt/csw # Blastwave
+        /opt
+)
+
+find_path(QMCKL_GPU_INCLUDE_DIR
+          NAMES qmckl_gpu_f.F90
+          HINTS $ENV{QMCKL_GPU_DIR}
+          PATH_SUFFIXES include
+          PATHS ${QMCKL_GPU_SEARCH_PATHS}
+          )
+
+
+# No need to specify platform-specific prefix (e.g. libqmckl_gpu on Unix) or
+# suffix (e.g. .so on Unix or .dylib on MacOS) in NAMES. CMake takes care of that.
+find_library(QMCKL_GPU_LIBRARY
+             NAMES qmckl_gpu
+             HINTS $ENV{QMCKL_GPU_DIR}
+             PATH_SUFFIXES lib64 lib
+             PATHS ${QMCKL_GPU_SEARCH_PATHS}
+             )
+
+get_filename_component(QMCKL_GPU_LIBRARY_DIR ${QMCKL_GPU_LIBRARY} DIRECTORY)
+
+set(QMCKL_GPU_LIBRARIES "-L${QMCKL_GPU_LIBRARY_DIR} -lqmckl_gpu")
+
+# Handle the QUIETLY and REQUIRED arguments and set QMCKL_GPU_FOUND to TRUE if
+# all listed variables are TRUE.
+INCLUDE(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(QMCKL_GPU DEFAULT_GPU_MSG QMCKL_GPU_LIBRARY QMCKL_GPU_INCLUDE_DIR )
+MARK_AS_ADVANCED(QMCKL_GPU_INCLUDE_DIR QMCKL_GPU_LIBRARY)
+
+# Mot setting _INCLUDE_DIR and _LIBRARIES is considered a bug,
+# see https://gitlab.kitware.com/cmake/community/-/wikis/doc/tutorials/How-To-Find-Libraries
+set(QMCKL_GPU_INCLUDE_DIRS ${QMCKL_GPU_INCLUDE_DIR})
+

cmake/compilers/GNU/CMakeLists.txt

@@ -30,6 +30,10 @@ macro(SET_GNU_FORTRAN_COMPILER)
 
   check_Fortran_compiler_flag("-ffree-line-length-none" _free_line_length_none_works)
 
+  if (${EXT_PROFILED})
+    add_fortran_flag_if_avail( "-pg" )
+  endif()
+
   if(EXT_GPU)
 	  add_fortran_flag_if_avail( "-fno-whole-file" "GNU Fortran compiler does not supports no-whole-flag, turn off GPU support" )
 	  add_fortran_flag_if_avail( "-fopenmp" )
@@ -58,6 +62,15 @@ macro(SET_GNU_FORTRAN_COMPILER)
         endif()
       endforeach()
   endif()
+
+  foreach( TARGET IN LISTS EXECUTABLES_S_L
+                           EXECUTABLES_P_L
+                           LIBRARIES_S_L
+                           LIBRARIES_P_L)
+    if (${EXT_PROFILED})
+      target_link_options( ${TARGET} PRIVATE "-pg" )
+    endif()
+  endforeach()
 
   add_fortran_flag_if_avail( "-march=native" )
   add_fortran_flag_if_avail( "-mtune=native" )
@@ -123,6 +136,10 @@ macro(SET_GNU_C_COMPILER)
 
   add_c_flag_if_avail( "-g" )
 
+  if (${EXT_PROFILED})
+    add_c_flag_if_avail( "-pg" )
+  endif()
+
   if( ${EXT_OPT} )
     add_c_flag_if_avail( "-O3" )
   else()

cmake/compilers/intel/CMakeLists.txt

@@ -16,18 +16,42 @@ macro(SET_INTEL_FORTRAN_COMPILER)
   endif()
 
   add_fortran_flag_if_avail( "-fpp"   "Intel (Legacy) Fortran compiler does not supports c preprocessor" )
-  add_fortran_flag_if_avail( "-qopenmp" )
+  add_fortran_flag_if_avail( "-march=core-avx2" )
+  add_fortran_flag_if_avail( "-qopt-report" )
+
   add_fortran_flag_if_avail( "-g" )
-
-  #message(${CMAKE_APPLE_SILICON_PROCESSOR})
-  #if( ${CMAKE_APPLE_SILICON_PROCESSOR} MATCHES x86_64 )
-  #  add_fortran_flag_if_avail( "-arch=x86_64" )
-  #endif()
+
+  if( ${EXT_PROFILED} )
+    add_fortran_flag_if_avail( "-pg" )
+  endif()
+
+  # If OpenMP is yes nad NATIVE_OMP_B is false, the compile with OpenMP
+  if( ${EXT_OMP} )
+    if( ${OPENMP} )
+      if( NOT ${NATIVE_OMP_B} )
+        add_fortran_flag_if_avail( "-qopenmp" )
+      endif()
+    endif()
+  endif()
 
   if ( ${APPLE} )
     add_fortran_flag_if_avail( "-fno-common" )
   endif()
 
+  foreach( LIBRARY IN LISTS LIBRARIES_S_L
+                            LIBRARIES_P_L 
+                            EXECUTABLES_S_L
+                            EXECUTABLES_P_L
+                            )
+    if( ${EXT_OMP} )
+      target_link_options(${LIBRARY} PRIVATE -qopenmp)
+    endif()
+    if( ${EXT_PROFILED} )
+      target_link_options(${LIBRARY} PRIVATE -pg)
+    endif()
+
+  endforeach()
+
   # Set optimization flags for all executables:
   foreach( LIBRARY IN LISTS LIBRARIES_S_L
                             LIBRARIES_P_L 

cmake/compilers/oneapi/CMakeLists.txt

@@ -9,6 +9,9 @@ macro(SET_ONEAPI_FORTRAN_COMPILER)
   message( STATUS "Using Intel oneAPI Compiler" )
 
   add_fortran_flag_if_avail( "-fpp"   "Intel LLVM Fortran compiler does not supports C preprocessor" )
+  add_fortran_flag_if_avail( "-xHost" )
+  add_fortran_flag_if_avail( "-finline-functions" )
+  add_fortran_flag_if_avail( "-qopt-report" )
 
   set(AGRESSIVE_F "-Ofast")
   if( ${EXT_OPT} )
@@ -20,6 +23,23 @@ macro(SET_ONEAPI_FORTRAN_COMPILER)
     set(PASIVE_F "-O0")
   endif()
 
+  if( ${EXT_PROFILED} )
+    add_fortran_flag_if_avail( "-pg" )
+  endif()
+
+  foreach( LIBRARY IN LISTS LIBRARIES_S_L
+                            LIBRARIES_P_L
+                            EXECUTABLES_S_L
+                            EXECUTABLES_P_L
+                            )
+    if( ${EXT_PROFILED} )
+      target_link_options( ${LIBRARY} PRIVATE "-pg" )
+    endif()
+    if( ${EXT_OMP} )
+      target_link_options( ${LIBRARY} PRIVATE "-qopenmp" )
+    endif()
+  endforeach()
+
   # Set optimization flags:
   foreach( LIBRARY IN LISTS LIBRARIES_S_L
                             LIBRARIES_P_L
@@ -68,6 +88,10 @@ macro(SET_ONEAPI_C_COMPILER)
 
   add_c_flag_if_avail( "-g" )
 
+  if( ${EXT_PROFILED} )
+    add_c_flag_if_avail( "-pg" )
+  endif()
+
   if( ${EXT_OPT} )
     add_c_flag_if_avail( "-O3" )
   else()

devel_tools/adjoints/make0branch.sh

@@ -16,25 +16,25 @@
 # Otto Kohulák created on 5th Nov. 2021.
 
 # copy in the present directory makefuns
-cp ../../src/makefun.f90 .
-cp ../../AD/reverse_cell/makefun_b.f90 .
-cp ../../src/makefun_pbc.f90 .
-cp ../../AD/reverse_cell/makefun_pbc_b.f90 .
-cp ../../src/makefun_bump.f90 .
-cp ../../AD/reverse_cell/makefun_bump_b.f90 .
+cp ../../src/c_adjoint_forward/makefun.f90 .
+cp ../../src/c_adjoint_backward/makefun_b.f90 .
+cp ../../src/c_adjoint_forward/makefun_pbc.f90 .
+cp ../../src/c_adjoint_backward/makefun_pbc_b.f90 .
+cp ../../src/c_adjoint_forward/makefun_bump.f90 .
+cp ../../src/c_adjoint_backward/makefun_bump_b.f90 .
 
 # process the mkefuns
-python preprocess.py -c makefun0:[i0=0,indtmin=0,indtm=0] -o makefun0.f90 makefun.f90
-python preprocess.py -c makefun0_b:[i0=0,indtmin=0,indtm=0] -o makefun0_b.f90 makefun_b.f90
-python preprocess.py -c makefun0_pbc:[i0=0,indtmin=0,indtm=0] -o makefun0_pbc.f90 makefun_pbc.f90
-python preprocess.py -c makefun0_pbc_b:[i0=0,indtmin=0,indtm=0] -o makefun0_pbc_b.f90 makefun_pbc_b.f90
-python preprocess.py -c makefun0_bump:[i0=0,indtmin=0,indtm=0] -o makefun0_bump.f90 makefun_bump.f90
-python preprocess.py -c makefun0_bump_b:[i0=0,indtmin=0,indtm=0] -o makefun0_bump_b.f90 makefun_bump_b.f90
+python3 preprocess.py -c makefun0:[i0=0,indtmin=0,indtm=0]        -o makefun0.f90        makefun.f90
+python3 preprocess.py -c makefun0_b:[i0=0,indtmin=0,indtm=0]      -o makefun0_b.f90      makefun_b.f90
+python3 preprocess.py -c makefun0_pbc:[i0=0,indtmin=0,indtm=0]    -o makefun0_pbc.f90    makefun_pbc.f90
+python3 preprocess.py -c makefun0_pbc_b:[i0=0,indtmin=0,indtm=0]  -o makefun0_pbc_b.f90  makefun_pbc_b.f90
+python3 preprocess.py -c makefun0_bump:[i0=0,indtmin=0,indtm=0]   -o makefun0_bump.f90   makefun_bump.f90
+python3 preprocess.py -c makefun0_bump_b:[i0=0,indtmin=0,indtm=0] -o makefun0_bump_b.f90 makefun_bump_b.f90
 
 # copy makefuns0
-cp makefun0.f90 ../../src/makefun0.f90
-cp makefun0_b.f90 ../../AD/reverse_cell/
-cp makefun0_pbc.f90 ../../src/makefun0_pbc.f90
-cp makefun0_pbc_b.f90 ../../AD/reverse_cell/
-cp makefun0_bump.f90 ../../src/makefun0_bump.f90
-cp makefun0_bump_b.f90 ../../AD/reverse_cell/
+cp makefun0.f90        ../../src/c_adjoint_forward/makefun0.f90
+cp makefun0_b.f90      ../../src/c_adjoint_backward/
+cp makefun0_pbc.f90    ../../src/c_adjoint_forward/makefun0_pbc.f90
+cp makefun0_pbc_b.f90  ../../src/c_adjoint_backward/
+cp makefun0_bump.f90   ../../src/c_adjoint_forward/makefun0_bump.f90
+cp makefun0_bump_b.f90 ../../src/c_adjoint_backward/