rocprofv3 OTF2 Output Support (#995)

* CMake support for OTF2 library

* Preliminary OTF2 generation implementation

* Completed OTF2 Support

- HSA API
- HIP API
- Marker API
- Async Memory Copies
- Kernel Dispatch

* Update lib/rocprofiler-sdk-tool/generateOTF2.cpp

- fix location type for dispatches

* Testing for OTF2 output

* Add OTF2 to requirements.txt

* Update lib/rocprofiler-sdk-tool/generateOTF2.cpp

- fix getting kernel name

* OTF2 testing with rocprofv3/tracing-hip-in-libraries

* Format external/otf2/CMakeLists.txt

* Update external/otf2/CMakeLists.txt

- guard CMP0135 for cmake < 3.24

* Update lib/rocprofiler-sdk-tool/generateOTF2.cpp

- fix duplicate string ref issue

* Update lib/rocprofiler-sdk-tool/generateOTF2.cpp

- fix header includes

* Update CI workflow

- sudo install pypi requirements for core-rpm for $HOME/.local installs

* Update pytest_utils/otf2_reader.py

- modifications for reading trace

* Update pytest_utils/otf2_reader.py

- misc cleanup

* Update CI workflow

- fix installer artifact naming

* Update pytest_utils/otf2_reader.py

- handle slightly overlapping kernel timestamps for MI300

* OTF2 attributes for category

* Testing with OTF2Reader category attributes

* Fix memory leak in OTF2 generation

- leaking OTF2_AttributeList

.github/workflows/continuous_integration.yml

@@ -77,7 +77,7 @@ jobs:
       shell: bash
       run: |
           echo 'ROCPROFILER_PC_SAMPLING_BETA_ENABLED=1' >> $GITHUB_ENV
-    
+
     - name: Configure, Build, and Test
       timeout-minutes: 30
       shell: bash
@@ -150,12 +150,12 @@ jobs:
       if: ${{ contains(matrix.runner, env.CORE_EXT_RUNNER) }}
       uses: actions/upload-artifact@v4
       with:
-        name: installers
+        name: installers-deb
         path: |
           ${{github.workspace}}/build/*.deb
           ${{github.workspace}}/build/*.rpm
           ${{github.workspace}}/build/*.tgz
-  
+
   core-rpm:
     # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix
     strategy:
@@ -181,6 +181,7 @@ jobs:
       shell: bash
       run: |
         python3 -m pip install -r requirements.txt
+        sudo python3 -m pip install -r requirements.txt
 
     - name: List Files
       shell: bash
@@ -190,7 +191,7 @@ jobs:
         which-realpath() { echo -e "\n$1 resolves to $(realpath $(which $1))"; echo "$($(which $1) --version &> /dev/stdout | head -n 1)"; }
         for i in python3 git cmake ctest gcc g++ gcov; do which-realpath $i; done
         ls -la
-    
+
     - name: Exclude PC Sampling Tests
       if: ${{ !contains(matrix.runner, 'mi200') && !contains(matrix.runner, 'mi300') }}
       shell: bash
@@ -275,7 +276,7 @@ jobs:
       if: ${{ contains(matrix.runner, env.CORE_EXT_RUNNER) }}
       uses: actions/upload-artifact@v4
       with:
-        name: installers
+        name: installers-rpm
         path: |
           ${{github.workspace}}/build/*.deb
           ${{github.workspace}}/build/*.rpm

cmake/rocprofiler_config_interfaces.cmake

@@ -276,3 +276,11 @@ target_link_libraries(rocprofiler-drm INTERFACE ${drm_LIBRARY} ${drm_amdgpu_LIBR
 # get_target_property(ELFIO_INCLUDE_DIR elfio::elfio INTERFACE_INCLUDE_DIRECTORIES)
 # target_include_directories(rocprofiler-elfio SYSTEM INTERFACE ${ELFIO_INCLUDE_DIR})
 target_link_libraries(rocprofiler-elfio INTERFACE elfio::elfio)
+
+# ----------------------------------------------------------------------------------------#
+#
+# OTF2
+#
+# ----------------------------------------------------------------------------------------#
+
+target_link_libraries(rocprofiler-otf2 INTERFACE otf2::otf2)

cmake/rocprofiler_interfaces.cmake

@@ -18,6 +18,7 @@ rocprofiler_add_interface_library(rocprofiler-threading "Enables multithreading
                                   INTERNAL)
 rocprofiler_add_interface_library(rocprofiler-perfetto "Enables Perfetto support"
                                   INTERNAL)
+rocprofiler_add_interface_library(rocprofiler-otf2 "Enables OTF2 support" INTERNAL)
 rocprofiler_add_interface_library(rocprofiler-cereal "Enables Cereal support" INTERNAL)
 rocprofiler_add_interface_library(rocprofiler-compile-definitions "Compile definitions"
                                   INTERNAL)

external/CMakeLists.txt

@@ -204,3 +204,6 @@ if(TARGET rocprofiler-elfio)
     get_target_property(ELFIO_INCLUDE_DIR elfio::elfio INTERFACE_INCLUDE_DIRECTORIES)
     target_include_directories(rocprofiler-elfio SYSTEM INTERFACE ${ELFIO_INCLUDE_DIR})
 endif()
+
+# OTF2
+add_subdirectory(otf2)

external/otf2/CMakeLists.txt

@@ -0,0 +1,72 @@
+# ======================================================================================
+# Builds OTF2
+# ======================================================================================
+
+set(ROCPROFILER_BINARY_DIR ${PROJECT_BINARY_DIR})
+set(OTF2_VERSION
+    "3.0.3"
+    CACHE STRING "OTF2 version")
+set(OTF2_URL_HASH
+    "SHA256=18a3905f7917340387e3edc8e5766f31ab1af41f4ecc5665da6c769ca21c4ee8"
+    CACHE STRING "OTF2 URL download hash")
+
+project(
+    OTF2
+    LANGUAGES C
+    VERSION ${OTF2_VERSION}
+    DESCRIPTION "Open Trace Format v2"
+    HOMEPAGE_URL "https://perftools.pages.jsc.fz-juelich.de/cicd/otf2")
+
+include(FetchContent)
+include(ExternalProject)
+include(ProcessorCount)
+
+if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.24)
+    cmake_policy(SET CMP0135 NEW)
+endif()
+
+set(FETCHCONTENT_BASE_DIR ${ROCPROFILER_BINARY_DIR}/external/packages)
+
+fetchcontent_declare(
+    otf2-source
+    URL https://perftools.pages.jsc.fz-juelich.de/cicd/otf2/tags/otf2-${OTF2_VERSION}/otf2-${OTF2_VERSION}.tar.gz
+    URL_HASH ${OTF2_URL_HASH})
+
+fetchcontent_getproperties(ot2f-source)
+
+if(NOT ot2f-source_POPULATED)
+    message(STATUS "Downloading OTF2...")
+    fetchcontent_populate(otf2-source)
+endif()
+
+set(_otf2_root ${ROCPROFILER_BINARY_DIR}/external/otf2)
+set(_otf2_inc_dirs $<BUILD_INTERFACE:${_otf2_root}/include>)
+set(_otf2_lib_dirs $<BUILD_INTERFACE:${_otf2_root}/lib>)
+set(_otf2_libs $<BUILD_INTERFACE:${_otf2_root}/lib/libotf2${CMAKE_STATIC_LIBRARY_SUFFIX}>)
+set(_otf2_build_byproducts "${_otf2_root}/lib/libotf2${CMAKE_STATIC_LIBRARY_SUFFIX}")
+
+find_program(
+    MAKE_COMMAND
+    NAMES make gmake
+    PATH_SUFFIXES bin REQUIRED)
+
+externalproject_add(
+    otf2-build
+    PREFIX ${_otf2_root}
+    SOURCE_DIR ${otf2-source_SOURCE_DIR}
+    BUILD_IN_SOURCE 1
+    DOWNLOAD_COMMAND ""
+    PATCH_COMMAND
+        ${CMAKE_COMMAND} -E env CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER}
+        <SOURCE_DIR>/configure -q --prefix=${_otf2_root} CFLAGS=-fPIC\ -O3\ -g
+        CXXFLAGS=-fPIC\ -O3\ -g PYTHON=: SPHINX=:
+    CONFIGURE_COMMAND ${MAKE_COMMAND} install -s
+    BUILD_COMMAND ""
+    BUILD_BYPRODUCTS "${_otf2_build_byproducts}"
+    INSTALL_COMMAND "")
+
+add_library(otf2 INTERFACE)
+add_library(otf2::otf2 ALIAS otf2)
+target_include_directories(otf2 SYSTEM INTERFACE ${_otf2_inc_dirs})
+target_link_directories(otf2 INTERFACE ${_otf2_lib_dirs})
+target_link_libraries(otf2 INTERFACE ${_otf2_libs})

requirements.txt

@@ -5,6 +5,7 @@ cmake>=3.21.0
 cmake-format
 dataclasses
 numpy
+otf2
 pandas
 perfetto
 pycobertura

source/bin/rocprofv3.py

@@ -196,10 +196,10 @@ def add_parser_bool_argument(*args, **kwargs):
     )
     parser.add_argument(
         "--output-format",
-        help="For adding output format (supported formats: csv, json, pftrace)",
+        help="For adding output format (supported formats: csv, json, pftrace, otf2)",
         nargs="+",
         default=None,
-        choices=("csv", "json", "pftrace"),
+        choices=("csv", "json", "pftrace", "otf2"),
         type=str.lower,
     )
     parser.add_argument(

source/include/rocprofiler-sdk/cxx/perfetto.hpp

@@ -53,7 +53,8 @@
     {                                                                                              \
     namespace NS                                                                                   \
     {                                                                                              \
-    struct VALUE;                                                                                  \
+    struct VALUE                                                                                   \
+    {};                                                                                            \
     }                                                                                              \
     }                                                                                              \
     }                                                                                              \

source/lib/common/utility.hpp

@@ -79,9 +79,11 @@ get_ticks(clockid_t clk_id_v) noexcept
     return (static_cast<uint64_t>(ts.tv_sec) * nanosec) + static_cast<uint64_t>(ts.tv_nsec);
 }
 
+static constexpr int default_clock_id = CLOCK_BOOTTIME;
+
 // CLOCK_MONOTONIC_RAW equates to HSA-runtime library implementation of os::ReadAccurateClock()
 // CLOCK_BOOTTIME equates to HSA-runtime library implementation of os::ReadSystemClock()
-template <int ClockT = CLOCK_BOOTTIME>
+template <int ClockT = default_clock_id>
 inline uint64_t
 timestamp_ns()
 {

source/lib/rocprofiler-sdk-tool/CMakeLists.txt

@@ -11,6 +11,7 @@ set(TOOL_HEADERS
     domain_type.hpp
     generateCSV.hpp
     generateJSON.hpp
+    generateOTF2.hpp
     generatePerfetto.hpp
     helper.hpp
     output_file.hpp
@@ -23,6 +24,7 @@ set(TOOL_SOURCES
     domain_type.cpp
     generateCSV.cpp
     generateJSON.cpp
+    generateOTF2.cpp
     generatePerfetto.cpp
     helper.cpp
     main.c
@@ -44,7 +46,8 @@ target_link_libraries(
             rocprofiler-sdk::rocprofiler-memcheck
             rocprofiler-sdk::rocprofiler-common-library
             rocprofiler-sdk::rocprofiler-cereal
-            rocprofiler-sdk::rocprofiler-perfetto)
+            rocprofiler-sdk::rocprofiler-perfetto
+            rocprofiler-sdk::rocprofiler-otf2)
 
 set_target_properties(
     rocprofiler-sdk-tool

source/lib/rocprofiler-sdk-tool/config.cpp

@@ -36,6 +36,7 @@
 
 #include <unistd.h>
 #include <algorithm>
+#include <chrono>
 #include <cstring>
 #include <ctime>
 #include <fstream>
@@ -51,24 +52,35 @@ namespace tool
 {
 namespace
 {
+template <typename Tp>
+auto
+as_pointer(Tp&& _val)
+{
+    return new Tp{_val};
+}
+
 std::string*
-get_local_datetime(const std::string& dt_format);
+get_local_datetime(const std::string& dt_format, std::time_t*& dt_curr);
 
-const auto* launch_datetime = get_local_datetime(get_env("ROCP_TIME_FORMAT", "%F_%H.%M"));
-const auto  env_regexes =
+std::time_t* launch_time  = nullptr;
+const auto*  launch_clock = as_pointer(std::chrono::system_clock::now());
+const auto*  launch_datetime =
+    get_local_datetime(get_env("ROCP_TIME_FORMAT", "%F_%H.%M"), launch_time);
+const auto env_regexes =
     new std::array<std::regex, 2>{std::regex{"(.*)%(env|ENV)\\{([A-Z0-9_]+)\\}%(.*)"},
                                   std::regex{"(.*)\\$(env|ENV)\\{([A-Z0-9_]+)\\}(.*)"}};
 
 std::string*
-get_local_datetime(const std::string& dt_format)
+get_local_datetime(const std::string& dt_format, std::time_t*& _dt_curr)
 {
     constexpr auto strsize = 512;
-    auto           dt_curr = std::time_t{std::time(nullptr)};
 
-    char mbstr[strsize];
+    if(!_dt_curr) _dt_curr = new std::time_t{std::time_t{std::time(nullptr)}};
+
+    char mbstr[strsize] = {};
     memset(mbstr, '\0', sizeof(mbstr) * sizeof(char));
 
-    if(std::strftime(mbstr, sizeof(mbstr) - 1, dt_format.c_str(), std::localtime(&dt_curr)) != 0)
+    if(std::strftime(mbstr, sizeof(mbstr) - 1, dt_format.c_str(), std::localtime(_dt_curr)) != 0)
         return new std::string{mbstr};
 
     return nullptr;
@@ -245,39 +257,23 @@ config::config()
       get_env("ROCPROF_KERNEL_FILTER_RANGE", std::string{}))}
 , counters{parse_counters(get_env("ROCPROF_COUNTERS", std::string{}))}
 {
-    auto output_format = get_env("ROCPROF_OUTPUT_FORMAT", "CSV");
-
-    for(auto& itr : output_format)
-        itr = toupper(itr);
-
-    for(auto itr : {',', ';', ':'})
-    {
-        auto pos = std::string::npos;
-        do
-        {
-            pos = output_format.find(itr);
-            if(pos != std::string::npos) output_format.replace(pos, 1, " ");
-        } while(pos != std::string::npos);
-    }
-
-    auto entries = std::set<std::string>{};
-    auto parser  = std::stringstream{output_format};
+    auto to_upper = [](std::string val) {
+        for(auto& vitr : val)
+            vitr = toupper(vitr);
+        return val;
+    };
 
-    while(true)
-    {
-        auto _val = std::string{};
-        parser >> _val;
-        if(!_val.empty())
-            entries.emplace(_val);
-        else
-            break;
-    }
+    auto output_format = get_env("ROCPROF_OUTPUT_FORMAT", "CSV");
+    auto entries       = std::set<std::string>{};
+    for(const auto& itr : sdk::parse::tokenize(output_format, " \t,;:"))
+        entries.emplace(to_upper(itr));
 
     csv_output     = entries.count("CSV") > 0 || entries.empty();
     json_output    = entries.count("JSON") > 0;
     pftrace_output = entries.count("PFTRACE") > 0;
+    otf2_output    = entries.count("OTF2") > 0;
 
-    const auto supported_formats = std::set<std::string_view>{"CSV", "JSON", "PFTRACE"};
+    const auto supported_formats = std::set<std::string_view>{"CSV", "JSON", "PFTRACE", "OTF2"};
     for(const auto& itr : entries)
     {
         LOG_IF(FATAL, supported_formats.count(itr) == 0)

source/lib/rocprofiler-sdk-tool/config.hpp

@@ -77,6 +77,7 @@ struct config
     bool        csv_output                  = false;
     bool        json_output                 = false;
     bool        pftrace_output              = false;
+    bool        otf2_output                 = false;
     bool        kernel_rename               = get_env("ROCPROF_KERNEL_RENAME", false);
     int         mpi_size                    = get_mpi_size();
     int         mpi_rank                    = get_mpi_rank();