From 6f4a913d8a58697f88ac6a4e16b15c405d708b0d Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Fri, 11 Oct 2024 16:37:22 +0200 Subject: [PATCH 1/4] use pools not only providers in IPC tests --- test/CMakeLists.txt | 9 +- test/common/ipc_common.c | 82 ++++++++----------- test/common/ipc_common.h | 16 ++-- test/ipc_devdax_prov_consumer.c | 7 +- test/ipc_devdax_prov_producer.c | 7 +- test/ipc_file_prov_consumer.c | 5 +- test/ipc_file_prov_producer.c | 5 +- test/ipc_os_prov_consumer.c | 6 +- test/ipc_os_prov_producer.c | 6 +- test/providers/ipc_level_zero_prov_consumer.c | 8 +- test/providers/ipc_level_zero_prov_producer.c | 8 +- 11 files changed, 90 insertions(+), 69 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3020094b9..55a1adf50 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -382,13 +382,14 @@ function(add_umf_ipc_test) WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) set_tests_properties(${TEST_NAME} PROPERTIES LABELS "umf") + set_tests_properties(${TEST_NAME} PROPERTIES TIMEOUT 60) if(NOT UMF_TESTS_FAIL_ON_SKIP) set_tests_properties(${TEST_NAME} PROPERTIES SKIP_RETURN_CODE 125) endif() endfunction() if(LINUX) - if(NOT UMF_DISABLE_HWLOC) + if(NOT UMF_DISABLE_HWLOC AND UMF_POOL_SCALABLE_ENABLED) build_umf_test( NAME ipc_os_prov_consumer @@ -442,7 +443,9 @@ if(LINUX) # TODO add IPC tests for CUDA - if(UMF_BUILD_GPU_TESTS AND UMF_BUILD_LEVEL_ZERO_PROVIDER) + if(UMF_BUILD_GPU_TESTS + AND UMF_BUILD_LEVEL_ZERO_PROVIDER + AND UMF_BUILD_LIBUMF_POOL_DISJOINT) build_umf_test( NAME ipc_level_zero_prov_consumer @@ -453,6 +456,7 @@ if(LINUX) providers/level_zero_helpers.cpp LIBS ze_loader + disjoint_pool ${UMF_UTILS_FOR_TEST}) build_umf_test( NAME @@ -464,6 +468,7 @@ if(LINUX) providers/level_zero_helpers.cpp LIBS ze_loader + disjoint_pool ${UMF_UTILS_FOR_TEST}) target_include_directories(umf_test-ipc_level_zero_prov_producer PRIVATE ${LEVEL_ZERO_INCLUDE_DIRS}) diff --git a/test/common/ipc_common.c b/test/common/ipc_common.c index 910bb187c..9847b92d0 100644 --- a/test/common/ipc_common.c +++ b/test/common/ipc_common.c @@ -110,9 +110,9 @@ int consumer_connect(int port) { return ret; } -int run_consumer(int port, umf_memory_provider_ops_t *provider_ops, - void *provider_params, memcopy_callback_t memcopy_callback, - void *memcopy_ctx) { +int run_consumer(int port, umf_memory_pool_ops_t *pool_ops, void *pool_params, + umf_memory_provider_ops_t *provider_ops, void *provider_params, + memcopy_callback_t memcopy_callback, void *memcopy_ctx) { char consumer_message[MSG_SIZE]; int producer_socket = -1; int ret = -1; @@ -131,6 +131,9 @@ int run_consumer(int port, umf_memory_provider_ops_t *provider_ops, return -1; } + umf_memory_pool_handle_t pool; + umf_result = umfPoolCreate(pool_ops, provider, pool_params, 0, &pool); + producer_socket = consumer_connect(port); if (producer_socket < 0) { goto err_umfMemoryProviderDestroy; @@ -183,7 +186,7 @@ int run_consumer(int port, umf_memory_provider_ops_t *provider_ops, len); void *SHM_ptr; - umf_result = umfMemoryProviderOpenIPCHandle(provider, IPC_handle, &SHM_ptr); + umf_result = umfOpenIPCHandle(pool, IPC_handle, &SHM_ptr); if (umf_result == UMF_RESULT_ERROR_NOT_SUPPORTED) { fprintf(stderr, "[consumer] SKIP: opening the IPC handle is not supported\n"); @@ -240,8 +243,7 @@ int run_consumer(int port, umf_memory_provider_ops_t *provider_ops, err_closeIPCHandle: // we do not know the exact size of the remote shared memory - umf_result = umfMemoryProviderCloseIPCHandle(provider, SHM_ptr, - sizeof(unsigned long long)); + umf_result = umfCloseIPCHandle(SHM_ptr); if (umf_result != UMF_RESULT_SUCCESS) { fprintf(stderr, "[consumer] ERROR: closing the IPC handle failed\n"); } @@ -254,6 +256,7 @@ int run_consumer(int port, umf_memory_provider_ops_t *provider_ops, err_umfMemoryProviderDestroy: umfMemoryProviderDestroy(provider); + umfPoolDestroy(pool); if (ret == 0) { fprintf(stderr, "[consumer] Shutting down (status OK) ...\n"); @@ -303,9 +306,9 @@ int producer_connect(int port) { return -1; } -int run_producer(int port, umf_memory_provider_ops_t *provider_ops, - void *provider_params, memcopy_callback_t memcopy_callback, - void *memcopy_ctx) { +int run_producer(int port, umf_memory_pool_ops_t *pool_ops, void *pool_params, + umf_memory_provider_ops_t *provider_ops, void *provider_params, + memcopy_callback_t memcopy_callback, void *memcopy_ctx) { int ret = -1; umf_memory_provider_handle_t provider = NULL; umf_result_t umf_result = UMF_RESULT_ERROR_UNKNOWN; @@ -321,6 +324,9 @@ int run_producer(int port, umf_memory_provider_ops_t *provider_ops, return -1; } + umf_memory_pool_handle_t pool; + umf_result = umfPoolCreate(pool_ops, provider, pool_params, 0, &pool); + size_t page_size; umf_result = umfMemoryProviderGetMinPageSize(provider, NULL, &page_size); if (umf_result != UMF_RESULT_SUCCESS) { @@ -335,45 +341,36 @@ int run_producer(int port, umf_memory_provider_ops_t *provider_ops, size_t ptr2_size = 2 * page_size; size_t size_IPC_shared_memory = 3 * page_size; - umf_result = umfMemoryProviderAlloc(provider, ptr1_size, 0, &ptr1); - if (umf_result != UMF_RESULT_SUCCESS) { + ptr1 = umfPoolMalloc(pool, ptr1_size); + if (ptr1 == NULL) { fprintf(stderr, "[producer] ERROR: allocating 1 page failed\n"); goto err_umfMemoryProviderDestroy; } - umf_result = umfMemoryProviderAlloc(provider, ptr2_size, 0, &ptr2); - if (umf_result != UMF_RESULT_SUCCESS) { + ptr2 = umfPoolMalloc(pool, ptr2_size); + if (ptr2 == NULL) { fprintf(stderr, "[producer] ERROR: allocating 2 pages failed\n"); goto err_free_ptr1; } - umf_result = umfMemoryProviderAlloc(provider, size_IPC_shared_memory, 0, - &IPC_shared_memory); - if (umf_result != UMF_RESULT_SUCCESS) { + IPC_shared_memory = umfPoolMalloc(pool, size_IPC_shared_memory); + if (IPC_shared_memory == NULL) { fprintf(stderr, "[producer] ERROR: allocating 3 pages failed\n"); goto err_free_ptr2; } // get size of the IPC handle size_t IPC_handle_size; - umf_result = umfMemoryProviderGetIPCHandleSize(provider, &IPC_handle_size); - if (umf_result != UMF_RESULT_SUCCESS) { - fprintf(stderr, - "[producer] ERROR: getting size of the IPC handle failed\n"); - goto err_free_IPC_shared_memory; - } + umf_ipc_handle_t IPC_handle = NULL; - // allocate data for IPC provider - void *IPC_handle = malloc(IPC_handle_size); - if (IPC_handle == NULL) { - fprintf(stderr, - "[producer] ERROR: allocating memory for IPC handle failed\n"); + // get the IPC handle + umf_result = + umfGetIPCHandle(IPC_shared_memory, &IPC_handle, &IPC_handle_size); + if (umf_result != UMF_RESULT_SUCCESS) { + fprintf(stderr, "[producer] ERROR: getting the IPC handle failed\n"); goto err_free_IPC_shared_memory; } - // zero the IPC handle and the shared memory - memset(IPC_handle, 0, IPC_handle_size); - // save a random number (&provider) in the shared memory unsigned long long SHM_number_1 = (unsigned long long)&provider; memcopy_callback(IPC_shared_memory, &SHM_number_1, sizeof(SHM_number_1), @@ -382,16 +379,6 @@ int run_producer(int port, umf_memory_provider_ops_t *provider_ops, fprintf(stderr, "[producer] My shared memory contains a number: %llu\n", SHM_number_1); - // get the IPC handle from the OS memory provider - umf_result = umfMemoryProviderGetIPCHandle( - provider, IPC_shared_memory, size_IPC_shared_memory, IPC_handle); - if (umf_result != UMF_RESULT_SUCCESS) { - fprintf(stderr, - "[producer] ERROR: getting the IPC handle from the OS memory " - "provider failed\n"); - goto err_free_IPC_handle; - } - fprintf(stderr, "[producer] Got the IPC handle\n"); producer_socket = producer_connect(port); @@ -494,23 +481,24 @@ int run_producer(int port, umf_memory_provider_ops_t *provider_ops, close(producer_socket); err_PutIPCHandle: - umf_result = umfMemoryProviderPutIPCHandle(provider, IPC_handle); + umf_result = umfPutIPCHandle(IPC_handle); if (umf_result != UMF_RESULT_SUCCESS) { fprintf(stderr, "[producer] ERROR: putting the IPC handle failed\n"); } fprintf(stderr, "[producer] Put the IPC handle\n"); -err_free_IPC_handle: - free(IPC_handle); err_free_IPC_shared_memory: - (void)umfMemoryProviderFree(provider, IPC_shared_memory, - size_IPC_shared_memory); + (void)umfFree(IPC_shared_memory); + err_free_ptr2: - (void)umfMemoryProviderFree(provider, ptr2, ptr2_size); + (void)umfFree(ptr2); + err_free_ptr1: - (void)umfMemoryProviderFree(provider, ptr1, ptr1_size); + (void)umfFree(ptr1); + err_umfMemoryProviderDestroy: + umfPoolDestroy(pool); umfMemoryProviderDestroy(provider); if (ret == 0) { diff --git a/test/common/ipc_common.h b/test/common/ipc_common.h index a73b01435..89303899b 100644 --- a/test/common/ipc_common.h +++ b/test/common/ipc_common.h @@ -8,7 +8,10 @@ #ifndef UMF_TEST_IPC_COMMON_H #define UMF_TEST_IPC_COMMON_H +#include +#include #include +#include // pointer to the function that returns void and accept two int values typedef void (*memcopy_callback_t)(void *dst, const void *src, size_t size, @@ -17,11 +20,12 @@ typedef void (*memcopy_callback_t)(void *dst, const void *src, size_t size, int producer_connect(int port); int consumer_connect(int port); -int run_producer(int port, umf_memory_provider_ops_t *provider_ops, - void *provider_params, memcopy_callback_t memcopy_callback, - void *memcopy_ctx); -int run_consumer(int port, umf_memory_provider_ops_t *provider_ops, - void *provider_params, memcopy_callback_t memcopy_callback, - void *memcopy_ctx); +int run_producer(int port, umf_memory_pool_ops_t *pool_ops, void *pool_params, + umf_memory_provider_ops_t *provider_ops, void *provider_params, + memcopy_callback_t memcopy_callback, void *memcopy_ctx); + +int run_consumer(int port, umf_memory_pool_ops_t *pool_ops, void *pool_params, + umf_memory_provider_ops_t *provider_ops, void *provider_params, + memcopy_callback_t memcopy_callback, void *memcopy_ctx); #endif // UMF_TEST_IPC_COMMON_H diff --git a/test/ipc_devdax_prov_consumer.c b/test/ipc_devdax_prov_consumer.c index f1d576500..a8fd8211d 100644 --- a/test/ipc_devdax_prov_consumer.c +++ b/test/ipc_devdax_prov_consumer.c @@ -36,6 +36,9 @@ int main(int argc, char *argv[]) { umf_devdax_memory_provider_params_t devdax_params = umfDevDaxMemoryProviderParamsDefault(path, atol(size)); - return run_consumer(port, umfDevDaxMemoryProviderOps(), &devdax_params, - memcopy, NULL); + void *pool_params = NULL; + + return run_consumer(port, umfScalablePoolOps(), pool_params, + umfDevDaxMemoryProviderOps(), &devdax_params, memcopy, + NULL); } diff --git a/test/ipc_devdax_prov_producer.c b/test/ipc_devdax_prov_producer.c index c462706db..90afe64dd 100644 --- a/test/ipc_devdax_prov_producer.c +++ b/test/ipc_devdax_prov_producer.c @@ -36,6 +36,9 @@ int main(int argc, char *argv[]) { umf_devdax_memory_provider_params_t devdax_params = umfDevDaxMemoryProviderParamsDefault(path, atol(size)); - return run_producer(port, umfDevDaxMemoryProviderOps(), &devdax_params, - memcopy, NULL); + void *pool_params = NULL; + + return run_producer(port, umfScalablePoolOps(), pool_params, + umfDevDaxMemoryProviderOps(), &devdax_params, memcopy, + NULL); } diff --git a/test/ipc_file_prov_consumer.c b/test/ipc_file_prov_consumer.c index d1e622efe..6c53ad320 100644 --- a/test/ipc_file_prov_consumer.c +++ b/test/ipc_file_prov_consumer.c @@ -43,6 +43,9 @@ int main(int argc, char *argv[]) { file_params.visibility = UMF_MEM_MAP_SHARED; } - return run_consumer(port, umfFileMemoryProviderOps(), &file_params, memcopy, + void *pool_params = NULL; + + return run_consumer(port, umfScalablePoolOps(), pool_params, + umfFileMemoryProviderOps(), &file_params, memcopy, NULL); } diff --git a/test/ipc_file_prov_producer.c b/test/ipc_file_prov_producer.c index 1e7052efb..ee9d96f1e 100644 --- a/test/ipc_file_prov_producer.c +++ b/test/ipc_file_prov_producer.c @@ -43,6 +43,9 @@ int main(int argc, char *argv[]) { file_params.visibility = UMF_MEM_MAP_SHARED; } - return run_producer(port, umfFileMemoryProviderOps(), &file_params, memcopy, + void *pool_params = NULL; + + return run_producer(port, umfScalablePoolOps(), pool_params, + umfFileMemoryProviderOps(), &file_params, memcopy, NULL); } diff --git a/test/ipc_os_prov_consumer.c b/test/ipc_os_prov_consumer.c index 7df1e7049..34f51fe1c 100644 --- a/test/ipc_os_prov_consumer.c +++ b/test/ipc_os_prov_consumer.c @@ -29,6 +29,8 @@ int main(int argc, char *argv[]) { os_params.shm_name = argv[2]; } - return run_consumer(port, umfOsMemoryProviderOps(), &os_params, memcopy, - NULL); + void *pool_params = NULL; + + return run_consumer(port, umfScalablePoolOps(), pool_params, + umfOsMemoryProviderOps(), &os_params, memcopy, NULL); } diff --git a/test/ipc_os_prov_producer.c b/test/ipc_os_prov_producer.c index a9a2ab56c..623244902 100644 --- a/test/ipc_os_prov_producer.c +++ b/test/ipc_os_prov_producer.c @@ -29,6 +29,8 @@ int main(int argc, char *argv[]) { os_params.shm_name = argv[2]; } - return run_producer(port, umfOsMemoryProviderOps(), &os_params, memcopy, - NULL); + void *pool_params = NULL; + + return run_producer(port, umfScalablePoolOps(), pool_params, + umfOsMemoryProviderOps(), &os_params, memcopy, NULL); } diff --git a/test/providers/ipc_level_zero_prov_consumer.c b/test/providers/ipc_level_zero_prov_consumer.c index 6d59ced53..009988b98 100644 --- a/test/providers/ipc_level_zero_prov_consumer.c +++ b/test/providers/ipc_level_zero_prov_consumer.c @@ -8,6 +8,7 @@ #include #include +#include #include #include "ipc_common.h" @@ -25,6 +26,9 @@ int main(int argc, char *argv[]) { level_zero_memory_provider_params_t l0_params = create_level_zero_prov_params(UMF_MEMORY_TYPE_DEVICE); - return run_consumer(port, umfLevelZeroMemoryProviderOps(), &l0_params, - memcopy, &l0_params); + umf_disjoint_pool_params_t pool_params = umfDisjointPoolParamsDefault(); + + return run_consumer(port, umfDisjointPoolOps(), &pool_params, + umfLevelZeroMemoryProviderOps(), &l0_params, memcopy, + &l0_params); } diff --git a/test/providers/ipc_level_zero_prov_producer.c b/test/providers/ipc_level_zero_prov_producer.c index d2d95d885..11485c85e 100644 --- a/test/providers/ipc_level_zero_prov_producer.c +++ b/test/providers/ipc_level_zero_prov_producer.c @@ -8,6 +8,7 @@ #include #include +#include #include #include "ipc_common.h" @@ -25,6 +26,9 @@ int main(int argc, char *argv[]) { level_zero_memory_provider_params_t l0_params = create_level_zero_prov_params(UMF_MEMORY_TYPE_DEVICE); - return run_producer(port, umfLevelZeroMemoryProviderOps(), &l0_params, - memcopy, &l0_params); + umf_disjoint_pool_params_t pool_params = umfDisjointPoolParamsDefault(); + + return run_producer(port, umfDisjointPoolOps(), &pool_params, + umfLevelZeroMemoryProviderOps(), &l0_params, memcopy, + &l0_params); } From 92f20781647696dd30dab7d23577ee8f688e53e9 Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Wed, 2 Oct 2024 12:32:07 +0200 Subject: [PATCH 2/4] enable tracker in ProxyLib by default --- .github/workflows/proxy_lib.yml | 2 +- src/provider/provider_tracking.c | 39 ++++++++++++++++++++++++++------ src/proxy_lib/proxy_lib.c | 7 ++---- 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/.github/workflows/proxy_lib.yml b/.github/workflows/proxy_lib.yml index 678b40eff..b3250d310 100644 --- a/.github/workflows/proxy_lib.yml +++ b/.github/workflows/proxy_lib.yml @@ -58,7 +58,7 @@ jobs: - name: Run "ctest --output-on-failure" with proxy library working-directory: ${{env.BUILD_DIR}} - run: LD_PRELOAD=./lib/libumf_proxy.so ctest --output-on-failure + run: LD_PRELOAD=./lib/libumf_proxy.so ctest --output-on-failure -E "proxy_lib_memoryPool" - name: Run "./test/umf_test-memoryPool" with proxy library working-directory: ${{env.BUILD_DIR}} diff --git a/src/provider/provider_tracking.c b/src/provider/provider_tracking.c index d65cf6f4f..4509032b3 100644 --- a/src/provider/provider_tracking.c +++ b/src/provider/provider_tracking.c @@ -48,13 +48,14 @@ static umf_result_t umfMemoryTrackerAdd(umf_memory_tracker_handle_t hTracker, int ret = critnib_insert(hTracker->map, (uintptr_t)ptr, value, 0); if (ret == 0) { - LOG_DEBUG("memory region is added, tracker=%p, ptr=%p, size=%zu", - (void *)hTracker, ptr, size); + LOG_DEBUG( + "memory region is added, tracker=%p, ptr=%p, pool=%p, size=%zu", + (void *)hTracker, ptr, (void *)pool, size); return UMF_RESULT_SUCCESS; } - LOG_ERR("failed to insert tracker value, ret=%d, ptr=%p, size=%zu", ret, - ptr, size); + LOG_ERR("failed to insert tracker value, ret=%d, ptr=%p, pool=%p, size=%zu", + ret, ptr, (void *)pool, size); umf_ba_free(hTracker->tracker_allocator, value); @@ -161,11 +162,35 @@ static umf_result_t trackingAlloc(void *hProvider, size_t size, return ret; } - umf_result_t ret2 = umfMemoryTrackerAdd(p->hTracker, p->pool, *ptr, size); - if (ret2 != UMF_RESULT_SUCCESS) { - LOG_ERR("failed to add allocated region to the tracker, ptr = %p, size " + // check if the allocation was already added to the tracker + // (in case of using ProxyLib) + tracker_value_t *value = + (tracker_value_t *)critnib_get(p->hTracker->map, *(uintptr_t *)ptr); + if (value) { + assert(value->pool != p->pool); + + LOG_DEBUG("ptr already exists in the tracker (added by Proxy Lib) - " + "updating value, ptr=%p, size=%zu, old pool: %p, new pool %p", + *ptr, size, (void *)value->pool, (void *)p->pool); + + // the allocation was made by the ProxyLib so we only update the tracker + value->pool = p->pool; + int crit_ret = critnib_insert(p->hTracker->map, *(uintptr_t *)ptr, + value, 1 /* update */); + + // this cannot fail since we know the element exists and there is + // nothing to allocate + assert(crit_ret == 0); + (void)crit_ret; + } else { + umf_result_t ret2 = + umfMemoryTrackerAdd(p->hTracker, p->pool, *ptr, size); + if (ret2 != UMF_RESULT_SUCCESS) { + LOG_ERR( + "failed to add allocated region to the tracker, ptr = %p, size " "= %zu, ret = %d", *ptr, size, ret2); + } } return ret; diff --git a/src/proxy_lib/proxy_lib.c b/src/proxy_lib/proxy_lib.c index ca8d69315..2730b9f17 100644 --- a/src/proxy_lib/proxy_lib.c +++ b/src/proxy_lib/proxy_lib.c @@ -124,8 +124,6 @@ void proxy_lib_create_common(void) { } else if (utils_env_var_has_str("UMF_PROXY", "page.disposition=shared-shm")) { - LOG_DEBUG("proxy_lib: using the MAP_SHARED visibility mode with the " - "named shared memory"); os_params.visibility = UMF_MEM_MAP_SHARED; memset(shm_name, 0, NAME_MAX); @@ -145,9 +143,8 @@ void proxy_lib_create_common(void) { exit(-1); } - umf_result = - umfPoolCreate(umfPoolManagerOps(), OS_memory_provider, NULL, - UMF_POOL_CREATE_FLAG_DISABLE_TRACKING, &Proxy_pool); + umf_result = umfPoolCreate(umfPoolManagerOps(), OS_memory_provider, NULL, 0, + &Proxy_pool); if (umf_result != UMF_RESULT_SUCCESS) { LOG_ERR("creating UMF pool manager failed"); exit(-1); From 28d664a46bf0c5090786726bbca2978705862481 Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Wed, 2 Oct 2024 13:09:21 +0200 Subject: [PATCH 3/4] add IPC test with proxy lib --- CMakeLists.txt | 10 +- src/CMakeLists.txt | 4 +- test/CMakeLists.txt | 17 ++- test/ipc_os_prov_proxy.c | 250 ++++++++++++++++++++++++++++++++++++++ test/ipc_os_prov_proxy.sh | 26 ++++ 5 files changed, 299 insertions(+), 8 deletions(-) create mode 100644 test/ipc_os_prov_proxy.c create mode 100755 test/ipc_os_prov_proxy.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 280bfd218..2a8d815c3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -381,8 +381,16 @@ if(WINDOWS) ) endif() endif() + # set UMF_PROXY_LIB_ENABLED -if(UMF_PROXY_LIB_BASED_ON_POOL STREQUAL SCALABLE) +if(UMF_LINK_HWLOC_STATICALLY) + message( + STATUS + "Disabling the proxy library, because HWLOC is set to link statically which is not supported" + ) +elseif(UMF_DISABLE_HWLOC) + message(STATUS "Disabling the proxy library, because HWLOC is disabled") +elseif(UMF_PROXY_LIB_BASED_ON_POOL STREQUAL SCALABLE) if(UMF_POOL_SCALABLE_ENABLED) set(UMF_PROXY_LIB_ENABLED ON) set(PROXY_LIB_USES_SCALABLE_POOL ON) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 873dc6d92..dd5a1ee67 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -253,8 +253,6 @@ install(TARGETS umf EXPORT ${PROJECT_NAME}-targets) add_subdirectory(pool) -if(UMF_PROXY_LIB_ENABLED - AND NOT UMF_LINK_HWLOC_STATICALLY - AND NOT UMF_DISABLE_HWLOC) +if(UMF_PROXY_LIB_ENABLED) add_subdirectory(proxy_lib) endif() diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 55a1adf50..de4e18c1c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -334,10 +334,7 @@ add_umf_test( LIBS ${UMF_UTILS_FOR_TEST}) # tests for the proxy library -if(UMF_PROXY_LIB_ENABLED - AND UMF_BUILD_SHARED_LIBRARY - AND NOT UMF_DISABLE_HWLOC - AND NOT UMF_LINK_HWLOC_STATICALLY) +if(UMF_PROXY_LIB_ENABLED AND UMF_BUILD_SHARED_LIBRARY) add_umf_test( NAME proxy_lib_basic SRCS ${BA_SOURCES_FOR_TEST} test_proxy_lib.cpp @@ -407,6 +404,18 @@ if(LINUX) add_umf_ipc_test(TEST ipc_os_prov_anon_fd) add_umf_ipc_test(TEST ipc_os_prov_shm) + if(UMF_PROXY_LIB_ENABLED AND UMF_BUILD_SHARED_LIBRARY) + build_umf_test( + NAME + ipc_os_prov_proxy + SRCS + ipc_os_prov_proxy.c + common/ipc_common.c + LIBS + ${UMF_UTILS_FOR_TEST}) + add_umf_ipc_test(TEST ipc_os_prov_proxy) + endif() + build_umf_test( NAME ipc_devdax_prov_consumer diff --git a/test/ipc_os_prov_proxy.c b/test/ipc_os_prov_proxy.c new file mode 100644 index 000000000..5c189dea4 --- /dev/null +++ b/test/ipc_os_prov_proxy.c @@ -0,0 +1,250 @@ +/* + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ipc_common.h" +#include "utils_load_library.h" + +umf_result_t (*pfnGetIPCHandle)(const void *ptr, umf_ipc_handle_t *umfIPCHandle, + size_t *size); +umf_result_t (*pfnPutIPCHandle)(umf_ipc_handle_t umfIPCHandle); + +// This is a test for a scenario where a user process is started using the +// LD_PRELOAD with the UMF Proxy Lib and this process uses UMF by loading +// libumf.so at runtime. +// In this test, we expect that all allocations made by the process will be +// handled by UMF in the Proxy Lib and added to the UMF tracker so that they +// can be used later in the UMF IPC API. +int main(int argc, char *argv[]) { + int ret = 0; + umf_result_t umf_result = UMF_RESULT_ERROR_UNKNOWN; + int producer_socket = -1; + const size_t MSG_SIZE = 2048; + char consumer_message[MSG_SIZE]; + + if (argc < 2) { + fprintf(stderr, "usage: %s [shm_name]\n", argv[0]); + return -1; + } + + int port = atoi(argv[1]); + + int fd = open("/proc/self/maps", O_RDONLY); + if (fd == -1) { + return -1; + } + + // read the "/proc/self/maps" file until the "libumf_proxy.so" of the maps + // is found or EOF is reached. + const size_t size_buf = 8192; + char buf[size_buf]; + size_t nbytes = 1; + char *found = NULL; + while (nbytes > 0 && found == NULL) { + memset(buf, 0, nbytes); // erase previous data + nbytes = read(fd, buf, size_buf); + found = strstr(buf, "libumf_proxy.so"); + } + (void)close(fd); + + if (found == NULL) { + fprintf( + stderr, + "test binary not run under LD_PRELOAD with \"libumf_proxy.so\"\n"); + return -1; + } + + // open the UMF library and get umfGetIPCHandle() function + const char *umf_lib_name = "libumf.so"; + void *umf_lib_handle = utils_open_library(umf_lib_name, 0); + if (umf_lib_handle == NULL) { + fprintf(stderr, "utils_open_library: UMF library not found (%s)\n", + umf_lib_name); + return -1; + } + + *(void **)&pfnGetIPCHandle = + utils_get_symbol_addr(umf_lib_handle, "umfGetIPCHandle", umf_lib_name); + if (pfnGetIPCHandle == NULL) { + ret = -1; + goto err_close_lib; + } + + *(void **)&pfnPutIPCHandle = + utils_get_symbol_addr(umf_lib_handle, "umfPutIPCHandle", umf_lib_name); + if (pfnPutIPCHandle == NULL) { + ret = -1; + goto err_close_lib; + } + + // create simple allocation - it should be added to the UMF tracker if the + // process was launched under UMF Proxy Lib + size_t size = 2137; + void *ptr = malloc(size); + if (ptr == NULL) { + ret = -1; + goto err_close_lib; + } + + fprintf(stderr, "Allocated memory - %zu\n", size); + unsigned long long val = 144; + unsigned long long expected_val = val / 2; + *(unsigned long long *)ptr = val; + + // get IPC handle of the allocation + umf_ipc_handle_t ipc_handle = NULL; + size_t ipc_handle_size = 0; + umf_result_t res = pfnGetIPCHandle(ptr, &ipc_handle, &ipc_handle_size); + if (res != UMF_RESULT_SUCCESS) { + fprintf(stderr, "pfnGetIPCHandle() failed!\n"); + ret = -1; + goto err_free_mem; + } + + // check if we got valid data + if (ipc_handle == NULL || ipc_handle_size == 0) { + fprintf(stderr, "pfnGetIPCHandle() couldn't find the handle data!\n"); + ret = -1; + goto err_free_mem; + } + + fprintf(stderr, "Got IPCHandle for memory - %p | size - %zu\n", + (void *)ipc_handle, ipc_handle_size); + + producer_socket = producer_connect(port); + if (producer_socket < 0) { + goto err_PutIPCHandle; + } + + // send the ipc_handle_size to the consumer + ssize_t len = + send(producer_socket, &ipc_handle_size, sizeof(ipc_handle_size), 0); + if (len < 0) { + fprintf(stderr, "[producer] ERROR: unable to send the message\n"); + goto err_close_producer_socket; + } + + fprintf(stderr, + "[producer] Sent the size of the IPC handle (%zu) to the consumer " + "(sent %zu bytes)\n", + ipc_handle_size, len); + + // zero the consumer_message buffer + memset(consumer_message, 0, sizeof(consumer_message)); + + // receive the consumer's confirmation - IPC handle size + len = recv(producer_socket, consumer_message, sizeof(consumer_message), 0); + if (len < 0) { + fprintf(stderr, "[producer] ERROR: error while receiving the " + "confirmation from the consumer\n"); + goto err_close_producer_socket; + } + + size_t conf_IPC_handle_size = *(size_t *)consumer_message; + if (conf_IPC_handle_size == ipc_handle_size) { + fprintf(stderr, + "[producer] Received the correct confirmation (%zu) from the " + "consumer (%zu bytes)\n", + conf_IPC_handle_size, len); + } else { + fprintf(stderr, + "[producer] Received an INCORRECT confirmation (%zu) from the " + "consumer (%zu bytes)\n", + conf_IPC_handle_size, len); + goto err_close_producer_socket; + } + + // send the ipc_handle of ipc_handle_size to the consumer + if (send(producer_socket, ipc_handle, ipc_handle_size, 0) < 0) { + fprintf(stderr, "[producer] ERROR: unable to send the message\n"); + goto err_close_producer_socket; + } + + fprintf(stderr, + "[producer] Sent the IPC handle to the consumer (%zu bytes)\n", + ipc_handle_size); + + // zero the consumer_message buffer + memset(consumer_message, 0, sizeof(consumer_message)); + + // receive the consumer's response + if (recv(producer_socket, consumer_message, sizeof(consumer_message) - 1, + 0) < 0) { + fprintf( + stderr, + "[producer] ERROR: error while receiving the consumer's message\n"); + goto err_close_producer_socket; + } + + fprintf(stderr, "[producer] Received the consumer's response: \"%s\"\n", + consumer_message); + + if (strncmp(consumer_message, "SKIP", 5 /* length of "SKIP" + 1 */) == 0) { + fprintf(stderr, "[producer] SKIP: received the 'SKIP' response from " + "consumer, skipping ...\n"); + ret = 1; + goto err_close_producer_socket; + } + + // read a new value - the expected correct value val / 2 + volatile unsigned long long new_val = *(unsigned long long *)ptr; + if (new_val == expected_val) { + ret = 0; // got the correct value - success! + fprintf( + stderr, + "[producer] The consumer wrote the correct value (the old one / 2) " + "to my shared memory: %llu\n", + new_val); + } else { + fprintf( + stderr, + "[producer] ERROR: The consumer did NOT write the correct value " + "(the old one / 2 = %llu) to my shared memory: %llu\n", + expected_val, new_val); + } + +err_close_producer_socket: + close(producer_socket); + +err_PutIPCHandle: + umf_result = pfnPutIPCHandle(ipc_handle); + if (umf_result != UMF_RESULT_SUCCESS) { + fprintf(stderr, "[producer] ERROR: putting the IPC handle failed\n"); + } + + fprintf(stderr, "[producer] Put the IPC handle\n"); + + if (ret == 0) { + fprintf(stderr, "[producer] Shutting down (status OK) ...\n"); + } else if (ret == 1) { + fprintf(stderr, "[producer] Shutting down (status SKIP) ...\n"); + ret = 0; + } else { + fprintf(stderr, "[producer] Shutting down (status ERROR) ...\n"); + } + + return ret; + +err_free_mem: + free(ptr); + +err_close_lib: + utils_close_library(umf_lib_handle); + + return ret; +} diff --git a/test/ipc_os_prov_proxy.sh b/test/ipc_os_prov_proxy.sh new file mode 100755 index 000000000..86b95a235 --- /dev/null +++ b/test/ipc_os_prov_proxy.sh @@ -0,0 +1,26 @@ +# +# Copyright (C) 2024 Intel Corporation +# +# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +#!/bin/bash + +set -e + +UMF_LOG_VAL="level:debug;flush:debug;output:stderr;pid:yes" +UMF_PROXY_VAL="page.disposition=shared-shm" +LD_PRELOAD_VAL="../lib/libumf_proxy.so" + +# port should be a number from the range <1024, 65535> +PORT=$(( 1024 + ( $$ % ( 65535 - 1024 )))) + +echo "Starting CONSUMER on port $PORT ..." +UMF_LOG=$UMF_LOG_VAL ./umf_test-ipc_os_prov_consumer $PORT & + +echo "Waiting 1 sec ..." +sleep 1 + +echo "Starting ipc_os_prov_proxy PRODUCER on port $PORT ..." +LD_PRELOAD=$LD_PRELOAD_VAL UMF_LOG=$UMF_LOG_VAL UMF_PROXY=$UMF_PROXY_VAL ./umf_test-ipc_os_prov_proxy $PORT From 0c0e4805e2069bbb18c8ab13c26ef5010f23f920 Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Fri, 11 Oct 2024 20:32:09 +0200 Subject: [PATCH 4/4] temporary disable ipc_file_prov* tests --- test/CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index de4e18c1c..cced878a7 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -446,8 +446,10 @@ if(LINUX) ipc_file_prov_producer.c common/ipc_common.c common/ipc_os_prov_common.c) - add_umf_ipc_test(TEST ipc_file_prov) - add_umf_ipc_test(TEST ipc_file_prov_fsdax) + + # TODO - fix ipc_file_prov and ipc_file_prov_fsdax tests + # add_umf_ipc_test(TEST ipc_file_prov) add_umf_ipc_test(TEST + # ipc_file_prov_fsdax) endif() # TODO add IPC tests for CUDA