From 5cd273ed236cd33ad340034900c0303fe19095a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Plewa?= Date: Thu, 19 Sep 2024 17:25:31 +0200 Subject: [PATCH] add memspace filter functions --- include/umf/base.h | 5 +- include/umf/memspace.h | 52 ++++++++++ include/umf/memtarget.h | 7 ++ src/libumf.def.in | 13 ++- src/libumf.map.in | 11 +- src/memspace.c | 143 ++++++++++++++++++++++++++ src/memspace_internal.h | 6 -- src/memtarget.c | 8 ++ src/memtarget_ops.h | 1 + src/memtargets/memtarget_numa.c | 10 ++ src/provider/provider_level_zero.c | 5 - test/memspaces/memspace_fixtures.hpp | 12 +++ test/memspaces/memspace_numa.cpp | 146 +++++++++++++++++++++++++++ test/memspaces/memtarget.cpp | 6 +- 14 files changed, 400 insertions(+), 25 deletions(-) diff --git a/include/umf/base.h b/include/umf/base.h index 854688a86..53378195d 100644 --- a/include/umf/base.h +++ b/include/umf/base.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2024 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -43,7 +43,8 @@ typedef enum umf_result_t { UMF_RESULT_ERROR_INVALID_ALIGNMENT = 4, ///< Invalid alignment of an argument UMF_RESULT_ERROR_NOT_SUPPORTED = 5, ///< Operation not supported - + UMF_RESULT_ERROR_USER_SPECIFIC = + 6, ///< Failure in user provider code (i.e in user provided callback) UMF_RESULT_ERROR_UNKNOWN = 0x7ffffffe ///< Unknown or internal error } umf_result_t; diff --git a/include/umf/memspace.h b/include/umf/memspace.h index 49eb5278e..45ae05eee 100644 --- a/include/umf/memspace.h +++ b/include/umf/memspace.h @@ -131,6 +131,58 @@ umf_result_t umfMemspaceMemtargetRemove(umf_memspace_handle_t hMemspace, umf_const_memtarget_handle_t hMemtarget); +/// \brief Clones memspace. +/// +/// \param hMemspace handle to memspace +/// \param hNewMemspace [out] handle to the newly created memspace +/// \return UMF_RESULT_SUCCESS on success or appropriate error code on failure. +/// +umf_result_t umfMemspaceClone(umf_const_memspace_handle_t hMemspace, + umf_memspace_handle_t *hNewMemspace); + +/// \brief Custom filter function for umfMemspaceUserFilter +/// +/// \param hMemspace handle to memspace +/// \param hMemtarget handle to memory target +/// \param args user provided arguments +/// \return zero if hMemtarget should be removed from memspace, positive otherwise, and negative on error +/// +typedef int (*umf_memspace_filter_func_t)( + umf_const_memspace_handle_t hMemspace, + umf_const_memtarget_handle_t hMemtarget, void *args); + +/// \brief Removes all memory targets with not matching numa node ids. +/// +/// \param hMemspace handle to memspace +/// \param ids array of numa node ids +/// \param size size of the array +/// \return UMF_RESULT_SUCCESS on success or appropriate error code on failure. +/// If the error code is UMF_RESULT_UNKNOWN the memspace is corrupted, otherwise the memspace is not modified. +/// +umf_result_t umfMemspaceFilterById(umf_memspace_handle_t hMemspace, + unsigned *ids, size_t size); + +/// \brief Filters out memory targets that capacity is less than specified size. +/// +/// \param hMemspace handle to memspace +/// \param size minimum capacity of memory target +/// \return UMF_RESULT_SUCCESS on success or appropriate error code on failure. +/// If the error code is UMF_RESULT_UNKNOWN the memspace is corrupted, otherwise the memspace is not modified. +/// +umf_result_t umfMemspaceFilterByCapacity(umf_memspace_handle_t hMemspace, + size_t size); + +/// \brief Filters out memory targets based on user provided function +/// +/// \param hMemspace handle to memspace +/// \param filter user provided function +/// \param args user provided arguments +/// \return UMF_RESULT_SUCCESS on success or appropriate error code on failure. +/// If the error code is UMF_RESULT_UNKNOWN the memspace is corrupted, otherwise the memspace is not modified. +/// +umf_result_t umfMemspaceUserFilter(umf_memspace_handle_t hMemspace, + umf_memspace_filter_func_t filter, + void *args); #ifdef __cplusplus } #endif diff --git a/include/umf/memtarget.h b/include/umf/memtarget.h index a4902b98d..d74947f14 100644 --- a/include/umf/memtarget.h +++ b/include/umf/memtarget.h @@ -39,6 +39,13 @@ umf_result_t umfMemtargetGetType(umf_const_memtarget_handle_t hMemtarget, umf_result_t umfMemtargetGetCapacity(umf_const_memtarget_handle_t hMemtarget, size_t *capacity); +/// \brief Get physical ID of the memory target. +/// \param hMemtarget handle to the memory target +/// \param id [out] id of the memory target +/// \return UMF_RESULT_SUCCESS on success or appropriate error code on failure. +umf_result_t umfMemtargetGetId(umf_const_memtarget_handle_t hMemtarget, + unsigned *id); + #ifdef __cplusplus } #endif diff --git a/src/libumf.def.in b/src/libumf.def.in index 8e2fe4b93..e0a43042e 100644 --- a/src/libumf.def.in +++ b/src/libumf.def.in @@ -40,13 +40,18 @@ EXPORTS umfMempolicyDestroy umfMempolicySetCustomSplitPartitions umfMempolicySetInterleavePartSize - umfMemspaceNew + umfMemspaceClone umfMemspaceDestroy - umfMemspaceMemtargetNum - umfMemspaceMemtargetGet - umfMemtargetGetCapacity + umfMemspaceFilterByCapacity + umfMemspaceFilterById umfMemspaceMemtargetAdd + umfMemspaceMemtargetGet + umfMemspaceMemtargetNum umfMemspaceMemtargetRemove + umfMemspaceNew + umfMemspaceUserFilter + umfMemtargetGetCapacity + umfMemtargetGetId umfMemtargetGetType umfOpenIPCHandle umfPoolAlignedMalloc diff --git a/src/libumf.map.in b/src/libumf.map.in index c078b24c9..59fbe3cf3 100644 --- a/src/libumf.map.in +++ b/src/libumf.map.in @@ -34,14 +34,19 @@ UMF_1.0 { umfMempolicyDestroy; umfMempolicySetCustomSplitPartitions; umfMempolicySetInterleavePartSize; - umfMemspaceNew; + umfMemspaceClone; umfMemspaceCreateFromNumaArray; umfMemspaceDestroy; + umfMemspaceFilterByCapacity; + umfMemspaceFilterById; umfMemspaceMemtargetAdd; - umfMemspaceMemtargetRemove; - umfMemspaceMemtargetNum; umfMemspaceMemtargetGet; + umfMemspaceMemtargetNum; + umfMemspaceMemtargetRemove; + umfMemspaceNew; + umfMemspaceUserFilter; umfMemtargetGetCapacity; + umfMemtargetGetId; umfMemtargetGetType; umfOpenIPCHandle; umfPoolAlignedMalloc; diff --git a/src/memspace.c b/src/memspace.c index f21ecbc4e..b51be6018 100644 --- a/src/memspace.c +++ b/src/memspace.c @@ -426,3 +426,146 @@ umfMemspaceMemtargetRemove(umf_memspace_handle_t hMemspace, hMemspace->size--; return UMF_RESULT_SUCCESS; } + +// Helper function - returns zero on success, negative in case of error in filter function +// and positive error code, in case of other errors. +static int umfMemspaceFilterHelper(umf_memspace_handle_t memspace, + umf_memspace_filter_func_t filter, + void *args) { + + if (!memspace || !filter) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + size_t idx = 0; + int ret; + umf_memtarget_handle_t *nodesToRemove = + umf_ba_global_alloc(sizeof(*nodesToRemove) * memspace->size); + if (!nodesToRemove) { + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + for (size_t i = 0; i < memspace->size; i++) { + ret = filter(memspace, memspace->nodes[i], args); + if (ret < 0) { + LOG_ERR("filter function failed"); + goto free_mem; + } else if (ret == 0) { + nodesToRemove[idx++] = memspace->nodes[i]; + } + } + + size_t i = 0; + for (; i < idx; i++) { + ret = umfMemspaceMemtargetRemove(memspace, nodesToRemove[i]); + if (ret != UMF_RESULT_SUCCESS) { + goto re_add; + } + } + + umf_ba_global_free(nodesToRemove); + return UMF_RESULT_SUCCESS; + +re_add: + // If target removal failed, add back previously removed targets. + for (size_t j = 0; j < i; j++) { + umf_result_t ret2 = umfMemspaceMemtargetAdd(memspace, nodesToRemove[j]); + if (ret2 != UMF_RESULT_SUCCESS) { + ret = + UMF_RESULT_ERROR_UNKNOWN; // indicate that memspace is corrupted + break; + } + } +free_mem: + umf_ba_global_free(nodesToRemove); + return ret; +} + +umf_result_t umfMemspaceUserFilter(umf_memspace_handle_t memspace, + umf_memspace_filter_func_t filter, + void *args) { + + if (!memspace || !filter) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + int ret = umfMemspaceFilterHelper(memspace, filter, args); + if (ret < 0) { + return UMF_RESULT_ERROR_USER_SPECIFIC; + } + + return ret; +} + +typedef struct filter_by_id_args { + unsigned *ids; // array of numa nodes ids + size_t size; // size of the array +} filter_by_id_args_t; + +/* + * The following predefined filter callbacks returns umf_result_t codes as negative value + * as negative only value are treated as an error. umfMemspaceFilter will passthru this error code + * so and calling filter function has to translate this code to positive umf_result_t code. + */ + +static int filterById(umf_const_memspace_handle_t memspace, + umf_const_memtarget_handle_t target, void *args) { + if (!memspace || !target || !args) { + return -UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + filter_by_id_args_t *filterArgs = args; + for (size_t i = 0; i < filterArgs->size; i++) { + unsigned id; + umf_result_t ret = umfMemtargetGetId(target, &id); + if (ret != UMF_RESULT_SUCCESS) { + return -ret; + } + + if (id == filterArgs->ids[i]) { + return 1; + } + } + return 0; +} + +static int filterByCapacity(umf_const_memspace_handle_t memspace, + umf_const_memtarget_handle_t target, void *args) { + if (!memspace || !target || !args) { + return -UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + size_t capacity; + umf_result_t ret = umfMemtargetGetCapacity(target, &capacity); + if (ret != UMF_RESULT_SUCCESS) { + return -ret; + } + + size_t *targetCapacity = args; + return (capacity >= *targetCapacity) ? 1 : 0; +} + +umf_result_t umfMemspaceFilterById(umf_memspace_handle_t memspace, + unsigned *ids, size_t size) { + if (!memspace || !ids || size == 0) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + filter_by_id_args_t args = {ids, size}; + int ret = umfMemspaceFilterHelper(memspace, &filterById, &args); + + // if umfMemspaceFilter() returned negative umf_result_t change it to positive + return ret < 0 ? -ret : ret; +} + +umf_result_t umfMemspaceFilterByCapacity(umf_memspace_handle_t memspace, + size_t capacity) { + if (!memspace) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + int ret = umfMemspaceFilterHelper(memspace, &filterByCapacity, &capacity); + + // if umfMemspaceFilter() returned negative umf_result_t change it to positive + return ret < 0 ? -ret : ret; +} diff --git a/src/memspace_internal.h b/src/memspace_internal.h index 8f678ff7d..e31898e84 100644 --- a/src/memspace_internal.h +++ b/src/memspace_internal.h @@ -24,12 +24,6 @@ struct umf_memspace_t { umf_memtarget_handle_t *nodes; }; -/// -/// \brief Clones memspace -/// -umf_result_t umfMemspaceClone(umf_const_memspace_handle_t hMemspace, - umf_memspace_handle_t *outHandle); - typedef umf_result_t (*umfGetPropertyFn)(umf_const_memtarget_handle_t, uint64_t *); diff --git a/src/memtarget.c b/src/memtarget.c index a79ec2a6c..9d32ceea6 100644 --- a/src/memtarget.c +++ b/src/memtarget.c @@ -107,6 +107,14 @@ umf_result_t umfMemtargetGetLatency(umf_memtarget_handle_t srcMemoryTarget, dstMemoryTarget->priv, latency); } +umf_result_t umfMemtargetGetId(umf_const_memtarget_handle_t hMemtarget, + unsigned *id) { + if (!hMemtarget || !id) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + return hMemtarget->ops->get_id(hMemtarget->priv, id); +} umf_result_t umfMemtargetGetType(umf_const_memtarget_handle_t memoryTarget, umf_memtarget_type_t *type) { if (!memoryTarget || !type) { diff --git a/src/memtarget_ops.h b/src/memtarget_ops.h index 43c66ce92..75e16447e 100644 --- a/src/memtarget_ops.h +++ b/src/memtarget_ops.h @@ -45,6 +45,7 @@ typedef struct umf_memtarget_ops_t { size_t *latency); umf_result_t (*get_type)(void *memoryTarget, umf_memtarget_type_t *type); + umf_result_t (*get_id)(void *memoryTarget, unsigned *type); umf_result_t (*compare)(void *memTarget, void *otherMemTarget, int *result); } umf_memtarget_ops_t; diff --git a/src/memtargets/memtarget_numa.c b/src/memtargets/memtarget_numa.c index 08be3f883..cb0e4ce00 100644 --- a/src/memtargets/memtarget_numa.c +++ b/src/memtargets/memtarget_numa.c @@ -326,6 +326,15 @@ static umf_result_t numa_get_type(void *memTarget, umf_memtarget_type_t *type) { return UMF_RESULT_SUCCESS; } +static umf_result_t numa_get_id(void *memTarget, unsigned *id) { + if (!memTarget || !id) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + *id = ((struct numa_memtarget_t *)memTarget)->physical_id; + return UMF_RESULT_SUCCESS; +} + static umf_result_t numa_compare(void *memTarget, void *otherMemTarget, int *result) { if (!memTarget || !otherMemTarget || !result) { @@ -350,6 +359,7 @@ struct umf_memtarget_ops_t UMF_MEMTARGET_NUMA_OPS = { .get_bandwidth = numa_get_bandwidth, .get_latency = numa_get_latency, .get_type = numa_get_type, + .get_id = numa_get_id, .compare = numa_compare, .memory_provider_create_from_memspace = numa_memory_provider_create_from_memspace}; diff --git a/src/provider/provider_level_zero.c b/src/provider/provider_level_zero.c index 6c2cf84b1..d9c4afe1f 100644 --- a/src/provider/provider_level_zero.c +++ b/src/provider/provider_level_zero.c @@ -188,12 +188,7 @@ static void ze_memory_provider_finalize(void *provider) { return; } - utils_init_once(&ze_is_initialized, init_ze_global_state); umf_ba_global_free(provider); - - // portable version of "ze_is_initialized = UTIL_ONCE_FLAG_INIT;" - static UTIL_ONCE_FLAG is_initialized = UTIL_ONCE_FLAG_INIT; - memcpy(&ze_is_initialized, &is_initialized, sizeof(ze_is_initialized)); } static bool use_relaxed_allocation(ze_memory_provider_t *ze_provider, diff --git a/test/memspaces/memspace_fixtures.hpp b/test/memspaces/memspace_fixtures.hpp index 116f3e5dc..d2886ee7e 100644 --- a/test/memspaces/memspace_fixtures.hpp +++ b/test/memspaces/memspace_fixtures.hpp @@ -102,6 +102,18 @@ struct memspaceProviderTest : ::memspaceGetTest { umf_memory_provider_handle_t hProvider = nullptr; }; +struct numaNodesCapacityTest : numaNodesTest { + void SetUp() override { + numaNodesTest::SetUp(); + + for (auto nodeId : nodeIds) { + capacities.push_back(numa_node_size64(nodeId, nullptr)); + } + } + + std::vector capacities; +}; + GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(memspaceGetTest); GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(memspaceProviderTest); diff --git a/test/memspaces/memspace_numa.cpp b/test/memspaces/memspace_numa.cpp index 0065f710f..b5eaddd61 100644 --- a/test/memspaces/memspace_numa.cpp +++ b/test/memspaces/memspace_numa.cpp @@ -235,3 +235,149 @@ TEST_F(memspaceNumaProviderTest, allocFree) { ret = umfMemoryProviderFree(hProvider, ptr, size); ASSERT_EQ(ret, UMF_RESULT_SUCCESS); } + +TEST_F(numaNodesCapacityTest, CapacityFilter) { + if (capacities.size() <= 1) { + GTEST_SKIP() << "Not enough numa nodes"; + } + + umf_memspace_handle_t hMemspace; + auto ret = umfMemspaceClone(umfMemspaceHostAllGet(), &hMemspace); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_NE(hMemspace, nullptr); + + std::sort(capacities.begin(), capacities.end()); + + size_t filter_size = capacities[capacities.size() / 2]; + ret = umfMemspaceFilterByCapacity(hMemspace, filter_size); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + + ASSERT_EQ(umfMemspaceMemtargetNum(hMemspace), (capacities.size() + 1) / 2); + for (size_t i = 0; i < umfMemspaceMemtargetNum(hMemspace); i++) { + auto hTarget = umfMemspaceMemtargetGet(hMemspace, i); + ASSERT_NE(hTarget, nullptr); + size_t capacity; + auto ret = umfMemtargetGetCapacity(hTarget, &capacity); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); + auto it = std::find(capacities.begin(), capacities.end(), capacity); + EXPECT_NE(it, capacities.end()); + EXPECT_GE(capacity, filter_size); + if (it != capacities.end()) { + capacities.erase(it); + } + } + umfMemspaceDestroy(hMemspace); +} + +TEST_F(numaNodesTest, idfilter) { + if (nodeIds.size() <= 1) { + GTEST_SKIP() << "Not enough numa nodes - skipping the test"; + } + + umf_memspace_handle_t hMemspace; + auto ret = umfMemspaceClone(umfMemspaceHostAllGet(), &hMemspace); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_NE(hMemspace, nullptr); + + std::vector ids = {nodeIds[0], nodeIds[1]}; + ret = umfMemspaceFilterById(hMemspace, ids.data(), 2); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + + ASSERT_EQ(umfMemspaceMemtargetNum(hMemspace), 2); + + for (size_t i = 0; i < umfMemspaceMemtargetNum(hMemspace); i++) { + auto hTarget = umfMemspaceMemtargetGet(hMemspace, i); + ASSERT_NE(hTarget, nullptr); + unsigned id; + auto ret = umfMemtargetGetId(hTarget, &id); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); + auto it = std::find(ids.begin(), ids.end(), id); + EXPECT_NE(it, ids.end()); + if (it != ids.end()) { + ids.erase(it); + } + } + umfMemspaceDestroy(hMemspace); +} + +int customfilter(umf_const_memspace_handle_t memspace, + umf_const_memtarget_handle_t memtarget, void *args) { + static unsigned customFilterCounter = 0; + + EXPECT_NE(args, nullptr); + EXPECT_NE(memspace, nullptr); + EXPECT_NE(memtarget, nullptr); + + auto ids = (std::vector *)args; + if (customFilterCounter++ % 2) { + ids->push_back(memtarget); + return 1; + } else { + return 0; + } +} + +TEST_F(numaNodesTest, customfilter) { + if (nodeIds.size() <= 1) { + GTEST_SKIP() << "Not enough numa nodes - skipping the test"; + } + + umf_memspace_handle_t hMemspace; + auto ret = umfMemspaceClone(umfMemspaceHostAllGet(), &hMemspace); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_NE(hMemspace, nullptr); + + std::vector vec; + ret = umfMemspaceUserFilter(hMemspace, &customfilter, &vec); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + + ASSERT_EQ(umfMemspaceMemtargetNum(hMemspace), nodeIds.size() / 2); + + for (size_t i = 0; i < umfMemspaceMemtargetNum(hMemspace); i++) { + auto hTarget = umfMemspaceMemtargetGet(hMemspace, i); + ASSERT_NE(hTarget, nullptr); + + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); + auto it = std::find(vec.begin(), vec.end(), hTarget); + EXPECT_NE(it, vec.end()); + if (it != vec.end()) { + vec.erase(it); + } + } + ASSERT_EQ(vec.size(), 0); + umfMemspaceDestroy(hMemspace); +} + +int invalidFilter(umf_const_memspace_handle_t memspace, + umf_const_memtarget_handle_t memtarget, void *args) { + EXPECT_EQ(args, nullptr); + EXPECT_NE(memspace, nullptr); + EXPECT_NE(memtarget, nullptr); + + return -1; +} + +TEST_F(numaNodesTest, invalidFilters) { + umf_memspace_handle_t hMemspace; + auto ret = umfMemspaceClone(umfMemspaceHostAllGet(), &hMemspace); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_NE(hMemspace, nullptr); + + ret = umfMemspaceFilterByCapacity(nullptr, 0); + ASSERT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + ret = umfMemspaceFilterById(hMemspace, nullptr, 0); + ASSERT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); + unsigned id = 0; + ret = umfMemspaceFilterById(nullptr, &id, 1); + ASSERT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + ret = umfMemspaceUserFilter(hMemspace, nullptr, nullptr); + ASSERT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ret = umfMemspaceUserFilter(nullptr, invalidFilter, nullptr); + ASSERT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + ret = umfMemspaceUserFilter(hMemspace, invalidFilter, nullptr); + ASSERT_EQ(ret, UMF_RESULT_ERROR_USER_SPECIFIC); + umfMemspaceDestroy(hMemspace); +} diff --git a/test/memspaces/memtarget.cpp b/test/memspaces/memtarget.cpp index c4cc80a8f..2b0e4a806 100644 --- a/test/memspaces/memtarget.cpp +++ b/test/memspaces/memtarget.cpp @@ -24,13 +24,9 @@ TEST_F(test, memTargetNuma) { } } -TEST_F(numaNodesTest, getCapacity) { +TEST_F(numaNodesCapacityTest, getCapacity) { auto memspace = umfMemspaceHostAllGet(); ASSERT_NE(memspace, nullptr); - std::vector capacities; - for (auto nodeId : nodeIds) { - capacities.push_back(numa_node_size64(nodeId, nullptr)); - } for (size_t i = 0; i < umfMemspaceMemtargetNum(memspace); i++) { auto hTarget = umfMemspaceMemtargetGet(memspace, i);