diff --git a/include/pando-lib-galois/containers/pod_local_storage.hpp b/include/pando-lib-galois/containers/pod_local_storage.hpp new file mode 100644 index 00000000..574e20a7 --- /dev/null +++ b/include/pando-lib-galois/containers/pod_local_storage.hpp @@ -0,0 +1,300 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2023. University of Texas at Austin. All rights reserved. + +// SPDX-License-Identifier: MIT +// Copyright (c) 2023-2024. University of Texas at Austin. All rights reserved. + +#ifndef PANDO_LIB_GALOIS_CONTAINERS_POD_LOCAL_STORAGE_HPP_ +#define PANDO_LIB_GALOIS_CONTAINERS_POD_LOCAL_STORAGE_HPP_ + +#include + +#include + +#include +#include +#include +#include +#include + +namespace galois { + +namespace PodLocalStorageHeap { + +constexpr std::uint64_t Size = 1 << 10; +constexpr std::uint64_t Granule = 128; +struct ModestArray { + std::byte arr[Size]; +}; + +extern pando::PodSpecificStorage heap; +extern pando::SlabMemoryResource* LocalHeapSlab; + +void HeapInit(); + +template +[[nodiscard]] pando::Expected> allocate() { + auto ptr = LocalHeapSlab->allocate(sizeof(T)); + if (ptr == nullptr) { + return pando::Status::BadAlloc; + } + pando::GlobalPtr ptrT = static_cast>(ptr); + auto heapAlias = pando::PodSpecificStorageAlias(heap); + return heapAlias.getStorageAliasAt(ptrT); +} + +template +void deallocate(pando::PodSpecificStorageAlias toDeAlloc) { + auto size = sizeof(T); + auto ptrStartTyped = toDeAlloc.getPointerAt(pando::NodeIndex{0}, pando::PodIndex{0, 0}); + pando::GlobalPtr ptrStartVoid = static_cast>(ptrStartTyped); + LocalHeapSlab->deallocate(ptrStartVoid, size); +} +} // namespace PodLocalStorageHeap + +template +class PodLocalStorageIt; + +template +class PodLocalStorage { + pando::PodSpecificStorageAlias m_items{}; + +public: + PodLocalStorage() noexcept = default; + PodLocalStorage(const PodLocalStorage&) = default; + PodLocalStorage(PodLocalStorage&&) = default; + + ~PodLocalStorage() = default; + + PodLocalStorage& operator=(const PodLocalStorage&) = default; + PodLocalStorage& operator=(PodLocalStorage&&) = default; + + using iterator = PodLocalStorageIt; + using reverse_iterator = std::reverse_iterator; + + [[nodiscard]] constexpr std::uint64_t getNumPods() const noexcept { + const auto p = pando::getPlaceDims(); + return static_cast(p.node.id * p.pod.x * p.pod.y); + } + + [[nodiscard]] constexpr std::uint64_t getCurrentPodIdx() const noexcept { + const auto dim = pando::getPlaceDims(); + const auto cur = pando::getCurrentPlace(); + return static_cast(cur.node.id * dim.pod.x * dim.pod.y + cur.pod.x * dim.pod.y + + cur.pod.y); + } + + [[nodiscard]] constexpr pando::Place getPlaceFromPodIdx(std::uint64_t idx) const noexcept { + const auto dim = pando::getPlaceDims(); + const auto pods = dim.pod.x * dim.pod.y; + const pando::NodeIndex node = pando::NodeIndex(idx / pods); + const std::int16_t localPodIdx = idx % pods; + const pando::PodIndex pod = pando::PodIndex(localPodIdx / dim.pod.x, localPodIdx % dim.pod.x); + return pando::Place(node, pod, pando::anyCore); + } + + std::uint64_t size() { + return getNumPods(); + } + + [[nodiscard]] pando::Status initialize() { + m_items = PANDO_EXPECT_RETURN(PodLocalStorageHeap::allocate()); + return pando::Status::Success; + } + + void deinitialize() { + PodLocalStorageHeap::deallocate(m_items); + } + + pando::GlobalRef getLocal() noexcept { + return *m_items.getPointer(); + } + + pando::GlobalRef get(std::uint64_t i) noexcept { + auto place = getPlaceFromPodIdx(i); + return *m_items.getPointerAt(place.node, place.pod); + } + + template + pando::GlobalRef getFromPtr(pando::GlobalPtr ptr) { + std::uint64_t i = static_cast(pando::localityOf(ptr).node.id); + return this->get(i); + } + + iterator begin() noexcept { + return iterator(*this, 0); + } + + iterator begin() const noexcept { + return iterator(*this, 0); + } + + iterator end() noexcept { + return iterator(*this, getNumPods()); + } + + iterator end() const noexcept { + return iterator(*this, getNumPods()); + } + + /** + * @brief reverse iterator to the first element + */ + reverse_iterator rbegin() noexcept { + return reverse_iterator(end()--); + } + + /** + * @copydoc rbegin() + */ + reverse_iterator rbegin() const noexcept { + return reverse_iterator(end()--); + } + + /** + * @brief reverse iterator to the last element + */ + reverse_iterator rend() noexcept { + return reverse_iterator(begin()--); + } + + /** + * @copydoc rend() + */ + reverse_iterator rend() const noexcept { + return reverse_iterator(begin()--); + } + + friend bool operator==(const PodLocalStorage& a, const PodLocalStorage& b) { + const pando::NodeIndex node(0); + const pando::PodIndex pod(0, 0); + return a.m_items.getPointerAt(node, pod) == b.m_items.getPointerAt(node, pod); + } + + friend bool operator!=(const PodLocalStorage& a, const PodLocalStorage& b) { + return !(a == b); + } +}; + +template +class PodLocalStorageIt { + PodLocalStorage m_curr; + std::uint64_t m_loc; + +public: + using iterator_category = std::random_access_iterator_tag; + using difference_type = std::int64_t; + using value_type = T; + using pointer = pando::GlobalPtr; + using reference = pando::GlobalRef; + + PodLocalStorageIt(PodLocalStorage curr, std::uint64_t loc) : m_curr(curr), m_loc(loc) {} + + constexpr PodLocalStorageIt() noexcept = default; + constexpr PodLocalStorageIt(PodLocalStorageIt&&) noexcept = default; + constexpr PodLocalStorageIt(const PodLocalStorageIt&) noexcept = default; + ~PodLocalStorageIt() = default; + + constexpr PodLocalStorageIt& operator=(const PodLocalStorageIt&) noexcept = default; + constexpr PodLocalStorageIt& operator=(PodLocalStorageIt&&) noexcept = default; + + reference operator*() const noexcept { + return m_curr.get(m_loc); + } + + reference operator*() noexcept { + return m_curr.get(m_loc); + } + + pointer operator->() { + return &m_curr.get(m_loc); + } + + PodLocalStorageIt& operator++() { + m_loc++; + return *this; + } + + PodLocalStorageIt operator++(int) { + PodLocalStorageIt tmp = *this; + ++(*this); + return tmp; + } + + PodLocalStorageIt& operator--() { + m_loc--; + return *this; + } + + PodLocalStorageIt operator--(int) { + PodLocalStorageIt tmp = *this; + --(*this); + return tmp; + } + + constexpr PodLocalStorageIt operator+(std::uint64_t n) const noexcept { + return PodLocalStorageIt(m_curr, m_loc + n); + } + + constexpr PodLocalStorageIt& operator+=(std::uint64_t n) noexcept { + m_loc += n; + return *this; + } + + constexpr PodLocalStorageIt operator-(std::uint64_t n) const noexcept { + return PodLocalStorageIt(m_curr, m_loc - n); + } + + constexpr difference_type operator-(PodLocalStorageIt b) const noexcept { + return m_loc - b.loc; + } + + friend bool operator==(const PodLocalStorageIt& a, const PodLocalStorageIt& b) { + return a.m_loc == b.m_loc; + } + + friend bool operator!=(const PodLocalStorageIt& a, const PodLocalStorageIt& b) { + return !(a == b); + } + + friend bool operator<(const PodLocalStorageIt& a, const PodLocalStorageIt& b) { + return a.m_loc < b.m_loc; + } + + friend bool operator<=(const PodLocalStorageIt& a, const PodLocalStorageIt& b) { + return a.m_loc <= b.m_loc; + } + + friend bool operator>(const PodLocalStorageIt& a, const PodLocalStorageIt& b) { + return a.m_loc > b.m_loc; + } + + friend bool operator>=(const PodLocalStorageIt& a, const PodLocalStorageIt& b) { + return a.m_loc >= b.m_loc; + } + + friend pando::Place localityOf(PodLocalStorageIt& a) { + return a.m_curr.getPlaceFromPodIdx(a.m_loc); + } +}; + +template +[[nodiscard]] pando::Expected> copyToAllPods(T& cont) { + galois::PodLocalStorage ret{}; + PANDO_CHECK_RETURN(ret.initialize()); + PANDO_CHECK_RETURN(galois::doAll( + cont, ret, +[](T cont, pando::GlobalRef refcopy) { + T copy; + const std::uint64_t size = cont.size(); + PANDO_CHECK(copy.initialize(size)); + for (std::uint64_t i = 0; i < cont.size(); i++) { + copy.get(i) = cont.get(i); + } + refcopy = copy; + })); + return ret; +} + +} // namespace galois + +#endif // PANDO_LIB_GALOIS_CONTAINERS_POD_LOCAL_STORAGE_HPP_ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 62c66a8d..77eb08b5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -25,7 +25,7 @@ target_sources(pando-lib-galois ${CMAKE_CURRENT_LIST_DIR}/wmd_graph_importer.cpp ${CMAKE_CURRENT_LIST_DIR}/ingest_wmd_csv.cpp ${CMAKE_CURRENT_LIST_DIR}/ingest_rmat_el.cpp - ${CMAKE_CURRENT_LIST_DIR}/host_local_storage.cpp + ${CMAKE_CURRENT_LIST_DIR}/local_storage.cpp ) pando_compiler_options(pando-lib-galois) diff --git a/src/host_local_storage.cpp b/src/local_storage.cpp similarity index 51% rename from src/host_local_storage.cpp rename to src/local_storage.cpp index f03238b4..8533744c 100644 --- a/src/host_local_storage.cpp +++ b/src/local_storage.cpp @@ -2,6 +2,7 @@ // Copyright (c) 2023. University of Texas at Austin. All rights reserved. #include +#include pando::NodeSpecificStorage galois::HostLocalStorageHeap::heap; @@ -24,3 +25,27 @@ void galois::HostLocalStorageHeap::HeapInit() { } LocalHeapSlab = new pando::SlabMemoryResource(heapStartByte, Size); } + +pando::PodSpecificStorage + galois::PodLocalStorageHeap::heap; + +pando::SlabMemoryResource* + galois::PodLocalStorageHeap::LocalHeapSlab; + +void galois::PodLocalStorageHeap::HeapInit() { + using galois::PodLocalStorageHeap::Granule; + using galois::PodLocalStorageHeap::heap; + using galois::PodLocalStorageHeap::LocalHeapSlab; + using galois::PodLocalStorageHeap::ModestArray; + using galois::PodLocalStorageHeap::Size; + pando::GlobalPtr heapStartTyped = + heap.getPointerAt(pando::NodeIndex{0}, pando::PodIndex{0, 0}); + pando::GlobalPtr heapStartNoType = static_cast>(heapStartTyped); + pando::GlobalPtr heapStartByte = + static_cast>(heapStartNoType); + std::uint64_t diff = heapStartNoType.address % Granule; + if (diff != 0) { + heapStartByte += Granule - diff; + } + LocalHeapSlab = new pando::SlabMemoryResource(heapStartByte, Size); +} diff --git a/test/containers/CMakeLists.txt b/test/containers/CMakeLists.txt index 69ff870d..774f222d 100644 --- a/test/containers/CMakeLists.txt +++ b/test/containers/CMakeLists.txt @@ -7,3 +7,4 @@ pando_add_driver_test(test_per_thread test_per_thread.cpp) pando_add_driver_test(test_stack test_stack.cpp) pando_add_driver_test(test_host_indexed_map test_host_indexed_map.cpp) pando_add_driver_test(test_host_local_storage test_host_local_storage.cpp) +pando_add_driver_test(test_pod_local_storage test_pod_local_storage.cpp) diff --git a/test/containers/test_pod_local_storage.cpp b/test/containers/test_pod_local_storage.cpp new file mode 100644 index 00000000..e7af81de --- /dev/null +++ b/test/containers/test_pod_local_storage.cpp @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2023. University of Texas at Austin. All rights reserved. + +#include +#include + +#include +#include +#include +#include + +TEST(PodLocalStorage, Init) { + galois::PodLocalStorage ph; + pando::Status err; + + EXPECT_EQ(ph.initialize(), pando::Status::Success); + std::uint64_t i = 0; + for (pando::GlobalRef val : ph) { + val = i; + i++; + } + auto f = +[](galois::PodLocalStorage ph, std::uint64_t i, + pando::NotificationHandle done) { + EXPECT_EQ(&ph.getLocal(), &ph.get(i)); + done.notify(); + }; + + pando::NotificationArray dones; + err = dones.init(ph.getNumPods()); + EXPECT_EQ(err, pando::Status::Success); + for (std::uint64_t i = 0; i < ph.getNumPods(); i++) { + auto place = ph.getPlaceFromPodIdx(i); + err = pando::executeOn(place, f, ph, i, dones.getHandle(i)); + EXPECT_EQ(err, pando::Status::Success); + } + dones.wait(); + + ph.deinitialize(); + + EXPECT_EQ(ph.initialize(), pando::Status::Success); + i = 0; + for (pando::GlobalRef val : ph) { + val = i; + i++; + } + + dones.reset(); + EXPECT_EQ(err, pando::Status::Success); + for (std::uint64_t i = 0; i < ph.getNumPods(); i++) { + auto place = ph.getPlaceFromPodIdx(i); + err = pando::executeOn(place, f, ph, i, dones.getHandle(i)); + EXPECT_EQ(err, pando::Status::Success); + } + dones.wait(); + + ph.deinitialize(); +} + +TEST(PodLocalStorage, DoAll) { + galois::PodLocalStorage ph; + pando::Status err; + + EXPECT_EQ(ph.initialize(), pando::Status::Success); + EXPECT_TRUE(ph == ph); + EXPECT_FALSE(ph != ph); + + for (std::uint64_t i = 0; i < ph.getNumPods(); i++) { + ph.get(i) = 0xDEADBEEF; + } + + auto g = +[](pando::GlobalRef val) { + val = static_cast(pando::getCurrentPlace().node.id); + }; + + err = galois::doAll(ph, g); + + EXPECT_EQ(err, pando::Status::Success); + + auto f = +[](galois::PodLocalStorage ph, pando::NotificationHandle done) { + EXPECT_EQ(ph.getLocal(), static_cast(pando::getCurrentPlace().node.id)); + done.notify(); + }; + pando::NotificationArray dones; + err = dones.init(ph.getNumPods()); + EXPECT_EQ(err, pando::Status::Success); + for (std::uint64_t i = 0; i < ph.getNumPods(); i++) { + const auto place = ph.getPlaceFromPodIdx(i); + err = pando::executeOn(place, f, ph, dones.getHandle(i)); + EXPECT_EQ(err, pando::Status::Success); + } + dones.wait(); + + ph.deinitialize(); +} + +TEST(PodLocalStorage, copyToAllHosts) { + const std::uint64_t SIZE = 100; + pando::Array arr; + EXPECT_EQ(pando::Status::Success, arr.initialize(100)); + for (std::uint64_t i = 0; i < SIZE; i++) { + arr[i] = i; + } + auto hlsarr = PANDO_EXPECT_CHECK(galois::copyToAllPods(arr)); + for (pando::Array tocheck : hlsarr) { + EXPECT_EQ(tocheck.size(), SIZE); + for (std::uint64_t i = 0; i < SIZE; i++) { + EXPECT_EQ(tocheck.get(i), i); + } + tocheck.deinitialize(); + } + hlsarr.deinitialize(); +} diff --git a/test/test_driver.cpp b/test/test_driver.cpp index adbf7e3a..96947f18 100644 --- a/test/test_driver.cpp +++ b/test/test_driver.cpp @@ -7,6 +7,7 @@ #include #include +#include #include int pandoMain(int argc, char** argv) { @@ -15,6 +16,7 @@ int pandoMain(int argc, char** argv) { int result = 0; if (pando::getCurrentPlace().node.id == 0) { galois::HostLocalStorageHeap::HeapInit(); + galois::PodLocalStorageHeap::HeapInit(); result = RUN_ALL_TESTS(); } pando::waitAll();