From 4c42b948ae26db338c393d33cbbf0e2b516d42bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 4 Dec 2023 17:20:10 +0100 Subject: [PATCH] Add loadChunkVariantAllocating --- include/openPMD/RecordComponent.hpp | 7 ++ include/openPMD/RecordComponent.tpp | 140 ++++++++++++++--------- include/openPMD/auxiliary/TypeTraits.hpp | 6 + test/SerialIOTest.cpp | 16 +++ 4 files changed, 115 insertions(+), 54 deletions(-) diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index e13245f5f5..5768468659 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -203,6 +203,9 @@ class RecordComponent : public BaseRecordComponent using shared_ptr_dataset_types = auxiliary::detail:: map_variant::type; + using raw_ptr_dataset_types = auxiliary::detail:: + map_variant::type; + /** std::variant-based version of allocating loadChunk(Offset, Extent) * * @return The same result that loadChunk() would return, but @@ -215,6 +218,10 @@ class RecordComponent : public BaseRecordComponent */ shared_ptr_dataset_types loadChunkVariant(Offset = {0u}, Extent = {-1u}); + template + raw_ptr_dataset_types + loadChunkVariantAllocating(Alloc &&alloc, Offset = {0u}, Extent = {-1u}); + /** Load a chunk of data into pre-allocated memory. * * @param data Preallocated, contiguous buffer, large enough to load the diff --git a/include/openPMD/RecordComponent.tpp b/include/openPMD/RecordComponent.tpp index e34013327f..bc4cd590e3 100644 --- a/include/openPMD/RecordComponent.tpp +++ b/include/openPMD/RecordComponent.tpp @@ -21,6 +21,8 @@ #pragma once +#include "openPMD/Dataset.hpp" +#include "openPMD/Error.hpp" #include "openPMD/RecordComponent.hpp" #include "openPMD/Span.hpp" #include "openPMD/auxiliary/Memory.hpp" @@ -54,27 +56,59 @@ inline RecordComponent &RecordComponent::makeEmpty(uint8_t dimensions) return makeEmpty(Dataset(determineDatatype(), Extent(dimensions, 0))); } -template -inline std::shared_ptr RecordComponent::loadChunk(Offset o, Extent e) +namespace detail { - uint8_t dim = getDimensionality(); + inline void + checkOffsetExtentForLoading(RecordComponent &rc, Offset &o, Extent &e) + { + uint8_t dim = rc.getDimensionality(); - // default arguments - // offset = {0u}: expand to right dim {0u, 0u, ...} - Offset offset = o; - if (o.size() == 1u && o.at(0) == 0u && dim > 1u) - offset = Offset(dim, 0u); + // default arguments + // offset = {0u}: expand to right dim {0u, 0u, ...} + Offset offset = o; + if (o.size() == 1u && o.at(0) == 0u && dim > 1u) + offset = Offset(dim, 0u); - // extent = {-1u}: take full size - Extent extent(dim, 1u); - if (e.size() == 1u && e.at(0) == -1u) - { - extent = getExtent(); - for (uint8_t i = 0u; i < dim; ++i) - extent[i] -= offset[i]; + // extent = {-1u}: take full size + Extent extent(dim, 1u); + if (e.size() == 1u && e.at(0) == -1u) + { + extent = rc.getExtent(); + for (uint8_t i = 0u; i < dim; ++i) + extent[i] -= offset[i]; + } + else + extent = e; + + if (extent.size() != dim || offset.size() != dim) + { + std::ostringstream oss; + oss << "Dimensionality of chunk (" + << "offset=" << offset.size() << "D, " + << "extent=" << extent.size() << "D) " + << "and record component (" << int(dim) << "D) " + << "do not match."; + throw std::runtime_error(oss.str()); + } + Extent dse = rc.getExtent(); + for (uint8_t i = 0; i < dim; ++i) + if (dse[i] < offset[i] + extent[i]) + throw std::runtime_error( + "Chunk does not reside inside dataset (Dimension on " + "index " + + std::to_string(i) + ". DS: " + std::to_string(dse[i]) + + " - Chunk: " + std::to_string(offset[i] + extent[i]) + ")"); + + o = offset; + e = extent; } - else - extent = e; +} // namespace detail + +template +inline std::shared_ptr +RecordComponent::loadChunk(Offset offset, Extent extent) +{ + detail::checkOffsetExtentForLoading(*this, offset, extent); uint64_t numPoints = 1u; for (auto const &dimensionSize : extent) @@ -94,8 +128,8 @@ inline std::shared_ptr RecordComponent::loadChunk(Offset o, Extent e) } template -inline void -RecordComponent::loadChunk(std::shared_ptr data, Offset o, Extent e) +inline void RecordComponent::loadChunk( + std::shared_ptr data, Offset offset, Extent extent) { Datatype dtype = determineDatatype(data); if (dtype != getDatatype()) @@ -113,42 +147,8 @@ RecordComponent::loadChunk(std::shared_ptr data, Offset o, Extent e) throw std::runtime_error(err_msg); } - uint8_t dim = getDimensionality(); - - // default arguments - // offset = {0u}: expand to right dim {0u, 0u, ...} - Offset offset = o; - if (o.size() == 1u && o.at(0) == 0u && dim > 1u) - offset = Offset(dim, 0u); - - // extent = {-1u}: take full size - Extent extent(dim, 1u); - if (e.size() == 1u && e.at(0) == -1u) - { - extent = getExtent(); - for (uint8_t i = 0u; i < dim; ++i) - extent[i] -= offset[i]; - } - else - extent = e; + detail::checkOffsetExtentForLoading(*this, offset, extent); - if (extent.size() != dim || offset.size() != dim) - { - std::ostringstream oss; - oss << "Dimensionality of chunk (" - << "offset=" << offset.size() << "D, " - << "extent=" << extent.size() << "D) " - << "and record component (" << int(dim) << "D) " - << "do not match."; - throw std::runtime_error(oss.str()); - } - Extent dse = getExtent(); - for (uint8_t i = 0; i < dim; ++i) - if (dse[i] < offset[i] + extent[i]) - throw std::runtime_error( - "Chunk does not reside inside dataset (Dimension on index " + - std::to_string(i) + ". DS: " + std::to_string(dse[i]) + - " - Chunk: " + std::to_string(offset[i] + extent[i]) + ")"); if (!data) throw std::runtime_error( "Unallocated pointer passed during chunk loading."); @@ -192,6 +192,38 @@ inline void RecordComponent::loadChunkRaw(T *ptr, Offset offset, Extent extent) loadChunk(auxiliary::shareRaw(ptr), std::move(offset), std::move(extent)); } +namespace detail +{ + template + struct LoadChunkVariantAllocating + { + template + static RecordComponent::raw_ptr_dataset_types + call(RecordComponent &self, Alloc &&alloc, Offset offset, Extent extent) + { + T *res; + auto dim = self.getDimensionality(); + size_t flat_extent = 1; + for (uint8_t i = 0; i < dim; ++i) + { + flat_extent *= extent[i]; + } + std::forward(alloc)(&res, flat_extent); + self.loadChunkRaw(res, std::move(offset), std::move(extent)); + return res; + } + }; +} // namespace detail + +template +auto RecordComponent::loadChunkVariantAllocating( + Alloc &&alloc, Offset offset, Extent extent) -> raw_ptr_dataset_types +{ + detail::checkOffsetExtentForLoading(*this, offset, extent); + return this->visit>( + std::forward(alloc), std::move(offset), std::move(extent)); +} + template inline void RecordComponent::storeChunk(std::shared_ptr data, Offset o, Extent e) diff --git a/include/openPMD/auxiliary/TypeTraits.hpp b/include/openPMD/auxiliary/TypeTraits.hpp index 64365d7bd9..334d494907 100644 --- a/include/openPMD/auxiliary/TypeTraits.hpp +++ b/include/openPMD/auxiliary/TypeTraits.hpp @@ -146,6 +146,12 @@ namespace detail using type = std::shared_ptr; }; + struct as_raw_pointer + { + template + using type = T *; + }; + template struct append_to_variant; diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index b7b91af422..d3ef6fc878 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -1,6 +1,7 @@ // expose private and protected members for invasive testing #include "openPMD/Datatype.hpp" #include "openPMD/IO/Access.hpp" +#include #if openPMD_USE_INVASIVE_TESTS #define OPENPMD_private public: #define OPENPMD_protected public: @@ -1696,6 +1697,21 @@ inline void write_test(const std::string &backend) << '\'' << std::endl; }, variantTypeDataset); + + std::vector serializationBuffer; + auto selfAllocatedVariant = rc.loadChunkVariantAllocating( + [&serializationBuffer](auto **ptr, size_t flat_extent) { + using type = std::remove_reference_t; + serializationBuffer.resize(sizeof(type) * flat_extent); + *ptr = reinterpret_cast(serializationBuffer.data()); + }); + rc.seriesFlush(); + std::visit( + [](auto *ptr) { + std::cout << "First value in loaded chunk: '" << ptr[0] << '\'' + << std::endl; + }, + selfAllocatedVariant); } TEST_CASE("write_test", "[serial]")