diff --git a/include/openPMD/CustomHierarchy.hpp b/include/openPMD/CustomHierarchy.hpp index 10ad959b13..fb3402cf23 100644 --- a/include/openPMD/CustomHierarchy.hpp +++ b/include/openPMD/CustomHierarchy.hpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -82,16 +83,6 @@ namespace internal Container m_embeddedDatasets; Container m_embeddedMeshes; Container m_embeddedParticles; - - /* - * Each call to operator[]() needs to check the Series if the meshes/ - * particlesPath has changed, so the Series gets buffered. - * - * Alternative: Require that the meshesPath/particlesPath is fixed as - * soon as operator[]() has been called for the first time, check - * at flush time. - */ - std::unique_ptr m_bufferedSeries; }; } // namespace internal @@ -168,19 +159,7 @@ class CustomHierarchy : public Container CustomHierarchy &operator=(CustomHierarchy const &) = default; CustomHierarchy &operator=(CustomHierarchy &&) = default; - mapped_type &operator[](key_type &&key); - mapped_type &operator[](key_type const &key); - template auto asContainerOf() -> Container &; - - Container meshes{}; - Container particles{}; - -private: - template - mapped_type &bracketOperatorImpl(KeyType &&); - - Series &getBufferedSeries(); }; } // namespace openPMD diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 2d89cc09ec..2c24d07883 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -145,7 +145,16 @@ class Iteration : public CustomHierarchy friend class SeriesIterator; Iteration(Iteration const &) = default; + Iteration(Iteration &&) = default; Iteration &operator=(Iteration const &) = default; + Iteration &operator=(Iteration &&) = default; + + // These use the openPMD Container class mainly for consistency. + // But they are in fact only aliases that don't actually exist + // in the backend. + // Hence meshes.written() and particles.written() will always be false. + Container meshes{}; + Container particles{}; /** * @tparam T Floating point type of user-selected precision (e.g. float, @@ -280,6 +289,12 @@ class Iteration : public CustomHierarchy * class. */ void flushIteration(internal::FlushParams const &); + + void sync_meshes_and_particles_from_alias_to_subgroups( + internal::MeshesParticlesPath const &); + void sync_meshes_and_particles_from_subgroups_to_alias( + internal::MeshesParticlesPath const &); + void deferParseAccess(internal::DeferredParseAccess); /* * Control flow for runDeferredParseAccess(), readFileBased(), @@ -388,6 +403,23 @@ class Iteration : public CustomHierarchy */ void setStepStatus(StepStatus); + /* + * @brief Check recursively whether this Iteration is dirty. + * It is dirty if any attribute or dataset is read from or written to + * the backend. + * + * @return true If dirty. + * @return false Otherwise. + */ + bool dirtyRecursive() const; + + /** + * @brief Link with parent. + * + * @param w The Writable representing the parent. + */ + void linkHierarchy(Writable &w); + /** * @brief Access an iteration in read mode that has potentially not been * parsed yet. diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp index d6d621d755..2035e2ea8a 100644 --- a/src/CustomHierarchy.cpp +++ b/src/CustomHierarchy.cpp @@ -45,6 +45,7 @@ #include #include #include +#include #include #include #include @@ -274,8 +275,6 @@ namespace internal CustomHierarchy::CustomHierarchy() { setData(std::make_shared()); - meshes.writable().ownKeyWithinParent = "meshes"; - particles.writable().ownKeyWithinParent = "particles"; } CustomHierarchy::CustomHierarchy(NoInit) : Container_t(NoInit()) {} @@ -533,16 +532,6 @@ void CustomHierarchy::flush_internal( auto &data = get(); if (access::write(IOHandler()->m_frontendAccess)) { - if (!meshes.empty()) - { - (*this)[mpp.m_defaultMeshesPath]; - } - - if (!particles.empty()) - { - (*this)[mpp.m_defaultParticlesPath]; - } - flushAttributes(flushParams); } @@ -615,43 +604,16 @@ void CustomHierarchy::flush_internal( } void CustomHierarchy::flush( - std::string const & /* path */, internal::FlushParams const &flushParams) + std::string const & /* path */, internal::FlushParams const &) { - /* - * Convention for CustomHierarchy::flush and CustomHierarchy::read: - * Path is created/opened already at entry point of method, method needs - * to create/open path for contained subpaths. - */ - - Series s = this->getBufferedSeries(); - std::vector meshesPaths = s.meshesPaths(), - particlesPaths = s.particlesPaths(); - internal::MeshesParticlesPath mpp(meshesPaths, particlesPaths); - std::vector currentPath; - flush_internal(flushParams, mpp, currentPath); - if (!mpp.collectNewMeshesPaths.empty() || - !mpp.collectNewParticlesPaths.empty()) - { - for (auto [newly_added_paths, vec] : - {std::make_pair(&mpp.collectNewMeshesPaths, &meshesPaths), - std::make_pair(&mpp.collectNewParticlesPaths, &particlesPaths)}) - { - std::transform( - newly_added_paths->begin(), - newly_added_paths->end(), - std::back_inserter(*vec), - [](auto const &pair) { return pair; }); - } - s.setMeshesPath(meshesPaths); - s.setParticlesPath(particlesPaths); - } + throw std::runtime_error( + "[CustomHierarchy::flush()] Don't use this method. Flushing should be " + "triggered via Iteration class."); } void CustomHierarchy::linkHierarchy(Writable &w) { Attributable::linkHierarchy(w); - meshes.linkHierarchy(this->writable()); - particles.linkHierarchy(this->writable()); } bool CustomHierarchy::dirtyRecursive() const @@ -672,26 +634,7 @@ bool CustomHierarchy::dirtyRecursive() const }; auto &data = get(); return check(data.m_embeddedMeshes) || check(data.m_embeddedParticles) || - - /* - * Need to check this, too. It might be that the `meshes` alias has not - * been synced yet with the "meshes" subgroup. - * The CustomHierarchy object needs to be flushed in order for that to - * happen (or the "meshes" group needs to be accessed explicitly via - * operator[]()). - */ - check(meshes) || check(particles) || check(data.m_embeddedDatasets) || - check(*this); -} - -auto CustomHierarchy::operator[](key_type &&key) -> mapped_type & -{ - return bracketOperatorImpl(std::move(key)); -} - -auto CustomHierarchy::operator[](key_type const &key) -> mapped_type & -{ - return bracketOperatorImpl(key); + check(data.m_embeddedDatasets) || check(*this); } template @@ -730,135 +673,6 @@ template auto CustomHierarchy::asContainerOf() template auto CustomHierarchy::asContainerOf() -> Container &; template auto CustomHierarchy::asContainerOf() -> Container &; - -/* - * This method implements the usual job of ::operator[](), but additionally - * ensures that returned entries are properly linked with ::particles and - * ::meshes. - */ -template -auto CustomHierarchy::bracketOperatorImpl(KeyType &&provided_key) - -> mapped_type & -{ - auto &cont = container(); - auto find_special_key = - [&cont, &provided_key, this]( - std::string const &special_key, - auto &alias, - auto &&embeddedAccessor) -> std::optional { - if (provided_key != special_key) - { - return std::nullopt; - } - if (auto it = cont.find(provided_key); it != cont.end()) - { - if (it->second.m_attri->get() != alias.m_attri->get() || - embeddedAccessor(it->second)->m_containerData.get() != - alias.m_containerData.get()) - { - /* - * This might happen if a user first creates a custom group - * "fields" and sets the default meshes path as "fields" - * only later. - * If the CustomHierarchy::meshes alias carries no data yet, - * we can just redirect it to that group now. - * Otherwise, we need to fail. - */ - if (alias.empty() && alias.attributes().empty()) - { - alias.m_containerData = - embeddedAccessor(it->second)->m_containerData; - alias.m_attri->asSharedPtrOfAttributable() = - it->second.m_attri->asSharedPtrOfAttributable(); - return &it->second; - } - throw error::WrongAPIUsage( - "Found a group '" + provided_key + "' at path '" + - myPath().printGroup() + - "' which is not synchronous with mesh/particles alias " - "despite '" + - special_key + - "' being the default meshes/particles path. This can " - "have happened because setting default " - "meshes/particles path too late (after first flush). " - "If that's not the case, this is likely an internal " - "bug."); - } - return &it->second; - } - else - { - auto *res = - &Container::operator[](std::forward(provided_key)); - embeddedAccessor(*res)->m_containerData = alias.m_containerData; - res->m_attri->asSharedPtrOfAttributable() = - alias.m_attri->asSharedPtrOfAttributable(); - res->m_customHierarchyData->syncAttributables(); - return res; - } - }; - - /* - * @todo Buffer this somehow while still ensuring that changed meshesPath - * or particlesPath will be recorded. - */ - struct - { - std::string m_defaultMeshesPath; - std::string m_defaultParticlesPath; - } defaultPaths; - - { - auto const &series = getBufferedSeries(); - auto meshes_paths = series.meshesPaths(); - auto particles_paths = series.particlesPaths(); - setDefaultMeshesParticlesPath( - meshes_paths, particles_paths, defaultPaths); - } - - if (auto res = find_special_key( - defaultPaths.m_defaultMeshesPath, - meshes, - [](auto &group) { - return &group.m_customHierarchyData->m_embeddedMeshes; - }); - res.has_value()) - { - return **res; - } - if (auto res = find_special_key( - defaultPaths.m_defaultParticlesPath, - particles, - [](auto &group) { - return &group.m_customHierarchyData->m_embeddedParticles; - }); - res.has_value()) - { - return **res; - } - else - { - return (*this).Container::operator[]( - std::forward(provided_key)); - } -} - -Series &CustomHierarchy::getBufferedSeries() -{ - auto &data = get(); - if (!data.m_bufferedSeries) - { - /* - * retrieveSeries() returns a non-owning Series handle anyway, but let's - * be explicit here that we need a non-owning Series to avoid creating - * a memory cycle. - */ - data.m_bufferedSeries = std::make_unique(); - data.m_bufferedSeries->setData(std::shared_ptr( - &retrieveSeries().get(), [](auto const *) {})); - } - return *data.m_bufferedSeries; -} } // namespace openPMD #undef OPENPMD_LEGAL_IDENTIFIER_CHARS diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 35825107ec..24bd527e39 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -32,7 +32,10 @@ #include #include +#include #include +#include +#include namespace openPMD { @@ -312,13 +315,117 @@ void Iteration::flushIteration(internal::FlushParams const &flushParams) { return; } - CustomHierarchy::flush("", flushParams); + + /* + * Convention for CustomHierarchy::flush and CustomHierarchy::read: + * Path is created/opened already at entry point of method, method needs + * to create/open path for contained subpaths. + */ + + Series s = retrieveSeries(); + std::vector meshesPaths = s.meshesPaths(), + particlesPaths = s.particlesPaths(); + internal::MeshesParticlesPath mpp(meshesPaths, particlesPaths); + + sync_meshes_and_particles_from_alias_to_subgroups(mpp); + + std::vector currentPath; + CustomHierarchy::flush_internal(flushParams, mpp, currentPath); + + sync_meshes_and_particles_from_subgroups_to_alias(mpp); + + if (!mpp.collectNewMeshesPaths.empty() || + !mpp.collectNewParticlesPaths.empty()) + { + for (auto [newly_added_paths, vec] : + {std::make_pair(&mpp.collectNewMeshesPaths, &meshesPaths), + std::make_pair(&mpp.collectNewParticlesPaths, &particlesPaths)}) + { + std::transform( + newly_added_paths->begin(), + newly_added_paths->end(), + std::back_inserter(*vec), + [](auto const &pair) { return pair; }); + } + s.setMeshesPath(meshesPaths); + s.setParticlesPath(particlesPaths); + } + if (access::write(IOHandler()->m_frontendAccess)) { flushAttributes(flushParams); } } +void Iteration::sync_meshes_and_particles_from_alias_to_subgroups( + internal::MeshesParticlesPath const &mpp) +{ + auto sync_meshes_and_particles = + [this](auto &m_or_p, std::string const &defaultPath) { + using type = + typename std::remove_reference_t::mapped_type; + + if (m_or_p.empty()) + { + return; + } + auto &container = (*this)[defaultPath].asContainerOf(); + + for (auto &[name, entry] : m_or_p) + { + if (auxiliary::contains(name, '/')) + { + throw std::runtime_error( + "Unimplemented: Multi-level paths in " + "Iteration::meshes/Iteration::particles"); + } + if (auto it = container.find(name); it != container.end()) + { + if (it->second.m_attri->asSharedPtrOfAttributable() == + entry.m_attri->asSharedPtrOfAttributable()) + { + continue; // has been emplaced previously + } + else + { + throw std::runtime_error("asdfasdfasdfasd"); + } + } + else + { + container.emplace(name, entry); + entry.linkHierarchy(container.writable()); + } + } + }; + + sync_meshes_and_particles(meshes, mpp.m_defaultMeshesPath); + sync_meshes_and_particles(particles, mpp.m_defaultParticlesPath); +} + +void Iteration::sync_meshes_and_particles_from_subgroups_to_alias( + internal::MeshesParticlesPath const &mpp) +{ + auto sync_meshes_and_particles = + [this](auto &m_or_p, std::string const &defaultPath) { + using type = + typename std::remove_reference_t::mapped_type; + auto it = this->find(defaultPath); + if (it == this->end()) + { + return; + } + auto &container = it->second.asContainerOf(); + for (auto &[name, entry] : container) + { + m_or_p.emplace(name, entry); + } + }; + + sync_meshes_and_particles(meshes, mpp.m_defaultMeshesPath); + sync_meshes_and_particles(particles, mpp.m_defaultParticlesPath); +} + void Iteration::deferParseAccess(DeferredParseAccess dr) { get().m_deferredParseAccess = @@ -445,6 +552,8 @@ void Iteration::read_impl(std::string const &groupPath) internal::MeshesParticlesPath mpp(s.meshesPaths(), s.particlesPaths()); CustomHierarchy::read(std::move(mpp)); + sync_meshes_and_particles_from_subgroups_to_alias(mpp); + #ifdef openPMD_USE_INVASIVE_TESTS if (containsAttribute("__openPMD_internal_fail")) { @@ -611,6 +720,36 @@ void Iteration::setStepStatus(StepStatus status) } } +bool Iteration::dirtyRecursive() const +{ + if (dirty() || CustomHierarchy::dirtyRecursive()) + { + return true; + } + for (auto const &pair : particles) + { + if (!pair.second.written()) + { + return true; + } + } + for (auto const &pair : meshes) + { + if (!pair.second.written()) + { + return true; + } + } + return false; +} + +void Iteration::linkHierarchy(Writable &w) +{ + Attributable::linkHierarchy(w); + meshes.linkHierarchy(this->writable()); + particles.linkHierarchy(this->writable()); +} + void Iteration::runDeferredParseAccess() { if (access::read(IOHandler()->m_frontendAccess)) diff --git a/src/binding/python/CustomHierarchy.cpp b/src/binding/python/CustomHierarchy.cpp index 3d7ac651b8..d618226d15 100644 --- a/src/binding/python/CustomHierarchy.cpp +++ b/src/binding/python/CustomHierarchy.cpp @@ -13,7 +13,9 @@ using namespace openPMD; void init_CustomHierarchy(py::module &m) { - auto py_ch_cont = declare_container(m, "Container_CustomHierarchy"); + auto py_ch_cont = + declare_container( + m, "Container_CustomHierarchy"); py::class_, Attributable>( m, "CustomHierarchy") @@ -23,20 +25,7 @@ void init_CustomHierarchy(py::module &m) .def("as_container_of_meshes", &CustomHierarchy::asContainerOf) .def( "as_container_of_particles", - &CustomHierarchy::asContainerOf) - - .def_readwrite( - "meshes", - &CustomHierarchy::meshes, - py::return_value_policy::copy, - // garbage collection: return value must be freed before Iteration - py::keep_alive<1, 0>()) - .def_readwrite( - "particles", - &CustomHierarchy::particles, - py::return_value_policy::copy, - // garbage collection: return value must be freed before Iteration - py::keep_alive<1, 0>()); + &CustomHierarchy::asContainerOf); finalize_container(py_ch_cont); } diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index 21709c117a..7cee1b47a5 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -215,9 +215,13 @@ TEST_CASE("custom_hierarchies", "[core]") write.setMeshesPath(std::vector{"fields/", "%%/meshes/"}); auto meshesManually = write.iterations[0]["fields"].asContainerOf(); + REQUIRE(meshesManually.size() == 0); + write.flush(); // Synchronized upon flushing REQUIRE(meshesManually.contains("E")); REQUIRE(meshesManually.size() == 1); meshesManually["B"]["x"].makeEmpty(2); + REQUIRE(meshesViaAlias.size() == 1); + write.flush(); REQUIRE(meshesViaAlias.contains("B")); REQUIRE(meshesViaAlias.size() == 2); @@ -303,12 +307,14 @@ TEST_CASE("custom_hierarchies", "[core]") { std::vector data(10, 3); - auto E_x = write.iterations[0]["custom_meshes"].meshes["E"]["x"]; + auto E_x = write.iterations[0]["custom_meshes"]["meshes"] + .asContainerOf()["E"]["x"]; E_x.resetDataset({Datatype::INT, {10}}); E_x.storeChunk(data, {0}, {10}); - auto e_pos_x = write.iterations[0]["custom_particles"] - .particles["e"]["position"]["x"]; + auto e_pos_x = + write.iterations[0]["custom_particles"]["particles"] + .asContainerOf()["e"]["position"]["x"]; e_pos_x.resetDataset({Datatype::INT, {10}}); e_pos_x.storeChunk(data, {0}, {10}); write.close(); @@ -318,17 +324,26 @@ TEST_CASE("custom_hierarchies", "[core]") { auto it0 = read.iterations[0]; auto custom_meshes = it0["custom_meshes"]; - REQUIRE(custom_meshes.meshes.size() == 1); - REQUIRE(read.iterations[0]["custom_meshes"].meshes.count("E") == 1); - auto E_x_loaded = read.iterations[0]["custom_meshes"] - .meshes["E"]["x"] + REQUIRE(custom_meshes["meshes"].asContainerOf().size() == 1); + REQUIRE( + read.iterations[0]["custom_meshes"]["meshes"] + .asContainerOf() + .count("E") == 1); + auto E_x_loaded = read.iterations[0]["custom_meshes"]["meshes"] + .asContainerOf()["E"]["x"] .loadChunk(); - REQUIRE(read.iterations[0]["custom_particles"].particles.size() == 1); REQUIRE( - read.iterations[0]["custom_particles"].particles.count("e") == 1); - auto e_pos_x_loaded = read.iterations[0]["custom_particles"] - .particles["e"]["position"]["x"] - .loadChunk(); + read.iterations[0]["custom_particles"]["particles"] + .asContainerOf() + .size() == 1); + REQUIRE( + read.iterations[0]["custom_particles"]["particles"] + .asContainerOf() + .count("e") == 1); + auto e_pos_x_loaded = + read.iterations[0]["custom_particles"]["particles"] + .asContainerOf()["e"]["position"]["x"] + .loadChunk(); read.flush(); for (size_t i = 0; i < 10; ++i) @@ -341,7 +356,7 @@ TEST_CASE("custom_hierarchies", "[core]") TEST_CASE("custom_hierarchies_no_rw", "[core]") { - std::string filePath = "../samples/custom_hierarchies_no_rw.json"; + std::string filePath = "../samples/custom_hierarchies_no_rw.bp"; Series write(filePath, Access::CREATE); write.setMeshesPath(std::vector{"%%/meshes/"}); write.iterations[0]["custom"]["hierarchy"]; @@ -368,12 +383,14 @@ TEST_CASE("custom_hierarchies_no_rw", "[core]") { std::vector data(10, 3); - auto E_x = write.iterations[0]["custom_meshes"].meshes["E"]["x"]; + auto E_x = write.iterations[0]["custom_meshes"]["meshes"] + .asContainerOf()["E"]["x"]; E_x.resetDataset({Datatype::INT, {10}}); E_x.storeChunk(data, {0}, {10}); - auto e_pos_x = write.iterations[0]["custom_particles"] - .particles["e"]["position"]["x"]; + auto e_pos_x = + write.iterations[0]["custom_particles"]["particles"] + .asContainerOf()["e"]["position"]["x"]; e_pos_x.resetDataset({Datatype::INT, {10}}); e_pos_x.storeChunk(data, {0}, {10}); write.close(); @@ -410,7 +427,9 @@ TEST_CASE("myPath", "[core]") recordComponent.template makeConstant(5678); }; - REQUIRE(pathOf(iteration.meshes) == vec_t{"iterations", "1234", "meshes"}); + // iteration.meshes is only an alias without a path of its own + // REQUIRE(pathOf(iteration.meshes) == vec_t{"iterations", "1234", + // "meshes"}); auto scalarMesh = iteration.meshes["e_chargeDensity"]; REQUIRE( @@ -429,9 +448,10 @@ TEST_CASE("myPath", "[core]") pathOf(vectorMeshComponent) == vec_t{"iterations", "1234", "meshes", "E", "x"}); - REQUIRE( - pathOf(iteration.particles) == - vec_t{"iterations", "1234", "particles"}); + // iteration.particles is only an alias without a path of its own + // REQUIRE( + // pathOf(iteration.particles) == + // vec_t{"iterations", "1234", "particles"}); auto speciesE = iteration.particles["e"]; REQUIRE(pathOf(speciesE) == vec_t{"iterations", "1234", "particles", "e"}); diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 6385432174..c4cc50eb3a 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -2611,9 +2611,17 @@ TEST_CASE("git_hdf5_sample_structure_test", "[serial][hdf5]") REQUIRE( o.iterations[100].meshes.parent() == getWritable(&o.iterations[100])); + REQUIRE( + getWritable( + &o.iterations[100]["fields"].asContainerOf()["E"]) == + getWritable(&o.iterations[100].meshes["E"])); + REQUIRE( + o.iterations[100]["fields"].asContainerOf()["E"].parent() == + getWritable(&o.iterations[100]["fields"].asContainerOf())); REQUIRE( o.iterations[100].meshes["E"].parent() == - getWritable(&o.iterations[100].meshes)); + // Iteration::meshes is only an alias, this is the actual parent + getWritable(&o.iterations[100]["fields"].asContainerOf())); REQUIRE( o.iterations[100].meshes["E"]["x"].parent() == getWritable(&o.iterations[100].meshes["E"])); @@ -2623,13 +2631,17 @@ TEST_CASE("git_hdf5_sample_structure_test", "[serial][hdf5]") REQUIRE( o.iterations[100].meshes["E"]["z"].parent() == getWritable(&o.iterations[100].meshes["E"])); + REQUIRE( + getWritable(&o.iterations[100].meshes["rho"]) == + getWritable( + &o.iterations[100]["fields"].asContainerOf()["rho"])); REQUIRE( o.iterations[100].meshes["rho"].parent() == - getWritable(&o.iterations[100].meshes)); + getWritable(&o.iterations[100]["fields"])); REQUIRE( o.iterations[100] .meshes["rho"][MeshRecordComponent::SCALAR] - .parent() == getWritable(&o.iterations[100].meshes)); + .parent() == getWritable(&o.iterations[100]["fields"])); REQUIRE_THROWS_AS( o.iterations[100].meshes["cherries"], std::out_of_range); REQUIRE( @@ -2637,7 +2649,11 @@ TEST_CASE("git_hdf5_sample_structure_test", "[serial][hdf5]") getWritable(&o.iterations[100])); REQUIRE( o.iterations[100].particles["electrons"].parent() == - getWritable(&o.iterations[100].particles)); + getWritable(&o.iterations[100]["particles"])); + REQUIRE( + getWritable(&o.iterations[100].particles["electrons"]) == + getWritable(&o.iterations[100]["particles"] + .asContainerOf()["electrons"])); REQUIRE( o.iterations[100].particles["electrons"]["charge"].parent() == getWritable(&o.iterations[100].particles["electrons"]));