From 0b42e2f8e795c564e0742902cfe42da9d42c8c7b Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Tue, 24 Sep 2024 17:16:31 -0700 Subject: [PATCH 001/100] Transitioning to support blueprint mesh in conduit::Node. --- src/axom/quest/IntersectionShaper.hpp | 235 ++++++++++++++---- src/axom/quest/Shaper.cpp | 31 ++- src/axom/quest/Shaper.hpp | 22 +- .../quest/examples/quest_shape_in_memory.cpp | 6 +- 4 files changed, 245 insertions(+), 49 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 506516f0ce..e48992771f 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -253,6 +253,86 @@ AXOM_HOST_DEVICE inline void GridFunctionView::finalize() } #endif +/*! + * \class TempArrayAccess + * + * \brief Given some array data, provides temporary version of + * that data but allocated with an alternate allocator id. + * This class performs data movement and clean-up to + * temporarily have the data with the desired allocator id. + */ +class TempArrayAccess +{ +public: + /*! + * \brief Constructor + * + * \param array + * \param writeBack Whether the data needs to be writen back to the + * original array device when this object goes out + * of scope. + */ + AXOM_HOST TempArrayAccess(axom::ArrayView& array, + int wantedAllocatorId, + bool writeBack = true) + { + m_origArray = array; + + if (array.getAllocatorID() != wantedAllocatorId) + { + m_tempArray = axom::Array(array, wantedAllocatorId); + m_accessible = m_tempArray.view(); + } + else + { + m_accessible = array; + } + m_writeBack = writeBack; + } + + /*! + * \brief Copy constructor, which is called to make a copy of the host + * object so it is accessible inside a RAJA kernel. Any data movement + * happened in the host constructor. This version sets hostData to + * nullptr so we know not to clean up in the destructor. + */ + AXOM_HOST_DEVICE TempArrayAccess(const TempArrayAccess& obj) + : m_origArray(obj.m_origArray) + , m_tempArray() + , m_accessible(obj.m_accessible) + , m_writeBack(obj.m_writeBack) + { } + + /*! + * \brief Destructor. On the host, this method may move data from the + device and deallocate device storage. + */ + AXOM_HOST_DEVICE ~TempArrayAccess() + { + if (m_writeBack && (m_accessible.data() != m_origArray.data())) + { + axom::copy(m_origArray.data(), m_accessible.data(), sizeof(double)*m_origArray.size()); + } + } + + /*! + * \brief Indexing operator for accessing the data. + * + * \param i The index at which to access the data. + * + * \return A reference to the data at index i. + */ + AXOM_HOST_DEVICE double& operator[](int i) { return m_accessible[i]; } + // non-const return on purpose. + AXOM_HOST_DEVICE double& operator[](int i) const { return m_accessible[i]; } + +private: + axom::ArrayView m_origArray; + axom::Array m_tempArray; + axom::ArrayView m_accessible; + bool m_writeBack; +}; + //--------------------------------------------------------------------------- /** * \class @@ -686,9 +766,10 @@ class IntersectionShaper : public Shaper // Create and register a scalar field for this shape's volume fractions // The Degrees of Freedom will be in correspondence with the elements - auto* volFrac = this->newVolFracGridFunction(); - auto volFracName = axom::fmt::format("shape_vol_frac_{}", shape.getName()); - this->getDC()->RegisterField(volFracName, volFrac); + std::string volFracName = axom::fmt::format("shape_vol_frac_{}", shape.getName()); + // auto* volFrac = this->newVolFracGridFunction(); + // this->getDC()->RegisterField(volFracName, volFrac); + auto volFrac = getScalarCellData(volFracName); // Initialize hexahedral elements m_hexes = axom::Array(NE, NE, device_allocator); @@ -715,7 +796,8 @@ class IntersectionShaper : public Shaper // Set each shape volume fraction to 1 double vf = 1.0; - (*volFrac)(i) = vf; + // (*volFrac)(i) = vf; + volFrac[i] = vf; // Get the coordinates for the vertices for(int j = 0; j < NUM_VERTS_PER_HEX; ++j) @@ -1040,7 +1122,8 @@ class IntersectionShaper : public Shaper * \return A pair containing the associated grid function and material * number (its order in the list). */ - std::pair getMaterial(const std::string& materialName) + // std::pair getMaterial(const std::string& materialName) + std::pair, int> getMaterial(const std::string& materialName) { // If we already know about the material, return it. for(size_t i = 0; i < m_vf_material_names.size(); i++) @@ -1053,17 +1136,15 @@ class IntersectionShaper : public Shaper // Get or create the volume fraction field for this shape's material auto materialVolFracName = materialNameToFieldName(materialName); - mfem::GridFunction* matVolFrac = nullptr; - if(this->getDC()->HasField(materialVolFracName)) - { - matVolFrac = this->getDC()->GetField(materialVolFracName); - } - else + + bool newData = !hasData(materialVolFracName); + + auto matVolFrac = getScalarCellData(materialVolFracName); + if (newData) { - matVolFrac = newVolFracGridFunction(); - this->getDC()->RegisterField(materialVolFracName, matVolFrac); // Zero out the volume fractions (on host). - memset(matVolFrac->begin(), 0, matVolFrac->Size() * sizeof(double)); + // memset(matVolFrac->begin(), 0, matVolFrac->Size() * sizeof(double)); + memset(matVolFrac.data(), 0, matVolFrac.size() * sizeof(double)); } // Add the material to our vectors. @@ -1117,34 +1198,32 @@ class IntersectionShaper : public Shaper * free space in each zone. */ template - mfem::GridFunction* getCompletelyFree() + axom::ArrayView getCompletelyFree() { - // Add the material prefix so the MFEMSidreDataCollection will automatically + // Add the material prefix so the mesh will automatically // consider the free material something it needs to write as a matset. const std::string fieldName(materialNameToFieldName(m_free_mat_name)); - mfem::GridFunction* cfgf = nullptr; - if(this->getDC()->HasField(fieldName)) - { - cfgf = this->getDC()->GetField(fieldName); - } - else - { - // Make the new grid function. - cfgf = newVolFracGridFunction(); - this->getDC()->RegisterField(fieldName, cfgf); + bool newData = !hasData(fieldName); + int execSpaceAllocatorID = ::getUmpireDeviceId(); + + axom::ArrayView cfgf = getScalarCellData(fieldName); + + if (newData) + { AXOM_ANNOTATE_SCOPE("compute_free"); - int dataSize = cfgf->Size(); - GridFunctionView cfView(cfgf); + int dataSize = cfgf.size(); + TempArrayAccess cfView(cfgf, execSpaceAllocatorID, true); + axom::for_all( dataSize, AXOM_LAMBDA(axom::IndexType i) { cfView[i] = 1.; }); // Iterate over all materials and subtract off their VFs from cfgf. - for(auto& gf : m_vf_grid_functions) + for(axom::ArrayView& gf : m_vf_grid_functions) { - GridFunctionView matVFView(gf, false); + TempArrayAccess matVFView(gf, execSpaceAllocatorID, false); axom::for_all( dataSize, AXOM_LAMBDA(axom::IndexType i) { @@ -1153,6 +1232,7 @@ class IntersectionShaper : public Shaper }); } } + return cfgf; } @@ -1213,17 +1293,20 @@ class IntersectionShaper : public Shaper populateMaterials(); // Get the free material so it is created first. - mfem::GridFunction* freeMat = getCompletelyFree(); + // mfem::GridFunction* freeMat = getCompletelyFree(); + axom::ArrayView freeMat = getCompletelyFree(); // Get this shape's material, creating the GridFunction if needed. auto matVF = getMaterial(shape.getMaterial()); - int dataSize = matVF.first->Size(); + // int dataSize = matVF.first->Size(); + int dataSize = matVF.first.size(); // Get this shape's array. auto shapeVolFracName = axom::fmt::format("shape_vol_frac_{}", shape.getName()); - auto* shapeVolFrac = this->getDC()->GetField(shapeVolFracName); - SLIC_ASSERT(shapeVolFrac != nullptr); + // auto* shapeVolFrac = this->getDC()->GetField(shapeVolFracName); + auto shapeVolFrac = getScalarCellData(shapeVolFracName); + // SLIC_ASSERT(shapeVolFrac != nullptr); // Allocate some memory for the replacement rule data arrays. int execSpaceAllocatorID = axom::execution_space::allocatorID(); @@ -1234,8 +1317,10 @@ class IntersectionShaper : public Shaper ArrayView vf_writable(vf_writable_array); // Determine which grid functions need to be considered for VF updates. - std::vector> gf_order_by_matnumber; - std::vector updateVFs, excludeVFs; + // std::vector> gf_order_by_matnumber; + // std::vector updateVFs, excludeVFs; + std::vector, int>> gf_order_by_matnumber; + std::vector> updateVFs, excludeVFs; if(!shape.getMaterialsReplaced().empty()) { // Include materials replaced in updateVFs. @@ -1284,8 +1369,8 @@ class IntersectionShaper : public Shaper // Sort eligible update materials by material number. std::sort(gf_order_by_matnumber.begin(), gf_order_by_matnumber.end(), - [&](const std::pair& lhs, - const std::pair& rhs) { + [&](const std::pair, int>& lhs, + const std::pair, int>& rhs) { return lhs.second < rhs.second; }); @@ -1316,7 +1401,8 @@ class IntersectionShaper : public Shaper for(const auto& name : shape.getMaterialsReplaced()) { auto mat = getMaterial(name); - GridFunctionView matVFView(mat.first, false); + // GridFunctionView matVFView(mat.first, false); + TempArrayAccess matVFView(mat.first, execSpaceAllocatorID, false); axom::for_all( dataSize, AXOM_LAMBDA(axom::IndexType i) { @@ -1335,7 +1421,8 @@ class IntersectionShaper : public Shaper for(auto& gf : excludeVFs) { - GridFunctionView matVFView(gf, false); + // GridFunctionView matVFView(gf, false); + TempArrayAccess matVFView(gf, execSpaceAllocatorID, false); axom::for_all( dataSize, AXOM_LAMBDA(axom::IndexType i) { @@ -1349,8 +1436,10 @@ class IntersectionShaper : public Shaper { AXOM_ANNOTATE_SCOPE("compute_vf"); - GridFunctionView matVFView(matVF.first); - GridFunctionView shapeVFView(shapeVolFrac); + // GridFunctionView matVFView(matVF.first); + // GridFunctionView shapeVFView(shapeVolFrac); + TempArrayAccess matVFView(matVF.first, execSpaceAllocatorID, true); + TempArrayAccess shapeVFView(shapeVolFrac, execSpaceAllocatorID, true); axom::ArrayView overlap_volumes_view = m_overlap_volumes.view(); axom::ArrayView hex_volumes_view = m_hex_volumes.view(); @@ -1380,7 +1469,8 @@ class IntersectionShaper : public Shaper AXOM_ANNOTATE_SCOPE("update_vf"); for(auto& gf : updateVFs) { - GridFunctionView matVFView(gf); + // GridFunctionView matVFView(gf); + TempArrayAccess matVFView(gf, execSpaceAllocatorID, true); axom::for_all( dataSize, AXOM_LAMBDA(axom::IndexType i) { @@ -2020,7 +2110,63 @@ class IntersectionShaper : public Shaper m_level = circleLevel; } + /// Whether the given field exists in the mesh. + bool hasData(const std::string& fieldName) + { + bool has = false; + if (m_dc) { + has = m_dc->HasField(fieldName); + } + else { + std::string fieldPath = axom::fmt::format("fields/{}", fieldName); + has = m_bpGrp->hasGroup(fieldPath); + } + return has; + } + + axom::ArrayView getScalarCellData(const std::string& fieldName) + { + axom::ArrayView rval; + + if (m_dc) { + + mfem::GridFunction* gridFunc = nullptr; + if (m_dc->HasField(fieldName)) + { + gridFunc = m_dc->GetField(fieldName); + } + else { + gridFunc = newVolFracGridFunction(); + m_dc->RegisterField(fieldName, gridFunc); + } + rval = axom::ArrayView(gridFunc->GetData(), + gridFunc->Size()); + } + else { + std::string fieldPath = axom::fmt::format("fields/{}", fieldName); + auto dtype = conduit::DataType::float64(m_cellCount); + axom::sidre::View* valuesView = nullptr; + if (m_bpGrp->hasGroup(fieldPath)) + { + auto* fieldsGrp = m_bpGrp->getGroup(fieldPath); + valuesView = fieldsGrp->getView("values"); + SLIC_ASSERT(valuesView->getNumElements() == m_cellCount); + SLIC_ASSERT(valuesView->getNode().dtype().id() == dtype.id()); + } + else + { + auto* fieldsGrp = m_bpGrp->createGroup(fieldPath); + valuesView = fieldsGrp->createView("values"); + valuesView->allocate(dtype); + } + rval = axom::ArrayView(static_cast(valuesView->getVoidPtr()), + m_cellCount); + } + return rval; + } + /// Create and return a new volume fraction grid function for the current mesh + // TODO: change to generic name. Nothing in here is about volume fractions. BTNG. mfem::GridFunction* newVolFracGridFunction() { mfem::Mesh* mesh = getDC()->GetMesh(); @@ -2066,7 +2212,8 @@ class IntersectionShaper : public Shaper axom::Array m_hexes; axom::Array m_hex_bbs; - std::vector m_vf_grid_functions; + // Views of volume-fraction data owned by grid. + std::vector> m_vf_grid_functions; std::vector m_vf_material_names; #endif }; diff --git a/src/axom/quest/Shaper.cpp b/src/axom/quest/Shaper.cpp index c3f0a9ca3e..46bd19401f 100644 --- a/src/axom/quest/Shaper.cpp +++ b/src/axom/quest/Shaper.cpp @@ -30,13 +30,31 @@ constexpr double Shaper::MINIMUM_PERCENT_ERROR; constexpr double Shaper::MAXIMUM_PERCENT_ERROR; constexpr double Shaper::DEFAULT_VERTEX_WELD_THRESHOLD; -Shaper::Shaper(const klee::ShapeSet& shapeSet, sidre::MFEMSidreDataCollection* dc) +Shaper::Shaper(const klee::ShapeSet& shapeSet, + sidre::MFEMSidreDataCollection* dc) : m_shapeSet(shapeSet) , m_dc(dc) { #if defined(AXOM_USE_MPI) && defined(MFEM_USE_MPI) m_comm = m_dc->GetComm(); #endif + m_cellCount = m_dc->GetMesh()->GetNE(); +} + +Shaper::Shaper(const klee::ShapeSet& shapeSet, + conduit::Node* bpNode, + const std::string& topo) + : m_shapeSet(shapeSet) + , m_bpNode(bpNode) + , m_bpTopo(topo) + , m_comm(MPI_COMM_WORLD) +{ + m_bpGrp = m_ds.getRoot()->createGroup("bpGrp"); + m_bpGrp->importConduitTreeExternal(*bpNode); + std::string coordsName = + m_bpGrp->getView(axom::fmt::format("topologies/{}/coordset", m_bpTopo))->getNode().as_string(); + auto* coordsView = m_bpGrp->getView(axom::fmt::format("coordsets/{}", coordsName)); + m_cellCount = coordsView->getNumElements(); } void Shaper::setSamplesPerKnotSpan(int nSamples) @@ -140,10 +158,17 @@ void Shaper::loadShapeInternal(const klee::Shape& shape, int Shaper::getRank() const { #if defined(AXOM_USE_MPI) && defined(MFEM_USE_MPI) - if(auto* pmesh = static_cast(m_dc->GetMesh())) + if (m_dc != nullptr) { - return pmesh->GetMyRank(); + if(auto* pmesh = static_cast(m_dc->GetMesh())) + { + return pmesh->GetMyRank(); + } } +#elif defined(AXOM_USE_MPI) + int rank = -1; + MPI_Comm_rank(m_comm, &rank); + return rank; #endif return 0; } diff --git a/src/axom/quest/Shaper.hpp b/src/axom/quest/Shaper.hpp index da8eed8341..3736b18752 100644 --- a/src/axom/quest/Shaper.hpp +++ b/src/axom/quest/Shaper.hpp @@ -37,8 +37,19 @@ namespace quest class Shaper { public: + /*! + @brief Construct Shaper to operate on an MFEM mesh. + */ Shaper(const klee::ShapeSet& shapeSet, sidre::MFEMSidreDataCollection* dc); + /*! + @brief Construct Shaper to operate on a blueprint-formatted mesh + stored in a Conduit Node. + */ + Shaper(const klee::ShapeSet& shapeSet, + conduit::Node* bpMesh, + const std::string& topo=""); + virtual ~Shaper() = default; public: @@ -155,7 +166,16 @@ class Shaper sidre::DataStore m_dataStore; const klee::ShapeSet& m_shapeSet; - sidre::MFEMSidreDataCollection* m_dc; + + // For mesh represented as MFEMSidreDataCollection + sidre::MFEMSidreDataCollection* m_dc{nullptr}; + + // For mesh represented in Conduit or sidre + conduit::Node* m_bpNode{nullptr}; + const std::string m_bpTopo; + sidre::DataStore m_ds; + axom::sidre::Group* m_bpGrp{nullptr}; + axom::IndexType m_cellCount; std::shared_ptr m_surfaceMesh; diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index ddfc5f04e4..aeec42f0a2 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -972,7 +972,11 @@ double sumMaterialVolumes(sidre::MFEMSidreDataCollection* dc, const std::string materialFieldName = axom::fmt::format("vol_frac_{}", material); mfem::GridFunction* volFracGf = dc->GetField(materialFieldName); - axom::quest::GridFunctionView volFracView(volFracGf); + // axom::quest::GridFunctionView volFracView(volFracGf); + axom::ArrayView volFracGfArrayView(volFracGf->GetData(), volFracGf->Size()); + axom::quest::TempArrayAccess volFracView(volFracGfArrayView, + ::getUmpireDeviceId(), + true); using ReducePolicy = typename axom::execution_space::reduce_policy; RAJA::ReduceSum localVol(0); From cc80862d2828ea3133c23e0b82872f9db4333ee2 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Tue, 24 Sep 2024 18:27:29 -0700 Subject: [PATCH 002/100] Remove obsolete GridFunctionView utility class. --- src/axom/quest/IntersectionShaper.hpp | 190 -------------------------- 1 file changed, 190 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index e48992771f..2ad09eb2c1 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -70,189 +70,6 @@ namespace axom { namespace quest { -/*! - * \class GridFunctionView - * - * \brief Provides a view over an MFEM grid function. MFEM grid functions are - * assumed to live in host memory. This class performs data movement - * needed to access the grid function data within a GPU device lambda. This - * view is limited in scope, though could be expanded in the future. - * - * \tparam ExecSpace The execution space where the grid function data will - * be accessed. - */ -template -class GridFunctionView -{ -public: - /*! - * \brief Host constructor that accepts the grid function. - * - * \param gf The grid function that will be accessed/modified by the view. - * \param _needResult Whether the data needs to be brought back to the host - * from the device. - */ - AXOM_HOST GridFunctionView(mfem::GridFunction* gf, bool _needResult = true) - { - initialize(gf->GetData(), gf->Size(), _needResult); - } - - /*! - * \brief Copy constructor, which is called to make a copy of the host - * object so it is accessible inside a RAJA kernel. Any data movement - * happened in the host constructor. This version sets hostData to - * nullptr so we know not to clean up in the destructor. - */ - AXOM_HOST_DEVICE GridFunctionView(const GridFunctionView& obj) - : m_hostData(nullptr) - , m_deviceData(obj.m_deviceData) - , m_numElements(obj.m_numElements) - , m_needResult(obj.m_needResult) - { } - - /*! - * \brief Destructor. On the host, this method may move data from the - device and deallocate device storage. - */ - AXOM_HOST_DEVICE ~GridFunctionView() { finalize(); } - - /*! - * \brief Indexing operator for accessing the data. - * - * \param i The index at which to access the data. - * - * \return A reference to the data at index i. - */ - AXOM_HOST_DEVICE double& operator[](int i) { return m_deviceData[i]; } - // non-const return on purpose. - AXOM_HOST_DEVICE double& operator[](int i) const { return m_deviceData[i]; } - -private: - /*! - * \brief Initializes members using data from the grid function. This method - * is called on the host. - * - * \param hostPtr The grid function data pointer on the host. - * \param nElem The grid function size. - * \param _needResult Whether any data are copied from device. - */ - AXOM_HOST void initialize(double* hostPtr, int nElem, bool _needResult) - { - m_hostData = m_deviceData = hostPtr; - m_numElements = nElem; - m_needResult = _needResult; - } - - /*! - * \brief Helps during destruction. - */ - AXOM_HOST_DEVICE void finalize() { m_deviceData = nullptr; } - -#if defined(AXOM_USE_CUDA) || defined(AXOM_USE_HIP) - /*! - * \brief Initializes members using data from the grid function. This method - * is called on the host and it copies data to the device. - * - * \param hostPtr The grid function data pointer on the host. - * \param nElem The grid function size. - * \param _needResult Whether any data are copied from device. - */ - AXOM_HOST void initializeDevice(double* hostPtr, int nElem, bool _needResult) - { - m_hostData = hostPtr; - m_numElements = nElem; - m_needResult = _needResult; - int execSpaceAllocatorID = axom::execution_space::allocatorID(); - - auto dataSize = sizeof(double) * m_numElements; - m_deviceData = axom::allocate(dataSize, execSpaceAllocatorID); - axom::copy(m_deviceData, m_hostData, dataSize); - } - - /*! - * \brief Helps during destruction. On the host, it copies device data back - * into the grid function on the host. - */ - AXOM_HOST_DEVICE void finalizeDevice() - { - #ifndef AXOM_DEVICE_CODE - // Only the host will do this work. - if(m_hostData != nullptr) - { - if(m_needResult) - { - auto dataSize = sizeof(double) * m_numElements; - axom::copy(m_hostData, m_deviceData, dataSize); - } - axom::deallocate(m_deviceData); - m_deviceData = nullptr; - } - #endif - } -#endif - -private: - double* m_hostData {nullptr}; - double* m_deviceData {nullptr}; - int m_numElements {0}; - bool m_needResult {false}; -}; - -#if defined(AXOM_USE_CUDA) -/*! - * \brief CUDA specialization that calls initializeDevice to copy data - * from the host to the device. - * - * \param hostPtr The grid function data pointer on the host. - * \param nElem The grid function size. - * \param _needResult Whether any data are copied from device. - */ -template <> -AXOM_HOST inline void GridFunctionView::initialize(double* hostPtr, - int nElem, - bool _needResult) -{ - initializeDevice(hostPtr, nElem, _needResult); -} - -/*! - * \brief CUDA specialization that may copy data back from the device - * and deallocate any associated device data. - */ -template <> -AXOM_HOST_DEVICE inline void GridFunctionView::finalize() -{ - finalizeDevice(); -} -#endif -#if defined(AXOM_USE_HIP) -/*! - * \brief HIP specialization that calls initializeDevice to copy data - * from the host to the device. - * - * \param hostPtr The grid function data pointer on the host. - * \param nElem The grid function size. - * \param _needResult Whether any data are copied from device. - */ -template <> -AXOM_HOST inline void GridFunctionView::initialize(double* hostPtr, - int nElem, - bool _needResult) -{ - initializeDevice(hostPtr, nElem, _needResult); -} - -/*! - * \brief HIP specialization that may copy data back from the device - * and deallocate any associated device data. - */ -template <> -AXOM_HOST_DEVICE inline void GridFunctionView::finalize() -{ - finalizeDevice(); -} -#endif - /*! * \class TempArrayAccess * @@ -767,8 +584,6 @@ class IntersectionShaper : public Shaper // Create and register a scalar field for this shape's volume fractions // The Degrees of Freedom will be in correspondence with the elements std::string volFracName = axom::fmt::format("shape_vol_frac_{}", shape.getName()); - // auto* volFrac = this->newVolFracGridFunction(); - // this->getDC()->RegisterField(volFracName, volFrac); auto volFrac = getScalarCellData(volFracName); // Initialize hexahedral elements @@ -1401,7 +1216,6 @@ class IntersectionShaper : public Shaper for(const auto& name : shape.getMaterialsReplaced()) { auto mat = getMaterial(name); - // GridFunctionView matVFView(mat.first, false); TempArrayAccess matVFView(mat.first, execSpaceAllocatorID, false); axom::for_all( dataSize, @@ -1421,7 +1235,6 @@ class IntersectionShaper : public Shaper for(auto& gf : excludeVFs) { - // GridFunctionView matVFView(gf, false); TempArrayAccess matVFView(gf, execSpaceAllocatorID, false); axom::for_all( dataSize, @@ -1436,8 +1249,6 @@ class IntersectionShaper : public Shaper { AXOM_ANNOTATE_SCOPE("compute_vf"); - // GridFunctionView matVFView(matVF.first); - // GridFunctionView shapeVFView(shapeVolFrac); TempArrayAccess matVFView(matVF.first, execSpaceAllocatorID, true); TempArrayAccess shapeVFView(shapeVolFrac, execSpaceAllocatorID, true); @@ -1469,7 +1280,6 @@ class IntersectionShaper : public Shaper AXOM_ANNOTATE_SCOPE("update_vf"); for(auto& gf : updateVFs) { - // GridFunctionView matVFView(gf); TempArrayAccess matVFView(gf, execSpaceAllocatorID, true); axom::for_all( dataSize, From 0017bfcaca74834fa82143038bed8a43d32a5721 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Wed, 25 Sep 2024 04:42:12 -0700 Subject: [PATCH 003/100] Add missing IntersectionShaper for using blueprint mesh in conduit::Node. --- src/axom/quest/IntersectionShaper.hpp | 15 +++++++++++++++ src/axom/quest/Shaper.cpp | 2 ++ 2 files changed, 17 insertions(+) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 2ad09eb2c1..b1a0a29ab4 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -204,6 +204,9 @@ class IntersectionShaper : public Shaper static constexpr double DEFAULT_REVOLVED_VOLUME {0.}; public: + /*! + @brief Construct Shaper to operate on an MFEM mesh. + */ IntersectionShaper(const klee::ShapeSet& shapeSet, sidre::MFEMSidreDataCollection* dc) : Shaper(shapeSet, dc) @@ -211,6 +214,18 @@ class IntersectionShaper : public Shaper m_free_mat_name = "free"; } + /*! + @brief Construct Shaper to operate on a blueprint-formatted mesh + stored in a Conduit Node. + */ + IntersectionShaper(const klee::ShapeSet& shapeSet, + conduit::Node* bpMesh, + const std::string& topo="") + : Shaper(shapeSet, bpMesh, topo) + , m_free_mat_name("free") + { + } + //@{ //! @name Functions to get and set shaping parameters related to intersection; supplements parameters in base class diff --git a/src/axom/quest/Shaper.cpp b/src/axom/quest/Shaper.cpp index 46bd19401f..5c78b7adf1 100644 --- a/src/axom/quest/Shaper.cpp +++ b/src/axom/quest/Shaper.cpp @@ -34,6 +34,7 @@ Shaper::Shaper(const klee::ShapeSet& shapeSet, sidre::MFEMSidreDataCollection* dc) : m_shapeSet(shapeSet) , m_dc(dc) + , m_bpNode(nullptr) { #if defined(AXOM_USE_MPI) && defined(MFEM_USE_MPI) m_comm = m_dc->GetComm(); @@ -45,6 +46,7 @@ Shaper::Shaper(const klee::ShapeSet& shapeSet, conduit::Node* bpNode, const std::string& topo) : m_shapeSet(shapeSet) + , m_dc(nullptr) , m_bpNode(bpNode) , m_bpTopo(topo) , m_comm(MPI_COMM_WORLD) From 280fc5b5a625d854c7f7ea68aa0d741b8d8d45e1 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Wed, 25 Sep 2024 06:00:49 -0700 Subject: [PATCH 004/100] Support shaping even without MFEM. This version compiles without MFEM, if conduit is supported. But it doesn't run because the conduit side is not fully written yet. --- src/axom/quest/CMakeLists.txt | 14 +- src/axom/quest/IntersectionShaper.hpp | 416 ++++++++++++------ src/axom/quest/Shaper.cpp | 32 +- src/axom/quest/Shaper.hpp | 35 +- .../quest/detail/shaping/shaping_helpers.cpp | 4 - .../quest/detail/shaping/shaping_helpers.hpp | 4 +- src/axom/quest/examples/shaping_driver.cpp | 6 +- 7 files changed, 339 insertions(+), 172 deletions(-) diff --git a/src/axom/quest/CMakeLists.txt b/src/axom/quest/CMakeLists.txt index c9dd6f67da..b7b9788bb8 100644 --- a/src/axom/quest/CMakeLists.txt +++ b/src/axom/quest/CMakeLists.txt @@ -130,18 +130,20 @@ blt_list_append( blt_list_append( TO quest_depends_on ELEMENTS conduit::conduit IF CONDUIT_FOUND ) -if(MFEM_FOUND AND AXOM_ENABLE_KLEE AND AXOM_ENABLE_SIDRE AND AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION) +if(AXOM_ENABLE_KLEE AND AXOM_ENABLE_SIDRE AND ((MFEM_FOUND AND AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION) OR CONDUIT_FOUND)) list(APPEND quest_headers Shaper.hpp - SamplingShaper.hpp IntersectionShaper.hpp - DiscreteShape.hpp - detail/shaping/shaping_helpers.hpp) + DiscreteShape.hpp) list(APPEND quest_sources Shaper.cpp - DiscreteShape.cpp - detail/shaping/shaping_helpers.cpp) + DiscreteShape.cpp) list(APPEND quest_depends_on klee) endif() +if(AXOM_ENABLE_KLEE AND AXOM_ENABLE_SIDRE AND (MFEM_FOUND AND AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION)) + list(APPEND quest_headers SamplingShaper.hpp detail/shaping/shaping_helpers.hpp) + list(APPEND quest_sources detail/shaping/shaping_helpers.cpp) +endif() + if(C2C_FOUND) list(APPEND quest_headers readers/C2CReader.hpp) list(APPEND quest_sources readers/C2CReader.cpp) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index b1a0a29ab4..45ee5966f6 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -20,19 +20,10 @@ #include "axom/mint.hpp" #include "axom/spin.hpp" #include "axom/klee.hpp" - -#ifndef AXOM_USE_MFEM - #error Shaping functionality requires Axom to be configured with MFEM and the AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION option -#endif - #include "axom/quest/Shaper.hpp" #include "axom/spin/BVH.hpp" #include "axom/quest/interface/internal/mpicomm_wrapper.hpp" #include "axom/quest/interface/internal/QuestHelpers.hpp" -#include "axom/quest/detail/shaping/shaping_helpers.hpp" - -#include "mfem.hpp" - #include "axom/fmt.hpp" // RAJA @@ -204,6 +195,7 @@ class IntersectionShaper : public Shaper static constexpr double DEFAULT_REVOLVED_VOLUME {0.}; public: +#if defined(AXOM_SHAPING_ON_MFEM_MESH) /*! @brief Construct Shaper to operate on an MFEM mesh. */ @@ -213,7 +205,9 @@ class IntersectionShaper : public Shaper { m_free_mat_name = "free"; } +#endif +#if defined(AXOM_SHAPING_ON_BLUEPRINT_MESH) /*! @brief Construct Shaper to operate on a blueprint-formatted mesh stored in a Conduit Node. @@ -225,6 +219,7 @@ class IntersectionShaper : public Shaper , m_free_mat_name("free") { } +#endif //@{ //! @name Functions to get and set shaping parameters related to intersection; supplements parameters in base class @@ -245,7 +240,7 @@ class IntersectionShaper : public Shaper { SLIC_ERROR("The free material name cannot contain underscores."); } - if(m_num_elements > 0) + if(m_cellCount > 0) { SLIC_ERROR( "The free material name cannot be set once shaping has occurred."); @@ -542,8 +537,6 @@ class IntersectionShaper : public Shaper axom::execution_space::allocatorID(); const int device_allocator = axom::execution_space::allocatorID(); - constexpr int NUM_VERTS_PER_HEX = 8; - constexpr int NUM_COMPS_PER_VERT = 3; constexpr int NUM_TETS_PER_HEX = 24; constexpr double ZERO_THRESHOLD = 1.e-10; @@ -568,126 +561,37 @@ class IntersectionShaper : public Shaper }); // Insert shapes' Bounding Boxes into BVH. - //bvh.setAllocatorID(poolID); spin::BVH<3, ExecSpace, double> bvh; bvh.initialize(aabbs_device_view, shape_count); SLIC_INFO(axom::fmt::format("{:-^80}", " Querying the BVH tree ")); - mfem::Mesh* mesh = getDC()->GetMesh(); - - // Intersection algorithm only works on linear elements - SLIC_ASSERT(mesh != nullptr); - int const NE = mesh->GetNE(); - m_num_elements = NE; - - if(this->isVerbose()) - { - SLIC_INFO(axom::fmt::format( - "{:-^80}", - axom::fmt::format( - " Initializing {} hexahedral elements from given mesh ", - m_num_elements))); - } - - if(NE > 0) - { - SLIC_ASSERT(mesh->GetNodes() == nullptr || - mesh->GetNodes()->FESpace()->GetOrder(0)); - } - // Create and register a scalar field for this shape's volume fractions // The Degrees of Freedom will be in correspondence with the elements std::string volFracName = axom::fmt::format("shape_vol_frac_{}", shape.getName()); auto volFrac = getScalarCellData(volFracName); - // Initialize hexahedral elements - m_hexes = axom::Array(NE, NE, device_allocator); + populateHexesFromMesh(); axom::ArrayView hexes_device_view = m_hexes.view(); - m_hex_bbs = axom::Array(NE, NE, device_allocator); + m_hex_bbs = axom::Array(m_cellCount, m_cellCount, device_allocator); axom::ArrayView hex_bbs_device_view = m_hex_bbs.view(); - // Initialize vertices from mfem mesh and - // set each shape volume fraction to 1 - // Allocation size is: - // # of elements * # of vertices per hex * # of components per vertex - axom::Array vertCoords_host( - NE * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT, - NE * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT, - host_allocator); - - for(int i = 0; i < NE; i++) - { - // Get the indices of this element's vertices - mfem::Array verts; - mesh->GetElementVertices(i, verts); - SLIC_ASSERT(verts.Size() == NUM_VERTS_PER_HEX); - - // Set each shape volume fraction to 1 - double vf = 1.0; - // (*volFrac)(i) = vf; - volFrac[i] = vf; - - // Get the coordinates for the vertices - for(int j = 0; j < NUM_VERTS_PER_HEX; ++j) - { - for(int k = 0; k < NUM_COMPS_PER_VERT; k++) - { - vertCoords_host[(i * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT) + - (j * NUM_COMPS_PER_VERT) + k] = - (mesh->GetVertex(verts[j]))[k]; - } - } - } - - axom::Array vertCoords_device = - axom::Array(vertCoords_host, device_allocator); - auto vertCoords_device_view = vertCoords_device.view(); - - // Initialize each hexahedral element and its bounding box + // Get bounding boxes for hexahedral elements axom::for_all( - NE, + m_cellCount, AXOM_LAMBDA(axom::IndexType i) { - // Set each hexahedral element vertices - hexes_device_view[i] = HexahedronType(); - for(int j = 0; j < NUM_VERTS_PER_HEX; ++j) - { - int vertIndex = (i * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT) + - j * NUM_COMPS_PER_VERT; - hexes_device_view[i][j] = - Point3D({vertCoords_device_view[vertIndex], - vertCoords_device_view[vertIndex + 1], - vertCoords_device_view[vertIndex + 2]}); - - // Set hexahedra components to zero if within threshold - if(axom::utilities::isNearlyEqual(hexes_device_view[i][j][0], - 0.0, - ZERO_THRESHOLD)) - { - hexes_device_view[i][j][0] = 0.0; - } - - if(axom::utilities::isNearlyEqual(hexes_device_view[i][j][1], - 0.0, - ZERO_THRESHOLD)) - { - hexes_device_view[i][j][1] = 0.0; - } - - if(axom::utilities::isNearlyEqual(hexes_device_view[i][j][2], - 0.0, - ZERO_THRESHOLD)) - { - hexes_device_view[i][j][2] = 0.0; - } - } - - // Get bounding box for hexahedral element hex_bbs_device_view[i] = primal::compute_bounding_box(hexes_device_view[i]); }); // end of loop to initialize hexahedral elements and bounding boxes + // Set each shape volume fraction to 1 + for(int i = 0; i < m_cellCount; i++) + { + double vf = 1.0; + volFrac[i] = vf; + } + // Set shape components to zero if within threshold axom::for_all( shape_count, @@ -722,10 +626,10 @@ class IntersectionShaper : public Shaper "{:-^80}", " Finding shape candidates for each hexahedral element ")); - axom::Array offsets(NE, NE, device_allocator); - axom::Array counts(NE, NE, device_allocator); + axom::Array offsets(m_cellCount, m_cellCount, device_allocator); + axom::Array counts(m_cellCount, m_cellCount, device_allocator); axom::Array candidates; - bvh.findBoundingBoxes(offsets, counts, candidates, NE, hex_bbs_device_view); + bvh.findBoundingBoxes(offsets, counts, candidates, m_cellCount, hex_bbs_device_view); // Get the total number of candidates using REDUCE_POL = typename axom::execution_space::reduce_policy; @@ -734,7 +638,7 @@ class IntersectionShaper : public Shaper const auto counts_device_view = counts.view(); RAJA::ReduceSum totalCandidates(0); axom::for_all( - NE, + m_cellCount, AXOM_LAMBDA(axom::IndexType i) { totalCandidates += counts_device_view[i]; }); @@ -753,8 +657,8 @@ class IntersectionShaper : public Shaper auto shape_candidates_device_view = shape_candidates_device.view(); // Tetrahedrons from hexes (24 for each hex) - axom::Array tets_from_hexes_device(NE * NUM_TETS_PER_HEX, - NE * NUM_TETS_PER_HEX, + axom::Array tets_from_hexes_device(m_cellCount * NUM_TETS_PER_HEX, + m_cellCount * NUM_TETS_PER_HEX, device_allocator); axom::ArrayView tets_from_hexes_device_view = tets_from_hexes_device.view(); @@ -782,7 +686,7 @@ class IntersectionShaper : public Shaper { AXOM_ANNOTATE_SCOPE("init_tets"); axom::for_all( - NE, + m_cellCount, AXOM_LAMBDA(axom::IndexType i) { TetHexArray cur_tets; hexes_device_view[i].triangulate(cur_tets); @@ -803,7 +707,7 @@ class IntersectionShaper : public Shaper { AXOM_ANNOTATE_SCOPE("init_candidates"); axom::for_all( - NE, + m_cellCount, AXOM_LAMBDA(axom::IndexType i) { for(int j = 0; j < counts_device_view[i]; j++) { @@ -824,10 +728,10 @@ class IntersectionShaper : public Shaper // Overlap volume is the volume of clip(oct,tet) for c2c // or clip(tet,tet) for Pro/E meshes - m_overlap_volumes = axom::Array(NE, NE, device_allocator); + m_overlap_volumes = axom::Array(m_cellCount, m_cellCount, device_allocator); // Hex volume is the volume of the hexahedron element - m_hex_volumes = axom::Array(NE, NE, device_allocator); + m_hex_volumes = axom::Array(m_cellCount, m_cellCount, device_allocator); axom::ArrayView overlap_volumes_device_view = m_overlap_volumes.view(); @@ -835,7 +739,7 @@ class IntersectionShaper : public Shaper // Set initial values to 0 axom::for_all( - NE, + m_cellCount, AXOM_LAMBDA(axom::IndexType i) { overlap_volumes_device_view[i] = 0; hex_volumes_device_view[i] = 0; @@ -847,7 +751,7 @@ class IntersectionShaper : public Shaper { AXOM_ANNOTATE_SCOPE("hex_volume"); axom::for_all( - NE, + m_cellCount, AXOM_LAMBDA(axom::IndexType i) { hex_volumes_device_view[i] = hexes_device_view[i].volume(); }); @@ -894,7 +798,7 @@ class IntersectionShaper : public Shaper RAJA::ReduceSum totalHex(0); axom::for_all( - NE, + m_cellCount, AXOM_LAMBDA(axom::IndexType i) { totalOverlap += overlap_volumes_device_view[i]; totalHex += hex_volumes_device_view[i]; @@ -995,6 +899,9 @@ class IntersectionShaper : public Shaper */ void populateMaterials() { +#if 1 + std::vector materialNames = getMaterialNames(); +#else std::vector materialNames; for(auto it : this->getDC()->GetFieldMap()) { @@ -1004,6 +911,7 @@ class IntersectionShaper : public Shaper materialNames.emplace_back(materialName); } } +#endif // Add any of these existing fields to this class' bookkeeping. for(const auto& materialName : materialNames) { @@ -1164,6 +1072,39 @@ class IntersectionShaper : public Shaper // Include all materials except those in "does_not_replace". // We'll also sort them by material number since the field map // sorts them by name rather than order added. +#if 1 + std::vector materialNames = getMaterialNames(); + for(auto name : materialNames) + { + // Check whether the field name is not the + // "free" field, which we handle specially) + if(name != m_free_mat_name) + { + // See if the field is in the exclusion list. For the normal + // case, the list is empty so we'd add the material. + auto it2 = std::find(shape.getMaterialsNotReplaced().cbegin(), + shape.getMaterialsNotReplaced().cend(), + name); + // The field is not in the exclusion list so add it to vfs. + if(it2 == shape.getMaterialsNotReplaced().cend()) + { + // Do not add the current shape material since it should + // not end up in updateVFs. + if(name != shape.getMaterial()) + { + gf_order_by_matnumber.emplace_back(getMaterial(name)); + } + } + else + { + // The material was in the exclusion list. This means that + // cannot write to materials that have volume fraction in + // that zone. + excludeVFs.emplace_back(getMaterial(name).first); + } + } + } +#else for(auto it : this->getDC()->GetFieldMap()) { // Check whether the field name looks like a VF field (and is not the @@ -1195,6 +1136,7 @@ class IntersectionShaper : public Shaper } } } +#endif } // Sort eligible update materials by material number. std::sort(gf_order_by_matnumber.begin(), @@ -1939,13 +1881,17 @@ class IntersectionShaper : public Shaper bool hasData(const std::string& fieldName) { bool has = false; - if (m_dc) { +#if defined(AXOM_SHAPING_ON_MFEM_MESH) + if (m_dc != nullptr) { has = m_dc->HasField(fieldName); } - else { +#endif +#if defined(AXOM_SHAPING_ON_BLUEPRINT_MESH) + if (m_bpGrp != nullptr) { std::string fieldPath = axom::fmt::format("fields/{}", fieldName); has = m_bpGrp->hasGroup(fieldPath); } +#endif return has; } @@ -1953,8 +1899,8 @@ class IntersectionShaper : public Shaper { axom::ArrayView rval; +#if defined(AXOM_SHAPING_ON_MFEM_MESH) if (m_dc) { - mfem::GridFunction* gridFunc = nullptr; if (m_dc->HasField(fieldName)) { @@ -1967,7 +1913,9 @@ class IntersectionShaper : public Shaper rval = axom::ArrayView(gridFunc->GetData(), gridFunc->Size()); } - else { +#endif +#if defined(AXOM_SHAPING_ON_BLUEPRINT_MESH) + if (m_bpGrp != nullptr) { std::string fieldPath = axom::fmt::format("fields/{}", fieldName); auto dtype = conduit::DataType::float64(m_cellCount); axom::sidre::View* valuesView = nullptr; @@ -1987,9 +1935,217 @@ class IntersectionShaper : public Shaper rval = axom::ArrayView(static_cast(valuesView->getVoidPtr()), m_cellCount); } +#endif return rval; } + std::vector getMaterialNames() + { + std::vector materialNames; +#if defined(AXOM_SHAPING_ON_MFEM_MESH) + if (m_dc) + { + for(auto it : this->getDC()->GetFieldMap()) + { + std::string materialName = fieldNameToMaterialName(it.first); + if(!materialName.empty()) + { + materialNames.emplace_back(materialName); + } + } + } +#elif defined(AXOM_SHAPING_ON_BLUEPRINT_MESH) + if (m_bpGrp) + { + auto fieldsGrp = m_bpGrp->getGroup("fields"); + for (auto& group : fieldsGrp->groups()) + { + std::string materialName = fieldNameToMaterialName(group.getName()); + if(!materialName.empty()) + { + materialNames.emplace_back(materialName); + } + } + } +#endif + return materialNames; + } + + template + void populateHexesFromMesh() + { + constexpr int NUM_VERTS_PER_HEX = 8; + constexpr int NUM_COMPS_PER_VERT = 3; + const int hostAllocator = + axom::execution_space::allocatorID(); + const int deviceAllocator = + axom::execution_space::allocatorID(); + + axom::Array vertCoords( + m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT, + m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT, + hostAllocator); + +#if defined(AXOM_SHAPING_ON_MFEM_MESH) + if (m_dc != nullptr) + { + populateVertCoordsFromMFEMMesh(vertCoords); + } +#endif +#if defined(AXOM_SHAPING_ON_BLUEPRINT_MESH) + if (m_bpGrp != nullptr) + { + populateVertCoordsFromBlueprintMesh(vertCoords); + } +#endif + + if (deviceAllocator != hostAllocator) + { + vertCoords = axom::Array(vertCoords, deviceAllocator); + } + + auto vertCoords_device_view = vertCoords.view(); + + m_hexes = axom::Array(m_cellCount, m_cellCount, deviceAllocator); + axom::ArrayView hexes_device_view = m_hexes.view(); + constexpr double ZERO_THRESHOLD = 1.e-10; + axom::for_all( + m_cellCount, + AXOM_LAMBDA(axom::IndexType i) { + // Set each hexahedral element vertices + hexes_device_view[i] = HexahedronType(); + for(int j = 0; j < NUM_VERTS_PER_HEX; ++j) + { + int vertIndex = (i * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT) + + j * NUM_COMPS_PER_VERT; + hexes_device_view[i][j] = + Point3D({vertCoords_device_view[vertIndex], + vertCoords_device_view[vertIndex + 1], + vertCoords_device_view[vertIndex + 2]}); + + // Set hexahedra components to zero if within threshold + if(axom::utilities::isNearlyEqual(hexes_device_view[i][j][0], + 0.0, + ZERO_THRESHOLD)) + { + hexes_device_view[i][j][0] = 0.0; + } + + if(axom::utilities::isNearlyEqual(hexes_device_view[i][j][1], + 0.0, + ZERO_THRESHOLD)) + { + hexes_device_view[i][j][1] = 0.0; + } + + if(axom::utilities::isNearlyEqual(hexes_device_view[i][j][2], + 0.0, + ZERO_THRESHOLD)) + { + hexes_device_view[i][j][2] = 0.0; + } + } + }); // end of loop to initialize hexahedral elements and bounding boxes + } + +#if defined(AXOM_SHAPING_ON_BLUEPRINT_MESH) + void populateVertCoordsFromBlueprintMesh(axom::Array& vertCoords_host) + { + // Initialize vertices from blueprint mesh and + // set each shape volume fraction to 1 + // Allocation size is: + // # of elements * # of vertices per hex * # of components per vertex + constexpr int NUM_VERTS_PER_HEX = 8; + constexpr int NUM_COMPS_PER_VERT = 3; + + const auto* topoGrp = m_bpGrp->getGroup(axom::fmt::format("topologies/{}", m_bpTopo)); + std::string coordsetName = topoGrp->getView("coordset")->getString(); + + // Assume unstructured and hexahedral + SLIC_ASSERT(std::string(topoGrp->getView("type")->getString()) == "unstructured"); + SLIC_ASSERT(std::string(topoGrp->getView("elements/shape")->getString()) == "hex"); + + const auto connGrp = topoGrp->getView("elements/connectivity"); + axom::ArrayView conn(connGrp->getNode().as_double_ptr(), + m_cellCount, NUM_VERTS_PER_HEX); + + const auto* coordGrp = m_bpGrp->getGroup(axom::fmt::format("coordsets/{}", coordsetName)); + const conduit::Node& coordValues = coordGrp->getView("values")->getNode(); + + // Assume explicit coordinates. + SLIC_ASSERT(std::string(coordGrp->getView("type")->getString()) == "explicit"); + + axom::IndexType vertexCount = coordValues["x"].dtype().number_of_elements(); + bool isInterleaved = conduit::blueprint::mcarray::is_interleaved(coordValues); + int stride = isInterleaved ? NUM_COMPS_PER_VERT : 1; + axom::StackArray, 3> coordArrays { + axom::ArrayView(coordValues["x"].as_double_ptr(), {vertexCount}, stride), + axom::ArrayView(coordValues["y"].as_double_ptr(), {vertexCount}, stride), + axom::ArrayView(coordValues["z"].as_double_ptr(), {vertexCount}, stride) }; + + for(int i = 0; i < m_cellCount; i++) + { + // Get the indices of this element's vertices + auto hexVerts = conn[i]; + SLIC_ASSERT(hexVerts.size() == NUM_VERTS_PER_HEX); + + // Get the coordinates for the vertices + for(int j = 0; j < NUM_VERTS_PER_HEX; ++j) + { + auto vertId = hexVerts[j]; + for(int k = 0; k < NUM_COMPS_PER_VERT; k++) + { + vertCoords_host[(i * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT) + + (j * NUM_COMPS_PER_VERT) + k] = + coordArrays[k][vertId]; + } + } + } + + } +#endif + +#if defined(AXOM_SHAPING_ON_MFEM_MESH) + void populateVertCoordsFromMFEMMesh(axom::Array& vertCoords_host) + { + mfem::Mesh* mesh = getDC()->GetMesh(); + // Intersection algorithm only works on linear elements + SLIC_ASSERT(mesh != nullptr); + + if(m_cellCount > 0) + { + SLIC_ASSERT(mesh->GetNodes() == nullptr || + mesh->GetNodes()->FESpace()->GetOrder(0)); + } + + // Initialize vertices from mfem mesh and + // set each shape volume fraction to 1 + // Allocation size is: + // # of elements * # of vertices per hex * # of components per vertex + constexpr int NUM_VERTS_PER_HEX = 8; + constexpr int NUM_COMPS_PER_VERT = 3; + + for(int i = 0; i < m_cellCount; i++) + { + // Get the indices of this element's vertices + mfem::Array verts; + mesh->GetElementVertices(i, verts); + SLIC_ASSERT(verts.Size() == NUM_VERTS_PER_HEX); + + // Get the coordinates for the vertices + for(int j = 0; j < NUM_VERTS_PER_HEX; ++j) + { + for(int k = 0; k < NUM_COMPS_PER_VERT; k++) + { + vertCoords_host[(i * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT) + + (j * NUM_COMPS_PER_VERT) + k] = + (mesh->GetVertex(verts[j]))[k]; + } + } + } + + } + /// Create and return a new volume fraction grid function for the current mesh // TODO: change to generic name. Nothing in here is about volume fractions. BTNG. mfem::GridFunction* newVolFracGridFunction() @@ -2007,6 +2163,7 @@ class IntersectionShaper : public Shaper return volFrac; } +#endif bool surfaceMeshIsTet() const { @@ -2020,7 +2177,6 @@ class IntersectionShaper : public Shaper RuntimePolicy m_execPolicy {RuntimePolicy::seq}; int m_level {DEFAULT_CIRCLE_REFINEMENT_LEVEL}; double m_revolvedVolume {DEFAULT_REVOLVED_VOLUME}; - int m_num_elements {0}; std::string m_free_mat_name; axom::Array m_hex_volumes; diff --git a/src/axom/quest/Shaper.cpp b/src/axom/quest/Shaper.cpp index 5c78b7adf1..6c0c6dff23 100644 --- a/src/axom/quest/Shaper.cpp +++ b/src/axom/quest/Shaper.cpp @@ -9,16 +9,11 @@ #include "axom/core.hpp" #include "axom/primal/operators/split.hpp" #include "axom/quest/interface/internal/QuestHelpers.hpp" +#include "axom/quest/Shaper.hpp" #include "axom/quest/DiscreteShape.hpp" #include "axom/fmt.hpp" -#ifndef AXOM_USE_MFEM - #error Shaping functionality requires Axom to be configured with MFEM and the AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION option -#endif - -#include "mfem.hpp" - namespace axom { namespace quest @@ -30,23 +25,24 @@ constexpr double Shaper::MINIMUM_PERCENT_ERROR; constexpr double Shaper::MAXIMUM_PERCENT_ERROR; constexpr double Shaper::DEFAULT_VERTEX_WELD_THRESHOLD; +#if defined(AXOM_SHAPING_ON_MFEM_MESH) Shaper::Shaper(const klee::ShapeSet& shapeSet, sidre::MFEMSidreDataCollection* dc) : m_shapeSet(shapeSet) , m_dc(dc) - , m_bpNode(nullptr) { #if defined(AXOM_USE_MPI) && defined(MFEM_USE_MPI) m_comm = m_dc->GetComm(); #endif m_cellCount = m_dc->GetMesh()->GetNE(); } +#endif +#if defined(AXOM_SHAPING_ON_BLUEPRINT_MESH) Shaper::Shaper(const klee::ShapeSet& shapeSet, conduit::Node* bpNode, const std::string& topo) : m_shapeSet(shapeSet) - , m_dc(nullptr) , m_bpNode(bpNode) , m_bpTopo(topo) , m_comm(MPI_COMM_WORLD) @@ -58,6 +54,7 @@ Shaper::Shaper(const klee::ShapeSet& shapeSet, auto* coordsView = m_bpGrp->getView(axom::fmt::format("coordsets/{}", coordsName)); m_cellCount = coordsView->getNumElements(); } +#endif void Shaper::setSamplesPerKnotSpan(int nSamples) { @@ -137,11 +134,9 @@ void Shaper::loadShapeInternal(const klee::Shape& shape, "{:-^80}", axom::fmt::format(" Loading shape '{}' ", shape.getName()))); - const axom::klee::Geometry& geometry = shape.getGeometry(); - const std::string& geometryFormat = geometry.getFormat(); SLIC_ASSERT_MSG( - this->isValidFormat(geometryFormat), - axom::fmt::format("Shape has unsupported format: '{}", geometryFormat)); + this->isValidFormat(shape.getGeometry().getFormat()), + axom::fmt::format("Shape has unsupported format: '{}", shape.getGeometry().getFormat())); // Code for discretizing shapes has been factored into DiscreteShape class. DiscreteShape discreteShape(shape, m_dataStore.getRoot(), m_shapeSet.getPath()); @@ -159,15 +154,7 @@ void Shaper::loadShapeInternal(const klee::Shape& shape, int Shaper::getRank() const { -#if defined(AXOM_USE_MPI) && defined(MFEM_USE_MPI) - if (m_dc != nullptr) - { - if(auto* pmesh = static_cast(m_dc->GetMesh())) - { - return pmesh->GetMyRank(); - } - } -#elif defined(AXOM_USE_MPI) +#if defined(AXOM_USE_MPI) int rank = -1; MPI_Comm_rank(m_comm, &rank); return rank; @@ -177,14 +164,11 @@ int Shaper::getRank() const double Shaper::allReduceSum(double val) const { - if(m_dc != nullptr && m_dc->GetNumProcs() > 1) - { #if defined(AXOM_USE_MPI) && defined(MFEM_USE_MPI) double global; MPI_Allreduce(&val, &global, 1, MPI_DOUBLE, MPI_SUM, m_comm); return global; #endif - } return val; } diff --git a/src/axom/quest/Shaper.hpp b/src/axom/quest/Shaper.hpp index 3736b18752..009db491b4 100644 --- a/src/axom/quest/Shaper.hpp +++ b/src/axom/quest/Shaper.hpp @@ -16,8 +16,18 @@ #ifndef AXOM_USE_KLEE #error Shaping functionality requires Axom to be configured with the Klee component #endif -#ifndef AXOM_USE_MFEM - #error Shaping functionality requires Axom to be configured with MFEM and the AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION option + +// #define AXOM_SHAPING_ON_MFEM_MESH (defined(AXOM_USE_MFEM) && defined(AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION)) +// #define AXOM_SHAPING_ON_BLUEPRINT_MESH defined(AXOM_USE_CONDUIT) +#if defined(AXOM_USE_MFEM) + #define AXOM_SHAPING_ON_MFEM_MESH 1 +#endif +#if defined(AXOM_USE_CONDUIT) + #define AXOM_SHAPING_ON_BLUEPRINT_MESH 1 +#endif + +#if !defined(AXOM_SHAPING_ON_MFEM_MESH) && !defined(AXOM_SHAPING_ON_BLUEPRINT_MESH) + #error Shaping functionality requires Axom to be configured with Conduit or MFEM and the AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION option #endif #include "axom/sidre.hpp" @@ -25,6 +35,13 @@ #include "axom/mint.hpp" #include "axom/quest/DiscreteShape.hpp" +#if defined(AXOM_SHAPING_ON_MFEM_MESH) +#include "mfem.hpp" +#endif +#if defined(AXOM_SHAPING_ON_BLUEPRINT_MESH) +#include "conduit_node.hpp" +#endif + #include "axom/quest/interface/internal/mpicomm_wrapper.hpp" namespace axom @@ -37,11 +54,14 @@ namespace quest class Shaper { public: +#if defined(AXOM_USE_MFEM) /*! @brief Construct Shaper to operate on an MFEM mesh. */ Shaper(const klee::ShapeSet& shapeSet, sidre::MFEMSidreDataCollection* dc); +#endif +#if defined(AXOM_USE_CONDUIT) /*! @brief Construct Shaper to operate on a blueprint-formatted mesh stored in a Conduit Node. @@ -49,6 +69,7 @@ class Shaper Shaper(const klee::ShapeSet& shapeSet, conduit::Node* bpMesh, const std::string& topo=""); +#endif virtual ~Shaper() = default; @@ -73,10 +94,13 @@ class Shaper //@} + mint::Mesh* getSurfaceMesh() const { return m_surfaceMesh.get(); } + bool isVerbose() const { return m_verboseOutput; } +#ifdef AXOM_SHAPING_ON_MFEM_MESH sidre::MFEMSidreDataCollection* getDC() { return m_dc; } - mint::Mesh* getSurfaceMesh() const { return m_surfaceMesh.get(); } +#endif /*! * \brief Predicate to determine if the specified format is valid @@ -167,14 +191,19 @@ class Shaper const klee::ShapeSet& m_shapeSet; +#if defined(AXOM_SHAPING_ON_MFEM_MESH) // For mesh represented as MFEMSidreDataCollection sidre::MFEMSidreDataCollection* m_dc{nullptr}; +#endif +#if defined(AXOM_SHAPING_ON_BLUEPRINT_MESH) // For mesh represented in Conduit or sidre conduit::Node* m_bpNode{nullptr}; const std::string m_bpTopo; sidre::DataStore m_ds; axom::sidre::Group* m_bpGrp{nullptr}; +#endif + axom::IndexType m_cellCount; std::shared_ptr m_surfaceMesh; diff --git a/src/axom/quest/detail/shaping/shaping_helpers.cpp b/src/axom/quest/detail/shaping/shaping_helpers.cpp index 0cac0b8958..735e91787c 100644 --- a/src/axom/quest/detail/shaping/shaping_helpers.cpp +++ b/src/axom/quest/detail/shaping/shaping_helpers.cpp @@ -11,10 +11,6 @@ #include "axom/fmt.hpp" -#ifndef AXOM_USE_MFEM - #error Shaping functionality requires Axom to be configured with MFEM and the AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION option -#endif - namespace axom { namespace quest diff --git a/src/axom/quest/detail/shaping/shaping_helpers.hpp b/src/axom/quest/detail/shaping/shaping_helpers.hpp index 4e223e3ab4..8e9550315a 100644 --- a/src/axom/quest/detail/shaping/shaping_helpers.hpp +++ b/src/axom/quest/detail/shaping/shaping_helpers.hpp @@ -14,8 +14,8 @@ #include "axom/config.hpp" -#ifndef AXOM_USE_MFEM - #error Shaping functionality requires Axom to be configured with MFEM and the AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION option +#if !defined(AXOM_USE_MFEM) + #error Sampling-shaping functionality requires Axom to be configured with MFEM and the AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION option #endif #include "mfem.hpp" diff --git a/src/axom/quest/examples/shaping_driver.cpp b/src/axom/quest/examples/shaping_driver.cpp index 31abd26041..7837d573c2 100644 --- a/src/axom/quest/examples/shaping_driver.cpp +++ b/src/axom/quest/examples/shaping_driver.cpp @@ -20,10 +20,10 @@ #include "axom/fmt.hpp" #include "axom/CLI11.hpp" -// NOTE: The shaping driver requires Axom to be configured with mfem as well as +// NOTE: The shaping driver requires Axom to be configured with conduit or mfem and // the AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION CMake option -#ifndef AXOM_USE_MFEM - #error Shaping functionality requires Axom to be configured with MFEM and the AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION option +#if !defined(AXOM_USE_MFEM) && !defined(AXOM_USE_CONDUIT) + #error Shaping functionality requires Axom to be configured with Conduit or MFEM and the AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION option #endif #include "mfem.hpp" From 56bad3dd647f1ed326b805af814238e7398ed6d7 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Wed, 2 Oct 2024 18:13:18 -0700 Subject: [PATCH 005/100] Avoid using removed util function getUmpireDeviceId. --- src/axom/quest/IntersectionShaper.hpp | 2 +- src/axom/quest/examples/quest_shape_in_memory.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 45ee5966f6..5cf5cc6064 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -943,7 +943,7 @@ class IntersectionShaper : public Shaper const std::string fieldName(materialNameToFieldName(m_free_mat_name)); bool newData = !hasData(fieldName); - int execSpaceAllocatorID = ::getUmpireDeviceId(); + int execSpaceAllocatorID = axom::execution_space::allocatorID(); axom::ArrayView cfgf = getScalarCellData(fieldName); diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index aeec42f0a2..ff9844e23d 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -975,7 +975,7 @@ double sumMaterialVolumes(sidre::MFEMSidreDataCollection* dc, // axom::quest::GridFunctionView volFracView(volFracGf); axom::ArrayView volFracGfArrayView(volFracGf->GetData(), volFracGf->Size()); axom::quest::TempArrayAccess volFracView(volFracGfArrayView, - ::getUmpireDeviceId(), + axom::execution_space::allocatorID(), true); using ReducePolicy = typename axom::execution_space::reduce_policy; From 323882502b823e10c9d5de2418b35422f592e99d Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Wed, 2 Oct 2024 18:14:00 -0700 Subject: [PATCH 006/100] Make private function populateHexesFromMesh public because NVCC complained. --- src/axom/quest/IntersectionShaper.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 5cf5cc6064..a9c45a4452 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -1170,6 +1170,7 @@ class IntersectionShaper : public Shaper axom::for_all( dataSize, AXOM_LAMBDA(axom::IndexType i) { vf_writable[i] = 0.; }); + for(const auto& name : shape.getMaterialsReplaced()) { auto mat = getMaterial(name); @@ -1971,6 +1972,7 @@ class IntersectionShaper : public Shaper return materialNames; } +public: template void populateHexesFromMesh() { @@ -2047,6 +2049,7 @@ class IntersectionShaper : public Shaper } }); // end of loop to initialize hexahedral elements and bounding boxes } +private: #if defined(AXOM_SHAPING_ON_BLUEPRINT_MESH) void populateVertCoordsFromBlueprintMesh(axom::Array& vertCoords_host) From 6603742c2c9835fab35ee449d8a48284774f2e78 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Wed, 2 Oct 2024 23:31:46 -0700 Subject: [PATCH 007/100] Revert to GridFunctionView because of bugs in TempArrayAccess utility. GridFunctionView is renamed TempArrayView to reflect that it doesn't work with just GridFunctions. --- src/axom/quest/IntersectionShaper.hpp | 214 +++++++++++++----- .../quest/examples/quest_shape_in_memory.cpp | 5 +- 2 files changed, 164 insertions(+), 55 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index a9c45a4452..a36ddfdfae 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -62,40 +62,39 @@ namespace axom namespace quest { /*! - * \class TempArrayAccess + * \class TempArrayView * - * \brief Given some array data, provides temporary version of - * that data but allocated with an alternate allocator id. - * This class performs data movement and clean-up to - * temporarily have the data with the desired allocator id. + * \brief Provides a view over an MFEM grid function. MFEM grid functions are + * assumed to live in host memory. This class performs data movement + * needed to access the grid function data within a GPU device lambda. This + * view is limited in scope, though could be expanded in the future. + * + * \tparam ExecSpace The execution space where the grid function data will + * be accessed. */ -class TempArrayAccess +template +class TempArrayView { public: /*! - * \brief Constructor + * \brief Host constructor that accepts the grid function. * - * \param array - * \param writeBack Whether the data needs to be writen back to the - * original array device when this object goes out - * of scope. + * \param gf The grid function that will be accessed/modified by the view. + * \param _needResult Whether the data needs to be brought back to the host + * from the device. */ - AXOM_HOST TempArrayAccess(axom::ArrayView& array, - int wantedAllocatorId, - bool writeBack = true) + AXOM_HOST TempArrayView(mfem::GridFunction* gf, bool _needResult = true) { - m_origArray = array; + initialize(gf->GetData(), gf->Size(), _needResult); + } - if (array.getAllocatorID() != wantedAllocatorId) - { - m_tempArray = axom::Array(array, wantedAllocatorId); - m_accessible = m_tempArray.view(); - } - else - { - m_accessible = array; - } - m_writeBack = writeBack; + AXOM_HOST TempArrayView(axom::Array& gf, bool _needResult = true) + { + initialize(gf.data(), gf.size(), _needResult); + } + AXOM_HOST TempArrayView(axom::ArrayView& gf, bool _needResult = true) + { + initialize(gf.data(), gf.size(), _needResult); } /*! @@ -104,24 +103,18 @@ class TempArrayAccess * happened in the host constructor. This version sets hostData to * nullptr so we know not to clean up in the destructor. */ - AXOM_HOST_DEVICE TempArrayAccess(const TempArrayAccess& obj) - : m_origArray(obj.m_origArray) - , m_tempArray() - , m_accessible(obj.m_accessible) - , m_writeBack(obj.m_writeBack) + AXOM_HOST_DEVICE TempArrayView(const TempArrayView& obj) + : m_hostData(nullptr) + , m_deviceData(obj.m_deviceData) + , m_numElements(obj.m_numElements) + , m_needResult(obj.m_needResult) { } /*! * \brief Destructor. On the host, this method may move data from the device and deallocate device storage. */ - AXOM_HOST_DEVICE ~TempArrayAccess() - { - if (m_writeBack && (m_accessible.data() != m_origArray.data())) - { - axom::copy(m_origArray.data(), m_accessible.data(), sizeof(double)*m_origArray.size()); - } - } + AXOM_HOST_DEVICE ~TempArrayView() { finalize(); } /*! * \brief Indexing operator for accessing the data. @@ -130,17 +123,136 @@ class TempArrayAccess * * \return A reference to the data at index i. */ - AXOM_HOST_DEVICE double& operator[](int i) { return m_accessible[i]; } + AXOM_HOST_DEVICE double& operator[](int i) { return m_deviceData[i]; } // non-const return on purpose. - AXOM_HOST_DEVICE double& operator[](int i) const { return m_accessible[i]; } + AXOM_HOST_DEVICE double& operator[](int i) const { return m_deviceData[i]; } private: - axom::ArrayView m_origArray; - axom::Array m_tempArray; - axom::ArrayView m_accessible; - bool m_writeBack; + /*! + * \brief Initializes members using data from the grid function. This method + * is called on the host. + * + * \param hostPtr The grid function data pointer on the host. + * \param nElem The grid function size. + * \param _needResult Whether any data are copied from device. + */ + AXOM_HOST void initialize(double* hostPtr, int nElem, bool _needResult) + { + m_hostData = m_deviceData = hostPtr; + m_numElements = nElem; + m_needResult = _needResult; + } + + /*! + * \brief Helps during destruction. + */ + AXOM_HOST_DEVICE void finalize() { m_deviceData = nullptr; } + +#if defined(AXOM_USE_CUDA) || defined(AXOM_USE_HIP) + /*! + * \brief Initializes members using data from the grid function. This method + * is called on the host and it copies data to the device. + * + * \param hostPtr The grid function data pointer on the host. + * \param nElem The grid function size. + * \param _needResult Whether any data are copied from device. + */ + AXOM_HOST void initializeDevice(double* hostPtr, int nElem, bool _needResult) + { + m_hostData = hostPtr; + m_numElements = nElem; + m_needResult = _needResult; + int execSpaceAllocatorID = axom::execution_space::allocatorID(); + + auto dataSize = sizeof(double) * m_numElements; + m_deviceData = axom::allocate(dataSize, execSpaceAllocatorID); + axom::copy(m_deviceData, m_hostData, dataSize); + } + + /*! + * \brief Helps during destruction. On the host, it copies device data back + * into the grid function on the host. + */ + AXOM_HOST_DEVICE void finalizeDevice() + { + #ifndef AXOM_DEVICE_CODE + // Only the host will do this work. + if(m_hostData != nullptr) + { + if(m_needResult) + { + auto dataSize = sizeof(double) * m_numElements; + axom::copy(m_hostData, m_deviceData, dataSize); + } + axom::deallocate(m_deviceData); + m_deviceData = nullptr; + } + #endif + } +#endif + +private: + double* m_hostData {nullptr}; + double* m_deviceData {nullptr}; + int m_numElements {0}; + bool m_needResult {false}; }; +#if defined(AXOM_USE_CUDA) +/*! + * \brief CUDA specialization that calls initializeDevice to copy data + * from the host to the device. + * + * \param hostPtr The grid function data pointer on the host. + * \param nElem The grid function size. + * \param _needResult Whether any data are copied from device. + */ +template <> +AXOM_HOST inline void TempArrayView::initialize(double* hostPtr, + int nElem, + bool _needResult) +{ + initializeDevice(hostPtr, nElem, _needResult); +} + +/*! + * \brief CUDA specialization that may copy data back from the device + * and deallocate any associated device data. + */ +template <> +AXOM_HOST_DEVICE inline void TempArrayView::finalize() +{ + finalizeDevice(); +} +#endif +#if defined(AXOM_USE_HIP) +/*! + * \brief HIP specialization that calls initializeDevice to copy data + * from the host to the device. + * + * \param hostPtr The grid function data pointer on the host. + * \param nElem The grid function size. + * \param _needResult Whether any data are copied from device. + */ +template <> +AXOM_HOST inline void TempArrayView::initialize(double* hostPtr, + int nElem, + bool _needResult) +{ + initializeDevice(hostPtr, nElem, _needResult); +} + +/*! + * \brief HIP specialization that may copy data back from the device + * and deallocate any associated device data. + */ +template <> +AXOM_HOST_DEVICE inline void TempArrayView::finalize() +{ + finalizeDevice(); +} +#endif + //--------------------------------------------------------------------------- /** * \class @@ -943,7 +1055,6 @@ class IntersectionShaper : public Shaper const std::string fieldName(materialNameToFieldName(m_free_mat_name)); bool newData = !hasData(fieldName); - int execSpaceAllocatorID = axom::execution_space::allocatorID(); axom::ArrayView cfgf = getScalarCellData(fieldName); @@ -952,7 +1063,7 @@ class IntersectionShaper : public Shaper AXOM_ANNOTATE_SCOPE("compute_free"); int dataSize = cfgf.size(); - TempArrayAccess cfView(cfgf, execSpaceAllocatorID, true); + TempArrayView cfView(cfgf, true); axom::for_all( dataSize, @@ -961,7 +1072,7 @@ class IntersectionShaper : public Shaper // Iterate over all materials and subtract off their VFs from cfgf. for(axom::ArrayView& gf : m_vf_grid_functions) { - TempArrayAccess matVFView(gf, execSpaceAllocatorID, false); + TempArrayView matVFView(gf, false); axom::for_all( dataSize, AXOM_LAMBDA(axom::IndexType i) { @@ -1174,7 +1285,7 @@ class IntersectionShaper : public Shaper for(const auto& name : shape.getMaterialsReplaced()) { auto mat = getMaterial(name); - TempArrayAccess matVFView(mat.first, execSpaceAllocatorID, false); + TempArrayView matVFView(mat.first, false); axom::for_all( dataSize, AXOM_LAMBDA(axom::IndexType i) { @@ -1193,7 +1304,7 @@ class IntersectionShaper : public Shaper for(auto& gf : excludeVFs) { - TempArrayAccess matVFView(gf, execSpaceAllocatorID, false); + TempArrayView matVFView(gf, false); axom::for_all( dataSize, AXOM_LAMBDA(axom::IndexType i) { @@ -1207,8 +1318,8 @@ class IntersectionShaper : public Shaper { AXOM_ANNOTATE_SCOPE("compute_vf"); - TempArrayAccess matVFView(matVF.first, execSpaceAllocatorID, true); - TempArrayAccess shapeVFView(shapeVolFrac, execSpaceAllocatorID, true); + TempArrayView matVFView(matVF.first, true); + TempArrayView shapeVFView(shapeVolFrac, true); axom::ArrayView overlap_volumes_view = m_overlap_volumes.view(); axom::ArrayView hex_volumes_view = m_hex_volumes.view(); @@ -1238,7 +1349,7 @@ class IntersectionShaper : public Shaper AXOM_ANNOTATE_SCOPE("update_vf"); for(auto& gf : updateVFs) { - TempArrayAccess matVFView(gf, execSpaceAllocatorID, true); + TempArrayView matVFView(gf, true); axom::for_all( dataSize, AXOM_LAMBDA(axom::IndexType i) { @@ -1973,6 +2084,7 @@ class IntersectionShaper : public Shaper } public: + // This should be private, but NVCC complains unless its public. template void populateHexesFromMesh() { diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index ff9844e23d..706771458e 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -972,11 +972,8 @@ double sumMaterialVolumes(sidre::MFEMSidreDataCollection* dc, const std::string materialFieldName = axom::fmt::format("vol_frac_{}", material); mfem::GridFunction* volFracGf = dc->GetField(materialFieldName); - // axom::quest::GridFunctionView volFracView(volFracGf); axom::ArrayView volFracGfArrayView(volFracGf->GetData(), volFracGf->Size()); - axom::quest::TempArrayAccess volFracView(volFracGfArrayView, - axom::execution_space::allocatorID(), - true); + axom::quest::TempArrayView volFracView(volFracGfArrayView, true); using ReducePolicy = typename axom::execution_space::reduce_policy; RAJA::ReduceSum localVol(0); From aaec48da1eedf3eee9b065a0cc8f932e77aee108 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Mon, 7 Oct 2024 12:07:57 -0700 Subject: [PATCH 008/100] Simplify include guards in shaping code. --- src/axom/quest/IntersectionShaper.hpp | 26 ++++++++++++++------------ src/axom/quest/Shaper.cpp | 4 ++-- src/axom/quest/Shaper.hpp | 21 ++++++--------------- 3 files changed, 22 insertions(+), 29 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index a36ddfdfae..bffc02c5ab 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -76,6 +76,7 @@ template class TempArrayView { public: +#if defined(AXOM_USE_MFEM) /*! * \brief Host constructor that accepts the grid function. * @@ -87,6 +88,7 @@ class TempArrayView { initialize(gf->GetData(), gf->Size(), _needResult); } +#endif AXOM_HOST TempArrayView(axom::Array& gf, bool _needResult = true) { @@ -307,7 +309,7 @@ class IntersectionShaper : public Shaper static constexpr double DEFAULT_REVOLVED_VOLUME {0.}; public: -#if defined(AXOM_SHAPING_ON_MFEM_MESH) +#if defined(AXOM_USE_MFEM) /*! @brief Construct Shaper to operate on an MFEM mesh. */ @@ -319,7 +321,7 @@ class IntersectionShaper : public Shaper } #endif -#if defined(AXOM_SHAPING_ON_BLUEPRINT_MESH) +#if defined(AXOM_USE_CONDUIT) /*! @brief Construct Shaper to operate on a blueprint-formatted mesh stored in a Conduit Node. @@ -1993,12 +1995,12 @@ class IntersectionShaper : public Shaper bool hasData(const std::string& fieldName) { bool has = false; -#if defined(AXOM_SHAPING_ON_MFEM_MESH) +#if defined(AXOM_USE_MFEM) if (m_dc != nullptr) { has = m_dc->HasField(fieldName); } #endif -#if defined(AXOM_SHAPING_ON_BLUEPRINT_MESH) +#if defined(AXOM_USE_CONDUIT) if (m_bpGrp != nullptr) { std::string fieldPath = axom::fmt::format("fields/{}", fieldName); has = m_bpGrp->hasGroup(fieldPath); @@ -2011,7 +2013,7 @@ class IntersectionShaper : public Shaper { axom::ArrayView rval; -#if defined(AXOM_SHAPING_ON_MFEM_MESH) +#if defined(AXOM_USE_MFEM) if (m_dc) { mfem::GridFunction* gridFunc = nullptr; if (m_dc->HasField(fieldName)) @@ -2026,7 +2028,7 @@ class IntersectionShaper : public Shaper gridFunc->Size()); } #endif -#if defined(AXOM_SHAPING_ON_BLUEPRINT_MESH) +#if defined(AXOM_USE_CONDUIT) if (m_bpGrp != nullptr) { std::string fieldPath = axom::fmt::format("fields/{}", fieldName); auto dtype = conduit::DataType::float64(m_cellCount); @@ -2054,7 +2056,7 @@ class IntersectionShaper : public Shaper std::vector getMaterialNames() { std::vector materialNames; -#if defined(AXOM_SHAPING_ON_MFEM_MESH) +#if defined(AXOM_USE_MFEM) if (m_dc) { for(auto it : this->getDC()->GetFieldMap()) @@ -2066,7 +2068,7 @@ class IntersectionShaper : public Shaper } } } -#elif defined(AXOM_SHAPING_ON_BLUEPRINT_MESH) +#elif defined(AXOM_USE_CONDUIT) if (m_bpGrp) { auto fieldsGrp = m_bpGrp->getGroup("fields"); @@ -2100,13 +2102,13 @@ class IntersectionShaper : public Shaper m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT, hostAllocator); -#if defined(AXOM_SHAPING_ON_MFEM_MESH) +#if defined(AXOM_USE_MFEM) if (m_dc != nullptr) { populateVertCoordsFromMFEMMesh(vertCoords); } #endif -#if defined(AXOM_SHAPING_ON_BLUEPRINT_MESH) +#if defined(AXOM_USE_CONDUIT) if (m_bpGrp != nullptr) { populateVertCoordsFromBlueprintMesh(vertCoords); @@ -2163,7 +2165,7 @@ class IntersectionShaper : public Shaper } private: -#if defined(AXOM_SHAPING_ON_BLUEPRINT_MESH) +#if defined(AXOM_USE_CONDUIT) void populateVertCoordsFromBlueprintMesh(axom::Array& vertCoords_host) { // Initialize vertices from blueprint mesh and @@ -2220,7 +2222,7 @@ class IntersectionShaper : public Shaper } #endif -#if defined(AXOM_SHAPING_ON_MFEM_MESH) +#if defined(AXOM_USE_MFEM) void populateVertCoordsFromMFEMMesh(axom::Array& vertCoords_host) { mfem::Mesh* mesh = getDC()->GetMesh(); diff --git a/src/axom/quest/Shaper.cpp b/src/axom/quest/Shaper.cpp index 6c0c6dff23..23be2be7df 100644 --- a/src/axom/quest/Shaper.cpp +++ b/src/axom/quest/Shaper.cpp @@ -25,7 +25,7 @@ constexpr double Shaper::MINIMUM_PERCENT_ERROR; constexpr double Shaper::MAXIMUM_PERCENT_ERROR; constexpr double Shaper::DEFAULT_VERTEX_WELD_THRESHOLD; -#if defined(AXOM_SHAPING_ON_MFEM_MESH) +#if defined(AXOM_USE_MFEM) Shaper::Shaper(const klee::ShapeSet& shapeSet, sidre::MFEMSidreDataCollection* dc) : m_shapeSet(shapeSet) @@ -38,7 +38,7 @@ Shaper::Shaper(const klee::ShapeSet& shapeSet, } #endif -#if defined(AXOM_SHAPING_ON_BLUEPRINT_MESH) +#if defined(AXOM_USE_CONDUIT) Shaper::Shaper(const klee::ShapeSet& shapeSet, conduit::Node* bpNode, const std::string& topo) diff --git a/src/axom/quest/Shaper.hpp b/src/axom/quest/Shaper.hpp index 009db491b4..04264c0d56 100644 --- a/src/axom/quest/Shaper.hpp +++ b/src/axom/quest/Shaper.hpp @@ -17,16 +17,7 @@ #error Shaping functionality requires Axom to be configured with the Klee component #endif -// #define AXOM_SHAPING_ON_MFEM_MESH (defined(AXOM_USE_MFEM) && defined(AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION)) -// #define AXOM_SHAPING_ON_BLUEPRINT_MESH defined(AXOM_USE_CONDUIT) -#if defined(AXOM_USE_MFEM) - #define AXOM_SHAPING_ON_MFEM_MESH 1 -#endif -#if defined(AXOM_USE_CONDUIT) - #define AXOM_SHAPING_ON_BLUEPRINT_MESH 1 -#endif - -#if !defined(AXOM_SHAPING_ON_MFEM_MESH) && !defined(AXOM_SHAPING_ON_BLUEPRINT_MESH) +#if !defined(AXOM_USE_MFEM) && !defined(AXOM_USE_CONDUIT) #error Shaping functionality requires Axom to be configured with Conduit or MFEM and the AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION option #endif @@ -35,10 +26,10 @@ #include "axom/mint.hpp" #include "axom/quest/DiscreteShape.hpp" -#if defined(AXOM_SHAPING_ON_MFEM_MESH) +#if defined(AXOM_USE_MFEM) #include "mfem.hpp" #endif -#if defined(AXOM_SHAPING_ON_BLUEPRINT_MESH) +#if defined(AXOM_USE_CONDUIT) #include "conduit_node.hpp" #endif @@ -98,7 +89,7 @@ class Shaper bool isVerbose() const { return m_verboseOutput; } -#ifdef AXOM_SHAPING_ON_MFEM_MESH +#ifdef AXOM_USE_MFEM sidre::MFEMSidreDataCollection* getDC() { return m_dc; } #endif @@ -191,12 +182,12 @@ class Shaper const klee::ShapeSet& m_shapeSet; -#if defined(AXOM_SHAPING_ON_MFEM_MESH) +#if defined(AXOM_USE_MFEM) // For mesh represented as MFEMSidreDataCollection sidre::MFEMSidreDataCollection* m_dc{nullptr}; #endif -#if defined(AXOM_SHAPING_ON_BLUEPRINT_MESH) +#if defined(AXOM_USE_CONDUIT) // For mesh represented in Conduit or sidre conduit::Node* m_bpNode{nullptr}; const std::string m_bpTopo; From f38509a499df1cdb2ca2a9c0ee5d4c83bae9e955 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Tue, 8 Oct 2024 23:45:28 -0700 Subject: [PATCH 009/100] Minor comment and white-space change. --- src/axom/quest/IntersectionShaper.hpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index bffc02c5ab..d259e24e10 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -2009,12 +2009,15 @@ class IntersectionShaper : public Shaper return has; } + /*! + @brief Get a scalar double-type field data from the mesh, creating it if it doesn't exist. + */ axom::ArrayView getScalarCellData(const std::string& fieldName) { axom::ArrayView rval; #if defined(AXOM_USE_MFEM) - if (m_dc) { + if (m_dc != nullptr) { mfem::GridFunction* gridFunc = nullptr; if (m_dc->HasField(fieldName)) { From 5c6b1a495a5efb4ca7c61de91030d47ddc449710 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Mon, 14 Oct 2024 12:04:40 -0700 Subject: [PATCH 010/100] Remove support for reading in MFEM mesh in the new test. The feature was never used and we are moving away from MFEM mesh in favor of blueprint mesh. --- .../quest/examples/quest_shape_in_memory.cpp | 43 ++++--------------- 1 file changed, 9 insertions(+), 34 deletions(-) diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index 706771458e..b212ae8e3f 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -70,7 +70,6 @@ using RuntimePolicy = axom::runtime_policy::Policy; struct Input { public: - std::string meshFile; std::string outputFile; std::vector center {0.0, 0.0, 0.0}; @@ -82,6 +81,9 @@ struct Input // Shape transformation parameters std::vector scaleFactors; + // Mesh format: mfem or blueprint + std::string meshFormat = "mfem"; + // Inline mesh parameters std::vector boxMins; std::vector boxMaxs; @@ -190,32 +192,16 @@ struct Input std::unique_ptr loadComputationalMesh() { constexpr bool dc_owns_data = true; - mfem::Mesh* mesh = meshFile.empty() ? createBoxMesh() : nullptr; - std::string name = meshFile.empty() ? "mesh" : getDCMeshName(); + mfem::Mesh* mesh = createBoxMesh(); + std::string name = "mesh"; auto dc = std::unique_ptr( new sidre::MFEMSidreDataCollection(name, mesh, dc_owns_data)); dc->SetComm(MPI_COMM_WORLD); - if(!meshFile.empty()) - { - dc->Load(meshFile, "sidre_hdf5"); - } - return dc; } - std::string getDCMeshName() const - { - using axom::utilities::string::removeSuffix; - - // Remove the parent directories and file suffix - std::string name = axom::Path(meshFile).baseName(); - name = removeSuffix(name, ".root"); - - return name; - } - void parse(int argc, char** argv, axom::CLI::App& app) { app.add_option("-o,--outputFile", outputFile) @@ -289,12 +275,6 @@ struct Input // use either an input mesh file or a simple inline Cartesian mesh { - auto* mesh_file = - app.add_option("-m,--mesh-file", meshFile) - ->description( - "Path to computational mesh (generated by MFEMSidreDataCollection)") - ->check(axom::CLI::ExistingFile); - auto* inline_mesh_subcommand = app.add_subcommand("inline_mesh") ->description("Options for setting up a simple inline mesh") @@ -314,15 +294,10 @@ struct Input ->expected(2, 3) ->required(); - auto* inline_mesh_dim = - inline_mesh_subcommand->add_option("-d,--dimension", boxDim) - ->description("Dimension of the box mesh") - ->check(axom::CLI::PositiveNumber) - ->required(); - - // we want either the mesh_file or an inline mesh - mesh_file->excludes(inline_mesh_dim); - inline_mesh_dim->excludes(mesh_file); + inline_mesh_subcommand->add_option("-d,--dimension", boxDim) + ->description("Dimension of the box mesh") + ->check(axom::CLI::PositiveNumber) + ->required(); } app.add_option("--background-material", backgroundMaterial) From 9e983fe462702593cf1e344b5c1a3ca9f22bedec Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Mon, 14 Oct 2024 12:28:01 -0700 Subject: [PATCH 011/100] Remove unused code (for sampling shaping) in the new test. --- src/axom/quest/examples/CMakeLists.txt | 2 +- .../quest/examples/quest_shape_in_memory.cpp | 127 +----------------- 2 files changed, 2 insertions(+), 127 deletions(-) diff --git a/src/axom/quest/examples/CMakeLists.txt b/src/axom/quest/examples/CMakeLists.txt index 75a358928c..34fc2a7d60 100644 --- a/src/axom/quest/examples/CMakeLists.txt +++ b/src/axom/quest/examples/CMakeLists.txt @@ -298,7 +298,7 @@ if(AXOM_ENABLE_MPI AND MFEM_FOUND AND MFEM_USE_MPI COMMAND quest_shape_in_memory_ex --policy ${_policy} --testShape ${_testshape} - --method intersection --refinements 2 + --refinements 2 --scale 0.75 0.75 0.75 inline_mesh --min -3 -3 -3 --max 3 3 3 --res 20 20 20 -d 3 NUM_MPI_TASKS ${_nranks}) diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index b212ae8e3f..a41d9d1e39 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -53,17 +53,8 @@ namespace quest = axom::quest; namespace slic = axom::slic; namespace sidre = axom::sidre; -using VolFracSampling = quest::shaping::VolFracSampling; - //------------------------------------------------------------------------------ -/// Struct to help choose our shaping method: sampling or intersection for now -enum class ShapingMethod : int -{ - Sampling, - Intersection -}; - using RuntimePolicy = axom::runtime_policy::Policy; /// Struct to parse and store the input parameters @@ -102,7 +93,6 @@ struct Input "hex", "plane"}; - ShapingMethod shapingMethod {ShapingMethod::Sampling}; RuntimePolicy policy {RuntimePolicy::seq}; int quadratureOrder {5}; int outputOrder {2}; @@ -113,8 +103,6 @@ struct Input std::string backgroundMaterial; - VolFracSampling vfSampling {VolFracSampling::SAMPLE_AT_QPTS}; - private: bool m_verboseOutput {false}; @@ -225,16 +213,6 @@ struct Input ->check(axom::CLI::PositiveNumber) ->capture_default_str(); - std::map methodMap { - {"sampling", ShapingMethod::Sampling}, - {"intersection", ShapingMethod::Intersection}}; - app.add_option("--method", shapingMethod) - ->description( - "Determines the shaping method -- either sampling or intersection") - ->capture_default_str() - ->transform( - axom::CLI::CheckedTransformer(methodMap, axom::CLI::ignore_case)); - app.add_option("-s,--testShape", testShape) ->description("The shape to run") ->check(axom::CLI::IsMember(availableShapes)); @@ -303,38 +281,6 @@ struct Input app.add_option("--background-material", backgroundMaterial) ->description("Sets the name of the background material"); - // parameters that only apply to the sampling method - { - auto* sampling_options = - app.add_option_group("sampling", - "Options related to sampling-based queries"); - - sampling_options->add_option("-o,--order", outputOrder) - ->description("Order of the output grid function") - ->capture_default_str() - ->check(axom::CLI::NonNegativeNumber); - - sampling_options->add_option("-q,--quadrature-order", quadratureOrder) - ->description( - "Quadrature order for sampling the inout field. \n" - "Determines number of samples per element in determining " - "volume fraction field") - ->capture_default_str() - ->check(axom::CLI::PositiveNumber); - - std::map vfsamplingMap { - {"qpts", VolFracSampling::SAMPLE_AT_QPTS}, - {"dofs", VolFracSampling::SAMPLE_AT_DOFS}}; - sampling_options->add_option("-s,--sampling-type", vfSampling) - ->description( - "Sampling strategy. \n" - "Sampling either at quadrature points or collocated with " - "degrees of freedom") - ->capture_default_str() - ->transform( - axom::CLI::CheckedTransformer(vfsamplingMap, axom::CLI::ignore_case)); - } - // parameters that only apply to the intersection method { auto* intersection_options = @@ -1129,15 +1075,7 @@ int main(int argc, char** argv) //--------------------------------------------------------------------------- AXOM_ANNOTATE_BEGIN("setup shaping problem"); quest::Shaper* shaper = nullptr; - switch(params.shapingMethod) - { - case ShapingMethod::Sampling: - shaper = new quest::SamplingShaper(shapeSet, &shapingDC); - break; - case ShapingMethod::Intersection: - shaper = new quest::IntersectionShaper(shapeSet, &shapingDC); - break; - } + shaper = new quest::IntersectionShaper(shapeSet, &shapingDC); SLIC_ASSERT_MSG(shaper != nullptr, "Invalid shaping method selected!"); // Set generic parameters for the base Shaper instance @@ -1153,25 +1091,6 @@ int main(int argc, char** argv) // the data collection is written, a matset will be created. shaper->getDC()->AssociateMaterialSet("vol_frac", "material"); - // Set specific parameters for a SamplingShaper, if appropriate - if(auto* samplingShaper = dynamic_cast(shaper)) - { - samplingShaper->setSamplingType(params.vfSampling); - samplingShaper->setQuadratureOrder(params.quadratureOrder); - samplingShaper->setVolumeFractionOrder(params.outputOrder); - - // register a point projector - if(shapingDC.GetMesh()->Dimension() == 3 && shapeDim == klee::Dimensions::Two) - { - samplingShaper->setPointProjector([](primal::Point pt) { - const double& x = pt[0]; - const double& y = pt[1]; - const double& z = pt[2]; - return primal::Point {z, sqrt(x * x + y * y)}; - }); - } - } - // Set specific parameters here for IntersectionShaper if(auto* intersectionShaper = dynamic_cast(shaper)) { @@ -1188,42 +1107,6 @@ int main(int argc, char** argv) } } - //--------------------------------------------------------------------------- - // Project initial volume fractions, if applicable - //--------------------------------------------------------------------------- - if(auto* samplingShaper = dynamic_cast(shaper)) - { - AXOM_ANNOTATE_SCOPE("import initial volume fractions"); - std::map initial_grid_functions; - - // Generate a background material (w/ volume fractions set to 1) if user provided a name - if(!params.backgroundMaterial.empty()) - { - auto material = params.backgroundMaterial; - auto name = axom::fmt::format("vol_frac_{}", material); - - const int order = params.outputOrder; - const int dim = shapingMesh->Dimension(); - const auto basis = mfem::BasisType::Positive; - - auto* coll = new mfem::L2_FECollection(order, dim, basis); - auto* fes = new mfem::FiniteElementSpace(shapingDC.GetMesh(), coll); - const int sz = fes->GetVSize(); - - auto* view = shapingDC.AllocNamedBuffer(name, sz); - auto* volFrac = new mfem::GridFunction(fes, view->getArray()); - volFrac->MakeOwner(coll); - - (*volFrac) = 1.; - - shapingDC.RegisterField(name, volFrac); - - initial_grid_functions[material] = shapingDC.GetField(name); - } - - // Project provided volume fraction grid functions as quadrature point data - samplingShaper->importInitialVolumeFractions(initial_grid_functions); - } AXOM_ANNOTATE_END("setup shaping problem"); AXOM_ANNOTATE_END("init"); @@ -1406,14 +1289,6 @@ int main(int argc, char** argv) //--------------------------------------------------------------------------- // Save meshes and fields //--------------------------------------------------------------------------- - if(params.isVerbose()) - { - if(auto* samplingShaper = dynamic_cast(shaper)) - { - SLIC_INFO(axom::fmt::format("{:-^80}", "")); - samplingShaper->printRegisteredFieldNames(" -- after shaping"); - } - } #ifdef MFEM_USE_MPI if(!params.outputFile.empty()) From 09056dafa9ecce6c76e564da004b87bd9e6bf349 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Tue, 15 Oct 2024 12:12:36 -0700 Subject: [PATCH 012/100] Autoformat. --- src/axom/quest/IntersectionShaper.hpp | 142 +++++++++++++++----------- src/axom/quest/Shaper.cpp | 26 ++--- src/axom/quest/Shaper.hpp | 12 +-- 3 files changed, 102 insertions(+), 78 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index d259e24e10..ea05b70f24 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -211,8 +211,8 @@ class TempArrayView */ template <> AXOM_HOST inline void TempArrayView::initialize(double* hostPtr, - int nElem, - bool _needResult) + int nElem, + bool _needResult) { initializeDevice(hostPtr, nElem, _needResult); } @@ -238,8 +238,8 @@ AXOM_HOST_DEVICE inline void TempArrayView::finalize() */ template <> AXOM_HOST inline void TempArrayView::initialize(double* hostPtr, - int nElem, - bool _needResult) + int nElem, + bool _needResult) { initializeDevice(hostPtr, nElem, _needResult); } @@ -328,11 +328,10 @@ class IntersectionShaper : public Shaper */ IntersectionShaper(const klee::ShapeSet& shapeSet, conduit::Node* bpMesh, - const std::string& topo="") + const std::string& topo = "") : Shaper(shapeSet, bpMesh, topo) , m_free_mat_name("free") - { - } + { } #endif //@{ @@ -682,13 +681,15 @@ class IntersectionShaper : public Shaper // Create and register a scalar field for this shape's volume fractions // The Degrees of Freedom will be in correspondence with the elements - std::string volFracName = axom::fmt::format("shape_vol_frac_{}", shape.getName()); + std::string volFracName = + axom::fmt::format("shape_vol_frac_{}", shape.getName()); auto volFrac = getScalarCellData(volFracName); populateHexesFromMesh(); axom::ArrayView hexes_device_view = m_hexes.view(); - m_hex_bbs = axom::Array(m_cellCount, m_cellCount, device_allocator); + m_hex_bbs = + axom::Array(m_cellCount, m_cellCount, device_allocator); axom::ArrayView hex_bbs_device_view = m_hex_bbs.view(); // Get bounding boxes for hexahedral elements @@ -743,7 +744,11 @@ class IntersectionShaper : public Shaper axom::Array offsets(m_cellCount, m_cellCount, device_allocator); axom::Array counts(m_cellCount, m_cellCount, device_allocator); axom::Array candidates; - bvh.findBoundingBoxes(offsets, counts, candidates, m_cellCount, hex_bbs_device_view); + bvh.findBoundingBoxes(offsets, + counts, + candidates, + m_cellCount, + hex_bbs_device_view); // Get the total number of candidates using REDUCE_POL = typename axom::execution_space::reduce_policy; @@ -771,9 +776,10 @@ class IntersectionShaper : public Shaper auto shape_candidates_device_view = shape_candidates_device.view(); // Tetrahedrons from hexes (24 for each hex) - axom::Array tets_from_hexes_device(m_cellCount * NUM_TETS_PER_HEX, - m_cellCount * NUM_TETS_PER_HEX, - device_allocator); + axom::Array tets_from_hexes_device( + m_cellCount * NUM_TETS_PER_HEX, + m_cellCount * NUM_TETS_PER_HEX, + device_allocator); axom::ArrayView tets_from_hexes_device_view = tets_from_hexes_device.view(); @@ -842,10 +848,12 @@ class IntersectionShaper : public Shaper // Overlap volume is the volume of clip(oct,tet) for c2c // or clip(tet,tet) for Pro/E meshes - m_overlap_volumes = axom::Array(m_cellCount, m_cellCount, device_allocator); + m_overlap_volumes = + axom::Array(m_cellCount, m_cellCount, device_allocator); // Hex volume is the volume of the hexahedron element - m_hex_volumes = axom::Array(m_cellCount, m_cellCount, device_allocator); + m_hex_volumes = + axom::Array(m_cellCount, m_cellCount, device_allocator); axom::ArrayView overlap_volumes_device_view = m_overlap_volumes.view(); @@ -988,7 +996,7 @@ class IntersectionShaper : public Shaper bool newData = !hasData(materialVolFracName); auto matVolFrac = getScalarCellData(materialVolFracName); - if (newData) + if(newData) { // Zero out the volume fractions (on host). // memset(matVolFrac->begin(), 0, matVolFrac->Size() * sizeof(double)); @@ -1013,9 +1021,9 @@ class IntersectionShaper : public Shaper */ void populateMaterials() { -#if 1 + #if 1 std::vector materialNames = getMaterialNames(); -#else + #else std::vector materialNames; for(auto it : this->getDC()->GetFieldMap()) { @@ -1025,7 +1033,7 @@ class IntersectionShaper : public Shaper materialNames.emplace_back(materialName); } } -#endif + #endif // Add any of these existing fields to this class' bookkeeping. for(const auto& materialName : materialNames) { @@ -1060,7 +1068,7 @@ class IntersectionShaper : public Shaper axom::ArrayView cfgf = getScalarCellData(fieldName); - if (newData) + if(newData) { AXOM_ANNOTATE_SCOPE("compute_free"); @@ -1182,10 +1190,10 @@ class IntersectionShaper : public Shaper } else { - // Include all materials except those in "does_not_replace". - // We'll also sort them by material number since the field map - // sorts them by name rather than order added. -#if 1 + // Include all materials except those in "does_not_replace". + // We'll also sort them by material number since the field map + // sorts them by name rather than order added. + #if 1 std::vector materialNames = getMaterialNames(); for(auto name : materialNames) { @@ -1217,7 +1225,7 @@ class IntersectionShaper : public Shaper } } } -#else + #else for(auto it : this->getDC()->GetFieldMap()) { // Check whether the field name looks like a VF field (and is not the @@ -1249,7 +1257,7 @@ class IntersectionShaper : public Shaper } } } -#endif + #endif } // Sort eligible update materials by material number. std::sort(gf_order_by_matnumber.begin(), @@ -1996,12 +2004,14 @@ class IntersectionShaper : public Shaper { bool has = false; #if defined(AXOM_USE_MFEM) - if (m_dc != nullptr) { + if(m_dc != nullptr) + { has = m_dc->HasField(fieldName); } #endif #if defined(AXOM_USE_CONDUIT) - if (m_bpGrp != nullptr) { + if(m_bpGrp != nullptr) + { std::string fieldPath = axom::fmt::format("fields/{}", fieldName); has = m_bpGrp->hasGroup(fieldPath); } @@ -2017,26 +2027,28 @@ class IntersectionShaper : public Shaper axom::ArrayView rval; #if defined(AXOM_USE_MFEM) - if (m_dc != nullptr) { + if(m_dc != nullptr) + { mfem::GridFunction* gridFunc = nullptr; - if (m_dc->HasField(fieldName)) + if(m_dc->HasField(fieldName)) { gridFunc = m_dc->GetField(fieldName); } - else { + else + { gridFunc = newVolFracGridFunction(); m_dc->RegisterField(fieldName, gridFunc); } - rval = axom::ArrayView(gridFunc->GetData(), - gridFunc->Size()); + rval = axom::ArrayView(gridFunc->GetData(), gridFunc->Size()); } #endif #if defined(AXOM_USE_CONDUIT) - if (m_bpGrp != nullptr) { + if(m_bpGrp != nullptr) + { std::string fieldPath = axom::fmt::format("fields/{}", fieldName); auto dtype = conduit::DataType::float64(m_cellCount); axom::sidre::View* valuesView = nullptr; - if (m_bpGrp->hasGroup(fieldPath)) + if(m_bpGrp->hasGroup(fieldPath)) { auto* fieldsGrp = m_bpGrp->getGroup(fieldPath); valuesView = fieldsGrp->getView("values"); @@ -2049,8 +2061,9 @@ class IntersectionShaper : public Shaper valuesView = fieldsGrp->createView("values"); valuesView->allocate(dtype); } - rval = axom::ArrayView(static_cast(valuesView->getVoidPtr()), - m_cellCount); + rval = + axom::ArrayView(static_cast(valuesView->getVoidPtr()), + m_cellCount); } #endif return rval; @@ -2060,7 +2073,7 @@ class IntersectionShaper : public Shaper { std::vector materialNames; #if defined(AXOM_USE_MFEM) - if (m_dc) + if(m_dc) { for(auto it : this->getDC()->GetFieldMap()) { @@ -2072,10 +2085,10 @@ class IntersectionShaper : public Shaper } } #elif defined(AXOM_USE_CONDUIT) - if (m_bpGrp) + if(m_bpGrp) { auto fieldsGrp = m_bpGrp->getGroup("fields"); - for (auto& group : fieldsGrp->groups()) + for(auto& group : fieldsGrp->groups()) { std::string materialName = fieldNameToMaterialName(group.getName()); if(!materialName.empty()) @@ -2097,8 +2110,7 @@ class IntersectionShaper : public Shaper constexpr int NUM_COMPS_PER_VERT = 3; const int hostAllocator = axom::execution_space::allocatorID(); - const int deviceAllocator = - axom::execution_space::allocatorID(); + const int deviceAllocator = axom::execution_space::allocatorID(); axom::Array vertCoords( m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT, @@ -2106,26 +2118,27 @@ class IntersectionShaper : public Shaper hostAllocator); #if defined(AXOM_USE_MFEM) - if (m_dc != nullptr) + if(m_dc != nullptr) { populateVertCoordsFromMFEMMesh(vertCoords); } #endif #if defined(AXOM_USE_CONDUIT) - if (m_bpGrp != nullptr) + if(m_bpGrp != nullptr) { populateVertCoordsFromBlueprintMesh(vertCoords); } #endif - if (deviceAllocator != hostAllocator) + if(deviceAllocator != hostAllocator) { vertCoords = axom::Array(vertCoords, deviceAllocator); } auto vertCoords_device_view = vertCoords.view(); - m_hexes = axom::Array(m_cellCount, m_cellCount, deviceAllocator); + m_hexes = + axom::Array(m_cellCount, m_cellCount, deviceAllocator); axom::ArrayView hexes_device_view = m_hexes.view(); constexpr double ZERO_THRESHOLD = 1.e-10; axom::for_all( @@ -2166,8 +2179,8 @@ class IntersectionShaper : public Shaper } }); // end of loop to initialize hexahedral elements and bounding boxes } -private: +private: #if defined(AXOM_USE_CONDUIT) void populateVertCoordsFromBlueprintMesh(axom::Array& vertCoords_host) { @@ -2178,30 +2191,42 @@ class IntersectionShaper : public Shaper constexpr int NUM_VERTS_PER_HEX = 8; constexpr int NUM_COMPS_PER_VERT = 3; - const auto* topoGrp = m_bpGrp->getGroup(axom::fmt::format("topologies/{}", m_bpTopo)); + const auto* topoGrp = + m_bpGrp->getGroup(axom::fmt::format("topologies/{}", m_bpTopo)); std::string coordsetName = topoGrp->getView("coordset")->getString(); // Assume unstructured and hexahedral - SLIC_ASSERT(std::string(topoGrp->getView("type")->getString()) == "unstructured"); - SLIC_ASSERT(std::string(topoGrp->getView("elements/shape")->getString()) == "hex"); + SLIC_ASSERT(std::string(topoGrp->getView("type")->getString()) == + "unstructured"); + SLIC_ASSERT(std::string(topoGrp->getView("elements/shape")->getString()) == + "hex"); const auto connGrp = topoGrp->getView("elements/connectivity"); axom::ArrayView conn(connGrp->getNode().as_double_ptr(), - m_cellCount, NUM_VERTS_PER_HEX); + m_cellCount, + NUM_VERTS_PER_HEX); - const auto* coordGrp = m_bpGrp->getGroup(axom::fmt::format("coordsets/{}", coordsetName)); + const auto* coordGrp = + m_bpGrp->getGroup(axom::fmt::format("coordsets/{}", coordsetName)); const conduit::Node& coordValues = coordGrp->getView("values")->getNode(); // Assume explicit coordinates. - SLIC_ASSERT(std::string(coordGrp->getView("type")->getString()) == "explicit"); + SLIC_ASSERT(std::string(coordGrp->getView("type")->getString()) == + "explicit"); axom::IndexType vertexCount = coordValues["x"].dtype().number_of_elements(); bool isInterleaved = conduit::blueprint::mcarray::is_interleaved(coordValues); int stride = isInterleaved ? NUM_COMPS_PER_VERT : 1; axom::StackArray, 3> coordArrays { - axom::ArrayView(coordValues["x"].as_double_ptr(), {vertexCount}, stride), - axom::ArrayView(coordValues["y"].as_double_ptr(), {vertexCount}, stride), - axom::ArrayView(coordValues["z"].as_double_ptr(), {vertexCount}, stride) }; + axom::ArrayView(coordValues["x"].as_double_ptr(), + {vertexCount}, + stride), + axom::ArrayView(coordValues["y"].as_double_ptr(), + {vertexCount}, + stride), + axom::ArrayView(coordValues["z"].as_double_ptr(), + {vertexCount}, + stride)}; for(int i = 0; i < m_cellCount; i++) { @@ -2216,12 +2241,10 @@ class IntersectionShaper : public Shaper for(int k = 0; k < NUM_COMPS_PER_VERT; k++) { vertCoords_host[(i * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT) + - (j * NUM_COMPS_PER_VERT) + k] = - coordArrays[k][vertId]; + (j * NUM_COMPS_PER_VERT) + k] = coordArrays[k][vertId]; } } } - } #endif @@ -2263,7 +2286,6 @@ class IntersectionShaper : public Shaper } } } - } /// Create and return a new volume fraction grid function for the current mesh diff --git a/src/axom/quest/Shaper.cpp b/src/axom/quest/Shaper.cpp index 23be2be7df..d48668dafe 100644 --- a/src/axom/quest/Shaper.cpp +++ b/src/axom/quest/Shaper.cpp @@ -26,14 +26,13 @@ constexpr double Shaper::MAXIMUM_PERCENT_ERROR; constexpr double Shaper::DEFAULT_VERTEX_WELD_THRESHOLD; #if defined(AXOM_USE_MFEM) -Shaper::Shaper(const klee::ShapeSet& shapeSet, - sidre::MFEMSidreDataCollection* dc) +Shaper::Shaper(const klee::ShapeSet& shapeSet, sidre::MFEMSidreDataCollection* dc) : m_shapeSet(shapeSet) , m_dc(dc) { -#if defined(AXOM_USE_MPI) && defined(MFEM_USE_MPI) + #if defined(AXOM_USE_MPI) && defined(MFEM_USE_MPI) m_comm = m_dc->GetComm(); -#endif + #endif m_cellCount = m_dc->GetMesh()->GetNE(); } #endif @@ -50,8 +49,11 @@ Shaper::Shaper(const klee::ShapeSet& shapeSet, m_bpGrp = m_ds.getRoot()->createGroup("bpGrp"); m_bpGrp->importConduitTreeExternal(*bpNode); std::string coordsName = - m_bpGrp->getView(axom::fmt::format("topologies/{}/coordset", m_bpTopo))->getNode().as_string(); - auto* coordsView = m_bpGrp->getView(axom::fmt::format("coordsets/{}", coordsName)); + m_bpGrp->getView(axom::fmt::format("topologies/{}/coordset", m_bpTopo)) + ->getNode() + .as_string(); + auto* coordsView = + m_bpGrp->getView(axom::fmt::format("coordsets/{}", coordsName)); m_cellCount = coordsView->getNumElements(); } #endif @@ -134,9 +136,9 @@ void Shaper::loadShapeInternal(const klee::Shape& shape, "{:-^80}", axom::fmt::format(" Loading shape '{}' ", shape.getName()))); - SLIC_ASSERT_MSG( - this->isValidFormat(shape.getGeometry().getFormat()), - axom::fmt::format("Shape has unsupported format: '{}", shape.getGeometry().getFormat())); + SLIC_ASSERT_MSG(this->isValidFormat(shape.getGeometry().getFormat()), + axom::fmt::format("Shape has unsupported format: '{}", + shape.getGeometry().getFormat())); // Code for discretizing shapes has been factored into DiscreteShape class. DiscreteShape discreteShape(shape, m_dataStore.getRoot(), m_shapeSet.getPath()); @@ -165,9 +167,9 @@ int Shaper::getRank() const double Shaper::allReduceSum(double val) const { #if defined(AXOM_USE_MPI) && defined(MFEM_USE_MPI) - double global; - MPI_Allreduce(&val, &global, 1, MPI_DOUBLE, MPI_SUM, m_comm); - return global; + double global; + MPI_Allreduce(&val, &global, 1, MPI_DOUBLE, MPI_SUM, m_comm); + return global; #endif return val; } diff --git a/src/axom/quest/Shaper.hpp b/src/axom/quest/Shaper.hpp index 04264c0d56..779e663013 100644 --- a/src/axom/quest/Shaper.hpp +++ b/src/axom/quest/Shaper.hpp @@ -27,10 +27,10 @@ #include "axom/quest/DiscreteShape.hpp" #if defined(AXOM_USE_MFEM) -#include "mfem.hpp" + #include "mfem.hpp" #endif #if defined(AXOM_USE_CONDUIT) -#include "conduit_node.hpp" + #include "conduit_node.hpp" #endif #include "axom/quest/interface/internal/mpicomm_wrapper.hpp" @@ -59,7 +59,7 @@ class Shaper */ Shaper(const klee::ShapeSet& shapeSet, conduit::Node* bpMesh, - const std::string& topo=""); + const std::string& topo = ""); #endif virtual ~Shaper() = default; @@ -184,15 +184,15 @@ class Shaper #if defined(AXOM_USE_MFEM) // For mesh represented as MFEMSidreDataCollection - sidre::MFEMSidreDataCollection* m_dc{nullptr}; + sidre::MFEMSidreDataCollection* m_dc {nullptr}; #endif #if defined(AXOM_USE_CONDUIT) // For mesh represented in Conduit or sidre - conduit::Node* m_bpNode{nullptr}; + conduit::Node* m_bpNode {nullptr}; const std::string m_bpTopo; sidre::DataStore m_ds; - axom::sidre::Group* m_bpGrp{nullptr}; + axom::sidre::Group* m_bpGrp {nullptr}; #endif axom::IndexType m_cellCount; From 4b005113a1f95076b656c5f64d69e09ab527ec54 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Tue, 15 Oct 2024 12:21:54 -0700 Subject: [PATCH 013/100] Add ability to generate cartesian mesh as an unstructured hex mesh. --- .../quest/examples/quest_shape_in_memory.cpp | 18 +- src/axom/quest/util/mesh_helpers.cpp | 158 ++++++++++++++++++ src/axom/quest/util/mesh_helpers.hpp | 39 +++++ 3 files changed, 214 insertions(+), 1 deletion(-) diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index a41d9d1e39..be22e143f0 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -383,6 +383,17 @@ void printMfemMeshInfo(mfem::Mesh* mesh, const std::string& prefixMessage = "") slic::flushStreams(); } +axom::sidre::Group* createBoxMesh(axom::sidre::Group* meshGrp) +{ + using BBox3D = primal::BoundingBox; + using Pt3D = primal::Point; + auto res = primal::NumericArray(params.boxResolution.data()); + auto bbox = BBox3D(Pt3D(params.boxMins.data()), Pt3D(params.boxMaxs.data())); + axom::quest::util::make_unstructured_blueprint_box_mesh(meshGrp, bbox, res); + + return meshGrp; +} + /// \brief Utility function to initialize the logger void initializeLogger() { @@ -893,7 +904,8 @@ double sumMaterialVolumes(sidre::MFEMSidreDataCollection* dc, const std::string materialFieldName = axom::fmt::format("vol_frac_{}", material); mfem::GridFunction* volFracGf = dc->GetField(materialFieldName); - axom::ArrayView volFracGfArrayView(volFracGf->GetData(), volFracGf->Size()); + axom::ArrayView volFracGfArrayView(volFracGf->GetData(), + volFracGf->Size()); axom::quest::TempArrayView volFracView(volFracGfArrayView, true); using ReducePolicy = typename axom::execution_space::reduce_policy; @@ -1070,6 +1082,10 @@ int main(int argc, char** argv) using ExecSpace = typename axom::SEQ_EXEC; using ReducePolicy = typename axom::execution_space::reduce_policy; + axom::sidre::Group* compMeshGrp = + createBoxMesh(ds.getRoot()->createGroup("compMesh")); + compMeshGrp->print(); + //--------------------------------------------------------------------------- // Initialize the shaping query object //--------------------------------------------------------------------------- diff --git a/src/axom/quest/util/mesh_helpers.cpp b/src/axom/quest/util/mesh_helpers.cpp index 0bdac364b8..be2ac0fc00 100644 --- a/src/axom/quest/util/mesh_helpers.cpp +++ b/src/axom/quest/util/mesh_helpers.cpp @@ -4,6 +4,11 @@ // SPDX-License-Identifier: (BSD-3-Clause) #include "mesh_helpers.hpp" +#include +#include +#include +#include +#include "axom/core/WhereMacro.hpp" namespace axom { @@ -83,6 +88,159 @@ mfem::Mesh* make_cartesian_mfem_mesh_3D(const primal::BoundingBox& bb #endif // AXOM_USE_MFEM +#if defined(AXOM_USE_SIDRE) + +axom::sidre::Group* make_structured_blueprint_box_mesh( + axom::sidre::Group* meshGrp, + const primal::BoundingBox& bbox, + const primal::NumericArray& res, + const std::string& topologyName, + const std::string& coordsetName) +{ + constexpr int DIM = 3; + + auto ni = res[0]; + auto nj = res[1]; + auto nk = res[2]; + + const axom::StackArray vertsShape {res[0] + 1, + res[1] + 1, + res[2] + 1}; + const auto numVerts = vertsShape[0] * vertsShape[1] * vertsShape[2]; + + auto* topoGrp = meshGrp->createGroup("topologies")->createGroup(topologyName); + auto* coordsetGrp = + meshGrp->createGroup("coordsets")->createGroup(coordsetName); + + topoGrp->createView("type")->setString("structured"); + topoGrp->createView("coordset")->setString(coordsetName); + auto* dimsGrp = topoGrp->createGroup("elements/dims"); + dimsGrp->createViewScalar("i", ni); + dimsGrp->createViewScalar("j", nj); + dimsGrp->createViewScalar("k", nk); + + coordsetGrp->createView("type")->setString("explicit"); + auto* valuesGrp = coordsetGrp->createGroup("values"); + auto* xVu = + valuesGrp->createViewAndAllocate("x", + axom::sidre::DataTypeId::FLOAT64_ID, + numVerts); + auto* yVu = + valuesGrp->createViewAndAllocate("y", + axom::sidre::DataTypeId::FLOAT64_ID, + numVerts); + auto* zVu = + valuesGrp->createViewAndAllocate("z", + axom::sidre::DataTypeId::FLOAT64_ID, + numVerts); + + const axom::MDMapping vertMapping(vertsShape, axom::ArrayStrideOrder::ROW); + axom::ArrayView xView(xVu->getData(), + vertsShape, + vertMapping.strides()); + axom::ArrayView yView(yVu->getData(), + vertsShape, + vertMapping.strides()); + axom::ArrayView zView(zVu->getData(), + vertsShape, + vertMapping.strides()); + + double dx = bbox.getMax()[0] - bbox.getMin()[0]; + double dy = bbox.getMax()[1] - bbox.getMin()[1]; + double dz = bbox.getMax()[2] - bbox.getMin()[2]; + for(int k = 0; k < nk + 1; ++k) + { + for(int j = 0; j < nj + 1; ++j) + { + for(int i = 0; i < ni + 1; ++i) + { + xView(i, j, k) = bbox.getMin()[0] + i * dx; + yView(i, j, k) = bbox.getMin()[1] + j * dy; + zView(i, j, k) = bbox.getMin()[2] + k * dz; + } + } + } + + #if defined(AXOM_DEBUG) && defined(AXOM_USE_CONDUIT) + { + conduit::Node info; + bool isValid = verifyBlueprintMesh(meshGrp, info); + SLIC_ASSERT_MSG(isValid, "Internal error: Generated mesh is invalid."); + } + #endif + + return meshGrp; +} + +#if defined(AXOM_USE_CONDUIT) +axom::sidre::Group* make_unstructured_blueprint_box_mesh( + axom::sidre::Group* meshGrp, + const primal::BoundingBox& bbox, + const primal::NumericArray& res, + const std::string& topologyName, + const std::string& coordsetName) +{ + make_structured_blueprint_box_mesh(meshGrp, bbox, res, topologyName, coordsetName); + convert_blueprint_structured_explicit_to_unstructured(meshGrp, topologyName); + return meshGrp; +} + +void convert_blueprint_structured_explicit_to_unstructured( + axom::sidre::Group* meshGrp, + const std::string& topoName) +{ + const std::string& coordsetName = + meshGrp->getView(axom::fmt::format("topologies/{}/coordset", topoName)) + ->getString(); + + // Convert mesh to conduit::Node to use conduit's blueprint support. + conduit::Node info; + conduit::Node curMesh; + meshGrp->createNativeLayout(curMesh); + const conduit::Node& curTopo = + curMesh.fetch_existing(axom::fmt::format("topologies/{}", topoName)); + SLIC_ASSERT( + conduit::blueprint::mesh::topology::structured::verify(curTopo, info)); + + // Use conduit to convert to unstructured. + conduit::Node newTopo; + conduit::Node newCoords; + conduit::blueprint::mesh::topology::structured::to_unstructured(curTopo, + newTopo, + newCoords); + + // Copy unstructured back into meshGrp. + meshGrp->getGroup("topologies")->destroyGroup(topoName); + meshGrp->getGroup("topologies")->createGroup(topoName)->importConduitTree(newTopo); + meshGrp->getGroup("coordsets")->destroyGroup(coordsetName); + meshGrp->getGroup("coordsets") + ->createGroup(coordsetName) + ->importConduitTree(newCoords); + + meshGrp->getView(axom::fmt::format("topologies/{}/coordset", topoName)) + ->setString(coordsetName); + + #if defined(AXOM_DEBUG) + conduit::Node newMesh; + meshGrp->createNativeLayout(newMesh); + SLIC_ASSERT(conduit::blueprint::mesh::verify(newMesh, info)); + #endif + + return; +} + +bool verifyBlueprintMesh(const axom::sidre::Group* meshGrp, conduit::Node info) +{ + conduit::Node meshNode; + meshGrp->createNativeLayout(meshNode); + bool isValid = conduit::blueprint::mesh::verify(meshNode, info); + if(!isValid) info.print(); + return isValid; +} +#endif + +#endif + } // namespace util } // namespace quest } // namespace axom diff --git a/src/axom/quest/util/mesh_helpers.hpp b/src/axom/quest/util/mesh_helpers.hpp index 9bafb4375b..a1096ca68e 100644 --- a/src/axom/quest/util/mesh_helpers.hpp +++ b/src/axom/quest/util/mesh_helpers.hpp @@ -8,6 +8,7 @@ #include "axom/config.hpp" #include "axom/primal.hpp" +#include "axom/sidre.hpp" #ifdef AXOM_USE_MFEM #include "mfem.hpp" @@ -55,6 +56,44 @@ mfem::Mesh* make_cartesian_mfem_mesh_3D(const primal::BoundingBox& bb const primal::NumericArray& res, int polynomial_order, bool reorder_space_filling = true); +#endif + +#if defined(AXOM_USE_SIDRE) +/*! + /brief Creates a 3D Cartesian mfem mesh lying within a given bounding box + + \param meshGrp Put the mesh in this Group + \param bbox The bounding box for the mesh + \param res The resolution of the mesh + \param topologyName Name of the blueprint topoloyy to use. + \param coordsetName Name of the blueprint coordset to use. + + \return The meshGrp pointer +*/ +axom::sidre::Group* make_structured_blueprint_box_mesh( + axom::sidre::Group* meshGrp, + const primal::BoundingBox& bbox, + const primal::NumericArray& res, + const std::string& topologyName = "mesh", + const std::string& coordsetName = "coords"); + +axom::sidre::Group* make_unstructured_blueprint_box_mesh( + axom::sidre::Group* meshGrp, + const primal::BoundingBox& bbox, + const primal::NumericArray& res, + const std::string& topologyName = "mesh", + const std::string& coordsetName = "coords"); + +void convert_blueprint_structured_explicit_to_unstructured( + axom::sidre::Group* meshGrp, + const std::string& topoName); + + #if defined(AXOM_USE_CONDUIT) +/*! + \brief Check if blueprint mesh is valid. +*/ +bool verifyBlueprintMesh(const axom::sidre::Group* meshGrp, conduit::Node info); + #endif #endif From d5e23b470222d6e7bd41805dd44b8318251f7e64 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Tue, 15 Oct 2024 16:12:42 -0700 Subject: [PATCH 014/100] Progress toward using blueprint mesh. --- src/axom/quest/IntersectionShaper.hpp | 4 +-- src/axom/quest/Shaper.cpp | 28 ++++++++++++++++++- src/axom/quest/Shaper.hpp | 14 ++++++++-- .../quest/examples/quest_shape_in_memory.cpp | 7 ++++- src/axom/sidre/core/Group.hpp | 2 ++ 5 files changed, 48 insertions(+), 7 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index ea05b70f24..48ea767793 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -327,9 +327,9 @@ class IntersectionShaper : public Shaper stored in a Conduit Node. */ IntersectionShaper(const klee::ShapeSet& shapeSet, - conduit::Node* bpMesh, + sidre::Group* bpGrp, const std::string& topo = "") - : Shaper(shapeSet, bpMesh, topo) + : Shaper(shapeSet, bpGrp, topo) , m_free_mat_name("free") { } #endif diff --git a/src/axom/quest/Shaper.cpp b/src/axom/quest/Shaper.cpp index d48668dafe..5dd75add36 100644 --- a/src/axom/quest/Shaper.cpp +++ b/src/axom/quest/Shaper.cpp @@ -37,13 +37,39 @@ Shaper::Shaper(const klee::ShapeSet& shapeSet, sidre::MFEMSidreDataCollection* d } #endif +Shaper::Shaper(const klee::ShapeSet& shapeSet, + sidre::Group* bpGrp, + const std::string& topo) + : m_shapeSet(shapeSet) + , m_bpGrp(bpGrp) + , m_bpTopo(topo.empty() ? bpGrp->getGroup("topologies")->getGroupName(0) : topo) + , m_bpNode(nullptr) + , m_comm(MPI_COMM_WORLD) +{ + SLIC_ASSERT(m_bpTopo != sidre::InvalidName); + + auto* topologies = m_bpGrp->getGroup("topologies"); + auto* topoGrp = topologies->getGroup(m_bpTopo); + auto* coordsetName = topoGrp->getView("coordset"); + std::string coordsName = coordsetName->getString(); +#if 0 + std::string coordsName = + m_bpGrp->getView(axom::fmt::format("topologies/{}/coordset", m_bpTopo)) + ->getString(); +#endif + auto* coordsView = + m_bpGrp->getView(axom::fmt::format("coordsets/{}/values/x", coordsName)); + m_cellCount = coordsView->getNumElements(); +} + #if defined(AXOM_USE_CONDUIT) Shaper::Shaper(const klee::ShapeSet& shapeSet, conduit::Node* bpNode, const std::string& topo) : m_shapeSet(shapeSet) - , m_bpNode(bpNode) + , m_bpGrp(nullptr) , m_bpTopo(topo) + , m_bpNode(bpNode) , m_comm(MPI_COMM_WORLD) { m_bpGrp = m_ds.getRoot()->createGroup("bpGrp"); diff --git a/src/axom/quest/Shaper.hpp b/src/axom/quest/Shaper.hpp index 779e663013..269d22e6c5 100644 --- a/src/axom/quest/Shaper.hpp +++ b/src/axom/quest/Shaper.hpp @@ -52,13 +52,21 @@ class Shaper Shaper(const klee::ShapeSet& shapeSet, sidre::MFEMSidreDataCollection* dc); #endif + /*! + @brief Construct Shaper to operate on a blueprint-formatted mesh + stored in a sidre Group. + */ + Shaper(const klee::ShapeSet& shapeSet, + sidre::Group* bpMesh, + const std::string& topo = ""); + #if defined(AXOM_USE_CONDUIT) /*! @brief Construct Shaper to operate on a blueprint-formatted mesh stored in a Conduit Node. */ Shaper(const klee::ShapeSet& shapeSet, - conduit::Node* bpMesh, + conduit::Node* bpNode, const std::string& topo = ""); #endif @@ -189,10 +197,10 @@ class Shaper #if defined(AXOM_USE_CONDUIT) // For mesh represented in Conduit or sidre - conduit::Node* m_bpNode {nullptr}; - const std::string m_bpTopo; sidre::DataStore m_ds; axom::sidre::Group* m_bpGrp {nullptr}; + const std::string m_bpTopo; + conduit::Node* m_bpNode {nullptr}; #endif axom::IndexType m_cellCount; diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index be22e143f0..4fd5223fbb 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -383,13 +383,17 @@ void printMfemMeshInfo(mfem::Mesh* mesh, const std::string& prefixMessage = "") slic::flushStreams(); } +const std::string topoName = "mesh"; +const std::string coordsetName = "coords"; + axom::sidre::Group* createBoxMesh(axom::sidre::Group* meshGrp) { using BBox3D = primal::BoundingBox; using Pt3D = primal::Point; auto res = primal::NumericArray(params.boxResolution.data()); auto bbox = BBox3D(Pt3D(params.boxMins.data()), Pt3D(params.boxMaxs.data())); - axom::quest::util::make_unstructured_blueprint_box_mesh(meshGrp, bbox, res); + axom::quest::util::make_unstructured_blueprint_box_mesh(meshGrp, bbox, res, + topoName, coordsetName); return meshGrp; } @@ -1091,6 +1095,7 @@ int main(int argc, char** argv) //--------------------------------------------------------------------------- AXOM_ANNOTATE_BEGIN("setup shaping problem"); quest::Shaper* shaper = nullptr; + // shaper = new quest::IntersectionShaper(shapeSet, compMeshGrp); shaper = new quest::IntersectionShaper(shapeSet, &shapingDC); SLIC_ASSERT_MSG(shaper != nullptr, "Invalid shaping method selected!"); diff --git a/src/axom/sidre/core/Group.hpp b/src/axom/sidre/core/Group.hpp index 02371765c7..9e55a64ac5 100644 --- a/src/axom/sidre/core/Group.hpp +++ b/src/axom/sidre/core/Group.hpp @@ -1789,11 +1789,13 @@ class Group /*! * \brief Detach Child Group with given name from this Group. + * \return the detached Group. */ Group* detachGroup(const std::string& name); /*! * \brief Detach Child Group with given index from this Group. + * \return the detached Group. */ Group* detachGroup(IndexType idx); From ccf29b0fe8584c6337108b277278f340c9cba9ee Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Wed, 16 Oct 2024 01:19:51 -0700 Subject: [PATCH 015/100] Remove code for MFEM mesh that's been replaced by more general code. --- src/axom/quest/IntersectionShaper.hpp | 52 ++------------------------- 1 file changed, 3 insertions(+), 49 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 48ea767793..e09fafcbe5 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -1021,19 +1021,7 @@ class IntersectionShaper : public Shaper */ void populateMaterials() { - #if 1 std::vector materialNames = getMaterialNames(); - #else - std::vector materialNames; - for(auto it : this->getDC()->GetFieldMap()) - { - std::string materialName = fieldNameToMaterialName(it.first); - if(!materialName.empty()) - { - materialNames.emplace_back(materialName); - } - } - #endif // Add any of these existing fields to this class' bookkeeping. for(const auto& materialName : materialNames) { @@ -1190,10 +1178,9 @@ class IntersectionShaper : public Shaper } else { - // Include all materials except those in "does_not_replace". - // We'll also sort them by material number since the field map - // sorts them by name rather than order added. - #if 1 + // Include all materials except those in "does_not_replace". + // We'll also sort them by material number since the field map + // sorts them by name rather than order added. std::vector materialNames = getMaterialNames(); for(auto name : materialNames) { @@ -1225,39 +1212,6 @@ class IntersectionShaper : public Shaper } } } - #else - for(auto it : this->getDC()->GetFieldMap()) - { - // Check whether the field name looks like a VF field (and is not the - // "free" field, which we handle specially) - std::string name = fieldNameToMaterialName(it.first); - if(!name.empty() && name != m_free_mat_name) - { - // See if the field is in the exclusion list. For the normal - // case, the list is empty so we'd add the material. - auto it2 = std::find(shape.getMaterialsNotReplaced().cbegin(), - shape.getMaterialsNotReplaced().cend(), - name); - // The field is not in the exclusion list so add it to vfs. - if(it2 == shape.getMaterialsNotReplaced().cend()) - { - // Do not add the current shape material since it should - // not end up in updateVFs. - if(name != shape.getMaterial()) - { - gf_order_by_matnumber.emplace_back(getMaterial(name)); - } - } - else - { - // The material was in the exclusion list. This means that - // cannot write to materials that have volume fraction in - // that zone. - excludeVFs.emplace_back(getMaterial(name).first); - } - } - } - #endif } // Sort eligible update materials by material number. std::sort(gf_order_by_matnumber.begin(), From add072e86047402a5729fb4d0d7c8a801eeafdb3 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Wed, 16 Oct 2024 07:54:46 -0700 Subject: [PATCH 016/100] Further progress toward using Blueprint mesh in test. Add blueprint support to shaper code and test code. --- src/axom/quest/IntersectionShaper.hpp | 146 ++++++------- src/axom/quest/Shaper.cpp | 22 -- src/axom/quest/Shaper.hpp | 11 +- .../quest/examples/quest_shape_in_memory.cpp | 196 ++++++++++++++++-- 4 files changed, 248 insertions(+), 127 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index e09fafcbe5..769d5acca8 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -970,47 +970,6 @@ class IntersectionShaper : public Shaper return name; } - /*! - * \brief Gets the grid function and material number for a material name. - * - * \param materialName The name of the material. - * - * \return A pair containing the associated grid function and material - * number (its order in the list). - */ - // std::pair getMaterial(const std::string& materialName) - std::pair, int> getMaterial(const std::string& materialName) - { - // If we already know about the material, return it. - for(size_t i = 0; i < m_vf_material_names.size(); i++) - { - if(m_vf_material_names[i] == materialName) - { - return std::make_pair(m_vf_grid_functions[i], static_cast(i)); - } - } - - // Get or create the volume fraction field for this shape's material - auto materialVolFracName = materialNameToFieldName(materialName); - - bool newData = !hasData(materialVolFracName); - - auto matVolFrac = getScalarCellData(materialVolFracName); - if(newData) - { - // Zero out the volume fractions (on host). - // memset(matVolFrac->begin(), 0, matVolFrac->Size() * sizeof(double)); - memset(matVolFrac.data(), 0, matVolFrac.size() * sizeof(double)); - } - - // Add the material to our vectors. - int idx = static_cast(m_vf_grid_functions.size()); - m_vf_grid_functions.push_back(matVolFrac); - m_vf_material_names.push_back(materialName); - - return std::make_pair(matVolFrac, idx); - } - /*! * \brief Scans the grid functions in the data collection and creates * a material entry for any that do not already exist. We maintain @@ -1498,6 +1457,79 @@ class IntersectionShaper : public Shaper // Implementation here -- not sure if this will require anything for intersection-based shaping } + std::vector getMaterialNames() const + { + std::vector materialNames; +#if defined(AXOM_USE_MFEM) + if(m_dc) + { + for(auto it : this->getDC()->GetFieldMap()) + { + std::string materialName = fieldNameToMaterialName(it.first); + if(!materialName.empty()) + { + materialNames.emplace_back(materialName); + } + } + } +#endif +#if defined(AXOM_USE_CONDUIT) + if(m_bpGrp) + { + auto fieldsGrp = m_bpGrp->getGroup("fields"); + for(auto& group : fieldsGrp->groups()) + { + std::string materialName = fieldNameToMaterialName(group.getName()); + if(!materialName.empty()) + { + materialNames.emplace_back(materialName); + } + } + } +#endif + return materialNames; + } + + /*! + * \brief Gets the grid function and material number for a material name. + * + * \param materialName The name of the material. + * + * \return A pair containing the associated grid function and material + * number (its order in the list). + */ + std::pair, int> getMaterial(const std::string& materialName) + { + // If we already know about the material, return it. + for(size_t i = 0; i < m_vf_material_names.size(); i++) + { + if(m_vf_material_names[i] == materialName) + { + return std::make_pair(m_vf_grid_functions[i], static_cast(i)); + } + } + + // Get or create the volume fraction field for this shape's material + auto materialVolFracName = materialNameToFieldName(materialName); + + bool newData = !hasData(materialVolFracName); + + auto matVolFrac = getScalarCellData(materialVolFracName); + if(newData) + { + // Zero out the volume fractions (on host). + // memset(matVolFrac->begin(), 0, matVolFrac->Size() * sizeof(double)); + memset(matVolFrac.data(), 0, matVolFrac.size() * sizeof(double)); + } + + // Add the material to our vectors. + int idx = static_cast(m_vf_grid_functions.size()); + m_vf_grid_functions.push_back(matVolFrac); + m_vf_material_names.push_back(materialName); + + return std::make_pair(matVolFrac, idx); + } + private: /*! * \brief Filter the input mesh so it does not have any degenerate segments @@ -2023,38 +2055,6 @@ class IntersectionShaper : public Shaper return rval; } - std::vector getMaterialNames() - { - std::vector materialNames; -#if defined(AXOM_USE_MFEM) - if(m_dc) - { - for(auto it : this->getDC()->GetFieldMap()) - { - std::string materialName = fieldNameToMaterialName(it.first); - if(!materialName.empty()) - { - materialNames.emplace_back(materialName); - } - } - } -#elif defined(AXOM_USE_CONDUIT) - if(m_bpGrp) - { - auto fieldsGrp = m_bpGrp->getGroup("fields"); - for(auto& group : fieldsGrp->groups()) - { - std::string materialName = fieldNameToMaterialName(group.getName()); - if(!materialName.empty()) - { - materialNames.emplace_back(materialName); - } - } - } -#endif - return materialNames; - } - public: // This should be private, but NVCC complains unless its public. template diff --git a/src/axom/quest/Shaper.cpp b/src/axom/quest/Shaper.cpp index 5dd75add36..163566a808 100644 --- a/src/axom/quest/Shaper.cpp +++ b/src/axom/quest/Shaper.cpp @@ -62,28 +62,6 @@ Shaper::Shaper(const klee::ShapeSet& shapeSet, m_cellCount = coordsView->getNumElements(); } -#if defined(AXOM_USE_CONDUIT) -Shaper::Shaper(const klee::ShapeSet& shapeSet, - conduit::Node* bpNode, - const std::string& topo) - : m_shapeSet(shapeSet) - , m_bpGrp(nullptr) - , m_bpTopo(topo) - , m_bpNode(bpNode) - , m_comm(MPI_COMM_WORLD) -{ - m_bpGrp = m_ds.getRoot()->createGroup("bpGrp"); - m_bpGrp->importConduitTreeExternal(*bpNode); - std::string coordsName = - m_bpGrp->getView(axom::fmt::format("topologies/{}/coordset", m_bpTopo)) - ->getNode() - .as_string(); - auto* coordsView = - m_bpGrp->getView(axom::fmt::format("coordsets/{}", coordsName)); - m_cellCount = coordsView->getNumElements(); -} -#endif - void Shaper::setSamplesPerKnotSpan(int nSamples) { using axom::utilities::clampLower; diff --git a/src/axom/quest/Shaper.hpp b/src/axom/quest/Shaper.hpp index 269d22e6c5..c6cf09a2f5 100644 --- a/src/axom/quest/Shaper.hpp +++ b/src/axom/quest/Shaper.hpp @@ -60,16 +60,6 @@ class Shaper sidre::Group* bpMesh, const std::string& topo = ""); -#if defined(AXOM_USE_CONDUIT) - /*! - @brief Construct Shaper to operate on a blueprint-formatted mesh - stored in a Conduit Node. - */ - Shaper(const klee::ShapeSet& shapeSet, - conduit::Node* bpNode, - const std::string& topo = ""); -#endif - virtual ~Shaper() = default; public: @@ -99,6 +89,7 @@ class Shaper #ifdef AXOM_USE_MFEM sidre::MFEMSidreDataCollection* getDC() { return m_dc; } + const sidre::MFEMSidreDataCollection* getDC() const { return m_dc; } #endif /*! diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index 4fd5223fbb..574de9f331 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -392,8 +392,11 @@ axom::sidre::Group* createBoxMesh(axom::sidre::Group* meshGrp) using Pt3D = primal::Point; auto res = primal::NumericArray(params.boxResolution.data()); auto bbox = BBox3D(Pt3D(params.boxMins.data()), Pt3D(params.boxMaxs.data())); - axom::quest::util::make_unstructured_blueprint_box_mesh(meshGrp, bbox, res, - topoName, coordsetName); + axom::quest::util::make_unstructured_blueprint_box_mesh(meshGrp, + bbox, + res, + topoName, + coordsetName); return meshGrp; } @@ -889,6 +892,100 @@ axom::sidre::View* getElementVolumes( return volSidreView; } +/*! + @brief Return the element volumes as a sidre::View. + + If it doesn't exist, allocate and compute it. + \post The volume data is in the blueprint field \c volFieldName. + + Most of this is lifted from IntersectionShaper::runShapeQueryImpl. +*/ +template +axom::sidre::View* getElementVolumes( + sidre::Group* meshGrp, + const std::string& volFieldName = std::string("elementVolumes")) +{ + std::unique_ptr> mesh{ + dynamic_cast*>(axom::mint::getMesh(meshGrp, topoName)) }; + using HexahedronType = axom::primal::Hexahedron; + + axom::sidre::View* volSidreView = meshGrp->getView(axom::fmt::format("fields/{}/values", volFieldName)); + if(volSidreView == nullptr) + { + const axom::IndexType cellCount = meshGrp->getView(axom::fmt::format("coordsets/{}/values/x", coordsetName))->getNumElements(); + + constexpr int NUM_VERTS_PER_HEX = 8; + constexpr int NUM_COMPS_PER_VERT = 3; + constexpr double ZERO_THRESHOLD = 1.e-10; + + axom::Array vertCoords(cellCount * NUM_VERTS_PER_HEX, + cellCount * NUM_VERTS_PER_HEX); + auto vertCoordsView = vertCoords.view(); + + // This runs only only on host, because the mfem::Mesh only uses host memory, I think. + for(axom::IndexType cellIdx = 0; cellIdx < cellCount; ++cellIdx) + { + // Get the indices of this element's vertices + axom::IndexType* verts = mesh->getCellNodeIDs(cellIdx); + mesh->getCellNodeIDs(cellIdx, verts); + + // Get the coordinates for the vertices + for(int j = 0; j < NUM_VERTS_PER_HEX; ++j) + { + int vertIdx = cellIdx * NUM_VERTS_PER_HEX + j; + for(int k = 0; k < NUM_COMPS_PER_VERT; k++) + { + vertCoordsView[vertIdx][k] = mesh->getNodeCoordinate(vertIdx, k); + } + } + } + + // Set vertex coords to zero if within threshold. + // (I don't know why we do this. I'm following examples.) + axom::ArrayView flatCoordsView( + (double*)vertCoords.data(), + vertCoords.size() * Point3D::dimension()); + assert(flatCoordsView.size() == cellCount * NUM_VERTS_PER_HEX * 3); + axom::for_all( + cellCount * 3, + AXOM_LAMBDA(axom::IndexType i) { + if(axom::utilities::isNearlyEqual(flatCoordsView[i], 0.0, ZERO_THRESHOLD)) + { + flatCoordsView[i] = 0.0; + } + }); + + // Initialize hexahedral elements. + axom::Array hexes(cellCount, cellCount); + auto hexesView = hexes.view(); + axom::for_all( + cellCount, + AXOM_LAMBDA(axom::IndexType cellIdx) { + // Set each hexahedral element vertices + hexesView[cellIdx] = HexahedronType(); + for(int j = 0; j < NUM_VERTS_PER_HEX; ++j) + { + int vertIndex = (cellIdx * NUM_VERTS_PER_HEX) + j; + auto& hex = hexesView[cellIdx]; + hex[j] = vertCoordsView[vertIndex]; + } + }); // end of loop to initialize hexahedral elements and bounding boxes + + // Allocate and populate cell volumes. + axom::sidre::Group* volGrp = meshGrp->createGroup(axom::fmt::format("fields/{}", volFieldName)); + volSidreView = volGrp->createViewAndAllocate("values", axom::sidre::DataTypeId::FLOAT64_ID, cellCount); + axom::ArrayView volView(volSidreView->getData(), + volSidreView->getNumElements()); + axom::for_all( + cellCount, + AXOM_LAMBDA(axom::IndexType cellIdx) { + volView[cellIdx] = hexesView[cellIdx].volume(); + }); + } + + return volSidreView; +} + /*! @brief Return global sum of volume of the given material. */ @@ -927,6 +1024,42 @@ double sumMaterialVolumes(sidre::MFEMSidreDataCollection* dc, return globalVol; } +template +double sumMaterialVolumes(sidre::Group* meshGrp, + const std::string& material) +{ + std::unique_ptr> mesh{ + dynamic_cast*>(axom::mint::getMesh(meshGrp, topoName)) }; + int const cellCount = mesh->getNumberOfCells(); + + // Get cell volumes from dc. + axom::sidre::View* elementVols = getElementVolumes(meshGrp); + axom::ArrayView elementVolsView(elementVols->getData(), + elementVols->getNumElements()); + + // Get material volume fractions + const std::string materialFieldName = + axom::fmt::format("vol_frac_{}", material); + axom::sidre::View* volFrac = meshGrp->getView(axom::fmt::format("field/{}/values", materialFieldName)); + axom::ArrayView volFracArrayView(volFrac->getArray(), + cellCount); + axom::quest::TempArrayView volFracView(volFracArrayView, true); + + using ReducePolicy = typename axom::execution_space::reduce_policy; + RAJA::ReduceSum localVol(0); + axom::for_all( + cellCount, + AXOM_LAMBDA(axom::IndexType i) { + localVol += volFracView[i] * elementVolsView[i]; + }); + + double globalVol = localVol.get(); +#ifdef AXOM_USE_MPI + MPI_Allreduce(MPI_IN_PLACE, &globalVol, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); +#endif + return globalVol; +} + //------------------------------------------------------------------------------ int main(int argc, char** argv) { @@ -1094,10 +1227,10 @@ int main(int argc, char** argv) // Initialize the shaping query object //--------------------------------------------------------------------------- AXOM_ANNOTATE_BEGIN("setup shaping problem"); - quest::Shaper* shaper = nullptr; - // shaper = new quest::IntersectionShaper(shapeSet, compMeshGrp); - shaper = new quest::IntersectionShaper(shapeSet, &shapingDC); - SLIC_ASSERT_MSG(shaper != nullptr, "Invalid shaping method selected!"); + bool useBp = false; + quest::IntersectionShaper* shaper = (useBp) ? + new quest::IntersectionShaper(shapeSet, compMeshGrp) : + new quest::IntersectionShaper(shapeSet, &shapingDC); // Set generic parameters for the base Shaper instance shaper->setVertexWeldThreshold(params.weldThresh); @@ -1110,7 +1243,12 @@ int main(int argc, char** argv) // Associate any fields that begin with "vol_frac" with "material" so when // the data collection is written, a matset will be created. - shaper->getDC()->AssociateMaterialSet("vol_frac", "material"); + if (useBp) + { + // TODO: What is the blueprint replacement for AssociateMaterialSet? + } else { + shaper->getDC()->AssociateMaterialSet("vol_frac", "material"); + } // Set specific parameters here for IntersectionShaper if(auto* intersectionShaper = dynamic_cast(shaper)) @@ -1182,31 +1320,44 @@ int main(int argc, char** argv) // Compute and print volumes of each material's volume fraction //--------------------------------------------------------------------------- using axom::utilities::string::startsWith; - for(auto& kv : shaper->getDC()->GetFieldMap()) + if (useBp) { - if(startsWith(kv.first, "vol_frac_")) + // TODO: What is the blueprint equivalent? + std::vector materialNames = shaper->getMaterialNames(); + for (const auto& materialName : materialNames) { - const auto mat_name = kv.first.substr(9); - auto* gf = kv.second; + auto p = shaper->getMaterial(materialName); + axom::ArrayView& materialField = p.first; + int materialIdx = p.second; + + } + } else { + for(auto& kv : shaper->getDC()->GetFieldMap()) + { + if(startsWith(kv.first, "vol_frac_")) + { + const auto mat_name = kv.first.substr(9); + auto* gf = kv.second; - mfem::ConstantCoefficient one(1.0); - mfem::LinearForm vol_form(gf->FESpace()); - vol_form.AddDomainIntegrator(new mfem::DomainLFIntegrator(one)); - vol_form.Assemble(); + mfem::ConstantCoefficient one(1.0); + mfem::LinearForm vol_form(gf->FESpace()); + vol_form.AddDomainIntegrator(new mfem::DomainLFIntegrator(one)); + vol_form.Assemble(); - const double volume = shaper->allReduceSum(*gf * vol_form); + const double volume = shaper->allReduceSum(*gf * vol_form); - SLIC_INFO(axom::fmt::format(axom::utilities::locale(), - "Volume of material '{}' is {:.6Lf}", - mat_name, - volume)); + SLIC_INFO(axom::fmt::format(axom::utilities::locale(), + "Volume of material '{}' is {:.6Lf}", + mat_name, + volume)); + } } } AXOM_ANNOTATE_END("adjust"); int failCounts = 0; - auto* volFracGroups = + axom::sidre::Group* volFracGroups = shapingDC.GetBPGroup()->getGroup("matsets/material/volume_fractions"); //--------------------------------------------------------------------------- @@ -1284,7 +1435,8 @@ int main(int argc, char** argv) shapeMesh->getNumberOfCells()))); const std::string& materialName = shape.getMaterial(); - double shapeVol = + double shapeVol = useBp ? + sumMaterialVolumes(compMeshGrp, materialName) : sumMaterialVolumes(&shapingDC, materialName); double correctShapeVol = params.testShape == "plane" ? params.boxMeshVolume() / 2 : shapeMeshVol; From 6d3baf101e6fe5a76f91e2d17c0356cd632d4644 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Thu, 17 Oct 2024 02:19:19 -0700 Subject: [PATCH 017/100] Debug new code supporting blueprint mesh. --- src/axom/quest/IntersectionShaper.hpp | 41 ++++++++++--------- src/axom/quest/Shaper.cpp | 21 +++++----- .../quest/examples/quest_shape_in_memory.cpp | 37 +++++++++-------- 3 files changed, 51 insertions(+), 48 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 769d5acca8..e4c7cb3dd6 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -2145,28 +2145,29 @@ class IntersectionShaper : public Shaper constexpr int NUM_VERTS_PER_HEX = 8; constexpr int NUM_COMPS_PER_VERT = 3; - const auto* topoGrp = - m_bpGrp->getGroup(axom::fmt::format("topologies/{}", m_bpTopo)); - std::string coordsetName = topoGrp->getView("coordset")->getString(); + conduit::Node meshNode; + m_bpGrp->createNativeLayout(meshNode); + + const conduit::Node& topoNode = meshNode.fetch_existing(axom::fmt::format("topologies/{}", m_bpTopo)); + const std::string coordsetName = topoNode.fetch_existing("coordset").as_string(); // Assume unstructured and hexahedral - SLIC_ASSERT(std::string(topoGrp->getView("type")->getString()) == - "unstructured"); - SLIC_ASSERT(std::string(topoGrp->getView("elements/shape")->getString()) == - "hex"); - - const auto connGrp = topoGrp->getView("elements/connectivity"); - axom::ArrayView conn(connGrp->getNode().as_double_ptr(), - m_cellCount, - NUM_VERTS_PER_HEX); - - const auto* coordGrp = - m_bpGrp->getGroup(axom::fmt::format("coordsets/{}", coordsetName)); - const conduit::Node& coordValues = coordGrp->getView("values")->getNode(); - - // Assume explicit coordinates. - SLIC_ASSERT(std::string(coordGrp->getView("type")->getString()) == - "explicit"); + SLIC_ASSERT(topoNode["type"].as_string() == "unstructured"); + SLIC_ASSERT(topoNode["elements/shape"].as_string() == "hex"); + + const auto connNode = topoNode["elements/connectivity"]; + SLIC_ASSERT_MSG(connNode.dtype().is_int32(), + "IntersectionShaper internal error: missing logic to handle multiple types."); + const std::int32_t* connPtr = connNode.as_int32_ptr(); + axom::ArrayView conn(connPtr, + m_cellCount, + NUM_VERTS_PER_HEX); + + // Put in Node so we can use conduit::blueprint utilities. + const conduit::Node& coordNode = meshNode[axom::fmt::format("coordsets/{}", coordsetName)]; + // coordGrp->createNativeLayout(coordNode); + + const conduit::Node& coordValues = coordNode.fetch_existing("values"); axom::IndexType vertexCount = coordValues["x"].dtype().number_of_elements(); bool isInterleaved = conduit::blueprint::mcarray::is_interleaved(coordValues); diff --git a/src/axom/quest/Shaper.cpp b/src/axom/quest/Shaper.cpp index 163566a808..e2365e47c0 100644 --- a/src/axom/quest/Shaper.cpp +++ b/src/axom/quest/Shaper.cpp @@ -11,6 +11,7 @@ #include "axom/quest/interface/internal/QuestHelpers.hpp" #include "axom/quest/Shaper.hpp" #include "axom/quest/DiscreteShape.hpp" +#include "conduit_blueprint_mesh.hpp" #include "axom/fmt.hpp" @@ -48,18 +49,16 @@ Shaper::Shaper(const klee::ShapeSet& shapeSet, { SLIC_ASSERT(m_bpTopo != sidre::InvalidName); - auto* topologies = m_bpGrp->getGroup("topologies"); - auto* topoGrp = topologies->getGroup(m_bpTopo); - auto* coordsetName = topoGrp->getView("coordset"); - std::string coordsName = coordsetName->getString(); -#if 0 - std::string coordsName = - m_bpGrp->getView(axom::fmt::format("topologies/{}/coordset", m_bpTopo)) - ->getString(); + conduit::Node meshNode; + bpGrp->createNativeLayout(meshNode); +#if defined(AXOM_DEBUG) + conduit::Node info; + SLIC_ASSERT(conduit::blueprint::mesh::verify(meshNode, info)); #endif - auto* coordsView = - m_bpGrp->getView(axom::fmt::format("coordsets/{}/values/x", coordsName)); - m_cellCount = coordsView->getNumElements(); + m_cellCount = + conduit::blueprint::mesh::topology::length( + meshNode.fetch_existing( + axom::fmt::format("topologies/{}", m_bpTopo))); } void Shaper::setSamplesPerKnotSpan(int nSamples) diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index 574de9f331..fff650e606 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -397,6 +397,12 @@ axom::sidre::Group* createBoxMesh(axom::sidre::Group* meshGrp) res, topoName, coordsetName); + #if defined(AXOM_DEBUG) + conduit::Node meshNode, info; + meshGrp->createNativeLayout(meshNode); +meshNode.print(); + SLIC_ASSERT(conduit::blueprint::mesh::verify(meshNode, info)); + #endif return meshGrp; } @@ -1228,9 +1234,9 @@ int main(int argc, char** argv) //--------------------------------------------------------------------------- AXOM_ANNOTATE_BEGIN("setup shaping problem"); bool useBp = false; - quest::IntersectionShaper* shaper = (useBp) ? - new quest::IntersectionShaper(shapeSet, compMeshGrp) : - new quest::IntersectionShaper(shapeSet, &shapingDC); + std::shared_ptr shaper = useBp ? + std::make_shared(shapeSet, compMeshGrp) : + std::make_shared(shapeSet, &shapingDC); // Set generic parameters for the base Shaper instance shaper->setVertexWeldThreshold(params.weldThresh); @@ -1251,19 +1257,16 @@ int main(int argc, char** argv) } // Set specific parameters here for IntersectionShaper - if(auto* intersectionShaper = dynamic_cast(shaper)) - { - intersectionShaper->setLevel(params.refinementLevel); - SLIC_INFO(axom::fmt::format( - "{:-^80}", - axom::fmt::format("Setting IntersectionShaper policy to '{}'", - axom::runtime_policy::policyToName(params.policy)))); - intersectionShaper->setExecPolicy(params.policy); + shaper->setLevel(params.refinementLevel); + SLIC_INFO(axom::fmt::format( + "{:-^80}", + axom::fmt::format("Setting IntersectionShaper policy to '{}'", + axom::runtime_policy::policyToName(params.policy)))); + shaper->setExecPolicy(params.policy); - if(!params.backgroundMaterial.empty()) - { - intersectionShaper->setFreeMaterialName(params.backgroundMaterial); - } + if(!params.backgroundMaterial.empty()) + { + shaper->setFreeMaterialName(params.backgroundMaterial); } AXOM_ANNOTATE_END("setup shaping problem"); @@ -1329,7 +1332,7 @@ int main(int argc, char** argv) auto p = shaper->getMaterial(materialName); axom::ArrayView& materialField = p.first; int materialIdx = p.second; - + assert(false); // Incomplete code. } } else { for(auto& kv : shaper->getDC()->GetFieldMap()) @@ -1472,7 +1475,7 @@ int main(int argc, char** argv) } #endif - delete shaper; + shaper.reset(); //--------------------------------------------------------------------------- // Cleanup and exit From 9205f47cdd7ed72da157fa8984389ab9f581c726 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Thu, 17 Oct 2024 02:21:48 -0700 Subject: [PATCH 018/100] Remove overly restrictive requirements for Mesh object. The code required what to Blueprint was optional. --- src/axom/mint/mesh/Mesh.cpp | 23 ++++++++----------- .../quest/examples/quest_shape_in_memory.cpp | 21 +++++++++++++---- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/src/axom/mint/mesh/Mesh.cpp b/src/axom/mint/mesh/Mesh.cpp index fc54c38395..c1ddc49846 100644 --- a/src/axom/mint/mesh/Mesh.cpp +++ b/src/axom/mint/mesh/Mesh.cpp @@ -123,22 +123,19 @@ Mesh::Mesh(sidre::Group* group, const std::string& topo) m_coordset = blueprint::getCoordsetGroup(m_group, getTopologyGroup())->getName(); - SLIC_ERROR_IF(!m_group->hasChildGroup("state"), - "root group does not have a state group."); - sidre::Group* state_group = m_group->getGroup("state"); - SLIC_ERROR_IF(!state_group->hasChildGroup(m_topology), - "state group has no " << m_topology << " child group."); - - state_group = state_group->getGroup(m_topology); - if(state_group->hasChildView("block_id")) + if (state_group != nullptr && state_group->hasChildGroup(m_topology)) { - m_block_idx = state_group->getView("block_id")->getScalar(); - } + state_group = state_group->getGroup(m_topology); + if(state_group->hasChildView("block_id")) + { + m_block_idx = state_group->getView("block_id")->getScalar(); + } - if(state_group->hasChildView("partition_id")) - { - m_part_idx = state_group->getView("partition_id")->getScalar(); + if(state_group->hasChildView("partition_id")) + { + m_part_idx = state_group->getView("partition_id")->getScalar(); + } } SLIC_ERROR_IF(!validMeshType(), "invalid mesh type=" << m_type); diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index fff650e606..b2cdf119fe 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -1034,9 +1034,17 @@ template double sumMaterialVolumes(sidre::Group* meshGrp, const std::string& material) { - std::unique_ptr> mesh{ - dynamic_cast*>(axom::mint::getMesh(meshGrp, topoName)) }; - int const cellCount = mesh->getNumberOfCells(); + conduit::Node meshNode; + meshGrp->createNativeLayout(meshNode); +#if defined(AXOM_DEBUG) + conduit::Node info; + conduit::blueprint::mesh::verify(meshNode, info); + SLIC_ASSERT(conduit::blueprint::mesh::verify(meshNode, info)); +#endif + const int cellCount = + conduit::blueprint::mesh::topology::length( + meshNode.fetch_existing( + axom::fmt::format("topologies/{}", topoName))); // Get cell volumes from dc. axom::sidre::View* elementVols = getElementVolumes(meshGrp); @@ -1332,7 +1340,12 @@ int main(int argc, char** argv) auto p = shaper->getMaterial(materialName); axom::ArrayView& materialField = p.first; int materialIdx = p.second; - assert(false); // Incomplete code. + // Compute and print volume of material. + const double volume = sumMaterialVolumes(compMeshGrp, materialName); + SLIC_INFO(axom::fmt::format(axom::utilities::locale(), + "Volume of material '{}' is {:.6Lf}", + materialName, + volume)); } } else { for(auto& kv : shaper->getDC()->GetFieldMap()) From dede781ef45f98e9c4e5af3404fe5776cf3b9e06 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Thu, 17 Oct 2024 02:29:49 -0700 Subject: [PATCH 019/100] Debug scalar field generation with blueprint mesh (add missing blueprint nodes). --- src/axom/quest/IntersectionShaper.hpp | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index e4c7cb3dd6..b77c26563a 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -2008,7 +2008,8 @@ class IntersectionShaper : public Shaper /*! @brief Get a scalar double-type field data from the mesh, creating it if it doesn't exist. */ - axom::ArrayView getScalarCellData(const std::string& fieldName) + axom::ArrayView getScalarCellData(const std::string& fieldName, + bool volumeDependent = false) { axom::ArrayView rval; @@ -2036,16 +2037,27 @@ class IntersectionShaper : public Shaper axom::sidre::View* valuesView = nullptr; if(m_bpGrp->hasGroup(fieldPath)) { - auto* fieldsGrp = m_bpGrp->getGroup(fieldPath); - valuesView = fieldsGrp->getView("values"); + auto* fieldGrp = m_bpGrp->getGroup(fieldPath); + valuesView = fieldGrp->getView("values"); + SLIC_ASSERT(fieldGrp->getView("association")->getString() == std::string("element")); + SLIC_ASSERT(fieldGrp->getView("topology")->getString() == m_bpTopo); SLIC_ASSERT(valuesView->getNumElements() == m_cellCount); SLIC_ASSERT(valuesView->getNode().dtype().id() == dtype.id()); } else { - auto* fieldsGrp = m_bpGrp->createGroup(fieldPath); - valuesView = fieldsGrp->createView("values"); - valuesView->allocate(dtype); + constexpr axom::IndexType componentCount = 1; + axom::IndexType shape[2] = {m_cellCount, componentCount}; + auto* fieldGrp = m_bpGrp->createGroup(fieldPath); + // valuesView = fieldGrp->createView("values"); + valuesView = fieldGrp->createViewWithShape("values", + axom::sidre::DataTypeId::FLOAT64_ID, + 2, shape); + fieldGrp->createView("association")->setString("element"); + fieldGrp->createView("topology")->setString(m_bpTopo); + fieldGrp->createView("volume_dependent")->setString( + std::string(volumeDependent ? "true" : "false")); + valuesView->allocate(); } rval = axom::ArrayView(static_cast(valuesView->getVoidPtr()), From a8114fdf76d16bcd1552a037b9633a2752f04d11 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Thu, 17 Oct 2024 08:48:38 -0700 Subject: [PATCH 020/100] Autoformat. --- src/axom/mint/mesh/Mesh.cpp | 2 +- src/axom/quest/Shaper.cpp | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/axom/mint/mesh/Mesh.cpp b/src/axom/mint/mesh/Mesh.cpp index c1ddc49846..e1df1fd075 100644 --- a/src/axom/mint/mesh/Mesh.cpp +++ b/src/axom/mint/mesh/Mesh.cpp @@ -124,7 +124,7 @@ Mesh::Mesh(sidre::Group* group, const std::string& topo) blueprint::getCoordsetGroup(m_group, getTopologyGroup())->getName(); sidre::Group* state_group = m_group->getGroup("state"); - if (state_group != nullptr && state_group->hasChildGroup(m_topology)) + if(state_group != nullptr && state_group->hasChildGroup(m_topology)) { state_group = state_group->getGroup(m_topology); if(state_group->hasChildView("block_id")) diff --git a/src/axom/quest/Shaper.cpp b/src/axom/quest/Shaper.cpp index e2365e47c0..0ee2ce3ab4 100644 --- a/src/axom/quest/Shaper.cpp +++ b/src/axom/quest/Shaper.cpp @@ -55,10 +55,8 @@ Shaper::Shaper(const klee::ShapeSet& shapeSet, conduit::Node info; SLIC_ASSERT(conduit::blueprint::mesh::verify(meshNode, info)); #endif - m_cellCount = - conduit::blueprint::mesh::topology::length( - meshNode.fetch_existing( - axom::fmt::format("topologies/{}", m_bpTopo))); + m_cellCount = conduit::blueprint::mesh::topology::length( + meshNode.fetch_existing(axom::fmt::format("topologies/{}", m_bpTopo))); } void Shaper::setSamplesPerKnotSpan(int nSamples) From f4c54ccc1553916dc16043d61bcf71663360bab4 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Thu, 17 Oct 2024 10:02:42 -0700 Subject: [PATCH 021/100] Allow reshaping sidre::View holding array data. --- src/axom/sidre/core/View.cpp | 38 +++++++++++++ src/axom/sidre/core/View.hpp | 10 ++++ src/axom/sidre/tests/sidre_view.cpp | 88 +++++++++++++++++++++++++++++ 3 files changed, 136 insertions(+) diff --git a/src/axom/sidre/core/View.cpp b/src/axom/sidre/core/View.cpp index aee158daa7..da342303fd 100644 --- a/src/axom/sidre/core/View.cpp +++ b/src/axom/sidre/core/View.cpp @@ -295,6 +295,44 @@ View* View::reallocate(const DataType& dtype) return this; } +/* + ************************************************************************* + * + * Reshape an array View. + * + ************************************************************************* + */ +View* View::reshapeArray(int ndims, const IndexType* shape) +{ + SLIC_ERROR_IF(m_state != BUFFER && m_state != EXTERNAL, + SIDRE_VIEW_LOG_PREPEND + << "Can only reshape array Views (BUFFER or EXTERNAL)."); + + IndexType oldSize = 1; + for(const auto& s : m_shape) + { + oldSize *= s; + } + IndexType newSize = 1; + for(int d = 0; d < ndims; ++d) + { + newSize *= shape[d]; + } + SLIC_ERROR_IF(newSize != oldSize, + SIDRE_VIEW_LOG_PREPEND + << "Reshape must not change the number of elements."); + + // If View was applied before reshape, then reapply it. + const bool is_applied = m_is_applied; + describe(getTypeID(), ndims, shape); + if(is_applied) + { + apply(); + } + + return this; +} + /* ************************************************************************* * diff --git a/src/axom/sidre/core/View.hpp b/src/axom/sidre/core/View.hpp index e4bf9cf165..6aa12563a6 100644 --- a/src/axom/sidre/core/View.hpp +++ b/src/axom/sidre/core/View.hpp @@ -431,6 +431,16 @@ class View //@} + /*! + * \brief Reshape the array without changing its size. + * + * \pre Old and new shapes must be the same size. + * \pre !isEmpty() && (isExternal() || hasBuffer()) + * \post getNumDimensions() == ndims + * \post getNumElements() is unchanged. + */ + View* reshapeArray(int ndims, const IndexType* shape); + /*! * \brief Attach Buffer object to data view. * diff --git a/src/axom/sidre/tests/sidre_view.cpp b/src/axom/sidre/tests/sidre_view.cpp index a210a72f99..a94071aca8 100644 --- a/src/axom/sidre/tests/sidre_view.cpp +++ b/src/axom/sidre/tests/sidre_view.cpp @@ -1852,6 +1852,94 @@ TEST(sidre_view, deep_copy_shape) //------------------------------------------------------------------------------ +TEST(sidre_view, reshape_array) +{ + using namespace axom; + DataStore ds; + Group* root = ds.getRoot(); + + constexpr int DMAX = 4; + IndexType shapeOutput[DMAX]; + int nDim = -1; + + // A view with array in a buffer. + auto* viewA = + root->createView("viewA", sidre::detail::SidreTT::id, 12); + + { + nDim = viewA->getShape(DMAX, shapeOutput); + EXPECT_EQ(viewA->getNumElements(), 12); + EXPECT_EQ(nDim, 1); + EXPECT_EQ(shapeOutput[0], 12); + } + + viewA->allocate(); + + { + IndexType shape2d[] = {3, 4}; + viewA->reshapeArray(2, shape2d); + nDim = viewA->getShape(DMAX, shapeOutput); + EXPECT_EQ(viewA->getNumElements(), 12); + EXPECT_EQ(nDim, 2); + EXPECT_EQ(shapeOutput[0], 3); + EXPECT_EQ(shapeOutput[1], 4); + } + + viewA->deallocate(); + + { + IndexType shape2d[] = {2, 6}; + viewA->reshapeArray(2, shape2d); + nDim = viewA->getShape(DMAX, shapeOutput); + EXPECT_EQ(viewA->getNumElements(), 12); + EXPECT_EQ(nDim, 2); + EXPECT_EQ(shapeOutput[0], 2); + EXPECT_EQ(shapeOutput[1], 6); + } + + viewA->allocate(); + + { + IndexType shape3d[] = {3, 2, 2}; + viewA->reshapeArray(3, shape3d); + nDim = viewA->getShape(DMAX, shapeOutput); + EXPECT_EQ(viewA->getNumElements(), 12); + EXPECT_EQ(nDim, 3); + EXPECT_EQ(shapeOutput[0], 3); + EXPECT_EQ(shapeOutput[1], 2); + EXPECT_EQ(shapeOutput[2], 2); + } + + // A view with external array data. + auto* viewB = root->createView("viewB"); + + std::int32_t extData[24]; + + { + viewB->setExternalDataPtr(sidre::detail::SidreTT::id, + 24, + extData); + nDim = viewB->getShape(DMAX, shapeOutput); + EXPECT_EQ(viewB->getNumElements(), 24); + EXPECT_EQ(nDim, 1); + EXPECT_EQ(shapeOutput[0], 24); + } + + { + IndexType shape4d[] = {1, 2, 3, 4}; + viewB->reshapeArray(4, shape4d); + nDim = viewB->getShape(DMAX, shapeOutput); + EXPECT_EQ(viewB->getNumElements(), 24); + EXPECT_EQ(nDim, 4); + EXPECT_EQ(shapeOutput[0], 1); + EXPECT_EQ(shapeOutput[1], 2); + EXPECT_EQ(shapeOutput[2], 3); + EXPECT_EQ(shapeOutput[3], 4); + } +} + +//------------------------------------------------------------------------------ + #ifdef AXOM_USE_UMPIRE class UmpireTest : public ::testing::TestWithParam From 71e3fdbd97c4207b93c6ba4b68451567a200a852 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Thu, 17 Oct 2024 10:13:07 -0700 Subject: [PATCH 022/100] More progress toward Blueprint support in IntersectionShaper. Facilitated by ability to reshape array data in sidre Views. --- src/axom/quest/IntersectionShaper.hpp | 27 +++--- .../quest/examples/quest_shape_in_memory.cpp | 89 +++++++++++-------- src/axom/quest/util/mesh_helpers.cpp | 66 +++++++++++--- 3 files changed, 123 insertions(+), 59 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index b77c26563a..cf82a47df3 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -2039,7 +2039,8 @@ class IntersectionShaper : public Shaper { auto* fieldGrp = m_bpGrp->getGroup(fieldPath); valuesView = fieldGrp->getView("values"); - SLIC_ASSERT(fieldGrp->getView("association")->getString() == std::string("element")); + SLIC_ASSERT(fieldGrp->getView("association")->getString() == + std::string("element")); SLIC_ASSERT(fieldGrp->getView("topology")->getString() == m_bpTopo); SLIC_ASSERT(valuesView->getNumElements() == m_cellCount); SLIC_ASSERT(valuesView->getNode().dtype().id() == dtype.id()); @@ -2050,13 +2051,15 @@ class IntersectionShaper : public Shaper axom::IndexType shape[2] = {m_cellCount, componentCount}; auto* fieldGrp = m_bpGrp->createGroup(fieldPath); // valuesView = fieldGrp->createView("values"); - valuesView = fieldGrp->createViewWithShape("values", - axom::sidre::DataTypeId::FLOAT64_ID, - 2, shape); + valuesView = + fieldGrp->createViewWithShape("values", + axom::sidre::DataTypeId::FLOAT64_ID, + 2, + shape); fieldGrp->createView("association")->setString("element"); fieldGrp->createView("topology")->setString(m_bpTopo); - fieldGrp->createView("volume_dependent")->setString( - std::string(volumeDependent ? "true" : "false")); + fieldGrp->createView("volume_dependent") + ->setString(std::string(volumeDependent ? "true" : "false")); valuesView->allocate(); } rval = @@ -2160,8 +2163,10 @@ class IntersectionShaper : public Shaper conduit::Node meshNode; m_bpGrp->createNativeLayout(meshNode); - const conduit::Node& topoNode = meshNode.fetch_existing(axom::fmt::format("topologies/{}", m_bpTopo)); - const std::string coordsetName = topoNode.fetch_existing("coordset").as_string(); + const conduit::Node& topoNode = + meshNode.fetch_existing(axom::fmt::format("topologies/{}", m_bpTopo)); + const std::string coordsetName = + topoNode.fetch_existing("coordset").as_string(); // Assume unstructured and hexahedral SLIC_ASSERT(topoNode["type"].as_string() == "unstructured"); @@ -2169,14 +2174,16 @@ class IntersectionShaper : public Shaper const auto connNode = topoNode["elements/connectivity"]; SLIC_ASSERT_MSG(connNode.dtype().is_int32(), - "IntersectionShaper internal error: missing logic to handle multiple types."); + "IntersectionShaper internal error: missing logic to " + "handle multiple types."); const std::int32_t* connPtr = connNode.as_int32_ptr(); axom::ArrayView conn(connPtr, m_cellCount, NUM_VERTS_PER_HEX); // Put in Node so we can use conduit::blueprint utilities. - const conduit::Node& coordNode = meshNode[axom::fmt::format("coordsets/{}", coordsetName)]; + const conduit::Node& coordNode = + meshNode[axom::fmt::format("coordsets/{}", coordsetName)]; // coordGrp->createNativeLayout(coordNode); const conduit::Node& coordValues = coordNode.fetch_existing("values"); diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index b2cdf119fe..ddeabf6ab7 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -397,12 +397,12 @@ axom::sidre::Group* createBoxMesh(axom::sidre::Group* meshGrp) res, topoName, coordsetName); - #if defined(AXOM_DEBUG) +#if defined(AXOM_DEBUG) conduit::Node meshNode, info; meshGrp->createNativeLayout(meshNode); -meshNode.print(); + meshNode.print(); SLIC_ASSERT(conduit::blueprint::mesh::verify(meshNode, info)); - #endif +#endif return meshGrp; } @@ -911,14 +911,20 @@ axom::sidre::View* getElementVolumes( sidre::Group* meshGrp, const std::string& volFieldName = std::string("elementVolumes")) { - std::unique_ptr> mesh{ - dynamic_cast*>(axom::mint::getMesh(meshGrp, topoName)) }; + std::unique_ptr> mesh { + dynamic_cast*>( + axom::mint::getMesh(meshGrp, topoName))}; using HexahedronType = axom::primal::Hexahedron; - axom::sidre::View* volSidreView = meshGrp->getView(axom::fmt::format("fields/{}/values", volFieldName)); - if(volSidreView == nullptr) + const auto valuesPath = axom::fmt::format("fields/{}/values", volFieldName); + axom::sidre::View* volSidreView = nullptr; + if(meshGrp->hasView(valuesPath)) { - const axom::IndexType cellCount = meshGrp->getView(axom::fmt::format("coordsets/{}/values/x", coordsetName))->getNumElements(); + volSidreView = meshGrp->getView(valuesPath); + } + else + { + const axom::IndexType cellCount = mesh->getNumberOfCells(); constexpr int NUM_VERTS_PER_HEX = 8; constexpr int NUM_COMPS_PER_VERT = 3; @@ -941,7 +947,7 @@ axom::sidre::View* getElementVolumes( int vertIdx = cellIdx * NUM_VERTS_PER_HEX + j; for(int k = 0; k < NUM_COMPS_PER_VERT; k++) { - vertCoordsView[vertIdx][k] = mesh->getNodeCoordinate(vertIdx, k); + vertCoordsView[vertIdx][k] = mesh->getNodeCoordinate(verts[j], k); } } } @@ -978,8 +984,12 @@ axom::sidre::View* getElementVolumes( }); // end of loop to initialize hexahedral elements and bounding boxes // Allocate and populate cell volumes. - axom::sidre::Group* volGrp = meshGrp->createGroup(axom::fmt::format("fields/{}", volFieldName)); - volSidreView = volGrp->createViewAndAllocate("values", axom::sidre::DataTypeId::FLOAT64_ID, cellCount); + const auto volFieldPath = axom::fmt::format("fields/{}", volFieldName); + axom::sidre::Group* volGrp = meshGrp->createGroup(volFieldPath); + volSidreView = + volGrp->createViewAndAllocate("values", + axom::sidre::DataTypeId::FLOAT64_ID, + cellCount); axom::ArrayView volView(volSidreView->getData(), volSidreView->getNumElements()); axom::for_all( @@ -1031,8 +1041,7 @@ double sumMaterialVolumes(sidre::MFEMSidreDataCollection* dc, } template -double sumMaterialVolumes(sidre::Group* meshGrp, - const std::string& material) +double sumMaterialVolumes(sidre::Group* meshGrp, const std::string& material) { conduit::Node meshNode; meshGrp->createNativeLayout(meshNode); @@ -1040,11 +1049,12 @@ double sumMaterialVolumes(sidre::Group* meshGrp, conduit::Node info; conduit::blueprint::mesh::verify(meshNode, info); SLIC_ASSERT(conduit::blueprint::mesh::verify(meshNode, info)); + meshGrp->print(); + meshNode.print(); #endif - const int cellCount = - conduit::blueprint::mesh::topology::length( - meshNode.fetch_existing( - axom::fmt::format("topologies/{}", topoName))); + std::string topoPath = axom::fmt::format("topologies/{}", topoName); + conduit::Node& topoNode = meshNode.fetch_existing(topoPath); + const int cellCount = conduit::blueprint::mesh::topology::length(topoNode); // Get cell volumes from dc. axom::sidre::View* elementVols = getElementVolumes(meshGrp); @@ -1052,11 +1062,11 @@ double sumMaterialVolumes(sidre::Group* meshGrp, elementVols->getNumElements()); // Get material volume fractions - const std::string materialFieldName = - axom::fmt::format("vol_frac_{}", material); - axom::sidre::View* volFrac = meshGrp->getView(axom::fmt::format("field/{}/values", materialFieldName)); - axom::ArrayView volFracArrayView(volFrac->getArray(), - cellCount); + const auto vfFieldName = axom::fmt::format("vol_frac_{}", material); + const auto vfFieldValuesPath = + axom::fmt::format("fields/{}/values", vfFieldName); + axom::sidre::View* volFrac = meshGrp->getView(vfFieldValuesPath); + axom::ArrayView volFracArrayView(volFrac->getArray(), cellCount); axom::quest::TempArrayView volFracView(volFracArrayView, true); using ReducePolicy = typename axom::execution_space::reduce_policy; @@ -1242,9 +1252,9 @@ int main(int argc, char** argv) //--------------------------------------------------------------------------- AXOM_ANNOTATE_BEGIN("setup shaping problem"); bool useBp = false; - std::shared_ptr shaper = useBp ? - std::make_shared(shapeSet, compMeshGrp) : - std::make_shared(shapeSet, &shapingDC); + std::shared_ptr shaper = useBp + ? std::make_shared(shapeSet, compMeshGrp) + : std::make_shared(shapeSet, &shapingDC); // Set generic parameters for the base Shaper instance shaper->setVertexWeldThreshold(params.weldThresh); @@ -1257,19 +1267,21 @@ int main(int argc, char** argv) // Associate any fields that begin with "vol_frac" with "material" so when // the data collection is written, a matset will be created. - if (useBp) + if(useBp) { // TODO: What is the blueprint replacement for AssociateMaterialSet? - } else { + } + else + { shaper->getDC()->AssociateMaterialSet("vol_frac", "material"); } // Set specific parameters here for IntersectionShaper shaper->setLevel(params.refinementLevel); SLIC_INFO(axom::fmt::format( - "{:-^80}", - axom::fmt::format("Setting IntersectionShaper policy to '{}'", - axom::runtime_policy::policyToName(params.policy)))); + "{:-^80}", + axom::fmt::format("Setting IntersectionShaper policy to '{}'", + axom::runtime_policy::policyToName(params.policy)))); shaper->setExecPolicy(params.policy); if(!params.backgroundMaterial.empty()) @@ -1331,23 +1343,26 @@ int main(int argc, char** argv) // Compute and print volumes of each material's volume fraction //--------------------------------------------------------------------------- using axom::utilities::string::startsWith; - if (useBp) + if(useBp) { // TODO: What is the blueprint equivalent? std::vector materialNames = shaper->getMaterialNames(); - for (const auto& materialName : materialNames) + for(const auto& materialName : materialNames) { auto p = shaper->getMaterial(materialName); axom::ArrayView& materialField = p.first; int materialIdx = p.second; // Compute and print volume of material. - const double volume = sumMaterialVolumes(compMeshGrp, materialName); + const double volume = + sumMaterialVolumes(compMeshGrp, materialName); SLIC_INFO(axom::fmt::format(axom::utilities::locale(), "Volume of material '{}' is {:.6Lf}", materialName, volume)); } - } else { + } + else + { for(auto& kv : shaper->getDC()->GetFieldMap()) { if(startsWith(kv.first, "vol_frac_")) @@ -1451,9 +1466,9 @@ int main(int argc, char** argv) shapeMesh->getNumberOfCells()))); const std::string& materialName = shape.getMaterial(); - double shapeVol = useBp ? - sumMaterialVolumes(compMeshGrp, materialName) : - sumMaterialVolumes(&shapingDC, materialName); + double shapeVol = useBp + ? sumMaterialVolumes(compMeshGrp, materialName) + : sumMaterialVolumes(&shapingDC, materialName); double correctShapeVol = params.testShape == "plane" ? params.boxMeshVolume() / 2 : shapeMeshVol; double diff = shapeVol - correctShapeVol; diff --git a/src/axom/quest/util/mesh_helpers.cpp b/src/axom/quest/util/mesh_helpers.cpp index be2ac0fc00..29018e7d02 100644 --- a/src/axom/quest/util/mesh_helpers.cpp +++ b/src/axom/quest/util/mesh_helpers.cpp @@ -8,6 +8,7 @@ #include #include #include +#include "axom/mint/mesh/UnstructuredMesh.hpp" #include "axom/core/WhereMacro.hpp" namespace axom @@ -172,7 +173,7 @@ axom::sidre::Group* make_structured_blueprint_box_mesh( return meshGrp; } -#if defined(AXOM_USE_CONDUIT) + #if defined(AXOM_USE_CONDUIT) axom::sidre::Group* make_unstructured_blueprint_box_mesh( axom::sidre::Group* meshGrp, const primal::BoundingBox& bbox, @@ -211,20 +212,61 @@ void convert_blueprint_structured_explicit_to_unstructured( // Copy unstructured back into meshGrp. meshGrp->getGroup("topologies")->destroyGroup(topoName); - meshGrp->getGroup("topologies")->createGroup(topoName)->importConduitTree(newTopo); - meshGrp->getGroup("coordsets")->destroyGroup(coordsetName); - meshGrp->getGroup("coordsets") - ->createGroup(coordsetName) - ->importConduitTree(newCoords); - - meshGrp->getView(axom::fmt::format("topologies/{}/coordset", topoName)) - ->setString(coordsetName); + auto* topoGrp = meshGrp->getGroup("topologies")->createGroup(topoName); + topoGrp->importConduitTree(newTopo); + topoGrp->getView("coordset") + ->setString(coordsetName); // Is this needed? Is coordset already set? - #if defined(AXOM_DEBUG) + meshGrp->getGroup("coordsets")->destroyGroup(coordsetName); + auto* coordsetGrp = meshGrp->getGroup("coordsets")->createGroup(coordsetName); + coordsetGrp->importConduitTree(newCoords); + + #define ADD_EXTRA_DATA_FOR_MINT 1 + #if ADD_EXTRA_DATA_FOR_MINT + /* + Constructing a mint mesh from meshGrp fails unless we add some + extra data. Blueprint doesn't require this extra data. (The mesh + passes conduit's Blueprint verification.) This should be fixed, + or we should write better blueprint support utilities. + */ + /* + Make the coordinate arrays 2D to use mint::Mesh. + For some reason, mint::Mesh requires the arrays to be + 2D, even though the second dimension is always 1. + */ + axom::IndexType curShape[2]; + int curDim; + auto* valuesGrp = coordsetGrp->getGroup("values"); + curDim = valuesGrp->getView("x")->getShape(2, curShape); + assert(curDim == 1); + axom::IndexType vertsShape[2] = {curShape[0], 1}; + valuesGrp->getView("x")->reshapeArray(2, vertsShape); + valuesGrp->getView("y")->reshapeArray(2, vertsShape); + valuesGrp->getView("z")->reshapeArray(2, vertsShape); + + // Make connectivity array 2D for the same reason. + auto* elementsGrp = topoGrp->getGroup("elements"); + auto* connView = elementsGrp->getView("connectivity"); + curDim = connView->getShape(2, curShape); + constexpr axom::IndexType NUM_VERTS_PER_HEX = 8; + SLIC_ASSERT(curDim == 1); + SLIC_ASSERT(curShape[0] % NUM_VERTS_PER_HEX == 0); + axom::IndexType connShape[2] = {curShape[0] / NUM_VERTS_PER_HEX, + NUM_VERTS_PER_HEX}; + connView->reshapeArray(2, connShape); + + // mint::Mesh requires connectivity strides, even though Blueprint doesn't. + elementsGrp->createViewScalar("stride", NUM_VERTS_PER_HEX); + + // mint::Mesh requires field group, even though Blueprint doesn't. + meshGrp->createGroup("fields"); + #endif + + #if defined(AXOM_DEBUG) conduit::Node newMesh; meshGrp->createNativeLayout(newMesh); SLIC_ASSERT(conduit::blueprint::mesh::verify(newMesh, info)); - #endif + #endif return; } @@ -237,7 +279,7 @@ bool verifyBlueprintMesh(const axom::sidre::Group* meshGrp, conduit::Node info) if(!isValid) info.print(); return isValid; } -#endif + #endif #endif From 12e8ea0ae9933f7b1bb115a79213f7466b6ae27b Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Fri, 18 Oct 2024 02:01:22 -0700 Subject: [PATCH 023/100] More progress toward testing Blueprint support. --- .../quest/examples/quest_shape_in_memory.cpp | 54 ++++++++++--------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index ddeabf6ab7..b437f6789e 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -400,7 +400,6 @@ axom::sidre::Group* createBoxMesh(axom::sidre::Group* meshGrp) #if defined(AXOM_DEBUG) conduit::Node meshNode, info; meshGrp->createNativeLayout(meshNode); - meshNode.print(); SLIC_ASSERT(conduit::blueprint::mesh::verify(meshNode, info)); #endif @@ -473,7 +472,6 @@ axom::klee::ShapeSet create2DShapeSet(sidre::DataStore& ds) axom::IndexType conn[3] = {0, 1, 2}; triangleMesh.appendCell(conn); - meshGroup->print(); SLIC_ASSERT(axom::mint::blueprint::isValidRootGroup(meshGroup)); axom::klee::TransformableGeometryProperties prop { @@ -899,7 +897,8 @@ axom::sidre::View* getElementVolumes( } /*! - @brief Return the element volumes as a sidre::View. + @brief Return the element volumes as a sidre::View containing + the volumes in an array. If it doesn't exist, allocate and compute it. \post The volume data is in the blueprint field \c volFieldName. @@ -911,20 +910,22 @@ axom::sidre::View* getElementVolumes( sidre::Group* meshGrp, const std::string& volFieldName = std::string("elementVolumes")) { - std::unique_ptr> mesh { - dynamic_cast*>( - axom::mint::getMesh(meshGrp, topoName))}; using HexahedronType = axom::primal::Hexahedron; - const auto valuesPath = axom::fmt::format("fields/{}/values", volFieldName); axom::sidre::View* volSidreView = nullptr; - if(meshGrp->hasView(valuesPath)) + + const auto fieldPath = axom::fmt::format("fields/{}", volFieldName); + if(meshGrp->hasGroup(fieldPath)) { - volSidreView = meshGrp->getView(valuesPath); + sidre::Group* fieldGrp = meshGrp->createGroup(fieldPath); + volSidreView = fieldGrp->getView(volFieldName); } else { - const axom::IndexType cellCount = mesh->getNumberOfCells(); + axom::mint::UnstructuredMesh mesh(meshGrp, + topoName); + + const axom::IndexType cellCount = mesh.getNumberOfCells(); constexpr int NUM_VERTS_PER_HEX = 8; constexpr int NUM_COMPS_PER_VERT = 3; @@ -938,8 +939,8 @@ axom::sidre::View* getElementVolumes( for(axom::IndexType cellIdx = 0; cellIdx < cellCount; ++cellIdx) { // Get the indices of this element's vertices - axom::IndexType* verts = mesh->getCellNodeIDs(cellIdx); - mesh->getCellNodeIDs(cellIdx, verts); + axom::IndexType* verts = mesh.getCellNodeIDs(cellIdx); + mesh.getCellNodeIDs(cellIdx, verts); // Get the coordinates for the vertices for(int j = 0; j < NUM_VERTS_PER_HEX; ++j) @@ -947,7 +948,7 @@ axom::sidre::View* getElementVolumes( int vertIdx = cellIdx * NUM_VERTS_PER_HEX + j; for(int k = 0; k < NUM_COMPS_PER_VERT; k++) { - vertCoordsView[vertIdx][k] = mesh->getNodeCoordinate(verts[j], k); + vertCoordsView[vertIdx][k] = mesh.getNodeCoordinate(verts[j], k); } } } @@ -984,12 +985,16 @@ axom::sidre::View* getElementVolumes( }); // end of loop to initialize hexahedral elements and bounding boxes // Allocate and populate cell volumes. - const auto volFieldPath = axom::fmt::format("fields/{}", volFieldName); - axom::sidre::Group* volGrp = meshGrp->createGroup(volFieldPath); + axom::sidre::Group* fieldGrp = meshGrp->createGroup(fieldPath); + fieldGrp->createViewString("topology", topoName); + fieldGrp->createViewString("association", "element"); + fieldGrp->createViewString("volume_dependent", "true"); volSidreView = - volGrp->createViewAndAllocate("values", - axom::sidre::DataTypeId::FLOAT64_ID, - cellCount); + fieldGrp->createViewAndAllocate("values", + axom::sidre::detail::SidreTT::id, + cellCount); + axom::IndexType shape2d[] = {cellCount, 1}; + volSidreView->reshapeArray(2, shape2d); axom::ArrayView volView(volSidreView->getData(), volSidreView->getNumElements()); axom::for_all( @@ -1049,15 +1054,15 @@ double sumMaterialVolumes(sidre::Group* meshGrp, const std::string& material) conduit::Node info; conduit::blueprint::mesh::verify(meshNode, info); SLIC_ASSERT(conduit::blueprint::mesh::verify(meshNode, info)); - meshGrp->print(); - meshNode.print(); #endif std::string topoPath = axom::fmt::format("topologies/{}", topoName); conduit::Node& topoNode = meshNode.fetch_existing(topoPath); const int cellCount = conduit::blueprint::mesh::topology::length(topoNode); - // Get cell volumes from dc. - axom::sidre::View* elementVols = getElementVolumes(meshGrp); + // Get cell volumes from meshGrp. + const std::string volsName = "vol_" + material; + axom::sidre::View* elementVols = + getElementVolumes(meshGrp, volsName); axom::ArrayView elementVolsView(elementVols->getData(), elementVols->getNumElements()); @@ -1388,8 +1393,9 @@ int main(int argc, char** argv) int failCounts = 0; - axom::sidre::Group* volFracGroups = - shapingDC.GetBPGroup()->getGroup("matsets/material/volume_fractions"); + axom::sidre::Group* volFracGroups = useBp + ? compMeshGrp->getGroup("matsets/material/volume_fractions") + : shapingDC.GetBPGroup()->getGroup("matsets/material/volume_fractions"); //--------------------------------------------------------------------------- // Correctness test: volume fractions should be in [0,1]. From 8c7efef90928ec47819bcecf6fa64d655649a8d6 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Fri, 18 Oct 2024 02:01:58 -0700 Subject: [PATCH 024/100] Slight refactor for readability. --- src/axom/quest/IntersectionShaper.hpp | 72 ++++++++++++++++----------- 1 file changed, 44 insertions(+), 28 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index cf82a47df3..726c289cb6 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -640,6 +640,15 @@ class IntersectionShaper : public Shaper #endif #if defined(AXOM_USE_RAJA) && defined(AXOM_USE_UMPIRE) + /*! + \tparam ShapeType either TetrahedeonType or OctahedronType. + depending on whether shape is tet or c2c. + \param shape the input shape to be superimposed on the mesh. + \param shapes either the array m_tets (if surface mesh is tet) + or m_octs (if surface mesh is c2c). + \param shape_count the count for the shapes array, maintained + separately from the array. + */ template void runShapeQueryImpl(const klee::Shape& shape, axom::Array& shapes, @@ -708,33 +717,9 @@ class IntersectionShaper : public Shaper } // Set shape components to zero if within threshold - axom::for_all( - shape_count, - AXOM_LAMBDA(axom::IndexType i) { - for(int j = 0; j < ShapeType::NUM_VERTS; j++) - { - if(axom::utilities::isNearlyEqual(shapes_device_view[i][j][0], - 0.0, - ZERO_THRESHOLD)) - { - shapes_device_view[i][j][0] = 0.0; - } - - if(axom::utilities::isNearlyEqual(shapes_device_view[i][j][1], - 0.0, - ZERO_THRESHOLD)) - { - shapes_device_view[i][j][1] = 0.0; - } - - if(axom::utilities::isNearlyEqual(shapes_device_view[i][j][2], - 0.0, - ZERO_THRESHOLD)) - { - shapes_device_view[i][j][2] = 0.0; - } - } - }); + snapShapeVerticesToZero(shapes, + shape_count, + ZERO_THRESHOLD); // Find which shape bounding boxes intersect hexahedron bounding boxes SLIC_INFO(axom::fmt::format( @@ -2071,7 +2056,8 @@ class IntersectionShaper : public Shaper } public: - // This should be private, but NVCC complains unless its public. + // These methods should be private, but NVCC complains unless its public. + template void populateHexesFromMesh() { @@ -2149,6 +2135,36 @@ class IntersectionShaper : public Shaper }); // end of loop to initialize hexahedral elements and bounding boxes } + //!@brief Set shape vertices to zero of within threshold. + template + void snapShapeVerticesToZero(axom::Array& shapes, + axom::IndexType shapeCount, + double zeroThreshold) + { + axom::ArrayView shapesView = shapes.view(); + axom::for_all( + shapeCount, + AXOM_LAMBDA(axom::IndexType i) { + for(int j = 0; j < ShapeType::NUM_VERTS; j++) + { + if(axom::utilities::isNearlyEqual(shapesView[i][j][0], 0.0, zeroThreshold)) + { + shapesView[i][j][0] = 0.0; + } + + if(axom::utilities::isNearlyEqual(shapesView[i][j][1], 0.0, zeroThreshold)) + { + shapesView[i][j][1] = 0.0; + } + + if(axom::utilities::isNearlyEqual(shapesView[i][j][2], 0.0, zeroThreshold)) + { + shapesView[i][j][2] = 0.0; + } + } + }); + } + private: #if defined(AXOM_USE_CONDUIT) void populateVertCoordsFromBlueprintMesh(axom::Array& vertCoords_host) From 8d0c2698a07b8b65d517925740f07096bbe19f8d Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Fri, 18 Oct 2024 14:54:44 -0700 Subject: [PATCH 025/100] Add missing matsets data to blueprint mesh. --- src/axom/quest/IntersectionShaper.hpp | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 726c289cb6..16903e766d 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -1990,8 +1990,11 @@ class IntersectionShaper : public Shaper return has; } - /*! - @brief Get a scalar double-type field data from the mesh, creating it if it doesn't exist. + /*! @brief Get a scalar double-type field data from the mesh, + "fields/fieldName/values", creating it if it doesn't exist. + + Also add the corresponding entry in the blueprint field + "matsets/fieldName/volume_fractions". */ axom::ArrayView getScalarCellData(const std::string& fieldName, bool volumeDependent = false) @@ -2046,6 +2049,24 @@ class IntersectionShaper : public Shaper fieldGrp->createView("volume_dependent") ->setString(std::string(volumeDependent ? "true" : "false")); valuesView->allocate(); + if (fieldName.rfind("vol_frac_", 0) == 0) + { + // This is a material volume fraction field. + // Shallow-copy valuesView to (uni-buffer) matsets. + const std::string matlName = fieldName.substr(9); + axom::sidre::Group* volFracGrp = nullptr; + if (m_bpGrp->hasGroup("matsets/material/volume_fractions")) + { + volFracGrp = m_bpGrp->getGroup("matsets/material/volume_fractions"); + } + else + { + volFracGrp = m_bpGrp->createGroup("matsets/material/volume_fractions"); + m_bpGrp->createViewString("matsets/material/topology", m_bpTopo); + } + auto* valuesViewInMatsets = volFracGrp->copyView(valuesView); + valuesViewInMatsets->rename(matlName); + } } rval = axom::ArrayView(static_cast(valuesView->getVoidPtr()), From 6dd0dcc4948155761cd9a5654fa76be7c92460f8 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Fri, 18 Oct 2024 15:04:35 -0700 Subject: [PATCH 026/100] Debug test code. First largely working version. --- src/axom/quest/examples/quest_shape_in_memory.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index b437f6789e..85276ea439 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -917,8 +917,8 @@ axom::sidre::View* getElementVolumes( const auto fieldPath = axom::fmt::format("fields/{}", volFieldName); if(meshGrp->hasGroup(fieldPath)) { - sidre::Group* fieldGrp = meshGrp->createGroup(fieldPath); - volSidreView = fieldGrp->getView(volFieldName); + sidre::Group* fieldGrp = meshGrp->getGroup(fieldPath); + volSidreView = fieldGrp->getView("values"); } else { @@ -1250,13 +1250,12 @@ int main(int argc, char** argv) axom::sidre::Group* compMeshGrp = createBoxMesh(ds.getRoot()->createGroup("compMesh")); - compMeshGrp->print(); //--------------------------------------------------------------------------- // Initialize the shaping query object //--------------------------------------------------------------------------- AXOM_ANNOTATE_BEGIN("setup shaping problem"); - bool useBp = false; + bool useBp = true; std::shared_ptr shaper = useBp ? std::make_shared(shapeSet, compMeshGrp) : std::make_shared(shapeSet, &shapingDC); From 5edb0cbe7945dd1ef8f4d4dfd2c163ee1d4a64a5 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Sun, 20 Oct 2024 07:43:57 -0700 Subject: [PATCH 027/100] Fix bug in new structured box mesh generator utility. --- src/axom/quest/util/mesh_helpers.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/axom/quest/util/mesh_helpers.cpp b/src/axom/quest/util/mesh_helpers.cpp index 29018e7d02..00178c71aa 100644 --- a/src/axom/quest/util/mesh_helpers.cpp +++ b/src/axom/quest/util/mesh_helpers.cpp @@ -135,7 +135,7 @@ axom::sidre::Group* make_structured_blueprint_box_mesh( axom::sidre::DataTypeId::FLOAT64_ID, numVerts); - const axom::MDMapping vertMapping(vertsShape, axom::ArrayStrideOrder::ROW); + const axom::MDMapping vertMapping(vertsShape, axom::ArrayStrideOrder::COLUMN); axom::ArrayView xView(xVu->getData(), vertsShape, vertMapping.strides()); @@ -146,9 +146,9 @@ axom::sidre::Group* make_structured_blueprint_box_mesh( vertsShape, vertMapping.strides()); - double dx = bbox.getMax()[0] - bbox.getMin()[0]; - double dy = bbox.getMax()[1] - bbox.getMin()[1]; - double dz = bbox.getMax()[2] - bbox.getMin()[2]; + double dx = (bbox.getMax()[0] - bbox.getMin()[0]) / res[0]; + double dy = (bbox.getMax()[1] - bbox.getMin()[1]) / res[1]; + double dz = (bbox.getMax()[2] - bbox.getMin()[2]) / res[2]; for(int k = 0; k < nk + 1; ++k) { for(int j = 0; j < nj + 1; ++j) From 8c7a75b60103634ce403463263351ebaed73ffec Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Mon, 21 Oct 2024 09:51:16 -0700 Subject: [PATCH 028/100] Fix non-MFEM builds and add switch to toggle MFEM mesh. Also add code to save results for visualization. --- src/axom/quest/IntersectionShaper.hpp | 2 +- src/axom/quest/examples/CMakeLists.txt | 36 +-- .../quest/examples/quest_shape_in_memory.cpp | 228 +++++++++++++----- 3 files changed, 189 insertions(+), 77 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 16903e766d..9da3303e1d 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -917,7 +917,7 @@ class IntersectionShaper : public Shaper SLIC_INFO(axom::fmt::format(axom::utilities::locale(), "Total mesh volume is {:.3Lf}", this->allReduceSum(totalHex))); - } // end of runShapeQuery() function + } // end of runShapeQueryImpl() function #endif #if defined(AXOM_USE_RAJA) && defined(AXOM_USE_UMPIRE) diff --git a/src/axom/quest/examples/CMakeLists.txt b/src/axom/quest/examples/CMakeLists.txt index 34fc2a7d60..194bf61f4e 100644 --- a/src/axom/quest/examples/CMakeLists.txt +++ b/src/axom/quest/examples/CMakeLists.txt @@ -265,14 +265,18 @@ if(AXOM_ENABLE_MPI AND MFEM_FOUND AND MFEM_USE_MPI endif() # Shaping in-memory example ------------------------------------------------------ -if(AXOM_ENABLE_MPI AND MFEM_FOUND AND MFEM_USE_MPI - AND AXOM_ENABLE_SIDRE AND AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION - AND AXOM_ENABLE_KLEE) +if((CONDUIT_FOUND OR + (AXOM_ENABLE_MPI AND MFEM_FOUND AND MFEM_USE_MPI + AND AXOM_ENABLE_SIDRE AND AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION)) + AND AXOM_ENABLE_KLEE) + if(MFEM_FOUND) + set(optional_dependency, "mfem") + endif() axom_add_executable( NAME quest_shape_in_memory_ex SOURCES quest_shape_in_memory.cpp OUTPUT_DIR ${EXAMPLE_OUTPUT_DIRECTORY} - DEPENDS_ON ${quest_example_depends} mfem + DEPENDS_ON ${quest_example_depends} ${optional_dependency} FOLDER axom/quest/examples ) @@ -289,19 +293,22 @@ if(AXOM_ENABLE_MPI AND MFEM_FOUND AND MFEM_USE_MPI endif() set(_testshapes "tetmesh" "tet" "hex" "sphere" "cyl" "cone" "vor" "plane") + set(_meshType "bp" "mfem") foreach(_policy ${_policies}) foreach(_testshape ${_testshapes}) - set(_testname "quest_shape_in_memory_${_policy}_${_testshape}") - axom_add_test( - NAME ${_testname} - COMMAND quest_shape_in_memory_ex - --policy ${_policy} - --testShape ${_testshape} - --refinements 2 - --scale 0.75 0.75 0.75 - inline_mesh --min -3 -3 -3 --max 3 3 3 --res 20 20 20 -d 3 - NUM_MPI_TASKS ${_nranks}) + foreach(_meshType ${_meshType}) + set(_testname "quest_shape_in_memory_${_policy}_${_meshType}_${_testshape}") + axom_add_test( + NAME ${_testname} + COMMAND quest_shape_in_memory_ex + --policy ${_policy} + --testShape ${_testshape} + --refinements 2 + --scale 0.75 0.75 0.75 + inline_mesh --min -3 -3 -3 --max 3 3 3 --res 20 20 20 -d 3 + NUM_MPI_TASKS ${_nranks}) + endforeach() endforeach() endforeach() endif() @@ -371,6 +378,7 @@ if(AXOM_ENABLE_MPI AND AXOM_ENABLE_SIDRE AND HDF5_FOUND) endforeach() endforeach() + unset(optional_dependency) unset(_nranks) unset(_policies) unset(_test) diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index 85276ea439..732e83186e 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -23,16 +23,16 @@ #include "axom/fmt.hpp" #include "axom/CLI11.hpp" +#if !defined(AXOM_USE_CONDUIT) + #error Shaping functionality requires Axom to be configured with Conduit +#endif + #include "conduit_relay_io_blueprint.hpp" -// NOTE: The shaping driver requires Axom to be configured with mfem as well as -// the AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION CMake option -#ifndef AXOM_USE_MFEM - #error Shaping functionality requires Axom to be configured with MFEM and the AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION option +#if defined(AXOM_USE_MFEM) + #include "mfem.hpp" #endif -#include "mfem.hpp" - #ifdef AXOM_USE_MPI #include "mpi.h" #endif @@ -103,6 +103,15 @@ struct Input std::string backgroundMaterial; + // clang-format off + enum class MeshType { bp = 0, mfem = 1 }; + const std::map meshTypeChoices + { {"bp", MeshType::bp} , {"mfem", MeshType::mfem} }; + // clang-format on + MeshType meshType {MeshType::bp}; + bool useMfem() { return meshType == MeshType::mfem; } + bool useBlueprint() { return meshType == MeshType::bp; } + private: bool m_verboseOutput {false}; @@ -119,6 +128,7 @@ struct Input return volume; } +#if defined(AXOM_USE_MFEM) && defined(AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION) /// Generate an mfem Cartesian mesh, scaled to the bounding box range mfem::Mesh* createBoxMesh() { @@ -189,6 +199,7 @@ struct Input return dc; } +#endif void parse(int argc, char** argv, axom::CLI::App& app) { @@ -199,6 +210,11 @@ struct Input ->description("Enable/disable verbose output") ->capture_default_str(); + app.add_flag("--meshType", meshType) + ->description("Use MFEM computational mesh instead of Blueprint") + ->capture_default_str() + ->transform(axom::CLI::CheckedTransformer(meshTypeChoices)); + app.add_option("-t,--weld-threshold", weldThresh) ->description("Threshold for welding") ->check(axom::CLI::NonNegativeNumber) @@ -323,6 +339,7 @@ struct Input }; // struct Input Input params; +#if defined(AXOM_USE_MFEM) && defined(AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION) /** * \brief Print some info about the mesh * @@ -382,6 +399,7 @@ void printMfemMeshInfo(mfem::Mesh* mesh, const std::string& prefixMessage = "") slic::flushStreams(); } +#endif const std::string topoName = "mesh"; const std::string coordsetName = "coords"; @@ -803,6 +821,7 @@ double volumeOfTetMesh( return meshVolume; } +#if defined(AXOM_USE_MFEM) && defined(AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION) /*! @brief Return the element volumes as a sidre::View. @@ -895,6 +914,7 @@ axom::sidre::View* getElementVolumes( return volSidreView; } +#endif /*! @brief Return the element volumes as a sidre::View containing @@ -1007,6 +1027,7 @@ axom::sidre::View* getElementVolumes( return volSidreView; } +#if defined(AXOM_USE_MFEM) && defined(AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION) /*! @brief Return global sum of volume of the given material. */ @@ -1044,6 +1065,7 @@ double sumMaterialVolumes(sidre::MFEMSidreDataCollection* dc, #endif return globalVol; } +#endif template double sumMaterialVolumes(sidre::Group* meshGrp, const std::string& material) @@ -1089,6 +1111,45 @@ double sumMaterialVolumes(sidre::Group* meshGrp, const std::string& material) return globalVol; } +/// Write blueprint mesh to disk +void saveMesh(const conduit::Node& mesh, const std::string& filename) +{ + AXOM_ANNOTATE_SCOPE("save mesh (conduit)"); + +#ifdef AXOM_USE_MPI + conduit::relay::mpi::io::blueprint::save_mesh(mesh, + filename, + "hdf5", + MPI_COMM_WORLD); +#else + conduit::relay::io::blueprint::save_mesh(mesh, filename, "hdf5"); +#endif +} + +/// Write blueprint mesh to disk +void saveMesh(const sidre::Group& mesh, const std::string& filename) +{ + AXOM_ANNOTATE_SCOPE("save mesh (sidre)"); + + conduit::Node tmpMesh; + mesh.createNativeLayout(tmpMesh); + { + conduit::Node info; +#ifdef AXOM_USE_MPI + if(!conduit::blueprint::mpi::verify("mesh", tmpMesh, info, MPI_COMM_WORLD)) +#else + if(!conduit::blueprint::verify("mesh", tmpMesh, info)) +#endif + { + SLIC_INFO("Invalid blueprint for mesh: \n" << info.to_yaml()); + slic::flushStreams(); + assert(false); + } + // info.print(); + } + saveMesh(tmpMesh, filename); +} + //------------------------------------------------------------------------------ int main(int argc, char** argv) { @@ -1211,54 +1272,78 @@ int main(int argc, char** argv) "the C2C library"); #endif - AXOM_ANNOTATE_BEGIN("load mesh"); - //--------------------------------------------------------------------------- - // Load the computational mesh - // originalMeshDC is the input MFEM mesh - // It's converted to shapingDC below, and that is used for shaping calls. - //--------------------------------------------------------------------------- - auto originalMeshDC = params.loadComputationalMesh(); + axom::IndexType cellCount = -1; - //--------------------------------------------------------------------------- - // Set up DataCollection for shaping - // shapingDC is a "copy" of originalMeshDC, the MFEM mesh. - // It's created empty then populated by the SetMesh call. - // shapingMesh and parallelMesh are some kind of temporary versions of originalMeshDC. - //--------------------------------------------------------------------------- - mfem::Mesh* shapingMesh = nullptr; - constexpr bool dc_owns_data = true; - sidre::MFEMSidreDataCollection shapingDC("shaping", shapingMesh, dc_owns_data); +#if defined(AXOM_USE_MFEM) && defined(AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION) + if(params.useMfem()) { - shapingDC.SetMeshNodesName("positions"); - - // With MPI, loadComputationalMesh returns a parallel mesh. - mfem::ParMesh* parallelMesh = - dynamic_cast(originalMeshDC->GetMesh()); - shapingMesh = (parallelMesh != nullptr) - ? new mfem::ParMesh(*parallelMesh) - : new mfem::Mesh(*originalMeshDC->GetMesh()); - shapingDC.SetMesh(shapingMesh); + AXOM_ANNOTATE_BEGIN("load mesh"); + //--------------------------------------------------------------------------- + // Load the computational mesh + // originalMeshDC is the input MFEM mesh + // It's converted to shapingDC below, and that is used for shaping calls. + //--------------------------------------------------------------------------- + auto originalMeshDC = params.loadComputationalMesh(); + + //--------------------------------------------------------------------------- + // Set up DataCollection for shaping + // shapingDC is a "copy" of originalMeshDC, the MFEM mesh. + // It's created empty then populated by the SetMesh call. + // shapingMesh and parallelMesh are some kind of temporary versions of originalMeshDC. + //--------------------------------------------------------------------------- + mfem::Mesh* shapingMesh = nullptr; + constexpr bool dc_owns_data = true; + sidre::MFEMSidreDataCollection shapingDC("shaping", shapingMesh, dc_owns_data); + { + shapingDC.SetMeshNodesName("positions"); + + // With MPI, loadComputationalMesh returns a parallel mesh. + mfem::ParMesh* parallelMesh = + dynamic_cast(originalMeshDC->GetMesh()); + shapingMesh = (parallelMesh != nullptr) + ? new mfem::ParMesh(*parallelMesh) + : new mfem::Mesh(*originalMeshDC->GetMesh()); + shapingDC.SetMesh(shapingMesh); + } + AXOM_ANNOTATE_END("load mesh"); + printMfemMeshInfo(shapingDC.GetMesh(), "After loading"); + + cellCount = shapingMesh->GetNE(); } - AXOM_ANNOTATE_END("load mesh"); - printMfemMeshInfo(shapingDC.GetMesh(), "After loading"); +#else + SLIC_ERROR_IF(params.useMfem(), "Cannot use MFEM mesh due to Axom configuration. Please use Blueprint mesh or configure with MFEM."); +#endif - const axom::IndexType cellCount = shapingMesh->GetNE(); + axom::sidre::Group* compMeshGrp = nullptr; + if(params.useBlueprint()) + { + compMeshGrp = createBoxMesh(ds.getRoot()->createGroup("compMesh")); + conduit::Node meshNode; + compMeshGrp->createNativeLayout(meshNode); + const conduit::Node& topoNode = meshNode["topologies"][topoName]; + cellCount = conduit::blueprint::mesh::topology::length(topoNode); + } // TODO Port to GPUs. Shaper should be data-parallel, but data may not be on devices yet. using ExecSpace = typename axom::SEQ_EXEC; using ReducePolicy = typename axom::execution_space::reduce_policy; - axom::sidre::Group* compMeshGrp = - createBoxMesh(ds.getRoot()->createGroup("compMesh")); - //--------------------------------------------------------------------------- // Initialize the shaping query object //--------------------------------------------------------------------------- AXOM_ANNOTATE_BEGIN("setup shaping problem"); - bool useBp = true; - std::shared_ptr shaper = useBp - ? std::make_shared(shapeSet, compMeshGrp) - : std::make_shared(shapeSet, &shapingDC); + std::shared_ptr shaper = nullptr; + if(params.useBlueprint()) + { + shaper = std::make_shared(shapeSet, compMeshGrp); + } +#if defined(AXOM_USE_MFEM) && defined(AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION) + if(params.useMfem()) + { + shaper = std::make_shared(shapeSet, &shapingDC); + } +#endif + SLIC_ASSERT( shaper != nullptr ); // Set generic parameters for the base Shaper instance shaper->setVertexWeldThreshold(params.weldThresh); @@ -1271,14 +1356,12 @@ int main(int argc, char** argv) // Associate any fields that begin with "vol_frac" with "material" so when // the data collection is written, a matset will be created. - if(useBp) - { - // TODO: What is the blueprint replacement for AssociateMaterialSet? - } - else +#if defined(AXOM_USE_MFEM) && defined(AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION) + if(params.useMfem()) { shaper->getDC()->AssociateMaterialSet("vol_frac", "material"); } +#endif // Set specific parameters here for IntersectionShaper shaper->setLevel(params.refinementLevel); @@ -1347,15 +1430,11 @@ int main(int argc, char** argv) // Compute and print volumes of each material's volume fraction //--------------------------------------------------------------------------- using axom::utilities::string::startsWith; - if(useBp) + if(params.useBlueprint()) { - // TODO: What is the blueprint equivalent? std::vector materialNames = shaper->getMaterialNames(); for(const auto& materialName : materialNames) { - auto p = shaper->getMaterial(materialName); - axom::ArrayView& materialField = p.first; - int materialIdx = p.second; // Compute and print volume of material. const double volume = sumMaterialVolumes(compMeshGrp, materialName); @@ -1365,7 +1444,8 @@ int main(int argc, char** argv) volume)); } } - else +#if defined(AXOM_USE_MFEM) && defined(AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION) + if(params.useMfem()) { for(auto& kv : shaper->getDC()->GetFieldMap()) { @@ -1388,13 +1468,22 @@ int main(int argc, char** argv) } } } +#endif AXOM_ANNOTATE_END("adjust"); int failCounts = 0; - axom::sidre::Group* volFracGroups = useBp - ? compMeshGrp->getGroup("matsets/material/volume_fractions") - : shapingDC.GetBPGroup()->getGroup("matsets/material/volume_fractions"); + axom::sidre::Group* volFracGroups = nullptr; + if(params.useBlueprint()) + { + volFracGroups = compMeshGrp->getGroup("matsets/material/volume_fractions"); + } +#if defined(AXOM_USE_MFEM) && defined(AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION) + if(params.useMfem()) + { + volFracGroups = shapingDC.GetBPGroup()->getGroup("matsets/material/volume_fractions"); + } +#endif //--------------------------------------------------------------------------- // Correctness test: volume fractions should be in [0,1]. @@ -1471,9 +1560,17 @@ int main(int argc, char** argv) shapeMesh->getNumberOfCells()))); const std::string& materialName = shape.getMaterial(); - double shapeVol = useBp - ? sumMaterialVolumes(compMeshGrp, materialName) - : sumMaterialVolumes(&shapingDC, materialName); + double shapeVol = -1; + if(params.useBlueprint()) + { + shapeVol = sumMaterialVolumes(compMeshGrp, materialName); + } +#if defined(AXOM_USE_MFEM) && defined(AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION) + if(params.useMfem()) + { + shapeVol = sumMaterialVolumes(&shapingDC, materialName); + } +#endif double correctShapeVol = params.testShape == "plane" ? params.boxMeshVolume() / 2 : shapeMeshVol; double diff = shapeVol - correctShapeVol; @@ -1499,14 +1596,21 @@ int main(int argc, char** argv) // Save meshes and fields //--------------------------------------------------------------------------- -#ifdef MFEM_USE_MPI if(!params.outputFile.empty()) { std::string fileName = params.outputFile + ".volfracs"; - shaper->getDC()->Save(fileName, sidre::Group::getDefaultIOProtocol()); - SLIC_INFO(axom::fmt::format("{:=^80}", "Wrote output mesh " + fileName)); - } + if (params.useBlueprint()) + { + saveMesh(*compMeshGrp, fileName); + SLIC_INFO(axom::fmt::format("{:=^80}", "Wrote output mesh " + fileName)); + } +#if defined(AXOM_USE_MFEM) && defined(AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION) + else + { + shaper->getDC()->Save(fileName, sidre::Group::getDefaultIOProtocol()); + } #endif + } shaper.reset(); From fea24a17bb5fc9bd879bcbeb7c5ff7a992425a12 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Mon, 21 Oct 2024 09:56:35 -0700 Subject: [PATCH 029/100] Remove debug code. --- src/axom/quest/util/mesh_helpers.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/axom/quest/util/mesh_helpers.cpp b/src/axom/quest/util/mesh_helpers.cpp index 00178c71aa..f0f50e7f12 100644 --- a/src/axom/quest/util/mesh_helpers.cpp +++ b/src/axom/quest/util/mesh_helpers.cpp @@ -9,7 +9,6 @@ #include #include #include "axom/mint/mesh/UnstructuredMesh.hpp" -#include "axom/core/WhereMacro.hpp" namespace axom { From 54548b3e236c116e99d31baae8f96ab760df31fc Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Mon, 21 Oct 2024 12:00:24 -0700 Subject: [PATCH 030/100] Autoformat. --- src/axom/quest/IntersectionShaper.hpp | 7 ++--- .../quest/examples/quest_shape_in_memory.cpp | 27 ++++++++++--------- src/axom/quest/util/mesh_helpers.cpp | 3 ++- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 9da3303e1d..8e212999f0 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -2049,19 +2049,20 @@ class IntersectionShaper : public Shaper fieldGrp->createView("volume_dependent") ->setString(std::string(volumeDependent ? "true" : "false")); valuesView->allocate(); - if (fieldName.rfind("vol_frac_", 0) == 0) + if(fieldName.rfind("vol_frac_", 0) == 0) { // This is a material volume fraction field. // Shallow-copy valuesView to (uni-buffer) matsets. const std::string matlName = fieldName.substr(9); axom::sidre::Group* volFracGrp = nullptr; - if (m_bpGrp->hasGroup("matsets/material/volume_fractions")) + if(m_bpGrp->hasGroup("matsets/material/volume_fractions")) { volFracGrp = m_bpGrp->getGroup("matsets/material/volume_fractions"); } else { - volFracGrp = m_bpGrp->createGroup("matsets/material/volume_fractions"); + volFracGrp = + m_bpGrp->createGroup("matsets/material/volume_fractions"); m_bpGrp->createViewString("matsets/material/topology", m_bpTopo); } auto* valuesViewInMatsets = volFracGrp->copyView(valuesView); diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index 732e83186e..0548135b03 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -172,7 +172,7 @@ struct Input } // Handle conversion to parallel mfem mesh -#if defined(AXOM_USE_MPI) && defined(MFEM_USE_MPI) + #if defined(AXOM_USE_MPI) && defined(MFEM_USE_MPI) { int* partitioning = nullptr; int part_method = 0; @@ -182,7 +182,7 @@ struct Input delete mesh; mesh = parallelMesh; } -#endif + #endif return mesh; } @@ -351,14 +351,14 @@ void printMfemMeshInfo(mfem::Mesh* mesh, const std::string& prefixMessage = "") namespace primal = axom::primal; int myRank = 0; -#ifdef AXOM_USE_MPI + #ifdef AXOM_USE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &myRank); -#endif + #endif int numElements = mesh->GetNE(); mfem::Vector mins, maxs; -#ifdef MFEM_USE_MPI + #ifdef MFEM_USE_MPI auto* parallelMesh = dynamic_cast(mesh); if(parallelMesh != nullptr) { @@ -367,7 +367,7 @@ void printMfemMeshInfo(mfem::Mesh* mesh, const std::string& prefixMessage = "") myRank = parallelMesh->GetMyRank(); } else -#endif + #endif { mesh->GetBoundingBox(mins, maxs); } @@ -1060,9 +1060,9 @@ double sumMaterialVolumes(sidre::MFEMSidreDataCollection* dc, }); double globalVol = localVol.get(); -#ifdef AXOM_USE_MPI + #ifdef AXOM_USE_MPI MPI_Allreduce(MPI_IN_PLACE, &globalVol, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); -#endif + #endif return globalVol; } #endif @@ -1311,7 +1311,9 @@ int main(int argc, char** argv) cellCount = shapingMesh->GetNE(); } #else - SLIC_ERROR_IF(params.useMfem(), "Cannot use MFEM mesh due to Axom configuration. Please use Blueprint mesh or configure with MFEM."); + SLIC_ERROR_IF(params.useMfem(), + "Cannot use MFEM mesh due to Axom configuration. Please use " + "Blueprint mesh or configure with MFEM."); #endif axom::sidre::Group* compMeshGrp = nullptr; @@ -1343,7 +1345,7 @@ int main(int argc, char** argv) shaper = std::make_shared(shapeSet, &shapingDC); } #endif - SLIC_ASSERT( shaper != nullptr ); + SLIC_ASSERT(shaper != nullptr); // Set generic parameters for the base Shaper instance shaper->setVertexWeldThreshold(params.weldThresh); @@ -1481,7 +1483,8 @@ int main(int argc, char** argv) #if defined(AXOM_USE_MFEM) && defined(AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION) if(params.useMfem()) { - volFracGroups = shapingDC.GetBPGroup()->getGroup("matsets/material/volume_fractions"); + volFracGroups = + shapingDC.GetBPGroup()->getGroup("matsets/material/volume_fractions"); } #endif @@ -1599,7 +1602,7 @@ int main(int argc, char** argv) if(!params.outputFile.empty()) { std::string fileName = params.outputFile + ".volfracs"; - if (params.useBlueprint()) + if(params.useBlueprint()) { saveMesh(*compMeshGrp, fileName); SLIC_INFO(axom::fmt::format("{:=^80}", "Wrote output mesh " + fileName)); diff --git a/src/axom/quest/util/mesh_helpers.cpp b/src/axom/quest/util/mesh_helpers.cpp index f0f50e7f12..7008a37b86 100644 --- a/src/axom/quest/util/mesh_helpers.cpp +++ b/src/axom/quest/util/mesh_helpers.cpp @@ -134,7 +134,8 @@ axom::sidre::Group* make_structured_blueprint_box_mesh( axom::sidre::DataTypeId::FLOAT64_ID, numVerts); - const axom::MDMapping vertMapping(vertsShape, axom::ArrayStrideOrder::COLUMN); + const axom::MDMapping vertMapping(vertsShape, + axom::ArrayStrideOrder::COLUMN); axom::ArrayView xView(xVu->getData(), vertsShape, vertMapping.strides()); From 6c6b7db56d6e87ec0dc8067e821c4b198fa93e22 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Tue, 22 Oct 2024 10:50:06 -0700 Subject: [PATCH 031/100] Fix bugs in handling meshType intput. --- src/axom/quest/examples/CMakeLists.txt | 5 +++-- src/axom/quest/examples/quest_shape_in_memory.cpp | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/axom/quest/examples/CMakeLists.txt b/src/axom/quest/examples/CMakeLists.txt index 194bf61f4e..bb9b6a79f9 100644 --- a/src/axom/quest/examples/CMakeLists.txt +++ b/src/axom/quest/examples/CMakeLists.txt @@ -293,16 +293,17 @@ if((CONDUIT_FOUND OR endif() set(_testshapes "tetmesh" "tet" "hex" "sphere" "cyl" "cone" "vor" "plane") - set(_meshType "bp" "mfem") + set(_meshTypes "bp" "mfem") foreach(_policy ${_policies}) foreach(_testshape ${_testshapes}) - foreach(_meshType ${_meshType}) + foreach(_meshType ${_meshTypes}) set(_testname "quest_shape_in_memory_${_policy}_${_meshType}_${_testshape}") axom_add_test( NAME ${_testname} COMMAND quest_shape_in_memory_ex --policy ${_policy} + --meshType ${_meshType} --testShape ${_testshape} --refinements 2 --scale 0.75 0.75 0.75 diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index 0548135b03..ff6cb32c70 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -210,8 +210,8 @@ struct Input ->description("Enable/disable verbose output") ->capture_default_str(); - app.add_flag("--meshType", meshType) - ->description("Use MFEM computational mesh instead of Blueprint") + app.add_option("--meshType", meshType) + ->description("Type of computational mesh to shape on") ->capture_default_str() ->transform(axom::CLI::CheckedTransformer(meshTypeChoices)); From 99258153634f2e96102af7a35dce7c4adcbc04b2 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Tue, 22 Oct 2024 10:56:30 -0700 Subject: [PATCH 032/100] Fix error in guards against using MFEM when it's not there. --- .../quest/examples/quest_shape_in_memory.cpp | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index ff6cb32c70..32faade3ec 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -128,7 +128,7 @@ struct Input return volume; } -#if defined(AXOM_USE_MFEM) && defined(AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION) +#if defined(AXOM_USE_MFEM) /// Generate an mfem Cartesian mesh, scaled to the bounding box range mfem::Mesh* createBoxMesh() { @@ -339,7 +339,7 @@ struct Input }; // struct Input Input params; -#if defined(AXOM_USE_MFEM) && defined(AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION) +#if defined(AXOM_USE_MFEM) /** * \brief Print some info about the mesh * @@ -821,7 +821,7 @@ double volumeOfTetMesh( return meshVolume; } -#if defined(AXOM_USE_MFEM) && defined(AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION) +#if defined(AXOM_USE_MFEM) /*! @brief Return the element volumes as a sidre::View. @@ -1027,7 +1027,7 @@ axom::sidre::View* getElementVolumes( return volSidreView; } -#if defined(AXOM_USE_MFEM) && defined(AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION) +#if defined(AXOM_USE_MFEM) /*! @brief Return global sum of volume of the given material. */ @@ -1274,7 +1274,7 @@ int main(int argc, char** argv) axom::IndexType cellCount = -1; -#if defined(AXOM_USE_MFEM) && defined(AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION) +#if defined(AXOM_USE_MFEM) if(params.useMfem()) { AXOM_ANNOTATE_BEGIN("load mesh"); @@ -1339,7 +1339,7 @@ int main(int argc, char** argv) { shaper = std::make_shared(shapeSet, compMeshGrp); } -#if defined(AXOM_USE_MFEM) && defined(AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION) +#if defined(AXOM_USE_MFEM) if(params.useMfem()) { shaper = std::make_shared(shapeSet, &shapingDC); @@ -1358,7 +1358,7 @@ int main(int argc, char** argv) // Associate any fields that begin with "vol_frac" with "material" so when // the data collection is written, a matset will be created. -#if defined(AXOM_USE_MFEM) && defined(AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION) +#if defined(AXOM_USE_MFEM) if(params.useMfem()) { shaper->getDC()->AssociateMaterialSet("vol_frac", "material"); @@ -1446,7 +1446,7 @@ int main(int argc, char** argv) volume)); } } -#if defined(AXOM_USE_MFEM) && defined(AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION) +#if defined(AXOM_USE_MFEM) if(params.useMfem()) { for(auto& kv : shaper->getDC()->GetFieldMap()) @@ -1480,7 +1480,7 @@ int main(int argc, char** argv) { volFracGroups = compMeshGrp->getGroup("matsets/material/volume_fractions"); } -#if defined(AXOM_USE_MFEM) && defined(AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION) +#if defined(AXOM_USE_MFEM) if(params.useMfem()) { volFracGroups = @@ -1568,7 +1568,7 @@ int main(int argc, char** argv) { shapeVol = sumMaterialVolumes(compMeshGrp, materialName); } -#if defined(AXOM_USE_MFEM) && defined(AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION) +#if defined(AXOM_USE_MFEM) if(params.useMfem()) { shapeVol = sumMaterialVolumes(&shapingDC, materialName); @@ -1607,7 +1607,7 @@ int main(int argc, char** argv) saveMesh(*compMeshGrp, fileName); SLIC_INFO(axom::fmt::format("{:=^80}", "Wrote output mesh " + fileName)); } -#if defined(AXOM_USE_MFEM) && defined(AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION) +#if defined(AXOM_USE_MFEM) else { shaper->getDC()->Save(fileName, sidre::Group::getDefaultIOProtocol()); From fa99f5b98e61c815258053e16ba0232014c379fd Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Tue, 22 Oct 2024 10:59:59 -0700 Subject: [PATCH 033/100] Fix scope of MFEM variable. --- .../quest/examples/quest_shape_in_memory.cpp | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index 32faade3ec..5ac9d1ce36 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -1275,6 +1275,7 @@ int main(int argc, char** argv) axom::IndexType cellCount = -1; #if defined(AXOM_USE_MFEM) + std::shared_ptr shapingDC; if(params.useMfem()) { AXOM_ANNOTATE_BEGIN("load mesh"); @@ -1293,9 +1294,11 @@ int main(int argc, char** argv) //--------------------------------------------------------------------------- mfem::Mesh* shapingMesh = nullptr; constexpr bool dc_owns_data = true; - sidre::MFEMSidreDataCollection shapingDC("shaping", shapingMesh, dc_owns_data); + shapingDC = std::make_shared("shaping", + shapingMesh, + dc_owns_data); { - shapingDC.SetMeshNodesName("positions"); + shapingDC->SetMeshNodesName("positions"); // With MPI, loadComputationalMesh returns a parallel mesh. mfem::ParMesh* parallelMesh = @@ -1303,17 +1306,18 @@ int main(int argc, char** argv) shapingMesh = (parallelMesh != nullptr) ? new mfem::ParMesh(*parallelMesh) : new mfem::Mesh(*originalMeshDC->GetMesh()); - shapingDC.SetMesh(shapingMesh); + shapingDC->SetMesh(shapingMesh); } AXOM_ANNOTATE_END("load mesh"); - printMfemMeshInfo(shapingDC.GetMesh(), "After loading"); + printMfemMeshInfo(shapingDC->GetMesh(), "Loaded MFEM mesh"); cellCount = shapingMesh->GetNE(); } #else SLIC_ERROR_IF(params.useMfem(), "Cannot use MFEM mesh due to Axom configuration. Please use " - "Blueprint mesh or configure with MFEM."); + "Blueprint mesh or configure with MFEM and " + "-DAXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION."); #endif axom::sidre::Group* compMeshGrp = nullptr; @@ -1322,6 +1326,8 @@ int main(int argc, char** argv) compMeshGrp = createBoxMesh(ds.getRoot()->createGroup("compMesh")); conduit::Node meshNode; compMeshGrp->createNativeLayout(meshNode); + SLIC_INFO(axom::fmt::format("{:-^80}", "Generated Blueprint mesh")); + meshNode.print(); const conduit::Node& topoNode = meshNode["topologies"][topoName]; cellCount = conduit::blueprint::mesh::topology::length(topoNode); } @@ -1342,7 +1348,8 @@ int main(int argc, char** argv) #if defined(AXOM_USE_MFEM) if(params.useMfem()) { - shaper = std::make_shared(shapeSet, &shapingDC); + shaper = + std::make_shared(shapeSet, shapingDC.get()); } #endif SLIC_ASSERT(shaper != nullptr); @@ -1484,7 +1491,7 @@ int main(int argc, char** argv) if(params.useMfem()) { volFracGroups = - shapingDC.GetBPGroup()->getGroup("matsets/material/volume_fractions"); + shapingDC->GetBPGroup()->getGroup("matsets/material/volume_fractions"); } #endif @@ -1571,7 +1578,8 @@ int main(int argc, char** argv) #if defined(AXOM_USE_MFEM) if(params.useMfem()) { - shapeVol = sumMaterialVolumes(&shapingDC, materialName); + shapeVol = + sumMaterialVolumes(shapingDC.get(), materialName); } #endif double correctShapeVol = @@ -1605,7 +1613,7 @@ int main(int argc, char** argv) if(params.useBlueprint()) { saveMesh(*compMeshGrp, fileName); - SLIC_INFO(axom::fmt::format("{:=^80}", "Wrote output mesh " + fileName)); + SLIC_INFO(axom::fmt::format("{:-^80}", "Wrote output mesh " + fileName)); } #if defined(AXOM_USE_MFEM) else From 78914f8d5607c1b4254574ef641b4ec3c851a58f Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Tue, 22 Oct 2024 11:00:38 -0700 Subject: [PATCH 034/100] Fix error setting up tet geometry. --- src/axom/quest/examples/quest_shape_in_memory.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index 5ac9d1ce36..29e9d1e9f0 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -63,10 +63,11 @@ struct Input public: std::string outputFile; - std::vector center {0.0, 0.0, 0.0}; + // Values for some shape geometries double radius {1.0}; double radius2 {0.3}; double length {2.0}; + std::vector center {0.0, 0.0, -length / 2}; std::vector direction {0.0, 0.0, 1.0}; // Shape transformation parameters @@ -706,8 +707,8 @@ axom::klee::Shape createShape_Tet() const double len = params.length; const Point3D a {0.0, 0.0, 0.0}; const Point3D b {len, 0.0, 0.0}; - const Point3D c {len, 1.0, 0.0}; - const Point3D d {0.0, 1.0, 0.0}; + const Point3D c {0.0, len, 0.0}; + const Point3D d {0.0, 0.0, len}; const primal::Tetrahedron tet {a, b, c, d}; axom::klee::Geometry tetGeometry(prop, tet, scaleOp); From c0d566afdb5f113cc29c75dd7d8f4d4e9cb107bb Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Tue, 22 Oct 2024 11:51:03 -0700 Subject: [PATCH 035/100] Tweak shapes to center them and reduce resolution-related bumps. --- src/axom/quest/examples/CMakeLists.txt | 2 +- .../quest/examples/quest_shape_in_memory.cpp | 89 ++++++++++++------- 2 files changed, 57 insertions(+), 34 deletions(-) diff --git a/src/axom/quest/examples/CMakeLists.txt b/src/axom/quest/examples/CMakeLists.txt index bb9b6a79f9..143fdb1c17 100644 --- a/src/axom/quest/examples/CMakeLists.txt +++ b/src/axom/quest/examples/CMakeLists.txt @@ -306,7 +306,7 @@ if((CONDUIT_FOUND OR --meshType ${_meshType} --testShape ${_testshape} --refinements 2 - --scale 0.75 0.75 0.75 + --scale 2.0 2.0 2.0 inline_mesh --min -3 -3 -3 --max 3 3 3 --res 20 20 20 -d 3 NUM_MPI_TASKS ${_nranks}) endforeach() diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index 29e9d1e9f0..84ac75feca 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -64,10 +64,11 @@ struct Input std::string outputFile; // Values for some shape geometries + // See createShape_*() for how specific shapes use them. double radius {1.0}; double radius2 {0.3}; double length {2.0}; - std::vector center {0.0, 0.0, -length / 2}; + std::vector center {0.1, 0.2, 0.3}; std::vector direction {0.0, 0.0, 1.0}; // Shape transformation parameters @@ -561,17 +562,20 @@ axom::klee::Shape createShape_TetMesh(sidre::DataStore& ds) topo, coordset); - double lll = 4.0; - - // Insert tet at origin. - tetMesh.appendNode(0.0, 0.0, 0.0); - tetMesh.appendNode(lll, 0.0, 0.0); - tetMesh.appendNode(0.0, lll, 0.0); - tetMesh.appendNode(0.0, 0.0, lll); - tetMesh.appendNode(lll, lll, 0.0); + double lll = params.length; + double h = lll/2; // To center the shape. + + tetMesh.appendNode(0.0 - h, 0.0 - h, 0.0 - h); + tetMesh.appendNode(lll - h, 0.0 - h, 0.0 - h); + tetMesh.appendNode(0.0 - h, lll - h, 0.0 - h); + tetMesh.appendNode(0.0 - h, 0.0 - h, lll - h); + tetMesh.appendNode(lll - h, lll - h, lll - h); + tetMesh.appendNode(lll - h, lll - h, 0.0 - h); + tetMesh.appendNode(0.0 - h, lll - h, lll - h); + tetMesh.appendNode(lll - h, 0.0 - h, lll - h); axom::IndexType conn0[4] = {0, 1, 2, 3}; tetMesh.appendCell(conn0); - axom::IndexType conn1[4] = {4, 1, 2, 3}; + axom::IndexType conn1[4] = {4, 5, 6, 7}; tetMesh.appendCell(conn1); SLIC_ASSERT(axom::mint::blueprint::isValidRootGroup(meshGroup)); @@ -629,15 +633,24 @@ axom::klee::Geometry createGeometry_Vor(axom::primal::Point& vorBase, axom::klee::Shape createShape_Vor() { - Point3D vorBase {params.center.data()}; + Point3D vorBase {0.0, 0.0, -params.length/2}; axom::primal::Vector vorDirection {params.direction.data()}; - axom::Array discreteFunction({3, 2}, axom::ArrayStrideOrder::ROW); - discreteFunction[0][0] = 0.0; - discreteFunction[0][1] = 1.0; - discreteFunction[1][0] = 0.5 * params.length; - discreteFunction[1][1] = 0.8; - discreteFunction[2][0] = params.length; - discreteFunction[2][1] = 1.0; + int numIntervals = 5; + axom::Array discreteFunction({numIntervals + 1, 2}, + axom::ArrayStrideOrder::ROW); + double dz = params.length / numIntervals; + discreteFunction[0][0] = 0 * dz; + discreteFunction[0][1] = params.radius; + discreteFunction[1][0] = 1 * dz; + discreteFunction[1][1] = params.radius; + discreteFunction[2][0] = 2 * dz; + discreteFunction[2][1] = params.radius2; + discreteFunction[3][0] = 3 * dz; + discreteFunction[3][1] = params.radius2; + discreteFunction[4][0] = 4 * dz; + discreteFunction[4][1] = params.radius; + discreteFunction[5][0] = 5 * dz; + discreteFunction[5][1] = 0.0; axom::klee::Geometry vorGeometry = createGeometry_Vor(vorBase, vorDirection, discreteFunction); @@ -649,7 +662,7 @@ axom::klee::Shape createShape_Vor() axom::klee::Shape createShape_Cylinder() { - Point3D vorBase {params.center.data()}; + Point3D vorBase {0.0, 0.0, -params.length/2}; axom::primal::Vector vorDirection {params.direction.data()}; axom::Array discreteFunction({2, 2}, axom::ArrayStrideOrder::ROW); double radius = params.radius; @@ -669,7 +682,7 @@ axom::klee::Shape createShape_Cylinder() axom::klee::Shape createShape_Cone() { - Point3D vorBase {params.center.data()}; + Point3D vorBase {0.0, 0.0, -params.length/2}; axom::primal::Vector vorDirection {params.direction.data()}; axom::Array discreteFunction({2, 2}, axom::ArrayStrideOrder::ROW); double baseRadius = params.radius; @@ -705,10 +718,11 @@ axom::klee::Shape createShape_Tet() } const double len = params.length; - const Point3D a {0.0, 0.0, 0.0}; - const Point3D b {len, 0.0, 0.0}; - const Point3D c {0.0, len, 0.0}; - const Point3D d {0.0, 0.0, len}; + const double h = len / 3; // To center the shape. + const Point3D a {0.0 - h, 0.0 - h, 0.0 - h}; + const Point3D b {len - h, 0.0 - h, 0.0 - h}; + const Point3D c {0.0 - h, len - h, 0.0 - h}; + const Point3D d {0.0 - h, 0.0 - h, len - h}; const primal::Tetrahedron tet {a, b, c, d}; axom::klee::Geometry tetGeometry(prop, tet, scaleOp); @@ -734,14 +748,19 @@ axom::klee::Shape createShape_Hex() } const double len = params.length; - const Point3D p {0.0, 0.0, 0.0}; - const Point3D q {len, 0.0, 0.0}; - const Point3D r {len, 1.0, 0.0}; - const Point3D s {0.0, 1.0, 0.0}; - const Point3D t {0.0, 0.0, 1.0}; - const Point3D u {len, 0.0, 1.0}; - const Point3D v {len, 1.0, 1.0}; - const Point3D w {0.0, 1.0, 1.0}; + const double xhl = 0.5 * len * 0.8; + const double yhl = 0.5 * len * 1.0; + const double zhl = 0.5 * len * 1.2; + // clang-format off + const Point3D p {-xhl, -yhl, -zhl}; + const Point3D q { xhl, -yhl, -zhl}; + const Point3D r { xhl, yhl, -zhl}; + const Point3D s {-xhl, yhl, -zhl}; + const Point3D t {-xhl, -yhl, zhl}; + const Point3D u { xhl, -yhl, zhl}; + const Point3D v { xhl, yhl, zhl}; + const Point3D w {-xhl, yhl, zhl}; + // clang-format on const primal::Hexahedron hex {p, q, r, s, t, u, v, w}; axom::klee::Geometry hexGeometry(prop, hex, scaleOp); @@ -1585,9 +1604,13 @@ int main(int argc, char** argv) #endif double correctShapeVol = params.testShape == "plane" ? params.boxMeshVolume() / 2 : shapeMeshVol; + SLIC_ASSERT(correctShapeVol > 0.0); // Indicates error in the test setup. double diff = shapeVol - correctShapeVol; - bool err = !axom::utilities::isNearlyEqual(shapeVol, correctShapeVol); + bool err = !axom::utilities::isNearlyEqualRelative(shapeVol, + correctShapeVol, + 1e-6, + 1e-8); failCounts += err; SLIC_INFO(axom::fmt::format( From b1d84e1d2cf87386a43d1480c03444eb276c0f2a Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Tue, 22 Oct 2024 16:51:52 -0700 Subject: [PATCH 036/100] Test non-grid-aligned VOR directin. --- src/axom/quest/examples/CMakeLists.txt | 2 +- src/axom/quest/examples/quest_shape_in_memory.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/axom/quest/examples/CMakeLists.txt b/src/axom/quest/examples/CMakeLists.txt index 143fdb1c17..135c66c611 100644 --- a/src/axom/quest/examples/CMakeLists.txt +++ b/src/axom/quest/examples/CMakeLists.txt @@ -307,7 +307,7 @@ if((CONDUIT_FOUND OR --testShape ${_testshape} --refinements 2 --scale 2.0 2.0 2.0 - inline_mesh --min -3 -3 -3 --max 3 3 3 --res 20 20 20 -d 3 + inline_mesh --min -4 -4 -4 --max 4 4 4 --res 20 20 20 -d 3 NUM_MPI_TASKS ${_nranks}) endforeach() endforeach() diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index 84ac75feca..a965124393 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -69,7 +69,7 @@ struct Input double radius2 {0.3}; double length {2.0}; std::vector center {0.1, 0.2, 0.3}; - std::vector direction {0.0, 0.0, 1.0}; + std::vector direction {0.0, 0.5, 1.0}; // Shape transformation parameters std::vector scaleFactors; From a02fc77a51d51230a471b77ac78e6cd31b6cdbdc Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Tue, 22 Oct 2024 16:52:26 -0700 Subject: [PATCH 037/100] Fix a weird numerical error, maybe caused by large floating point values. --- src/axom/quest/DiscreteShape.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/axom/quest/DiscreteShape.cpp b/src/axom/quest/DiscreteShape.cpp index 10ec8b5e1f..1cc5575e99 100644 --- a/src/axom/quest/DiscreteShape.cpp +++ b/src/axom/quest/DiscreteShape.cpp @@ -374,7 +374,7 @@ void DiscreteShape::createPlaneRepresentation() const auto& plane = geometry.getPlane(); // Generate a big bounding hex on the positive side of the plane. axom::primal::Hexahedron boundingHex; - const double len = 1e6; // Big enough to contain anticipated mesh. + const double len = 1e4; // Big enough to contain anticipated mesh. // We should compute based on the mesh. // clang-format off boundingHex[0] = Point3D{0.0, -len, -len}; From a26dc9bbcf5f1723565443525d65720710c971ab Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Tue, 22 Oct 2024 17:11:08 -0700 Subject: [PATCH 038/100] Tilt the plane shape away from the axes. --- src/axom/quest/examples/quest_shape_in_memory.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index a965124393..cec2d48e8e 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -69,7 +69,7 @@ struct Input double radius2 {0.3}; double length {2.0}; std::vector center {0.1, 0.2, 0.3}; - std::vector direction {0.0, 0.5, 1.0}; + std::vector direction {0.1, 0.2, 0.4}; // Shape transformation parameters std::vector scaleFactors; @@ -790,7 +790,7 @@ axom::klee::Shape createShape_Plane() Point3D center {0.5 * (primal::NumericArray(params.boxMins.data()) + primal::NumericArray(params.boxMaxs.data()))}; - primal::Vector normal {1.0, 0.0, 0.0}; + primal::Vector normal(params.direction.data()); const primal::Plane plane {normal, center, true}; axom::klee::Geometry planeGeometry(prop, plane, scaleOp); From 10f97fc32cd00b1ac5534434afe629237dfa6732 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Tue, 22 Oct 2024 17:35:28 -0700 Subject: [PATCH 039/100] Don't test MFEM mesh if no MFEM. --- src/axom/quest/examples/CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/axom/quest/examples/CMakeLists.txt b/src/axom/quest/examples/CMakeLists.txt index 135c66c611..945d3237b4 100644 --- a/src/axom/quest/examples/CMakeLists.txt +++ b/src/axom/quest/examples/CMakeLists.txt @@ -293,7 +293,10 @@ if((CONDUIT_FOUND OR endif() set(_testshapes "tetmesh" "tet" "hex" "sphere" "cyl" "cone" "vor" "plane") - set(_meshTypes "bp" "mfem") + set(_meshTypes "bp") + if(MFEM_FOUND) + list(APPEND _meshTypes "mfem") + endif() foreach(_policy ${_policies}) foreach(_testshape ${_testshapes}) From 00a3074708840e41f81997ed598ddc1a4bbc6648 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Wed, 23 Oct 2024 11:06:16 -0700 Subject: [PATCH 040/100] Autoformat. --- src/axom/quest/examples/quest_shape_in_memory.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index cec2d48e8e..096c0bfdcc 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -563,7 +563,7 @@ axom::klee::Shape createShape_TetMesh(sidre::DataStore& ds) coordset); double lll = params.length; - double h = lll/2; // To center the shape. + double h = lll / 2; // To center the shape. tetMesh.appendNode(0.0 - h, 0.0 - h, 0.0 - h); tetMesh.appendNode(lll - h, 0.0 - h, 0.0 - h); @@ -633,7 +633,7 @@ axom::klee::Geometry createGeometry_Vor(axom::primal::Point& vorBase, axom::klee::Shape createShape_Vor() { - Point3D vorBase {0.0, 0.0, -params.length/2}; + Point3D vorBase {0.0, 0.0, -params.length / 2}; axom::primal::Vector vorDirection {params.direction.data()}; int numIntervals = 5; axom::Array discreteFunction({numIntervals + 1, 2}, @@ -662,7 +662,7 @@ axom::klee::Shape createShape_Vor() axom::klee::Shape createShape_Cylinder() { - Point3D vorBase {0.0, 0.0, -params.length/2}; + Point3D vorBase {0.0, 0.0, -params.length / 2}; axom::primal::Vector vorDirection {params.direction.data()}; axom::Array discreteFunction({2, 2}, axom::ArrayStrideOrder::ROW); double radius = params.radius; @@ -682,7 +682,7 @@ axom::klee::Shape createShape_Cylinder() axom::klee::Shape createShape_Cone() { - Point3D vorBase {0.0, 0.0, -params.length/2}; + Point3D vorBase {0.0, 0.0, -params.length / 2}; axom::primal::Vector vorDirection {params.direction.data()}; axom::Array discreteFunction({2, 2}, axom::ArrayStrideOrder::ROW); double baseRadius = params.radius; From ec2e1d0dfecd5c461cc1272a0dd6a82cb822af6f Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Thu, 24 Oct 2024 06:31:46 -0700 Subject: [PATCH 041/100] Debug device runtime policies in shaper and test code. --- src/axom/quest/IntersectionShaper.hpp | 43 ++++- .../quest/examples/quest_shape_in_memory.cpp | 96 ++++++++-- src/axom/quest/util/mesh_helpers.cpp | 181 +++++++++++++++--- src/axom/quest/util/mesh_helpers.hpp | 37 +++- 4 files changed, 311 insertions(+), 46 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 8e212999f0..fc20c081f5 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -710,10 +710,16 @@ class IntersectionShaper : public Shaper }); // end of loop to initialize hexahedral elements and bounding boxes // Set each shape volume fraction to 1 - for(int i = 0; i < m_cellCount; i++) + // volFrac may be on the host (MFEM) or device (Sidre). + auto fillVolFrac = AXOM_LAMBDA(axom::IndexType i) { volFrac[i] = 1.0; }; + if(axom::detail::getAllocatorSpace(volFrac.getAllocatorID()) == + MemorySpace::Host) { - double vf = 1.0; - volFrac[i] = vf; + axom::for_all(m_cellCount, fillVolFrac); + } + else + { + axom::for_all(m_cellCount, fillVolFrac); } // Set shape components to zero if within threshold @@ -1503,8 +1509,35 @@ class IntersectionShaper : public Shaper if(newData) { // Zero out the volume fractions (on host). - // memset(matVolFrac->begin(), 0, matVolFrac->Size() * sizeof(double)); - memset(matVolFrac.data(), 0, matVolFrac.size() * sizeof(double)); + auto allocId = matVolFrac.getAllocatorID(); + const axom::MemorySpace memorySpace = + axom::detail::getAllocatorSpace(allocId); + const bool onDevice = memorySpace == axom::MemorySpace::Device || + memorySpace == axom::MemorySpace::Unified; + if(onDevice) + { +#if defined(AXOM_USE_CUDA) + if(m_execPolicy == RuntimePolicy::cuda) + { + axom::for_all>( + matVolFrac.size(), + AXOM_LAMBDA(axom::IndexType i) { matVolFrac[i] = 0.0; }); + } +#endif +#if defined(AXOM_USE_HIP) + if(m_execPolicy == RuntimePolicy::hip) + { + axom::for_all>( + matVolFrac.size(), + AXOM_LAMBDA(axom::IndexType i) { matVolFrac[i] = 0.0; }); + } +#endif + } + else + { + // memset(matVolFrac->begin(), 0, matVolFrac->Size() * sizeof(double)); + memset(matVolFrac.data(), 0, matVolFrac.size() * sizeof(double)); + } } // Add the material to our vectors. diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index 096c0bfdcc..64875e5180 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -416,7 +416,8 @@ axom::sidre::Group* createBoxMesh(axom::sidre::Group* meshGrp) bbox, res, topoName, - coordsetName); + coordsetName, + params.policy); #if defined(AXOM_DEBUG) conduit::Node meshNode, info; meshGrp->createNativeLayout(meshNode); @@ -971,11 +972,14 @@ axom::sidre::View* getElementVolumes( constexpr int NUM_COMPS_PER_VERT = 3; constexpr double ZERO_THRESHOLD = 1.e-10; + /* + Get vertex coordinates. We use UnstructuredMesh for this, + so get it on host first then transfer to device if needed. + */ axom::Array vertCoords(cellCount * NUM_VERTS_PER_HEX, cellCount * NUM_VERTS_PER_HEX); auto vertCoordsView = vertCoords.view(); - // This runs only only on host, because the mfem::Mesh only uses host memory, I think. for(axom::IndexType cellIdx = 0; cellIdx < cellCount; ++cellIdx) { // Get the indices of this element's vertices @@ -992,6 +996,12 @@ axom::sidre::View* getElementVolumes( } } } + if(vertCoords.getAllocatorID() != meshGrp->getDefaultAllocatorID()) + { + vertCoords = + axom::Array(vertCoords, meshGrp->getDefaultAllocatorID()); + vertCoordsView = vertCoords.view(); + } // Set vertex coords to zero if within threshold. // (I don't know why we do this. I'm following examples.) @@ -1009,7 +1019,9 @@ axom::sidre::View* getElementVolumes( }); // Initialize hexahedral elements. - axom::Array hexes(cellCount, cellCount); + axom::Array hexes(cellCount, + cellCount, + meshGrp->getDefaultAllocatorID()); auto hexesView = hexes.view(); axom::for_all( cellCount, @@ -1088,14 +1100,19 @@ double sumMaterialVolumes(sidre::MFEMSidreDataCollection* dc, #endif template -double sumMaterialVolumes(sidre::Group* meshGrp, const std::string& material) +double sumMaterialVolumesImpl(sidre::Group* meshGrp, const std::string& material) { conduit::Node meshNode; meshGrp->createNativeLayout(meshNode); #if defined(AXOM_DEBUG) - conduit::Node info; - conduit::blueprint::mesh::verify(meshNode, info); - SLIC_ASSERT(conduit::blueprint::mesh::verify(meshNode, info)); + // Conduit can verify Blueprint mesh, but only if data is on host. + if(axom::detail::getAllocatorSpace(meshGrp->getDefaultAllocatorID()) == + axom::MemorySpace::Host) + { + conduit::Node info; + conduit::blueprint::mesh::verify(meshNode, info); + SLIC_ASSERT(conduit::blueprint::mesh::verify(meshNode, info)); + } #endif std::string topoPath = axom::fmt::format("topologies/{}", topoName); conduit::Node& topoNode = meshNode.fetch_existing(topoPath); @@ -1131,6 +1148,34 @@ double sumMaterialVolumes(sidre::Group* meshGrp, const std::string& material) return globalVol; } +double sumMaterialVolumes(sidre::Group* meshGrp, const std::string& material) +{ + double rval = 0.0; + if(params.policy == RuntimePolicy::seq) + { + rval = sumMaterialVolumesImpl(meshGrp, material); + } +#if defined(AXOM_USE_OPENMP) + if(params.policy == RuntimePolicy::omp) + { + rval = sumMaterialVolumesImpl(meshGrp, material); + } +#endif +#if defined(AXOM_USE_CUDA) + if(params.policy == RuntimePolicy::cuda) + { + rval = sumMaterialVolumesImpl>(meshGrp, material); + } +#endif +#if defined(AXOM_USE_HIP) + if(params.policy == RuntimePolicy::hip) + { + rval = sumMaterialVolumesImpl>(meshGrp, material); + } +#endif + return rval; +} + /// Write blueprint mesh to disk void saveMesh(const conduit::Node& mesh, const std::string& filename) { @@ -1206,6 +1251,26 @@ int main(int argc, char** argv) axom::utilities::raii::AnnotationsWrapper annotations_raii_wrapper( params.annotationMode); + // We will use array memory compatible with the specified runtime policy. + int defaultAllocId = axom::execution_space::allocatorID(); +#if defined(AXOM_USE_CUDA) + if(params.policy == RuntimePolicy::cuda) + { + defaultAllocId = axom::execution_space>::allocatorID(); + } +#endif +#if defined(AXOM_USE_HIP) + if(params.policy == RuntimePolicy::hip) + { + defaultAllocId = axom::execution_space>::allocatorID(); + } +#endif + const axom::MemorySpace memorySpace = + axom::detail::getAllocatorSpace(defaultAllocId); + const bool onHost = memorySpace == axom::MemorySpace::Host || + memorySpace == axom::MemorySpace::Dynamic || + memorySpace == axom::MemorySpace::Unified; + AXOM_ANNOTATE_BEGIN("quest example for shaping primals"); AXOM_ANNOTATE_BEGIN("init"); @@ -1343,11 +1408,17 @@ int main(int argc, char** argv) axom::sidre::Group* compMeshGrp = nullptr; if(params.useBlueprint()) { - compMeshGrp = createBoxMesh(ds.getRoot()->createGroup("compMesh")); + compMeshGrp = ds.getRoot()->createGroup("compMesh"); + compMeshGrp->setDefaultAllocator(defaultAllocId); + + createBoxMesh(compMeshGrp); conduit::Node meshNode; compMeshGrp->createNativeLayout(meshNode); SLIC_INFO(axom::fmt::format("{:-^80}", "Generated Blueprint mesh")); - meshNode.print(); + if(onHost) + { + meshNode.print(); + } const conduit::Node& topoNode = meshNode["topologies"][topoName]; cellCount = conduit::blueprint::mesh::topology::length(topoNode); } @@ -1373,6 +1444,7 @@ int main(int argc, char** argv) } #endif SLIC_ASSERT(shaper != nullptr); + shaper->setExecPolicy(params.policy); // Set generic parameters for the base Shaper instance shaper->setVertexWeldThreshold(params.weldThresh); @@ -1398,7 +1470,6 @@ int main(int argc, char** argv) "{:-^80}", axom::fmt::format("Setting IntersectionShaper policy to '{}'", axom::runtime_policy::policyToName(params.policy)))); - shaper->setExecPolicy(params.policy); if(!params.backgroundMaterial.empty()) { @@ -1465,8 +1536,7 @@ int main(int argc, char** argv) for(const auto& materialName : materialNames) { // Compute and print volume of material. - const double volume = - sumMaterialVolumes(compMeshGrp, materialName); + const double volume = sumMaterialVolumes(compMeshGrp, materialName); SLIC_INFO(axom::fmt::format(axom::utilities::locale(), "Volume of material '{}' is {:.6Lf}", materialName, @@ -1593,7 +1663,7 @@ int main(int argc, char** argv) double shapeVol = -1; if(params.useBlueprint()) { - shapeVol = sumMaterialVolumes(compMeshGrp, materialName); + shapeVol = sumMaterialVolumes(compMeshGrp, materialName); } #if defined(AXOM_USE_MFEM) if(params.useMfem()) diff --git a/src/axom/quest/util/mesh_helpers.cpp b/src/axom/quest/util/mesh_helpers.cpp index 7008a37b86..0f0c283521 100644 --- a/src/axom/quest/util/mesh_helpers.cpp +++ b/src/axom/quest/util/mesh_helpers.cpp @@ -95,8 +95,26 @@ axom::sidre::Group* make_structured_blueprint_box_mesh( const primal::BoundingBox& bbox, const primal::NumericArray& res, const std::string& topologyName, - const std::string& coordsetName) + const std::string& coordsetName, + axom::runtime_policy::Policy runtimePolicy) { + auto* topoGrp = meshGrp->createGroup("topologies")->createGroup(topologyName); + SLIC_ERROR_IF(topoGrp == nullptr, + "Cannot allocate topology '" + topologyName + + "' in blueprint mesh '" + meshGrp->getName() + + "'. It already exists."); + + auto* coordsetGrp = + meshGrp->createGroup("coordsets")->createGroup(coordsetName); + SLIC_ERROR_IF(coordsetGrp == nullptr, + "Cannot allocate coordset '" + coordsetName + + "' in blueprint mesh '" + meshGrp->getName() + + "'. It already exists."); + + topoGrp->createView("type")->setString("structured"); + topoGrp->createView("coordset")->setString(coordsetName); + auto* dimsGrp = topoGrp->createGroup("elements/dims"); + constexpr int DIM = 3; auto ni = res[0]; @@ -108,13 +126,6 @@ axom::sidre::Group* make_structured_blueprint_box_mesh( res[2] + 1}; const auto numVerts = vertsShape[0] * vertsShape[1] * vertsShape[2]; - auto* topoGrp = meshGrp->createGroup("topologies")->createGroup(topologyName); - auto* coordsetGrp = - meshGrp->createGroup("coordsets")->createGroup(coordsetName); - - topoGrp->createView("type")->setString("structured"); - topoGrp->createView("coordset")->setString(coordsetName); - auto* dimsGrp = topoGrp->createGroup("elements/dims"); dimsGrp->createViewScalar("i", ni); dimsGrp->createViewScalar("j", nj); dimsGrp->createViewScalar("k", nk); @@ -146,21 +157,7 @@ axom::sidre::Group* make_structured_blueprint_box_mesh( vertsShape, vertMapping.strides()); - double dx = (bbox.getMax()[0] - bbox.getMin()[0]) / res[0]; - double dy = (bbox.getMax()[1] - bbox.getMin()[1]) / res[1]; - double dz = (bbox.getMax()[2] - bbox.getMin()[2]) / res[2]; - for(int k = 0; k < nk + 1; ++k) - { - for(int j = 0; j < nj + 1; ++j) - { - for(int i = 0; i < ni + 1; ++i) - { - xView(i, j, k) = bbox.getMin()[0] + i * dx; - yView(i, j, k) = bbox.getMin()[1] + j * dy; - zView(i, j, k) = bbox.getMin()[2] + k * dz; - } - } - } + fill_cartesian_coords_3d(runtimePolicy, bbox, xView, yView, zView); #if defined(AXOM_DEBUG) && defined(AXOM_USE_CONDUIT) { @@ -179,9 +176,15 @@ axom::sidre::Group* make_unstructured_blueprint_box_mesh( const primal::BoundingBox& bbox, const primal::NumericArray& res, const std::string& topologyName, - const std::string& coordsetName) + const std::string& coordsetName, + axom::runtime_policy::Policy runtimePolicy) { - make_structured_blueprint_box_mesh(meshGrp, bbox, res, topologyName, coordsetName); + make_structured_blueprint_box_mesh(meshGrp, + bbox, + res, + topologyName, + coordsetName, + runtimePolicy); convert_blueprint_structured_explicit_to_unstructured(meshGrp, topologyName); return meshGrp; } @@ -194,6 +197,27 @@ void convert_blueprint_structured_explicit_to_unstructured( meshGrp->getView(axom::fmt::format("topologies/{}/coordset", topoName)) ->getString(); + sidre::Group* coordsetGrp = nullptr; + #if defined(AXOM_USE_UMPIRE) + /* If using device memory, temporarily copy coords to host + so we can use conduit's blueprint::mesh utilities. + When we re-import the newCoords, we will get the data + back to meshGrp memory space. + */ + coordsetGrp = meshGrp->getGroup("coordsets")->getGroup(coordsetName); + int coordsetAllocId = coordsetGrp->getDefaultAllocatorID(); + MemorySpace memSpace = detail::getAllocatorSpace(coordsetAllocId); + sidre::Group* stashGrp = nullptr; + sidre::Group* stashedValuesGrp = nullptr; + if(memSpace == MemorySpace::Device) + { + int hostAllocId = execution_space::allocatorID(); + stashGrp = meshGrp->createGroup("tempStash"); + stashedValuesGrp = stashGrp->moveGroup(coordsetGrp->getGroup("values")); + coordsetGrp->deepCopyGroup(stashedValuesGrp, hostAllocId); + } + #endif + // Convert mesh to conduit::Node to use conduit's blueprint support. conduit::Node info; conduit::Node curMesh; @@ -218,7 +242,7 @@ void convert_blueprint_structured_explicit_to_unstructured( ->setString(coordsetName); // Is this needed? Is coordset already set? meshGrp->getGroup("coordsets")->destroyGroup(coordsetName); - auto* coordsetGrp = meshGrp->getGroup("coordsets")->createGroup(coordsetName); + coordsetGrp = meshGrp->getGroup("coordsets")->createGroup(coordsetName); coordsetGrp->importConduitTree(newCoords); #define ADD_EXTRA_DATA_FOR_MINT 1 @@ -283,6 +307,111 @@ bool verifyBlueprintMesh(const axom::sidre::Group* meshGrp, conduit::Node info) #endif +void fill_cartesian_coords_3d(axom::runtime_policy::Policy runtimePolicy, + const primal::BoundingBox& domainBox, + axom::ArrayView& xView, + axom::ArrayView& yView, + axom::ArrayView& zView) +{ + if(runtimePolicy == axom::runtime_policy::Policy::seq) + { + fill_cartesian_coords_3d_impl(domainBox, xView, yView, zView); + } +#if defined(AXOM_RUNTIME_POLICY_USE_OPENMP) + if(runtimePolicy == axom::runtime_policy::Policy::omp) + { + fill_cartesian_coords_3d_impl(domainBox, xView, yView, zView); + } +#endif +#if defined(AXOM_RUNTIME_POLICY_USE_CUDA) + if(runtimePolicy == axom::runtime_policy::Policy::cuda) + { + fill_cartesian_coords_3d_impl>(domainBox, + xView, + yView, + zView); + } +#endif +#if defined(AXOM_RUNTIME_POLICY_USE_HIP) + if(runtimePolicy == axom::runtime_policy::Policy::hip) + { + fill_cartesian_coords_3d_impl>(domainBox, + xView, + yView, + zView); + } +#endif +} + +template +void fill_cartesian_coords_3d_impl(const primal::BoundingBox& domainBox, + axom::ArrayView& xView, + axom::ArrayView& yView, + axom::ArrayView& zView) +{ + const auto& shape = xView.shape(); + const auto& mapping = xView.mapping(); + auto order = mapping.getStrideOrder(); + + SLIC_ASSERT(shape == yView.shape()); + SLIC_ASSERT(shape == zView.shape()); + SLIC_ASSERT(mapping == yView.mapping()); + SLIC_ASSERT(mapping == zView.mapping()); + + // Mesh resolution + const axom::primal::NumericArray res {shape[0] - 1, + shape[1] - 1, + shape[2] - 1}; + + // Mesh spacings. + double dx = (domainBox.getMax()[0] - domainBox.getMin()[0]) / res[0]; + double dy = (domainBox.getMax()[1] - domainBox.getMin()[1]) / res[1]; + double dz = (domainBox.getMax()[2] - domainBox.getMin()[2]) / res[2]; + +#if defined(AXOM_USE_RAJA) + RAJA::RangeSegment kRange(0, shape[2]); + RAJA::RangeSegment jRange(0, shape[1]); + RAJA::RangeSegment iRange(0, shape[0]); + using EXEC_POL = + typename axom::internal::nested_for_exec::loop3d_policy; + if(int(order) & int(axom::ArrayStrideOrder::COLUMN)) + { + RAJA::kernel( + RAJA::make_tuple(iRange, jRange, kRange), + AXOM_LAMBDA(axom::IndexType i, axom::IndexType j, axom::IndexType k) { + xView(i, j, k) = domainBox.getMin()[0] + i * dx; + yView(i, j, k) = domainBox.getMin()[1] + j * dy; + zView(i, j, k) = domainBox.getMin()[2] + k * dz; + }); + } + else + { + RAJA::kernel( + RAJA::make_tuple(kRange, jRange, iRange), + AXOM_LAMBDA(axom::IndexType k, axom::IndexType j, axom::IndexType i) { + xView(i, j, k) = domainBox.getMin()[0] + i * dx; + yView(i, j, k) = domainBox.getMin()[1] + j * dy; + zView(i, j, k) = domainBox.getMin()[2] + k * dz; + }); + } +#else + for(int k = 0; k < res[2]; ++k) + { + for(int j = 0; j < res[1]; ++j) + { + for(int i = 0; i < res[0]; ++i) + { + xView(i, j, k) = domainBox.getMin()[0] + i * dx; + yView(i, j, k) = domainBox.getMin()[1] + j * dy; + zView(i, j, k) = domainBox.getMin()[2] + k * dz; + } + } + } +#endif + + return; +} + } // namespace util } // namespace quest } // namespace axom diff --git a/src/axom/quest/util/mesh_helpers.hpp b/src/axom/quest/util/mesh_helpers.hpp index a1096ca68e..b646e4d801 100644 --- a/src/axom/quest/util/mesh_helpers.hpp +++ b/src/axom/quest/util/mesh_helpers.hpp @@ -67,6 +67,9 @@ mfem::Mesh* make_cartesian_mfem_mesh_3D(const primal::BoundingBox& bb \param res The resolution of the mesh \param topologyName Name of the blueprint topoloyy to use. \param coordsetName Name of the blueprint coordset to use. + \param runtimePolicy Runtime policy, see axom::runtime_policy. + Memory in \c meshGrp must be compatible with the + specified policy. \return The meshGrp pointer */ @@ -75,14 +78,16 @@ axom::sidre::Group* make_structured_blueprint_box_mesh( const primal::BoundingBox& bbox, const primal::NumericArray& res, const std::string& topologyName = "mesh", - const std::string& coordsetName = "coords"); + const std::string& coordsetName = "coords", + axom::runtime_policy::Policy runtimePolicy = axom::runtime_policy::Policy::seq); axom::sidre::Group* make_unstructured_blueprint_box_mesh( axom::sidre::Group* meshGrp, const primal::BoundingBox& bbox, const primal::NumericArray& res, const std::string& topologyName = "mesh", - const std::string& coordsetName = "coords"); + const std::string& coordsetName = "coords", + axom::runtime_policy::Policy runtimePolicy = axom::runtime_policy::Policy::seq); void convert_blueprint_structured_explicit_to_unstructured( axom::sidre::Group* meshGrp, @@ -97,6 +102,34 @@ bool verifyBlueprintMesh(const axom::sidre::Group* meshGrp, conduit::Node info); #endif +/*! + @brief Fill in structured mesh cartesian coordinates. + \param xView Vertex x-values array + \param yView Vertex y-values array + \param zView Vertex z-values array + \param domainBox Physical domain +*/ +void fill_cartesian_coords_3d(axom::runtime_policy::Policy runtimePolicy, + const primal::BoundingBox& domainBox, + axom::ArrayView& xView, + axom::ArrayView& yView, + axom::ArrayView& zView); + +/*! + @brief Fill in structured mesh cartesian coordinates. + \tparam ExecSpace Execution space, e.g. \c axom::SEQ_EXEC. + @see axom::execution_space + \param xView Vertex x-values array + \param yView Vertex y-values array + \param zView Vertex z-values array + \param domainBox Physical domain +*/ +template +void fill_cartesian_coords_3d_impl(const primal::BoundingBox& domainBox, + axom::ArrayView& xView, + axom::ArrayView& yView, + axom::ArrayView& zView); + } // namespace util } // namespace quest } // namespace axom From 6f745d4affb7456b94540fcd603d285a7abd2c36 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Fri, 25 Oct 2024 07:02:18 -0700 Subject: [PATCH 042/100] Add conduit::Node mesh support, not tested yet. --- src/axom/quest/IntersectionShaper.hpp | 13 ++++++++++++- src/axom/quest/Shaper.cpp | 25 ++++++++++++++++++++++++- src/axom/quest/Shaper.hpp | 8 ++++++++ 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index fc20c081f5..fc0ac23f9a 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -324,7 +324,7 @@ class IntersectionShaper : public Shaper #if defined(AXOM_USE_CONDUIT) /*! @brief Construct Shaper to operate on a blueprint-formatted mesh - stored in a Conduit Node. + stored in a sidre Group. */ IntersectionShaper(const klee::ShapeSet& shapeSet, sidre::Group* bpGrp, @@ -332,6 +332,17 @@ class IntersectionShaper : public Shaper : Shaper(shapeSet, bpGrp, topo) , m_free_mat_name("free") { } + + /*! + @brief Construct Shaper to operate on a blueprint-formatted mesh + stored in a Conduit Node. + */ + IntersectionShaper(const klee::ShapeSet& shapeSet, + conduit::Node* bpNode, + const std::string& topo = "") + : Shaper(shapeSet, bpNode, topo) + , m_free_mat_name("free") + { } #endif //@{ diff --git a/src/axom/quest/Shaper.cpp b/src/axom/quest/Shaper.cpp index 0ee2ce3ab4..362c08c962 100644 --- a/src/axom/quest/Shaper.cpp +++ b/src/axom/quest/Shaper.cpp @@ -55,8 +55,31 @@ Shaper::Shaper(const klee::ShapeSet& shapeSet, conduit::Node info; SLIC_ASSERT(conduit::blueprint::mesh::verify(meshNode, info)); #endif + + m_cellCount = conduit::blueprint::mesh::topology::length( + meshNode.fetch_existing("topologies").fetch_existing(m_bpTopo)); +} + +Shaper::Shaper(const klee::ShapeSet& shapeSet, + conduit::Node* bpNode, + const std::string& topo) + : m_shapeSet(shapeSet) + , m_bpGrp(nullptr) + , m_bpTopo(topo.empty() ? bpNode->fetch_existing("topologies").child(0).name() + : topo) + , m_bpNode(bpNode) + , m_comm(MPI_COMM_WORLD) +{ +#if defined(AXOM_DEBUG) + conduit::Node info; + SLIC_ASSERT(conduit::blueprint::mesh::verify(*bpNode, info)); +#endif + + m_bpGrp = m_ds.getRoot()->createGroup("internalGrp"); + m_bpGrp->importConduitTreeExternal(*bpNode); + m_cellCount = conduit::blueprint::mesh::topology::length( - meshNode.fetch_existing(axom::fmt::format("topologies/{}", m_bpTopo))); + m_bpNode->fetch_existing("topologies").fetch_existing(m_bpTopo)); } void Shaper::setSamplesPerKnotSpan(int nSamples) diff --git a/src/axom/quest/Shaper.hpp b/src/axom/quest/Shaper.hpp index c6cf09a2f5..71f97e6862 100644 --- a/src/axom/quest/Shaper.hpp +++ b/src/axom/quest/Shaper.hpp @@ -60,6 +60,14 @@ class Shaper sidre::Group* bpMesh, const std::string& topo = ""); + /*! + @brief Construct Shaper to operate on a blueprint-formatted mesh + stored in a conduit Node. + */ + Shaper(const klee::ShapeSet& shapeSet, + conduit::Node* bpNode, + const std::string& topo = ""); + virtual ~Shaper() = default; public: From 12f0c833bd4c71ff8fb265fc9828f674c2a687a1 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Sun, 27 Oct 2024 09:08:50 -0700 Subject: [PATCH 043/100] Add utilities to check memory spaces against execution spaces. This is often done for verifying user provides data that are accessible in the execution spaces. Also useful for debugging Axom. --- src/axom/core/execution/execution_space.hpp | 13 ++++++++++++ .../core/execution/internal/cuda_exec.hpp | 12 +++++++++++ src/axom/core/execution/internal/hip_exec.hpp | 12 +++++++++++ src/axom/core/execution/internal/omp_exec.hpp | 16 +++++++++++++++ src/axom/core/execution/internal/seq_exec.hpp | 16 +++++++++++++++ src/axom/core/memory_management.hpp | 20 +++++++++++++++++++ 6 files changed, 89 insertions(+) diff --git a/src/axom/core/execution/execution_space.hpp b/src/axom/core/execution/execution_space.hpp index 1c23a74d90..2dbab4777e 100644 --- a/src/axom/core/execution/execution_space.hpp +++ b/src/axom/core/execution/execution_space.hpp @@ -8,6 +8,7 @@ #include "axom/config.hpp" #include "axom/core/memory_management.hpp" +#include "axom/core/execution/runtime_policy.hpp" /*! * \file execution_space.hpp @@ -88,6 +89,18 @@ struct execution_space static constexpr bool onDevice() noexcept { return false; } static constexpr char* name() noexcept { return (char*)"[UNDEFINED]"; } static int allocatorID() noexcept { return axom::INVALID_ALLOCATOR_ID; } + static constexpr runtime_policy::Policy runtimePolicy() noexcept + { + return runtime_policy::Policy::seq; + } + static bool usesMemorySpace(axom::MemorySpace m) noexcept + { + return m == memory_space; + } + static bool usesAllocId(int allocId) noexcept + { + return usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); + } }; } // namespace axom diff --git a/src/axom/core/execution/internal/cuda_exec.hpp b/src/axom/core/execution/internal/cuda_exec.hpp index e4ede04544..7fbe66f50f 100644 --- a/src/axom/core/execution/internal/cuda_exec.hpp +++ b/src/axom/core/execution/internal/cuda_exec.hpp @@ -65,6 +65,18 @@ struct execution_space> { return axom::getUmpireResourceAllocatorID(umpire::resource::Device); } + static constexpr runtime_policy::Policy runtimePolicy() noexcept + { + return runtime_policy::Policy::cuda; + } + static bool usesMemorySpace(axom::MemorySpace m) noexcept + { + return m == memory_space || m == MemorySpace::Unified; + } + static bool usesAllocId(int allocId) noexcept + { + return usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); + } }; /*! diff --git a/src/axom/core/execution/internal/hip_exec.hpp b/src/axom/core/execution/internal/hip_exec.hpp index 42b43a3eba..c03e434fda 100644 --- a/src/axom/core/execution/internal/hip_exec.hpp +++ b/src/axom/core/execution/internal/hip_exec.hpp @@ -63,6 +63,18 @@ struct execution_space> { return axom::getUmpireResourceAllocatorID(umpire::resource::Device); } + static constexpr runtime_policy::Policy runtimePolicy() noexcept + { + return runtime_policy::Policy::hip; + } + static bool usesMemorySpace(axom::MemorySpace m) noexcept + { + return m == memory_space || m == MemorySpace::Unified; + } + static bool usesAllocId(int allocId) noexcept + { + return usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); + } }; /*! diff --git a/src/axom/core/execution/internal/omp_exec.hpp b/src/axom/core/execution/internal/omp_exec.hpp index 448b0fc923..aea108fad6 100644 --- a/src/axom/core/execution/internal/omp_exec.hpp +++ b/src/axom/core/execution/internal/omp_exec.hpp @@ -60,6 +60,22 @@ struct execution_space return axom::getDefaultAllocatorID(); #endif } + static constexpr runtime_policy::Policy runtimePolicy() noexcept + { + return runtime_policy::Policy::omp; + } + static bool usesMemorySpace(axom::MemorySpace m) noexcept + { + return m == memory_space +#ifdef AXOM_USE_UMPIRE + || m == MemorySpace::Unified +#endif + ; + } + static bool usesAllocId(int allocId) noexcept + { + return usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); + } }; } // namespace axom diff --git a/src/axom/core/execution/internal/seq_exec.hpp b/src/axom/core/execution/internal/seq_exec.hpp index 9c50f80ee8..1f33fd2e6c 100644 --- a/src/axom/core/execution/internal/seq_exec.hpp +++ b/src/axom/core/execution/internal/seq_exec.hpp @@ -69,6 +69,22 @@ struct execution_space return axom::getDefaultAllocatorID(); #endif } + static constexpr runtime_policy::Policy runtimePolicy() noexcept + { + return runtime_policy::Policy::seq; + } + static bool usesMemorySpace(axom::MemorySpace m) noexcept + { + return m == memory_space +#ifdef AXOM_USE_UMPIRE + || m == MemorySpace::Unified +#endif + ; + } + static bool usesAllocId(int allocId) noexcept + { + return usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); + } }; } // namespace axom diff --git a/src/axom/core/memory_management.hpp b/src/axom/core/memory_management.hpp index 1feb63bee6..c90206a68d 100644 --- a/src/axom/core/memory_management.hpp +++ b/src/axom/core/memory_management.hpp @@ -110,6 +110,26 @@ inline int getDefaultAllocatorID() #endif } +/*! + * \brief Get the allocator id from which data has been allocated. + * \return Allocator id. If Umpire doesn't have an allocator for + * the pointer, or Axom wasn't configured with Umpire, return 0. + * + * \pre ptr has a valid pointer value. + */ +inline int getAllocatorIDFromPointer(const void* ptr) +{ +#ifdef AXOM_USE_UMPIRE + umpire::ResourceManager& rm = umpire::ResourceManager::getInstance(); + if(rm.hasAllocator(const_cast(ptr))) + { + umpire::Allocator allocator = rm.getAllocator(const_cast(ptr)); + return allocator.getId(); + } +#endif + return 0; +} + /*! * \brief Allocates a chunk of memory of type T. * From 2ff4ac0235480287028ae1e98810cdc0b132913f Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Sun, 27 Oct 2024 09:11:56 -0700 Subject: [PATCH 044/100] Importing Conduit array into View should use the parent Group's memory space. This makes importing Conduit array behave similar to deep copying from sidre. --- src/axom/sidre/core/View.cpp | 7 ++++--- src/axom/sidre/core/View.hpp | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/axom/sidre/core/View.cpp b/src/axom/sidre/core/View.cpp index da342303fd..4fd7a98ff5 100644 --- a/src/axom/sidre/core/View.cpp +++ b/src/axom/sidre/core/View.cpp @@ -1533,7 +1533,7 @@ void View::importFrom(conduit::Node& data_holder, * ************************************************************************* */ -View* View::importArrayNode(const Node& array) +View* View::importArrayNode(const Node& array, int allocID) { conduit::DataType array_dtype = array.dtype(); @@ -1550,14 +1550,15 @@ View* View::importArrayNode(const Node& array) conduit::index_t num_ele = array_dtype.number_of_elements(); conduit::index_t ele_bytes = DataType::default_bytes(array_dtype.id()); - buff->allocate((TypeID)array_dtype.id(), num_ele); + allocID = getValidAllocatorID(allocID); + buff->allocate((TypeID)array_dtype.id(), num_ele, allocID); // copy the data in a way that matches // to compact representation of the buffer conduit::uint8* data_ptr = (conduit::uint8*)buff->getVoidPtr(); for(conduit::index_t i = 0; i < num_ele; i++) { - memcpy(data_ptr, array.element_ptr(i), ele_bytes); + axom::copy(data_ptr, array.element_ptr(i), ele_bytes); data_ptr += ele_bytes; } diff --git a/src/axom/sidre/core/View.hpp b/src/axom/sidre/core/View.hpp index 6aa12563a6..693e713e3f 100644 --- a/src/axom/sidre/core/View.hpp +++ b/src/axom/sidre/core/View.hpp @@ -687,6 +687,7 @@ class View /* * \brief set the View to hold an array in a Buffer + * \param allocID Allocator id for array data allocation. * * This takes a Node that holds an array of data and sets up the View to * hold a copy of that data in an attached Buffer. @@ -702,7 +703,7 @@ class View * * \return pointer to this View object */ - View* importArrayNode(const Node& array); + View* importArrayNode(const Node& array, int allocID = INVALID_ALLOCATOR_ID); // // RDH -- Add an overload of the following that takes a const char *. From 5d08506fc1437851c47c49c59414186a80ba16d0 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Sun, 27 Oct 2024 09:18:26 -0700 Subject: [PATCH 045/100] Fix mesh_helper inconsistencies in mixing host and device data. mesh_helper should stick to one memory space when generating the mesh, not output sometimes host data and sometimes device data. Some of this inconsisteny was driven by the need to use Conduit's blueprint utilities, which can't work with device data. I fixed by doing all data moves in local scopes so the calling scope doesn't have to keep track of where the data is. --- .../quest/examples/quest_shape_in_memory.cpp | 10 +- src/axom/quest/util/mesh_helpers.cpp | 107 ++++++++++-------- src/axom/quest/util/mesh_helpers.hpp | 8 ++ 3 files changed, 79 insertions(+), 46 deletions(-) diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index 64875e5180..f693154368 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -1196,8 +1196,16 @@ void saveMesh(const sidre::Group& mesh, const std::string& filename) { AXOM_ANNOTATE_SCOPE("save mesh (sidre)"); + axom::sidre::DataStore ds; + const sidre::Group* meshOnHost = &mesh; + if(mesh.getDefaultAllocatorID() != axom::execution_space::allocatorID()) + { + meshOnHost = ds.getRoot()->deepCopyGroup( + &mesh, + axom::execution_space::allocatorID()); + } conduit::Node tmpMesh; - mesh.createNativeLayout(tmpMesh); + meshOnHost->createNativeLayout(tmpMesh); { conduit::Node info; #ifdef AXOM_USE_MPI diff --git a/src/axom/quest/util/mesh_helpers.cpp b/src/axom/quest/util/mesh_helpers.cpp index 0f0c283521..3892db986d 100644 --- a/src/axom/quest/util/mesh_helpers.cpp +++ b/src/axom/quest/util/mesh_helpers.cpp @@ -144,6 +144,9 @@ axom::sidre::Group* make_structured_blueprint_box_mesh( valuesGrp->createViewAndAllocate("z", axom::sidre::DataTypeId::FLOAT64_ID, numVerts); + SLIC_ASSERT(axom::getAllocatorIDFromPointer(xVu->getVoidPtr()) == meshGrp->getDefaultAllocatorID()); + SLIC_ASSERT(axom::getAllocatorIDFromPointer(yVu->getVoidPtr()) == meshGrp->getDefaultAllocatorID()); + SLIC_ASSERT(axom::getAllocatorIDFromPointer(zVu->getVoidPtr()) == meshGrp->getDefaultAllocatorID()); const axom::MDMapping vertMapping(vertsShape, axom::ArrayStrideOrder::COLUMN); @@ -238,58 +241,61 @@ void convert_blueprint_structured_explicit_to_unstructured( meshGrp->getGroup("topologies")->destroyGroup(topoName); auto* topoGrp = meshGrp->getGroup("topologies")->createGroup(topoName); topoGrp->importConduitTree(newTopo); - topoGrp->getView("coordset") - ->setString(coordsetName); // Is this needed? Is coordset already set? + topoGrp->getView("coordset")->setString(coordsetName); meshGrp->getGroup("coordsets")->destroyGroup(coordsetName); coordsetGrp = meshGrp->getGroup("coordsets")->createGroup(coordsetName); + SLIC_ASSERT(coordsetGrp->getDefaultAllocatorID() == meshGrp->getDefaultAllocatorID()); coordsetGrp->importConduitTree(newCoords); - #define ADD_EXTRA_DATA_FOR_MINT 1 - #if ADD_EXTRA_DATA_FOR_MINT - /* - Constructing a mint mesh from meshGrp fails unless we add some - extra data. Blueprint doesn't require this extra data. (The mesh - passes conduit's Blueprint verification.) This should be fixed, - or we should write better blueprint support utilities. - */ - /* - Make the coordinate arrays 2D to use mint::Mesh. - For some reason, mint::Mesh requires the arrays to be - 2D, even though the second dimension is always 1. - */ - axom::IndexType curShape[2]; - int curDim; - auto* valuesGrp = coordsetGrp->getGroup("values"); - curDim = valuesGrp->getView("x")->getShape(2, curShape); - assert(curDim == 1); - axom::IndexType vertsShape[2] = {curShape[0], 1}; - valuesGrp->getView("x")->reshapeArray(2, vertsShape); - valuesGrp->getView("y")->reshapeArray(2, vertsShape); - valuesGrp->getView("z")->reshapeArray(2, vertsShape); - - // Make connectivity array 2D for the same reason. - auto* elementsGrp = topoGrp->getGroup("elements"); - auto* connView = elementsGrp->getView("connectivity"); - curDim = connView->getShape(2, curShape); - constexpr axom::IndexType NUM_VERTS_PER_HEX = 8; - SLIC_ASSERT(curDim == 1); - SLIC_ASSERT(curShape[0] % NUM_VERTS_PER_HEX == 0); - axom::IndexType connShape[2] = {curShape[0] / NUM_VERTS_PER_HEX, - NUM_VERTS_PER_HEX}; - connView->reshapeArray(2, connShape); - - // mint::Mesh requires connectivity strides, even though Blueprint doesn't. - elementsGrp->createViewScalar("stride", NUM_VERTS_PER_HEX); - - // mint::Mesh requires field group, even though Blueprint doesn't. - meshGrp->createGroup("fields"); - #endif + auto* elemGrp = topoGrp->getGroup("elements"); + auto* connView = elemGrp->getView("connectivity"); + + const bool addExtraDataForMint = true; + if(addExtraDataForMint) + { + /* + Constructing a mint mesh from meshGrp fails unless we add some + extra data. Blueprint doesn't require this extra data. (The mesh + passes conduit's Blueprint verification.) This should be fixed, + or we should write better blueprint support utilities. + */ + /* + Make the coordinate arrays 2D to use mint::Mesh. + For some reason, mint::Mesh requires the arrays to be + 2D, even though the second dimension is always 1. + */ + axom::IndexType curShape[2]; + int curDim; + auto* valuesGrp = coordsetGrp->getGroup("values"); + curDim = valuesGrp->getView("x")->getShape(2, curShape); + assert(curDim == 1); + axom::IndexType vertsShape[2] = {curShape[0], 1}; + valuesGrp->getView("x")->reshapeArray(2, vertsShape); + valuesGrp->getView("y")->reshapeArray(2, vertsShape); + valuesGrp->getView("z")->reshapeArray(2, vertsShape); + + // Make connectivity array 2D for the same reason. + auto* elementsGrp = topoGrp->getGroup("elements"); + auto* connView = elementsGrp->getView("connectivity"); + curDim = connView->getShape(2, curShape); + constexpr axom::IndexType NUM_VERTS_PER_HEX = 8; + SLIC_ASSERT(curDim == 1); + SLIC_ASSERT(curShape[0] % NUM_VERTS_PER_HEX == 0); + axom::IndexType connShape[2] = {curShape[0] / NUM_VERTS_PER_HEX, + NUM_VERTS_PER_HEX}; + connView->reshapeArray(2, connShape); + + // mint::Mesh requires connectivity strides, even though Blueprint doesn't. + elementsGrp->createViewScalar("stride", NUM_VERTS_PER_HEX); + + // mint::Mesh requires field group, even though Blueprint doesn't. + meshGrp->createGroup("fields"); + } #if defined(AXOM_DEBUG) - conduit::Node newMesh; - meshGrp->createNativeLayout(newMesh); - SLIC_ASSERT(conduit::blueprint::mesh::verify(newMesh, info)); + bool isValid = verifyBlueprintMesh(meshGrp, info); + SLIC_ASSERT_MSG(isValid, "Internal error: Generated mesh is invalid."); #endif return; @@ -349,6 +355,17 @@ void fill_cartesian_coords_3d_impl(const primal::BoundingBox& domainB axom::ArrayView& yView, axom::ArrayView& zView) { + using XS = axom::execution_space; + SLIC_ASSERT_MSG( + XS::usesAllocId(xView.getAllocatorID()) && + XS::usesAllocId(yView.getAllocatorID()) && + XS::usesAllocId(zView.getAllocatorID()), + std::string("fill_cartesian_coords_3d_impl: alloc ids ") + + std::to_string(xView.getAllocatorID()) + + ", " + std::to_string(yView.getAllocatorID()) + + " and " + std::to_string(zView.getAllocatorID()) + + " are not all compatible with execution space " + XS::name()); + const auto& shape = xView.shape(); const auto& mapping = xView.mapping(); auto order = mapping.getStrideOrder(); diff --git a/src/axom/quest/util/mesh_helpers.hpp b/src/axom/quest/util/mesh_helpers.hpp index b646e4d801..994899e820 100644 --- a/src/axom/quest/util/mesh_helpers.hpp +++ b/src/axom/quest/util/mesh_helpers.hpp @@ -89,6 +89,14 @@ axom::sidre::Group* make_unstructured_blueprint_box_mesh( const std::string& coordsetName = "coords", axom::runtime_policy::Policy runtimePolicy = axom::runtime_policy::Policy::seq); +/*! + \brief Convert a structured explicit blueprint mesh to unstructured. + + All input mesh data are expected to have the allocator id of + meshGrp->getDefaultAllocatorID(). On output, they will also have + the same allocator id, even though some transfers to and from host + memory are used in intermediate steps. +*/ void convert_blueprint_structured_explicit_to_unstructured( axom::sidre::Group* meshGrp, const std::string& topoName); From 34debeba712b738290497ec300c5abbe766cb7ea Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Sun, 27 Oct 2024 09:22:06 -0700 Subject: [PATCH 046/100] Fix IntersectionShaper to work in a single memory space. Previously, it was working in both device and host space at the same time, which was hard to keep track of. All required data movement between host and device are now transparent and confined to the scopes of methods and utilities that IntersectionShaper uses. --- src/axom/quest/IntersectionShaper.hpp | 75 ++++++++++++++------------- src/axom/sidre/core/View.hpp | 6 +++ 2 files changed, 44 insertions(+), 37 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index fc0ac23f9a..b44998731a 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -2127,16 +2127,17 @@ class IntersectionShaper : public Shaper template void populateHexesFromMesh() { + using XS = axom::execution_space; + constexpr int NUM_VERTS_PER_HEX = 8; constexpr int NUM_COMPS_PER_VERT = 3; - const int hostAllocator = - axom::execution_space::allocatorID(); - const int deviceAllocator = axom::execution_space::allocatorID(); + // const int hostAllocator = axom::execution_space::allocatorID(); + const int allocId = XS::allocatorID(); axom::Array vertCoords( m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT, m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT, - hostAllocator); + allocId); #if defined(AXOM_USE_MFEM) if(m_dc != nullptr) @@ -2147,19 +2148,14 @@ class IntersectionShaper : public Shaper #if defined(AXOM_USE_CONDUIT) if(m_bpGrp != nullptr) { - populateVertCoordsFromBlueprintMesh(vertCoords); + populateVertCoordsFromBlueprintMesh(vertCoords); } #endif - if(deviceAllocator != hostAllocator) - { - vertCoords = axom::Array(vertCoords, deviceAllocator); - } - auto vertCoords_device_view = vertCoords.view(); m_hexes = - axom::Array(m_cellCount, m_cellCount, deviceAllocator); + axom::Array(m_cellCount, m_cellCount, allocId); axom::ArrayView hexes_device_view = m_hexes.view(); constexpr double ZERO_THRESHOLD = 1.e-10; axom::for_all( @@ -2231,10 +2227,15 @@ class IntersectionShaper : public Shaper }); } -private: #if defined(AXOM_USE_CONDUIT) - void populateVertCoordsFromBlueprintMesh(axom::Array& vertCoords_host) + template + void populateVertCoordsFromBlueprintMesh(axom::Array& vertCoords) { + using XS = axom::execution_space; + + SLIC_ASSERT_MSG(XS::usesAllocId(vertCoords.getAllocatorID()), + std::string(XS::name()) + " cannot use the vertCoords allocator id"); + // Initialize vertices from blueprint mesh and // set each shape volume fraction to 1 // Allocation size is: @@ -2242,11 +2243,12 @@ class IntersectionShaper : public Shaper constexpr int NUM_VERTS_PER_HEX = 8; constexpr int NUM_COMPS_PER_VERT = 3; + // Put mesh in Node so we can use conduit::blueprint utilities. conduit::Node meshNode; m_bpGrp->createNativeLayout(meshNode); const conduit::Node& topoNode = - meshNode.fetch_existing(axom::fmt::format("topologies/{}", m_bpTopo)); + meshNode.fetch_existing("topologies").fetch_existing(m_bpTopo); const std::string coordsetName = topoNode.fetch_existing("coordset").as_string(); @@ -2254,7 +2256,7 @@ class IntersectionShaper : public Shaper SLIC_ASSERT(topoNode["type"].as_string() == "unstructured"); SLIC_ASSERT(topoNode["elements/shape"].as_string() == "hex"); - const auto connNode = topoNode["elements/connectivity"]; + const auto& connNode = topoNode["elements/connectivity"]; SLIC_ASSERT_MSG(connNode.dtype().is_int32(), "IntersectionShaper internal error: missing logic to " "handle multiple types."); @@ -2262,17 +2264,15 @@ class IntersectionShaper : public Shaper axom::ArrayView conn(connPtr, m_cellCount, NUM_VERTS_PER_HEX); + SLIC_ASSERT_MSG(XS::usesAllocId(conn.getAllocatorID()), + std::string(XS::name()) + " cannot use the connectivity allocator id"); - // Put in Node so we can use conduit::blueprint utilities. - const conduit::Node& coordNode = - meshNode[axom::fmt::format("coordsets/{}", coordsetName)]; - // coordGrp->createNativeLayout(coordNode); - + const conduit::Node& coordNode = meshNode["coordsets"][coordsetName]; const conduit::Node& coordValues = coordNode.fetch_existing("values"); - axom::IndexType vertexCount = coordValues["x"].dtype().number_of_elements(); bool isInterleaved = conduit::blueprint::mcarray::is_interleaved(coordValues); int stride = isInterleaved ? NUM_COMPS_PER_VERT : 1; + axom::StackArray, 3> coordArrays { axom::ArrayView(coordValues["x"].as_double_ptr(), {vertexCount}, @@ -2284,26 +2284,27 @@ class IntersectionShaper : public Shaper {vertexCount}, stride)}; - for(int i = 0; i < m_cellCount; i++) - { - // Get the indices of this element's vertices - auto hexVerts = conn[i]; - SLIC_ASSERT(hexVerts.size() == NUM_VERTS_PER_HEX); + auto vertCoordsView = vertCoords.view(); - // Get the coordinates for the vertices - for(int j = 0; j < NUM_VERTS_PER_HEX; ++j) - { - auto vertId = hexVerts[j]; - for(int k = 0; k < NUM_COMPS_PER_VERT; k++) + axom::for_all(m_cellCount, AXOM_LAMBDA(axom::IndexType i) { + // Get the indices of this element's vertices + auto hexVerts = conn[i]; + + // Get the coordinates for the vertices + for(int j = 0; j < NUM_VERTS_PER_HEX; ++j) { - vertCoords_host[(i * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT) + - (j * NUM_COMPS_PER_VERT) + k] = coordArrays[k][vertId]; + auto vertId = hexVerts[j]; + for(int k = 0; k < NUM_COMPS_PER_VERT; k++) + { + vertCoordsView[(i * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT) + + (j * NUM_COMPS_PER_VERT) + k] = coordArrays[k][vertId]; + } } - } - } + }); } -#endif +#endif // AXOM_USE_CONDUIT +private: #if defined(AXOM_USE_MFEM) void populateVertCoordsFromMFEMMesh(axom::Array& vertCoords_host) { @@ -2361,7 +2362,7 @@ class IntersectionShaper : public Shaper return volFrac; } -#endif +#endif // AXOM_USE_MFEM bool surfaceMeshIsTet() const { diff --git a/src/axom/sidre/core/View.hpp b/src/axom/sidre/core/View.hpp index 693e713e3f..5bddebe66f 100644 --- a/src/axom/sidre/core/View.hpp +++ b/src/axom/sidre/core/View.hpp @@ -823,6 +823,12 @@ class View return getData(); } + /// \overload + Node::ConstValue getArray() const + { + return getData(); + } + /*! * \brief Returns a pointer to the string contained in the view. * From 64c0c315e5a7d58295a6f502f5c21e46bba41604 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Sun, 27 Oct 2024 20:36:24 -0700 Subject: [PATCH 047/100] Fix bug in populating vertices array with cuda + mfem. --- src/axom/quest/IntersectionShaper.hpp | 42 ++++++++--- .../quest/examples/quest_shape_in_memory.cpp | 70 ++++++++++++++++--- src/axom/quest/util/mesh_helpers.cpp | 3 - 3 files changed, 91 insertions(+), 24 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index b44998731a..632b8b1cc7 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -2131,7 +2131,6 @@ class IntersectionShaper : public Shaper constexpr int NUM_VERTS_PER_HEX = 8; constexpr int NUM_COMPS_PER_VERT = 3; - // const int hostAllocator = axom::execution_space::allocatorID(); const int allocId = XS::allocatorID(); axom::Array vertCoords( @@ -2142,7 +2141,7 @@ class IntersectionShaper : public Shaper #if defined(AXOM_USE_MFEM) if(m_dc != nullptr) { - populateVertCoordsFromMFEMMesh(vertCoords); + populateVertCoordsFromMFEMMesh(vertCoords); } #endif #if defined(AXOM_USE_CONDUIT) @@ -2233,8 +2232,7 @@ class IntersectionShaper : public Shaper { using XS = axom::execution_space; - SLIC_ASSERT_MSG(XS::usesAllocId(vertCoords.getAllocatorID()), - std::string(XS::name()) + " cannot use the vertCoords allocator id"); + const int allocId = XS::allocatorID(); // Initialize vertices from blueprint mesh and // set each shape volume fraction to 1 @@ -2284,6 +2282,9 @@ class IntersectionShaper : public Shaper {vertexCount}, stride)}; + vertCoords = axom::Array(m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT, + m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT, + allocId); auto vertCoordsView = vertCoords.view(); axom::for_all(m_cellCount, AXOM_LAMBDA(axom::IndexType i) { @@ -2304,10 +2305,13 @@ class IntersectionShaper : public Shaper } #endif // AXOM_USE_CONDUIT -private: #if defined(AXOM_USE_MFEM) - void populateVertCoordsFromMFEMMesh(axom::Array& vertCoords_host) + template + void populateVertCoordsFromMFEMMesh(axom::Array& vertCoords) { + using XS = axom::execution_space; + const int allocId = XS::allocatorID(); + mfem::Mesh* mesh = getDC()->GetMesh(); // Intersection algorithm only works on linear elements SLIC_ASSERT(mesh != nullptr); @@ -2318,13 +2322,24 @@ class IntersectionShaper : public Shaper mesh->GetNodes()->FESpace()->GetOrder(0)); } - // Initialize vertices from mfem mesh and - // set each shape volume fraction to 1 // Allocation size is: // # of elements * # of vertices per hex * # of components per vertex constexpr int NUM_VERTS_PER_HEX = 8; constexpr int NUM_COMPS_PER_VERT = 3; + // The MFEM mesh interface works only on host. + // If on device, fill temporary host array then copy to device. + axom::Array tmpVertCoords; + + axom::Array& fillVertCoords = + axom::execution_space::onDevice() ? tmpVertCoords : vertCoords; + fillVertCoords = axom::Array(m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT, + m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT); + + // Initialize vertices from mfem mesh and + // set each shape volume fraction to 1 + + auto fillVertCoordsView = fillVertCoords.view(); for(int i = 0; i < m_cellCount; i++) { // Get the indices of this element's vertices @@ -2337,14 +2352,21 @@ class IntersectionShaper : public Shaper { for(int k = 0; k < NUM_COMPS_PER_VERT; k++) { - vertCoords_host[(i * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT) + - (j * NUM_COMPS_PER_VERT) + k] = + fillVertCoordsView[(i * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT) + + (j * NUM_COMPS_PER_VERT) + k] = (mesh->GetVertex(verts[j]))[k]; } } } + if(vertCoords.data() != fillVertCoords.data()) + { + axom::copy(vertCoords.data(), + fillVertCoords.data(), + sizeof(double)*vertCoords.size()); + } } +private: /// Create and return a new volume fraction grid function for the current mesh // TODO: change to generic name. Nothing in here is about volume fractions. BTNG. mfem::GridFunction* newVolFracGridFunction() diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index f693154368..3d8c3c25c5 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -405,6 +405,7 @@ void printMfemMeshInfo(mfem::Mesh* mesh, const std::string& prefixMessage = "") const std::string topoName = "mesh"; const std::string coordsetName = "coords"; +int cellCount = -1; axom::sidre::Group* createBoxMesh(axom::sidre::Group* meshGrp) { @@ -823,10 +824,10 @@ double volumeOfTetMesh( axom::ArrayView x(tetMesh.getCoordinateArray(0), nodesShape); axom::ArrayView y(tetMesh.getCoordinateArray(1), nodesShape); axom::ArrayView z(tetMesh.getCoordinateArray(2), nodesShape); - const axom::IndexType cellCount = tetMesh.getNumberOfCells(); - axom::Array tetVolumes(cellCount, cellCount); + const axom::IndexType tetCount = tetMesh.getNumberOfCells(); + axom::Array tetVolumes(tetCount, tetCount); double meshVolume = 0.0; - for(axom::IndexType ic = 0; ic < cellCount; ++ic) + for(axom::IndexType ic = 0; ic < tetCount; ++ic) { const axom::IndexType* nodeIds = tetMesh.getCellNodeIDs(ic); TetType tet; @@ -862,7 +863,6 @@ axom::sidre::View* getElementVolumes( if(volSidreView == nullptr) { mfem::Mesh* mesh = dc->GetMesh(); - const axom::IndexType cellCount = mesh->GetNE(); constexpr int NUM_VERTS_PER_HEX = 8; constexpr int NUM_COMPS_PER_VERT = 3; @@ -951,6 +951,7 @@ axom::sidre::View* getElementVolumes( sidre::Group* meshGrp, const std::string& volFieldName = std::string("elementVolumes")) { + using XS = axom::execution_space; using HexahedronType = axom::primal::Hexahedron; axom::sidre::View* volSidreView = nullptr; @@ -966,8 +967,6 @@ axom::sidre::View* getElementVolumes( axom::mint::UnstructuredMesh mesh(meshGrp, topoName); - const axom::IndexType cellCount = mesh.getNumberOfCells(); - constexpr int NUM_VERTS_PER_HEX = 8; constexpr int NUM_COMPS_PER_VERT = 3; constexpr double ZERO_THRESHOLD = 1.e-10; @@ -976,10 +975,63 @@ axom::sidre::View* getElementVolumes( Get vertex coordinates. We use UnstructuredMesh for this, so get it on host first then transfer to device if needed. */ +#if 1 + auto* connData = meshGrp->getGroup("topologies")->getGroup(topoName) + ->getGroup("elements")->getView("connectivity"); + SLIC_ASSERT(connData->getNode().dtype().is_int32()); + + conduit::Node coordNode; + meshGrp->getGroup("coordsets")->getGroup(coordsetName)->createNativeLayout(coordNode); + const conduit::Node& coordValues = coordNode.fetch_existing("values"); + axom::IndexType vertexCount = coordValues["x"].dtype().number_of_elements(); + bool isInterleaved = conduit::blueprint::mcarray::is_interleaved(coordValues); + int stride = isInterleaved ? NUM_COMPS_PER_VERT : 1; + axom::StackArray, 3> coordArrays { + axom::ArrayView(coordValues["x"].as_double_ptr(), + {vertexCount}, + stride), + axom::ArrayView(coordValues["y"].as_double_ptr(), + {vertexCount}, + stride), + axom::ArrayView(coordValues["z"].as_double_ptr(), + {vertexCount}, + stride)}; + + const std::int32_t* connPtr = connData->getArray(); + SLIC_ASSERT(connPtr != nullptr); + axom::ArrayView conn(connPtr, + cellCount, + NUM_VERTS_PER_HEX); + axom::Array vertCoords(cellCount * NUM_VERTS_PER_HEX, + cellCount * NUM_VERTS_PER_HEX, + XS::allocatorID()); + auto vertCoordsView = vertCoords.view(); + + axom::for_all( + cellCount, AXOM_LAMBDA(axom::IndexType cellIdx) + { + // Get the indices of this element's vertices + auto verts = conn[cellIdx]; + + // Get the coordinates for the vertices + for(int j = 0; j < NUM_VERTS_PER_HEX; ++j) + { + int vertIdx = cellIdx * NUM_VERTS_PER_HEX + j; + for(int k = 0; k < NUM_COMPS_PER_VERT; k++) + { + vertCoordsView[vertIdx][k] = coordArrays[k][verts[j]]; + // vertCoordsView[vertIdx][k] = mesh.getNodeCoordinate(verts[j], k); + } + } + }); +#else axom::Array vertCoords(cellCount * NUM_VERTS_PER_HEX, cellCount * NUM_VERTS_PER_HEX); auto vertCoordsView = vertCoords.view(); +// DEBUG to here: This code doesn't work when the mesh is on device. +// Fix this error-checking code. Or copy the mesh to host for checking. +// NOTE: This method uses a mix of host and device loops. Make more consistent. for(axom::IndexType cellIdx = 0; cellIdx < cellCount; ++cellIdx) { // Get the indices of this element's vertices @@ -1002,6 +1054,7 @@ axom::sidre::View* getElementVolumes( axom::Array(vertCoords, meshGrp->getDefaultAllocatorID()); vertCoordsView = vertCoords.view(); } +#endif // Set vertex coords to zero if within threshold. // (I don't know why we do this. I'm following examples.) @@ -1067,9 +1120,6 @@ template double sumMaterialVolumes(sidre::MFEMSidreDataCollection* dc, const std::string& material) { - mfem::Mesh* mesh = dc->GetMesh(); - int const cellCount = mesh->GetNE(); - // Get cell volumes from dc. axom::sidre::View* elementVols = getElementVolumes(dc); axom::ArrayView elementVolsView(elementVols->getData(), @@ -1365,8 +1415,6 @@ int main(int argc, char** argv) "the C2C library"); #endif - axom::IndexType cellCount = -1; - #if defined(AXOM_USE_MFEM) std::shared_ptr shapingDC; if(params.useMfem()) diff --git a/src/axom/quest/util/mesh_helpers.cpp b/src/axom/quest/util/mesh_helpers.cpp index 3892db986d..41c9d91c07 100644 --- a/src/axom/quest/util/mesh_helpers.cpp +++ b/src/axom/quest/util/mesh_helpers.cpp @@ -248,9 +248,6 @@ void convert_blueprint_structured_explicit_to_unstructured( SLIC_ASSERT(coordsetGrp->getDefaultAllocatorID() == meshGrp->getDefaultAllocatorID()); coordsetGrp->importConduitTree(newCoords); - auto* elemGrp = topoGrp->getGroup("elements"); - auto* connView = elemGrp->getView("connectivity"); - const bool addExtraDataForMint = true; if(addExtraDataForMint) { From 8ea5baab9ba34c72ea83c34795857f92c35fd36f Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Sun, 27 Oct 2024 20:39:34 -0700 Subject: [PATCH 048/100] Autoformat. --- src/axom/core/execution/internal/omp_exec.hpp | 4 +-- src/axom/core/execution/internal/seq_exec.hpp | 4 +-- src/axom/quest/IntersectionShaper.hpp | 30 +++++++++++-------- .../quest/examples/quest_shape_in_memory.cpp | 23 ++++++++------ src/axom/quest/util/mesh_helpers.cpp | 29 ++++++++++-------- src/axom/sidre/core/View.hpp | 5 +--- 6 files changed, 52 insertions(+), 43 deletions(-) diff --git a/src/axom/core/execution/internal/omp_exec.hpp b/src/axom/core/execution/internal/omp_exec.hpp index aea108fad6..bbc0fec051 100644 --- a/src/axom/core/execution/internal/omp_exec.hpp +++ b/src/axom/core/execution/internal/omp_exec.hpp @@ -68,9 +68,9 @@ struct execution_space { return m == memory_space #ifdef AXOM_USE_UMPIRE - || m == MemorySpace::Unified + || m == MemorySpace::Unified #endif - ; + ; } static bool usesAllocId(int allocId) noexcept { diff --git a/src/axom/core/execution/internal/seq_exec.hpp b/src/axom/core/execution/internal/seq_exec.hpp index 1f33fd2e6c..2049416ae1 100644 --- a/src/axom/core/execution/internal/seq_exec.hpp +++ b/src/axom/core/execution/internal/seq_exec.hpp @@ -77,9 +77,9 @@ struct execution_space { return m == memory_space #ifdef AXOM_USE_UMPIRE - || m == MemorySpace::Unified + || m == MemorySpace::Unified #endif - ; + ; } static bool usesAllocId(int allocId) noexcept { diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 632b8b1cc7..1c499ef954 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -2153,8 +2153,7 @@ class IntersectionShaper : public Shaper auto vertCoords_device_view = vertCoords.view(); - m_hexes = - axom::Array(m_cellCount, m_cellCount, allocId); + m_hexes = axom::Array(m_cellCount, m_cellCount, allocId); axom::ArrayView hexes_device_view = m_hexes.view(); constexpr double ZERO_THRESHOLD = 1.e-10; axom::for_all( @@ -2262,8 +2261,9 @@ class IntersectionShaper : public Shaper axom::ArrayView conn(connPtr, m_cellCount, NUM_VERTS_PER_HEX); - SLIC_ASSERT_MSG(XS::usesAllocId(conn.getAllocatorID()), - std::string(XS::name()) + " cannot use the connectivity allocator id"); + SLIC_ASSERT_MSG( + XS::usesAllocId(conn.getAllocatorID()), + std::string(XS::name()) + " cannot use the connectivity allocator id"); const conduit::Node& coordNode = meshNode["coordsets"][coordsetName]; const conduit::Node& coordValues = coordNode.fetch_existing("values"); @@ -2282,12 +2282,15 @@ class IntersectionShaper : public Shaper {vertexCount}, stride)}; - vertCoords = axom::Array(m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT, - m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT, - allocId); + vertCoords = + axom::Array(m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT, + m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT, + allocId); auto vertCoordsView = vertCoords.view(); - axom::for_all(m_cellCount, AXOM_LAMBDA(axom::IndexType i) { + axom::for_all( + m_cellCount, + AXOM_LAMBDA(axom::IndexType i) { // Get the indices of this element's vertices auto hexVerts = conn[i]; @@ -2303,7 +2306,7 @@ class IntersectionShaper : public Shaper } }); } -#endif // AXOM_USE_CONDUIT +#endif // AXOM_USE_CONDUIT #if defined(AXOM_USE_MFEM) template @@ -2333,8 +2336,9 @@ class IntersectionShaper : public Shaper axom::Array& fillVertCoords = axom::execution_space::onDevice() ? tmpVertCoords : vertCoords; - fillVertCoords = axom::Array(m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT, - m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT); + fillVertCoords = + axom::Array(m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT, + m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT); // Initialize vertices from mfem mesh and // set each shape volume fraction to 1 @@ -2362,7 +2366,7 @@ class IntersectionShaper : public Shaper { axom::copy(vertCoords.data(), fillVertCoords.data(), - sizeof(double)*vertCoords.size()); + sizeof(double) * vertCoords.size()); } } @@ -2384,7 +2388,7 @@ class IntersectionShaper : public Shaper return volFrac; } -#endif // AXOM_USE_MFEM +#endif // AXOM_USE_MFEM bool surfaceMeshIsTet() const { diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index 3d8c3c25c5..f6b9f5ecc7 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -976,12 +976,16 @@ axom::sidre::View* getElementVolumes( so get it on host first then transfer to device if needed. */ #if 1 - auto* connData = meshGrp->getGroup("topologies")->getGroup(topoName) - ->getGroup("elements")->getView("connectivity"); + auto* connData = meshGrp->getGroup("topologies") + ->getGroup(topoName) + ->getGroup("elements") + ->getView("connectivity"); SLIC_ASSERT(connData->getNode().dtype().is_int32()); conduit::Node coordNode; - meshGrp->getGroup("coordsets")->getGroup(coordsetName)->createNativeLayout(coordNode); + meshGrp->getGroup("coordsets") + ->getGroup(coordsetName) + ->createNativeLayout(coordNode); const conduit::Node& coordValues = coordNode.fetch_existing("values"); axom::IndexType vertexCount = coordValues["x"].dtype().number_of_elements(); bool isInterleaved = conduit::blueprint::mcarray::is_interleaved(coordValues); @@ -1008,8 +1012,8 @@ axom::sidre::View* getElementVolumes( auto vertCoordsView = vertCoords.view(); axom::for_all( - cellCount, AXOM_LAMBDA(axom::IndexType cellIdx) - { + cellCount, + AXOM_LAMBDA(axom::IndexType cellIdx) { // Get the indices of this element's vertices auto verts = conn[cellIdx]; @@ -1029,9 +1033,9 @@ axom::sidre::View* getElementVolumes( cellCount * NUM_VERTS_PER_HEX); auto vertCoordsView = vertCoords.view(); -// DEBUG to here: This code doesn't work when the mesh is on device. -// Fix this error-checking code. Or copy the mesh to host for checking. -// NOTE: This method uses a mix of host and device loops. Make more consistent. + // DEBUG to here: This code doesn't work when the mesh is on device. + // Fix this error-checking code. Or copy the mesh to host for checking. + // NOTE: This method uses a mix of host and device loops. Make more consistent. for(axom::IndexType cellIdx = 0; cellIdx < cellCount; ++cellIdx) { // Get the indices of this element's vertices @@ -1248,7 +1252,8 @@ void saveMesh(const sidre::Group& mesh, const std::string& filename) axom::sidre::DataStore ds; const sidre::Group* meshOnHost = &mesh; - if(mesh.getDefaultAllocatorID() != axom::execution_space::allocatorID()) + if(mesh.getDefaultAllocatorID() != + axom::execution_space::allocatorID()) { meshOnHost = ds.getRoot()->deepCopyGroup( &mesh, diff --git a/src/axom/quest/util/mesh_helpers.cpp b/src/axom/quest/util/mesh_helpers.cpp index 41c9d91c07..926da0be65 100644 --- a/src/axom/quest/util/mesh_helpers.cpp +++ b/src/axom/quest/util/mesh_helpers.cpp @@ -144,9 +144,12 @@ axom::sidre::Group* make_structured_blueprint_box_mesh( valuesGrp->createViewAndAllocate("z", axom::sidre::DataTypeId::FLOAT64_ID, numVerts); - SLIC_ASSERT(axom::getAllocatorIDFromPointer(xVu->getVoidPtr()) == meshGrp->getDefaultAllocatorID()); - SLIC_ASSERT(axom::getAllocatorIDFromPointer(yVu->getVoidPtr()) == meshGrp->getDefaultAllocatorID()); - SLIC_ASSERT(axom::getAllocatorIDFromPointer(zVu->getVoidPtr()) == meshGrp->getDefaultAllocatorID()); + SLIC_ASSERT(axom::getAllocatorIDFromPointer(xVu->getVoidPtr()) == + meshGrp->getDefaultAllocatorID()); + SLIC_ASSERT(axom::getAllocatorIDFromPointer(yVu->getVoidPtr()) == + meshGrp->getDefaultAllocatorID()); + SLIC_ASSERT(axom::getAllocatorIDFromPointer(zVu->getVoidPtr()) == + meshGrp->getDefaultAllocatorID()); const axom::MDMapping vertMapping(vertsShape, axom::ArrayStrideOrder::COLUMN); @@ -245,7 +248,8 @@ void convert_blueprint_structured_explicit_to_unstructured( meshGrp->getGroup("coordsets")->destroyGroup(coordsetName); coordsetGrp = meshGrp->getGroup("coordsets")->createGroup(coordsetName); - SLIC_ASSERT(coordsetGrp->getDefaultAllocatorID() == meshGrp->getDefaultAllocatorID()); + SLIC_ASSERT(coordsetGrp->getDefaultAllocatorID() == + meshGrp->getDefaultAllocatorID()); coordsetGrp->importConduitTree(newCoords); const bool addExtraDataForMint = true; @@ -353,15 +357,14 @@ void fill_cartesian_coords_3d_impl(const primal::BoundingBox& domainB axom::ArrayView& zView) { using XS = axom::execution_space; - SLIC_ASSERT_MSG( - XS::usesAllocId(xView.getAllocatorID()) && - XS::usesAllocId(yView.getAllocatorID()) && - XS::usesAllocId(zView.getAllocatorID()), - std::string("fill_cartesian_coords_3d_impl: alloc ids ") + - std::to_string(xView.getAllocatorID()) + - ", " + std::to_string(yView.getAllocatorID()) + - " and " + std::to_string(zView.getAllocatorID()) + - " are not all compatible with execution space " + XS::name()); + SLIC_ASSERT_MSG(XS::usesAllocId(xView.getAllocatorID()) && + XS::usesAllocId(yView.getAllocatorID()) && + XS::usesAllocId(zView.getAllocatorID()), + std::string("fill_cartesian_coords_3d_impl: alloc ids ") + + std::to_string(xView.getAllocatorID()) + ", " + + std::to_string(yView.getAllocatorID()) + " and " + + std::to_string(zView.getAllocatorID()) + + " are not all compatible with execution space " + XS::name()); const auto& shape = xView.shape(); const auto& mapping = xView.mapping(); diff --git a/src/axom/sidre/core/View.hpp b/src/axom/sidre/core/View.hpp index 5bddebe66f..c423659f7d 100644 --- a/src/axom/sidre/core/View.hpp +++ b/src/axom/sidre/core/View.hpp @@ -824,10 +824,7 @@ class View } /// \overload - Node::ConstValue getArray() const - { - return getData(); - } + Node::ConstValue getArray() const { return getData(); } /*! * \brief Returns a pointer to the string contained in the view. From 58d0d6d3e8584d4fd68520b0051ce32bd3d03fe4 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Sun, 27 Oct 2024 22:37:35 -0700 Subject: [PATCH 049/100] Silence warnings. --- src/axom/multimat/multimat.cpp | 4 ++++ src/axom/quest/util/mesh_helpers.cpp | 2 +- src/axom/slam/tests/slam_map_BivariateMap.cpp | 1 - 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/axom/multimat/multimat.cpp b/src/axom/multimat/multimat.cpp index c51cdf9c50..2c8de6eaa5 100644 --- a/src/axom/multimat/multimat.cpp +++ b/src/axom/multimat/multimat.cpp @@ -49,6 +49,7 @@ bool AllocatorOnDevice(int allocatorId) return true; } #endif + AXOM_UNUSED_VAR(allocatorId); return false; } @@ -509,6 +510,9 @@ void ScanRelationOffsetsRAJA(const axom::ArrayView counts, } }); #else + AXOM_UNUSED_VAR(counts); + AXOM_UNUSED_VAR(begins); + AXOM_UNUSED_VAR(firstIndices); SLIC_ASSERT_MSG( false, "Calling ScanRelationOffsetsRAJA requires support for RAJA and Umpire."); diff --git a/src/axom/quest/util/mesh_helpers.cpp b/src/axom/quest/util/mesh_helpers.cpp index 926da0be65..eb13484ee1 100644 --- a/src/axom/quest/util/mesh_helpers.cpp +++ b/src/axom/quest/util/mesh_helpers.cpp @@ -368,7 +368,6 @@ void fill_cartesian_coords_3d_impl(const primal::BoundingBox& domainB const auto& shape = xView.shape(); const auto& mapping = xView.mapping(); - auto order = mapping.getStrideOrder(); SLIC_ASSERT(shape == yView.shape()); SLIC_ASSERT(shape == zView.shape()); @@ -391,6 +390,7 @@ void fill_cartesian_coords_3d_impl(const primal::BoundingBox& domainB RAJA::RangeSegment iRange(0, shape[0]); using EXEC_POL = typename axom::internal::nested_for_exec::loop3d_policy; + auto order = mapping.getStrideOrder(); if(int(order) & int(axom::ArrayStrideOrder::COLUMN)) { RAJA::kernel( diff --git a/src/axom/slam/tests/slam_map_BivariateMap.cpp b/src/axom/slam/tests/slam_map_BivariateMap.cpp index 7ebed71bcd..cbe2cda3d2 100644 --- a/src/axom/slam/tests/slam_map_BivariateMap.cpp +++ b/src/axom/slam/tests/slam_map_BivariateMap.cpp @@ -375,7 +375,6 @@ void constructAndTestBivariateMapIterator(int stride) { int idx2 = 0; int compIdx = 0; - auto begin_iter = m.begin(idx1); for(auto iter = m.begin(idx1); iter != m.end(idx1); ++iter) { EXPECT_EQ(*iter, getVal(idx1, idx2, compIdx)); From 40921229e09e6e32d2c296cc390a4a858620a723 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Mon, 28 Oct 2024 01:53:21 -0700 Subject: [PATCH 050/100] Silence more warnings. --- src/axom/core/memory_management.hpp | 1 + src/axom/quest/IntersectionShaper.hpp | 30 ++++++------------- .../examples/quest_marching_cubes_example.cpp | 1 + 3 files changed, 11 insertions(+), 21 deletions(-) diff --git a/src/axom/core/memory_management.hpp b/src/axom/core/memory_management.hpp index c90206a68d..43fddf261d 100644 --- a/src/axom/core/memory_management.hpp +++ b/src/axom/core/memory_management.hpp @@ -127,6 +127,7 @@ inline int getAllocatorIDFromPointer(const void* ptr) return allocator.getId(); } #endif + AXOM_UNUSED_VAR(ptr); return 0; } diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 1c499ef954..1b1e71e3db 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -13,6 +13,8 @@ #define AXOM_QUEST_INTERSECTION_SHAPER__HPP_ #include "axom/config.hpp" +#if defined (AXOM_USE_RAJA) && defined (AXOM_USE_UMPIRE) + #include "axom/core.hpp" #include "axom/slic.hpp" #include "axom/slam.hpp" @@ -407,8 +409,6 @@ class IntersectionShaper : public Shaper //@{ //! @name Functions related to the stages for a given shape -#if defined(AXOM_USE_RAJA) && defined(AXOM_USE_UMPIRE) - // Prepares the tet mesh mesh cells for the spatial index template void prepareTetCells() @@ -648,9 +648,7 @@ class IntersectionShaper : public Shaper axom::fmt::format("The shape format {} is unsupported", shapeFormat)); } } -#endif -#if defined(AXOM_USE_RAJA) && defined(AXOM_USE_UMPIRE) /*! \tparam ShapeType either TetrahedeonType or OctahedronType. depending on whether shape is tet or c2c. @@ -935,9 +933,7 @@ class IntersectionShaper : public Shaper "Total mesh volume is {:.3Lf}", this->allReduceSum(totalHex))); } // end of runShapeQueryImpl() function -#endif -#if defined(AXOM_USE_RAJA) && defined(AXOM_USE_UMPIRE) // These methods are private in support of replacement rules. private: /*! @@ -1290,7 +1286,6 @@ class IntersectionShaper : public Shaper } } } -#endif /*! * \brief Apply material replacement rules for the current shape, using @@ -1302,7 +1297,6 @@ class IntersectionShaper : public Shaper switch(m_execPolicy) { -#if defined(AXOM_USE_RAJA) && defined(AXOM_USE_UMPIRE) case RuntimePolicy::seq: applyReplacementRulesImpl(shape); break; @@ -1321,7 +1315,6 @@ class IntersectionShaper : public Shaper applyReplacementRulesImpl(shape); break; #endif // AXOM_USE_HIP -#endif // AXOM_USE_RAJA && AXOM_USE_UMPIRE } AXOM_UNUSED_VAR(shape); } @@ -1358,7 +1351,6 @@ class IntersectionShaper : public Shaper // Now that the mesh is refined, dispatch to device implementations. switch(m_execPolicy) { -#if defined(AXOM_USE_RAJA) && defined(AXOM_USE_UMPIRE) case RuntimePolicy::seq: prepareShapeQueryImpl(shapeDimension, shape); break; @@ -1377,7 +1369,6 @@ class IntersectionShaper : public Shaper prepareShapeQueryImpl(shapeDimension, shape); break; #endif // AXOM_USE_HIP -#endif // AXOM_USE_RAJA && AXOM_USE_UMPIRE } AXOM_UNUSED_VAR(shapeDimension); AXOM_UNUSED_VAR(shape); @@ -1399,7 +1390,6 @@ class IntersectionShaper : public Shaper { switch(m_execPolicy) { -#if defined(AXOM_USE_RAJA) && defined(AXOM_USE_UMPIRE) case RuntimePolicy::seq: runShapeQueryImpl(shape, m_tets, m_tetcount); break; @@ -1418,14 +1408,12 @@ class IntersectionShaper : public Shaper runShapeQueryImpl(shape, m_tets, m_tetcount); break; #endif // AXOM_USE_HIP -#endif // AXOM_USE_RAJA && AXOM_USE_UMPIRE } } else if(shapeFormat == "c2c") { switch(m_execPolicy) { -#if defined(AXOM_USE_RAJA) && defined(AXOM_USE_UMPIRE) case RuntimePolicy::seq: runShapeQueryImpl(shape, m_octs, m_octcount); break; @@ -1444,7 +1432,6 @@ class IntersectionShaper : public Shaper runShapeQueryImpl(shape, m_octs, m_octcount); break; #endif // AXOM_USE_HIP -#endif // AXOM_USE_RAJA && AXOM_USE_UMPIRE } } else @@ -1520,11 +1507,15 @@ class IntersectionShaper : public Shaper if(newData) { // Zero out the volume fractions (on host). +#ifdef AXOM_USE_UMPIRE auto allocId = matVolFrac.getAllocatorID(); const axom::MemorySpace memorySpace = axom::detail::getAllocatorSpace(allocId); const bool onDevice = memorySpace == axom::MemorySpace::Device || memorySpace == axom::MemorySpace::Unified; +#else + const bool onDevice = false; +#endif if(onDevice) { #if defined(AXOM_USE_CUDA) @@ -1546,7 +1537,6 @@ class IntersectionShaper : public Shaper } else { - // memset(matVolFrac->begin(), 0, matVolFrac->Size() * sizeof(double)); memset(matVolFrac.data(), 0, matVolFrac.size() * sizeof(double)); } } @@ -2312,9 +2302,6 @@ class IntersectionShaper : public Shaper template void populateVertCoordsFromMFEMMesh(axom::Array& vertCoords) { - using XS = axom::execution_space; - const int allocId = XS::allocatorID(); - mfem::Mesh* mesh = getDC()->GetMesh(); // Intersection algorithm only works on linear elements SLIC_ASSERT(mesh != nullptr); @@ -2406,8 +2393,8 @@ class IntersectionShaper : public Shaper axom::Array m_hex_volumes; axom::Array m_overlap_volumes; -#if defined(AXOM_USE_RAJA) && defined(AXOM_USE_UMPIRE) double m_vertexWeldThreshold {1.e-10}; + // Guard these to prevent warnings. int m_octcount {0}; int m_tetcount {0}; @@ -2421,10 +2408,11 @@ class IntersectionShaper : public Shaper // Views of volume-fraction data owned by grid. std::vector> m_vf_grid_functions; std::vector m_vf_material_names; -#endif }; } // end namespace quest } // end namespace axom +#endif // AXOM_USE_RAJA && AXOM_USE_UMPIRE + #endif // AXOM_QUEST_INTERSECTION_SHAPER__HPP_ diff --git a/src/axom/quest/examples/quest_marching_cubes_example.cpp b/src/axom/quest/examples/quest_marching_cubes_example.cpp index 3527c5e90a..0be3f80673 100644 --- a/src/axom/quest/examples/quest_marching_cubes_example.cpp +++ b/src/axom/quest/examples/quest_marching_cubes_example.cpp @@ -1663,6 +1663,7 @@ int allocatorIdToTest(axom::runtime_policy::Policy policy) #endif axom::INVALID_ALLOCATOR_ID; #else + AXOM_UNUSED_VAR(policy); int allocatorID = axom::getDefaultAllocatorID(); #endif return allocatorID; From 6affea027b40c78caa1c61e41006a1bd5406f022 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Mon, 28 Oct 2024 02:30:20 -0700 Subject: [PATCH 051/100] Mods and guards for building without RAJA or Umpire. --- src/axom/quest/CMakeLists.txt | 4 +++- src/axom/quest/examples/CMakeLists.txt | 3 ++- .../quest/examples/quest_shape_in_memory.cpp | 20 ++++++++++++------- src/axom/quest/examples/shaping_driver.cpp | 6 ++++++ .../quest/tests/quest_intersection_shaper.cpp | 2 ++ src/axom/quest/util/mesh_helpers.cpp | 2 ++ src/axom/sidre/core/Group.hpp | 17 +++++++++++++++- 7 files changed, 44 insertions(+), 10 deletions(-) diff --git a/src/axom/quest/CMakeLists.txt b/src/axom/quest/CMakeLists.txt index b7b9788bb8..bac906450f 100644 --- a/src/axom/quest/CMakeLists.txt +++ b/src/axom/quest/CMakeLists.txt @@ -132,11 +132,13 @@ blt_list_append( TO quest_depends_on ELEMENTS conduit::conduit IF CONDUIT_FOUND if(AXOM_ENABLE_KLEE AND AXOM_ENABLE_SIDRE AND ((MFEM_FOUND AND AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION) OR CONDUIT_FOUND)) list(APPEND quest_headers Shaper.hpp - IntersectionShaper.hpp DiscreteShape.hpp) list(APPEND quest_sources Shaper.cpp DiscreteShape.cpp) list(APPEND quest_depends_on klee) + if(RAJA_FOUND) + list(APPEND quest_headers IntersectionShaper.hpp) + endif() endif() if(AXOM_ENABLE_KLEE AND AXOM_ENABLE_SIDRE AND (MFEM_FOUND AND AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION)) diff --git a/src/axom/quest/examples/CMakeLists.txt b/src/axom/quest/examples/CMakeLists.txt index 945d3237b4..3a494db5b1 100644 --- a/src/axom/quest/examples/CMakeLists.txt +++ b/src/axom/quest/examples/CMakeLists.txt @@ -268,7 +268,8 @@ endif() if((CONDUIT_FOUND OR (AXOM_ENABLE_MPI AND MFEM_FOUND AND MFEM_USE_MPI AND AXOM_ENABLE_SIDRE AND AXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION)) - AND AXOM_ENABLE_KLEE) + AND AXOM_ENABLE_KLEE + AND UMPIRE_FOUND AND RAJA_FOUND) if(MFEM_FOUND) set(optional_dependency, "mfem") endif() diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index f6b9f5ecc7..b89d158b54 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -38,9 +38,10 @@ #endif // RAJA -#ifdef AXOM_USE_RAJA - #include "RAJA/RAJA.hpp" +#if !defined(AXOM_USE_RAJA) +#error quest_shape_in_memory example and IntersectionShaper require RAJA #endif +#include "RAJA/RAJA.hpp" // C/C++ includes #include @@ -1135,7 +1136,9 @@ double sumMaterialVolumes(sidre::MFEMSidreDataCollection* dc, mfem::GridFunction* volFracGf = dc->GetField(materialFieldName); axom::ArrayView volFracGfArrayView(volFracGf->GetData(), volFracGf->Size()); - axom::quest::TempArrayView volFracView(volFracGfArrayView, true); + axom::ArrayvolFracGfArray(volFracGfArrayView, + axom::execution_space::allocatorID()); + auto volFracView = volFracGfArray.view(); using ReducePolicy = typename axom::execution_space::reduce_policy; RAJA::ReduceSum localVol(0); @@ -1160,8 +1163,7 @@ double sumMaterialVolumesImpl(sidre::Group* meshGrp, const std::string& material meshGrp->createNativeLayout(meshNode); #if defined(AXOM_DEBUG) // Conduit can verify Blueprint mesh, but only if data is on host. - if(axom::detail::getAllocatorSpace(meshGrp->getDefaultAllocatorID()) == - axom::MemorySpace::Host) + if(axom::execution_space::usesAllocId(meshGrp->getDefaultAllocatorID())) { conduit::Node info; conduit::blueprint::mesh::verify(meshNode, info); @@ -1184,8 +1186,7 @@ double sumMaterialVolumesImpl(sidre::Group* meshGrp, const std::string& material const auto vfFieldValuesPath = axom::fmt::format("fields/{}/values", vfFieldName); axom::sidre::View* volFrac = meshGrp->getView(vfFieldValuesPath); - axom::ArrayView volFracArrayView(volFrac->getArray(), cellCount); - axom::quest::TempArrayView volFracView(volFracArrayView, true); + axom::ArrayView volFracView(volFrac->getArray(), cellCount); using ReducePolicy = typename axom::execution_space::reduce_policy; RAJA::ReduceSum localVol(0); @@ -1328,11 +1329,16 @@ int main(int argc, char** argv) defaultAllocId = axom::execution_space>::allocatorID(); } #endif + +#ifdef AXOM_USE_UMPIRE const axom::MemorySpace memorySpace = axom::detail::getAllocatorSpace(defaultAllocId); const bool onHost = memorySpace == axom::MemorySpace::Host || memorySpace == axom::MemorySpace::Dynamic || memorySpace == axom::MemorySpace::Unified; +#else + const bool onHost = true; +#endif AXOM_ANNOTATE_BEGIN("quest example for shaping primals"); AXOM_ANNOTATE_BEGIN("init"); diff --git a/src/axom/quest/examples/shaping_driver.cpp b/src/axom/quest/examples/shaping_driver.cpp index 7837d573c2..5b8ea32445 100644 --- a/src/axom/quest/examples/shaping_driver.cpp +++ b/src/axom/quest/examples/shaping_driver.cpp @@ -566,7 +566,11 @@ int main(int argc, char** argv) shaper = new quest::SamplingShaper(params.shapeSet, &shapingDC); break; case ShapingMethod::Intersection: +#if defined(AXOM_USE_RAJA) && defined(AXOM_USE_UMPIRE) shaper = new quest::IntersectionShaper(params.shapeSet, &shapingDC); +#else + SLIC_ERROR("IntersectionShaper requires Axom to be configured with Umpire."); +#endif break; } SLIC_ASSERT_MSG(shaper != nullptr, "Invalid shaping method selected!"); @@ -604,6 +608,7 @@ int main(int argc, char** argv) } } +#if defined(AXOM_USE_RAJA) && defined(AXOM_USE_UMPIRE) // Set specific parameters here for IntersectionShaper if(auto* intersectionShaper = dynamic_cast(shaper)) { @@ -615,6 +620,7 @@ int main(int argc, char** argv) intersectionShaper->setFreeMaterialName(params.backgroundMaterial); } } +#endif //--------------------------------------------------------------------------- // Project initial volume fractions, if applicable diff --git a/src/axom/quest/tests/quest_intersection_shaper.cpp b/src/axom/quest/tests/quest_intersection_shaper.cpp index 6d32f07c9f..dfc1d0a066 100644 --- a/src/axom/quest/tests/quest_intersection_shaper.cpp +++ b/src/axom/quest/tests/quest_intersection_shaper.cpp @@ -284,6 +284,7 @@ bool loadBaseline(const std::string &filename, conduit::Node &n) return loaded; } +#if defined(AXOM_USE_RAJA) && defined(AXOM_USE_UMPIRE) void replacementRuleTest(const std::string &shapeFile, const std::string &policyName, RuntimePolicy policy, @@ -771,6 +772,7 @@ dimensions: 3 policy); } } +#endif //--------------------------------------------------------------------------- // Define testing functions for different modes. diff --git a/src/axom/quest/util/mesh_helpers.cpp b/src/axom/quest/util/mesh_helpers.cpp index eb13484ee1..416709618a 100644 --- a/src/axom/quest/util/mesh_helpers.cpp +++ b/src/axom/quest/util/mesh_helpers.cpp @@ -144,12 +144,14 @@ axom::sidre::Group* make_structured_blueprint_box_mesh( valuesGrp->createViewAndAllocate("z", axom::sidre::DataTypeId::FLOAT64_ID, numVerts); +#ifdef AXOM_USE_UMPIRE SLIC_ASSERT(axom::getAllocatorIDFromPointer(xVu->getVoidPtr()) == meshGrp->getDefaultAllocatorID()); SLIC_ASSERT(axom::getAllocatorIDFromPointer(yVu->getVoidPtr()) == meshGrp->getDefaultAllocatorID()); SLIC_ASSERT(axom::getAllocatorIDFromPointer(zVu->getVoidPtr()) == meshGrp->getDefaultAllocatorID()); +#endif const axom::MDMapping vertMapping(vertsShape, axom::ArrayStrideOrder::COLUMN); diff --git a/src/axom/sidre/core/Group.hpp b/src/axom/sidre/core/Group.hpp index 9e55a64ac5..ba8245b1e0 100644 --- a/src/axom/sidre/core/Group.hpp +++ b/src/axom/sidre/core/Group.hpp @@ -275,7 +275,6 @@ class Group bool isRoot() const { return m_parent == this; } #ifdef AXOM_USE_UMPIRE - /*! * \brief Return the ID of the default umpire::Allocator associated with this * Group. @@ -308,6 +307,22 @@ class Group m_default_allocator_id = allocId; return this; } +#else + /*! + * \brief Return the ID of the default umpire::Allocator associated with this + * Group. + */ + int getDefaultAllocatorID() const { return axom::getDefaultAllocatorID(); } + + /*! + * \brief Set the default umpire::Allocator associated with this Group. + */ + Group* setDefaultAllocator(int allocId) + { + AXOM_UNUSED_VAR(allocId); + SLIC_ASSERT(allocId == axom::getDefaultAllocatorID()); + return this; + } #endif /*! From af8ba2bf91ce25cc6acda2c7466d03356fe96086 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Mon, 28 Oct 2024 07:04:00 -0700 Subject: [PATCH 052/100] Autoformat. --- src/axom/quest/IntersectionShaper.hpp | 126 +++++++++--------- .../quest/examples/quest_shape_in_memory.cpp | 10 +- src/axom/quest/examples/shaping_driver.cpp | 3 +- .../quest/tests/quest_intersection_shaper.cpp | 16 +-- src/axom/quest/util/mesh_helpers.cpp | 4 +- 5 files changed, 81 insertions(+), 78 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 1b1e71e3db..fdbce182c0 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -13,27 +13,27 @@ #define AXOM_QUEST_INTERSECTION_SHAPER__HPP_ #include "axom/config.hpp" -#if defined (AXOM_USE_RAJA) && defined (AXOM_USE_UMPIRE) - -#include "axom/core.hpp" -#include "axom/slic.hpp" -#include "axom/slam.hpp" -#include "axom/primal.hpp" -#include "axom/mint.hpp" -#include "axom/spin.hpp" -#include "axom/klee.hpp" -#include "axom/quest/Shaper.hpp" -#include "axom/spin/BVH.hpp" -#include "axom/quest/interface/internal/mpicomm_wrapper.hpp" -#include "axom/quest/interface/internal/QuestHelpers.hpp" -#include "axom/fmt.hpp" - -// RAJA -#if defined(AXOM_USE_RAJA) - #include "RAJA/RAJA.hpp" -#endif +#if defined(AXOM_USE_RAJA) && defined(AXOM_USE_UMPIRE) + + #include "axom/core.hpp" + #include "axom/slic.hpp" + #include "axom/slam.hpp" + #include "axom/primal.hpp" + #include "axom/mint.hpp" + #include "axom/spin.hpp" + #include "axom/klee.hpp" + #include "axom/quest/Shaper.hpp" + #include "axom/spin/BVH.hpp" + #include "axom/quest/interface/internal/mpicomm_wrapper.hpp" + #include "axom/quest/interface/internal/QuestHelpers.hpp" + #include "axom/fmt.hpp" + + // RAJA + #if defined(AXOM_USE_RAJA) + #include "RAJA/RAJA.hpp" + #endif -// clang-format off + // clang-format off #if defined (AXOM_USE_RAJA) && defined (AXOM_USE_UMPIRE) using seq_exec = axom::SEQ_EXEC; @@ -78,7 +78,7 @@ template class TempArrayView { public: -#if defined(AXOM_USE_MFEM) + #if defined(AXOM_USE_MFEM) /*! * \brief Host constructor that accepts the grid function. * @@ -90,7 +90,7 @@ class TempArrayView { initialize(gf->GetData(), gf->Size(), _needResult); } -#endif + #endif AXOM_HOST TempArrayView(axom::Array& gf, bool _needResult = true) { @@ -152,7 +152,7 @@ class TempArrayView */ AXOM_HOST_DEVICE void finalize() { m_deviceData = nullptr; } -#if defined(AXOM_USE_CUDA) || defined(AXOM_USE_HIP) + #if defined(AXOM_USE_CUDA) || defined(AXOM_USE_HIP) /*! * \brief Initializes members using data from the grid function. This method * is called on the host and it copies data to the device. @@ -179,7 +179,7 @@ class TempArrayView */ AXOM_HOST_DEVICE void finalizeDevice() { - #ifndef AXOM_DEVICE_CODE + #ifndef AXOM_DEVICE_CODE // Only the host will do this work. if(m_hostData != nullptr) { @@ -191,9 +191,9 @@ class TempArrayView axom::deallocate(m_deviceData); m_deviceData = nullptr; } - #endif + #endif } -#endif + #endif private: double* m_hostData {nullptr}; @@ -202,7 +202,7 @@ class TempArrayView bool m_needResult {false}; }; -#if defined(AXOM_USE_CUDA) + #if defined(AXOM_USE_CUDA) /*! * \brief CUDA specialization that calls initializeDevice to copy data * from the host to the device. @@ -228,8 +228,8 @@ AXOM_HOST_DEVICE inline void TempArrayView::finalize() { finalizeDevice(); } -#endif -#if defined(AXOM_USE_HIP) + #endif + #if defined(AXOM_USE_HIP) /*! * \brief HIP specialization that calls initializeDevice to copy data * from the host to the device. @@ -255,7 +255,7 @@ AXOM_HOST_DEVICE inline void TempArrayView::finalize() { finalizeDevice(); } -#endif + #endif //--------------------------------------------------------------------------- /** @@ -311,7 +311,7 @@ class IntersectionShaper : public Shaper static constexpr double DEFAULT_REVOLVED_VOLUME {0.}; public: -#if defined(AXOM_USE_MFEM) + #if defined(AXOM_USE_MFEM) /*! @brief Construct Shaper to operate on an MFEM mesh. */ @@ -321,9 +321,9 @@ class IntersectionShaper : public Shaper { m_free_mat_name = "free"; } -#endif + #endif -#if defined(AXOM_USE_CONDUIT) + #if defined(AXOM_USE_CONDUIT) /*! @brief Construct Shaper to operate on a blueprint-formatted mesh stored in a sidre Group. @@ -345,7 +345,7 @@ class IntersectionShaper : public Shaper : Shaper(shapeSet, bpNode, topo) , m_free_mat_name("free") { } -#endif + #endif //@{ //! @name Functions to get and set shaping parameters related to intersection; supplements parameters in base class @@ -1449,7 +1449,7 @@ class IntersectionShaper : public Shaper std::vector getMaterialNames() const { std::vector materialNames; -#if defined(AXOM_USE_MFEM) + #if defined(AXOM_USE_MFEM) if(m_dc) { for(auto it : this->getDC()->GetFieldMap()) @@ -1461,8 +1461,8 @@ class IntersectionShaper : public Shaper } } } -#endif -#if defined(AXOM_USE_CONDUIT) + #endif + #if defined(AXOM_USE_CONDUIT) if(m_bpGrp) { auto fieldsGrp = m_bpGrp->getGroup("fields"); @@ -1475,7 +1475,7 @@ class IntersectionShaper : public Shaper } } } -#endif + #endif return materialNames; } @@ -1506,34 +1506,34 @@ class IntersectionShaper : public Shaper auto matVolFrac = getScalarCellData(materialVolFracName); if(newData) { - // Zero out the volume fractions (on host). -#ifdef AXOM_USE_UMPIRE + // Zero out the volume fractions (on host). + #ifdef AXOM_USE_UMPIRE auto allocId = matVolFrac.getAllocatorID(); const axom::MemorySpace memorySpace = axom::detail::getAllocatorSpace(allocId); const bool onDevice = memorySpace == axom::MemorySpace::Device || memorySpace == axom::MemorySpace::Unified; -#else + #else const bool onDevice = false; -#endif + #endif if(onDevice) { -#if defined(AXOM_USE_CUDA) + #if defined(AXOM_USE_CUDA) if(m_execPolicy == RuntimePolicy::cuda) { axom::for_all>( matVolFrac.size(), AXOM_LAMBDA(axom::IndexType i) { matVolFrac[i] = 0.0; }); } -#endif -#if defined(AXOM_USE_HIP) + #endif + #if defined(AXOM_USE_HIP) if(m_execPolicy == RuntimePolicy::hip) { axom::for_all>( matVolFrac.size(), AXOM_LAMBDA(axom::IndexType i) { matVolFrac[i] = 0.0; }); } -#endif + #endif } else { @@ -2008,19 +2008,19 @@ class IntersectionShaper : public Shaper bool hasData(const std::string& fieldName) { bool has = false; -#if defined(AXOM_USE_MFEM) + #if defined(AXOM_USE_MFEM) if(m_dc != nullptr) { has = m_dc->HasField(fieldName); } -#endif -#if defined(AXOM_USE_CONDUIT) + #endif + #if defined(AXOM_USE_CONDUIT) if(m_bpGrp != nullptr) { std::string fieldPath = axom::fmt::format("fields/{}", fieldName); has = m_bpGrp->hasGroup(fieldPath); } -#endif + #endif return has; } @@ -2035,7 +2035,7 @@ class IntersectionShaper : public Shaper { axom::ArrayView rval; -#if defined(AXOM_USE_MFEM) + #if defined(AXOM_USE_MFEM) if(m_dc != nullptr) { mfem::GridFunction* gridFunc = nullptr; @@ -2050,8 +2050,8 @@ class IntersectionShaper : public Shaper } rval = axom::ArrayView(gridFunc->GetData(), gridFunc->Size()); } -#endif -#if defined(AXOM_USE_CONDUIT) + #endif + #if defined(AXOM_USE_CONDUIT) if(m_bpGrp != nullptr) { std::string fieldPath = axom::fmt::format("fields/{}", fieldName); @@ -2107,7 +2107,7 @@ class IntersectionShaper : public Shaper axom::ArrayView(static_cast(valuesView->getVoidPtr()), m_cellCount); } -#endif + #endif return rval; } @@ -2128,18 +2128,18 @@ class IntersectionShaper : public Shaper m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT, allocId); -#if defined(AXOM_USE_MFEM) + #if defined(AXOM_USE_MFEM) if(m_dc != nullptr) { populateVertCoordsFromMFEMMesh(vertCoords); } -#endif -#if defined(AXOM_USE_CONDUIT) + #endif + #if defined(AXOM_USE_CONDUIT) if(m_bpGrp != nullptr) { populateVertCoordsFromBlueprintMesh(vertCoords); } -#endif + #endif auto vertCoords_device_view = vertCoords.view(); @@ -2215,7 +2215,7 @@ class IntersectionShaper : public Shaper }); } -#if defined(AXOM_USE_CONDUIT) + #if defined(AXOM_USE_CONDUIT) template void populateVertCoordsFromBlueprintMesh(axom::Array& vertCoords) { @@ -2296,9 +2296,9 @@ class IntersectionShaper : public Shaper } }); } -#endif // AXOM_USE_CONDUIT + #endif // AXOM_USE_CONDUIT -#if defined(AXOM_USE_MFEM) + #if defined(AXOM_USE_MFEM) template void populateVertCoordsFromMFEMMesh(axom::Array& vertCoords) { @@ -2375,7 +2375,7 @@ class IntersectionShaper : public Shaper return volFrac; } -#endif // AXOM_USE_MFEM + #endif // AXOM_USE_MFEM bool surfaceMeshIsTet() const { @@ -2413,6 +2413,6 @@ class IntersectionShaper : public Shaper } // end namespace quest } // end namespace axom -#endif // AXOM_USE_RAJA && AXOM_USE_UMPIRE +#endif // AXOM_USE_RAJA && AXOM_USE_UMPIRE #endif // AXOM_QUEST_INTERSECTION_SHAPER__HPP_ diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index b89d158b54..5aa22968c1 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -39,7 +39,7 @@ // RAJA #if !defined(AXOM_USE_RAJA) -#error quest_shape_in_memory example and IntersectionShaper require RAJA + #error quest_shape_in_memory example and IntersectionShaper require RAJA #endif #include "RAJA/RAJA.hpp" @@ -1136,8 +1136,9 @@ double sumMaterialVolumes(sidre::MFEMSidreDataCollection* dc, mfem::GridFunction* volFracGf = dc->GetField(materialFieldName); axom::ArrayView volFracGfArrayView(volFracGf->GetData(), volFracGf->Size()); - axom::ArrayvolFracGfArray(volFracGfArrayView, - axom::execution_space::allocatorID()); + axom::Array volFracGfArray( + volFracGfArrayView, + axom::execution_space::allocatorID()); auto volFracView = volFracGfArray.view(); using ReducePolicy = typename axom::execution_space::reduce_policy; @@ -1163,7 +1164,8 @@ double sumMaterialVolumesImpl(sidre::Group* meshGrp, const std::string& material meshGrp->createNativeLayout(meshNode); #if defined(AXOM_DEBUG) // Conduit can verify Blueprint mesh, but only if data is on host. - if(axom::execution_space::usesAllocId(meshGrp->getDefaultAllocatorID())) + if(axom::execution_space::usesAllocId( + meshGrp->getDefaultAllocatorID())) { conduit::Node info; conduit::blueprint::mesh::verify(meshNode, info); diff --git a/src/axom/quest/examples/shaping_driver.cpp b/src/axom/quest/examples/shaping_driver.cpp index 5b8ea32445..f20ef0be46 100644 --- a/src/axom/quest/examples/shaping_driver.cpp +++ b/src/axom/quest/examples/shaping_driver.cpp @@ -569,7 +569,8 @@ int main(int argc, char** argv) #if defined(AXOM_USE_RAJA) && defined(AXOM_USE_UMPIRE) shaper = new quest::IntersectionShaper(params.shapeSet, &shapingDC); #else - SLIC_ERROR("IntersectionShaper requires Axom to be configured with Umpire."); + SLIC_ERROR( + "IntersectionShaper requires Axom to be configured with Umpire."); #endif break; } diff --git a/src/axom/quest/tests/quest_intersection_shaper.cpp b/src/axom/quest/tests/quest_intersection_shaper.cpp index dfc1d0a066..03fe202283 100644 --- a/src/axom/quest/tests/quest_intersection_shaper.cpp +++ b/src/axom/quest/tests/quest_intersection_shaper.cpp @@ -317,13 +317,13 @@ void replacementRuleTest(const std::string &shapeFile, // Need to do the pipeline of the shaping driver. SLIC_INFO(axom::fmt::format("Shaping materials...")); const int refinementLevel = 7; -#ifdef AXOM_USE_MPI + #ifdef AXOM_USE_MPI // This has to happen here because the shaper gets its communicator from it. // If we do it before the mfem mesh is added to the data collection then the // data collection communicator gets set to MPI_COMM_NULL, which is bad for // the C2C reader. dc.SetComm(MPI_COMM_WORLD); -#endif + #endif quest::IntersectionShaper shaper(shapeSet, &dc); shaper.setLevel(refinementLevel); shaper.setExecPolicy(policy); @@ -361,10 +361,10 @@ void replacementRuleTest(const std::string &shapeFile, conduit::Node current; dcToConduit(dc, current); -#ifdef VISUALIZE_DATASETS + #ifdef VISUALIZE_DATASETS saveVisIt("", baselineName, dc); -#endif -#ifdef GENERATE_BASELINES + #endif + #ifdef GENERATE_BASELINES for(const auto &path : baselinePaths) { SLIC_INFO(axom::fmt::format("Saving baseline to {}", path)); @@ -372,7 +372,7 @@ void replacementRuleTest(const std::string &shapeFile, std::string filename(pjoin(path, baselineName)); saveBaseline(filename, current); } -#endif + #endif // TODO: I might want an auto compare for generating baselines so I know if I need a policy-specific baseline. @@ -453,13 +453,13 @@ void IntersectionWithErrorTolerances(const std::string &filebase, // Need to do the pipeline of the shaping driver. SLIC_INFO(axom::fmt::format("Shaping materials...")); -#ifdef AXOM_USE_MPI + #ifdef AXOM_USE_MPI // This has to happen here because the shaper gets its communicator from it. // If we do it before the mfem mesh is added to the data collection then the // data collection communicator gets set to MPI_COMM_NULL, which is bad for // the C2C reader. dc.SetComm(MPI_COMM_WORLD); -#endif + #endif quest::IntersectionShaper shaper(shapeSet, &dc); shaper.setLevel(refinementLevel); shaper.setPercentError(targetPercentError); diff --git a/src/axom/quest/util/mesh_helpers.cpp b/src/axom/quest/util/mesh_helpers.cpp index 416709618a..07f03e8235 100644 --- a/src/axom/quest/util/mesh_helpers.cpp +++ b/src/axom/quest/util/mesh_helpers.cpp @@ -144,14 +144,14 @@ axom::sidre::Group* make_structured_blueprint_box_mesh( valuesGrp->createViewAndAllocate("z", axom::sidre::DataTypeId::FLOAT64_ID, numVerts); -#ifdef AXOM_USE_UMPIRE + #ifdef AXOM_USE_UMPIRE SLIC_ASSERT(axom::getAllocatorIDFromPointer(xVu->getVoidPtr()) == meshGrp->getDefaultAllocatorID()); SLIC_ASSERT(axom::getAllocatorIDFromPointer(yVu->getVoidPtr()) == meshGrp->getDefaultAllocatorID()); SLIC_ASSERT(axom::getAllocatorIDFromPointer(zVu->getVoidPtr()) == meshGrp->getDefaultAllocatorID()); -#endif + #endif const axom::MDMapping vertMapping(vertsShape, axom::ArrayStrideOrder::COLUMN); From 13b8f9138078c1148ddfd860967dcddd09dd73ab Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Tue, 29 Oct 2024 12:05:11 -0700 Subject: [PATCH 053/100] Add guards for non-sidre builds. --- .../examples/quest_marching_cubes_example.cpp | 22 ++++++++++++++----- src/axom/quest/util/mesh_helpers.cpp | 6 +++-- src/axom/quest/util/mesh_helpers.hpp | 4 +++- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/axom/quest/examples/quest_marching_cubes_example.cpp b/src/axom/quest/examples/quest_marching_cubes_example.cpp index 0be3f80673..b8158e1a04 100644 --- a/src/axom/quest/examples/quest_marching_cubes_example.cpp +++ b/src/axom/quest/examples/quest_marching_cubes_example.cpp @@ -30,7 +30,9 @@ #include "axom/core/MDMapping.hpp" #include "axom/quest/MarchingCubes.hpp" #include "axom/quest/MeshViewUtil.hpp" -#include "axom/sidre.hpp" +#if defined(AXOM_USE_SIDRE) + #include "axom/sidre.hpp" +#endif #include "axom/core/Types.hpp" #include "axom/core/numerics/floating_point_limits.hpp" @@ -56,7 +58,9 @@ namespace quest = axom::quest; namespace slic = axom::slic; +#if defined(AXOM_USE_SIDRE) namespace sidre = axom::sidre; +#endif namespace primal = axom::primal; namespace mint = axom::mint; namespace numerics = axom::numerics; @@ -667,6 +671,7 @@ void saveMesh(const conduit::Node& mesh, const std::string& filename) #endif } +#if defined(AXOM_USE_SIDRE) /// Write blueprint mesh to disk void saveMesh(const sidre::Group& mesh, const std::string& filename) { @@ -676,11 +681,11 @@ void saveMesh(const sidre::Group& mesh, const std::string& filename) mesh.createNativeLayout(tmpMesh); { conduit::Node info; -#ifdef AXOM_USE_MPI + #ifdef AXOM_USE_MPI if(!conduit::blueprint::mpi::verify("mesh", tmpMesh, info, MPI_COMM_WORLD)) -#else + #else if(!conduit::blueprint::verify("mesh", tmpMesh, info)) -#endif + #endif { SLIC_INFO("Invalid blueprint for mesh: \n" << info.to_yaml()); slic::flushStreams(); @@ -690,6 +695,7 @@ void saveMesh(const sidre::Group& mesh, const std::string& filename) } saveMesh(tmpMesh, filename); } +#endif template T product(const axom::StackArray& a) @@ -914,11 +920,13 @@ struct ContourTestBase AXOM_ANNOTATE_BEGIN("error checking"); AXOM_ANNOTATE_BEGIN("convert to mint mesh"); +#if defined(AXOM_USE_SIDRE) std::string sidreGroupName = "contour_mesh"; sidre::DataStore objectDS; // While awaiting fix for PR #1271, don't use Sidre storage in contourMesh. auto* meshGroup = objectDS.getRoot()->createGroup(sidreGroupName); AXOM_UNUSED_VAR(meshGroup); // variable is only referenced in debug configs +#endif axom::mint::UnstructuredMesh contourMesh( DIM, @@ -954,6 +962,7 @@ struct ContourTestBase checkCellsContainingContour(computationalMesh, contourMesh); } +#if defined(AXOM_USE_SIDRE) if(contourMesh.hasSidreGroup()) { assert(contourMesh.getSidreGroup() == meshGroup); @@ -962,9 +971,10 @@ struct ContourTestBase saveMesh(*contourMesh.getSidreGroup(), outputName); SLIC_INFO(axom::fmt::format("Wrote contour mesh to {}", outputName)); } - AXOM_ANNOTATE_END("error checking"); - objectDS.getRoot()->destroyGroupAndData(sidreGroupName); +#endif + + AXOM_ANNOTATE_END("error checking"); return localErrCount; } diff --git a/src/axom/quest/util/mesh_helpers.cpp b/src/axom/quest/util/mesh_helpers.cpp index 07f03e8235..b03245d246 100644 --- a/src/axom/quest/util/mesh_helpers.cpp +++ b/src/axom/quest/util/mesh_helpers.cpp @@ -4,7 +4,9 @@ // SPDX-License-Identifier: (BSD-3-Clause) #include "mesh_helpers.hpp" -#include +#if defined(AXOM_USE_SIDRE) + #include +#endif #include #include #include @@ -314,7 +316,7 @@ bool verifyBlueprintMesh(const axom::sidre::Group* meshGrp, conduit::Node info) } #endif -#endif +#endif // AXOM_USE_SIDRE void fill_cartesian_coords_3d(axom::runtime_policy::Policy runtimePolicy, const primal::BoundingBox& domainBox, diff --git a/src/axom/quest/util/mesh_helpers.hpp b/src/axom/quest/util/mesh_helpers.hpp index 994899e820..c61bb714eb 100644 --- a/src/axom/quest/util/mesh_helpers.hpp +++ b/src/axom/quest/util/mesh_helpers.hpp @@ -8,7 +8,9 @@ #include "axom/config.hpp" #include "axom/primal.hpp" -#include "axom/sidre.hpp" +#if defined(AXOM_USE_SIDRE) + #include "axom/sidre.hpp" +#endif #ifdef AXOM_USE_MFEM #include "mfem.hpp" From 8fbb4b4c757c7ecb374ddfb39e8248acdef9092a Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Thu, 7 Nov 2024 17:03:37 -0800 Subject: [PATCH 054/100] Silence compiler warnings: unused variables and shadow declarations. --- src/axom/primal/geometry/KnotVector.hpp | 36 ++++++++++---------- src/axom/primal/geometry/NURBSCurve.hpp | 16 ++++----- src/axom/primal/tests/primal_nurbs_curve.cpp | 1 - src/axom/quest/DiscreteShape.cpp | 1 + src/axom/quest/interface/inout.cpp | 2 ++ src/axom/quest/util/mesh_helpers.cpp | 2 ++ src/axom/sidre/core/View.cpp | 2 +- src/axom/sidre/core/View.hpp | 3 +- src/axom/slam/DynamicConstantRelation.hpp | 4 +-- 9 files changed, 35 insertions(+), 32 deletions(-) diff --git a/src/axom/primal/geometry/KnotVector.hpp b/src/axom/primal/geometry/KnotVector.hpp index 36ea5396d0..6a5af1d3da 100644 --- a/src/axom/primal/geometry/KnotVector.hpp +++ b/src/axom/primal/geometry/KnotVector.hpp @@ -523,23 +523,23 @@ class KnotVector { SLIC_ASSERT(isValidSpan(span, t)); - const int m_deg = getDegree(); + const int deg = getDegree(); - axom::Array> ndu(m_deg + 1), a(2); - axom::Array left(m_deg + 1), right(m_deg + 1); - for(int j = 0; j <= m_deg; j++) + axom::Array> ndu(deg + 1), a(2); + axom::Array left(deg + 1), right(deg + 1); + for(int j = 0; j <= deg; j++) { - ndu[j].resize(m_deg + 1); + ndu[j].resize(deg + 1); } for(int j = 0; j <= n; j++) { - ders[j].resize(m_deg + 1); + ders[j].resize(deg + 1); } - a[0].resize(m_deg + 1); - a[1].resize(m_deg + 1); + a[0].resize(deg + 1); + a[1].resize(deg + 1); ndu[0][0] = 1.; - for(int j = 1; j <= m_deg; j++) + for(int j = 1; j <= deg; j++) { left[j] = t - m_knots[span + 1 - j]; right[j] = m_knots[span + j] - t; @@ -556,15 +556,15 @@ class KnotVector ndu[j][j] = saved; } // Load basis functions - for(int j = 0; j <= m_deg; j++) + for(int j = 0; j <= deg; j++) { - ders[0][j] = ndu[j][m_deg]; + ders[0][j] = ndu[j][deg]; } // This section computes the derivatives (Eq. [2.9]) // Loop over function index. - for(int r = 0; r <= m_deg; r++) + for(int r = 0; r <= deg; r++) { int s1 = 0, s2 = 1; // Alternate rows in array a a[0][0] = 1.; @@ -573,14 +573,14 @@ class KnotVector { T d = 0.; int rk = r - k; - int pk = m_deg - k; + int pk = deg - k; if(r >= k) { a[s2][0] = a[s1][0] / ndu[pk + 1][rk]; d = a[s2][0] * ndu[rk][pk]; } int j1 = (rk >= -1) ? 1 : -rk; - int j2 = (r - 1 <= pk) ? (k - 1) : (m_deg - r); + int j2 = (r - 1 <= pk) ? (k - 1) : (deg - r); for(int j = j1; j <= j2; j++) { a[s2][j] = (a[s1][j] - a[s1][j - 1]) / ndu[pk + 1][rk + j]; @@ -597,14 +597,14 @@ class KnotVector } } // Multiply through by the correct factors (Eq. [2.9]) - T r = static_cast(m_deg); + T r = static_cast(deg); for(int k = 1; k <= n; k++) { - for(int j = 0; j <= m_deg; j++) + for(int j = 0; j <= deg; j++) { ders[k][j] *= r; } - r *= static_cast(m_deg - k); + r *= static_cast(deg - k); } } @@ -797,4 +797,4 @@ template struct axom::fmt::formatter> : ostream_formatter { }; -#endif // AXOM_PRIMAL_KNOTVECTOR_HPP \ No newline at end of file +#endif // AXOM_PRIMAL_KNOTVECTOR_HPP diff --git a/src/axom/primal/geometry/NURBSCurve.hpp b/src/axom/primal/geometry/NURBSCurve.hpp index b6f983aed7..ce53c06cf0 100644 --- a/src/axom/primal/geometry/NURBSCurve.hpp +++ b/src/axom/primal/geometry/NURBSCurve.hpp @@ -918,18 +918,18 @@ class NURBSCurve } // Split the curve at each knot value - int numBeziers = 1; + // int numBeziers = 1; NURBSCurve n1(*this); for(int i = p + 1; i < m_knotvec.getNumKnots() - p - 1; ++i) { - auto old_knot_count = n1.getNumKnots(); + n1.getNumKnots(); n1.insertKnot(m_knotvec[i], p); - auto new_knot_count = n1.getNumKnots(); + n1.getNumKnots(); - if(new_knot_count != old_knot_count) - { - numBeziers++; - } + // if(new_knot_count != old_knot_count) + // { + // numBeziers++; + // } } // For each Bezier, copy the control nodes into Bezier curves @@ -1299,4 +1299,4 @@ template struct axom::fmt::formatter> : ostream_formatter { }; -#endif // AXOM_PRIMAL_NURBSCURVE_HPP_ \ No newline at end of file +#endif // AXOM_PRIMAL_NURBSCURVE_HPP_ diff --git a/src/axom/primal/tests/primal_nurbs_curve.cpp b/src/axom/primal/tests/primal_nurbs_curve.cpp index c76e2f6ddf..afe4837831 100644 --- a/src/axom/primal/tests/primal_nurbs_curve.cpp +++ b/src/axom/primal/tests/primal_nurbs_curve.cpp @@ -218,7 +218,6 @@ TEST(primal_nurbscurve, evaluate) { SLIC_INFO("Testing NURBS evaluation"); - const int MAX_DIM = 3; using CoordType = double; using Point1D = primal::Point; using Point2D = primal::Point; diff --git a/src/axom/quest/DiscreteShape.cpp b/src/axom/quest/DiscreteShape.cpp index f5a5ac9fa3..49ea2b2906 100644 --- a/src/axom/quest/DiscreteShape.cpp +++ b/src/axom/quest/DiscreteShape.cpp @@ -530,6 +530,7 @@ void DiscreteShape::createVORRepresentation() m_shape.getGeometry().getLevelOfRefinement(), octs, octCount); + AXOM_UNUSED_VAR(good); SLIC_ASSERT(good); // Rotate to the VOR axis direction and translate to the base location. diff --git a/src/axom/quest/interface/inout.cpp b/src/axom/quest/interface/inout.cpp index 05316e103b..c3e0a992b1 100644 --- a/src/axom/quest/interface/inout.cpp +++ b/src/axom/quest/interface/inout.cpp @@ -97,7 +97,9 @@ struct InOutHelper // load the mesh int rc = QUEST_INOUT_FAILED; +#ifdef AXOM_USE_C2C double revolvedVolume = 0.; +#endif switch(DIM) { case 2: diff --git a/src/axom/quest/util/mesh_helpers.cpp b/src/axom/quest/util/mesh_helpers.cpp index b03245d246..06d46bd908 100644 --- a/src/axom/quest/util/mesh_helpers.cpp +++ b/src/axom/quest/util/mesh_helpers.cpp @@ -360,6 +360,7 @@ void fill_cartesian_coords_3d_impl(const primal::BoundingBox& domainB axom::ArrayView& yView, axom::ArrayView& zView) { +#if defined(AXOM_DEBUG) using XS = axom::execution_space; SLIC_ASSERT_MSG(XS::usesAllocId(xView.getAllocatorID()) && XS::usesAllocId(yView.getAllocatorID()) && @@ -369,6 +370,7 @@ void fill_cartesian_coords_3d_impl(const primal::BoundingBox& domainB std::to_string(yView.getAllocatorID()) + " and " + std::to_string(zView.getAllocatorID()) + " are not all compatible with execution space " + XS::name()); +#endif const auto& shape = xView.shape(); const auto& mapping = xView.mapping(); diff --git a/src/axom/sidre/core/View.cpp b/src/axom/sidre/core/View.cpp index 109951fad8..07baca1642 100644 --- a/src/axom/sidre/core/View.cpp +++ b/src/axom/sidre/core/View.cpp @@ -1533,7 +1533,7 @@ void View::importFrom(conduit::Node& data_holder, * ************************************************************************* */ -View* View::importArrayNode(const Node& array, int allocID) +View* View::importArrayNode(const Node& array) { conduit::DataType array_dtype = array.dtype(); diff --git a/src/axom/sidre/core/View.hpp b/src/axom/sidre/core/View.hpp index dacc29d5bf..02a4f1b8c1 100644 --- a/src/axom/sidre/core/View.hpp +++ b/src/axom/sidre/core/View.hpp @@ -692,7 +692,6 @@ class View /* * \brief set the View to hold an array in a Buffer - * \param allocID Allocator id for array data allocation. * * This takes a Node that holds an array of data and sets up the View to * hold a copy of that data in an attached Buffer. @@ -708,7 +707,7 @@ class View * * \return pointer to this View object */ - View* importArrayNode(const Node& array, int allocID = INVALID_ALLOCATOR_ID); + View* importArrayNode(const Node& array); // // RDH -- Add an overload of the following that takes a const char *. diff --git a/src/axom/slam/DynamicConstantRelation.hpp b/src/axom/slam/DynamicConstantRelation.hpp index 7b78dcab25..2011926df4 100644 --- a/src/axom/slam/DynamicConstantRelation.hpp +++ b/src/axom/slam/DynamicConstantRelation.hpp @@ -304,9 +304,9 @@ class DynamicConstantRelation : public /*Relation,*/ CardinalityPolicy { const auto SZ = relationCardinality(); const auto beg_idx = idx * SZ; - for(auto idx = beg_idx; idx < (beg_idx + SZ); ++idx) + for(auto i = beg_idx; i < (beg_idx + SZ); ++i) { - if(m_relationsVec[idx] != INVALID_INDEX) + if(m_relationsVec[i] != INVALID_INDEX) { return true; } From fd9fa7a4b58549a6db4477eefe4c9800701e98bb Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Thu, 14 Nov 2024 15:50:49 -0800 Subject: [PATCH 055/100] Support external BP mesh as a Conduit Node. Previously, external BP mesh can only be a sidre Group. --- src/axom/quest/IntersectionShaper.hpp | 36 +++++-- src/axom/quest/Shaper.cpp | 93 ++++++++++++++++--- src/axom/quest/Shaper.hpp | 21 ++++- .../quest/examples/quest_shape_in_memory.cpp | 5 + src/axom/quest/util/mesh_helpers.cpp | 2 +- src/axom/quest/util/mesh_helpers.hpp | 4 +- 6 files changed, 134 insertions(+), 27 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index e7a3c0ed46..bf495cacf9 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -28,11 +28,11 @@ #include "axom/quest/interface/internal/QuestHelpers.hpp" #include "axom/fmt.hpp" -#ifdef AXOM_USE_MFEM -#include "mfem.hpp" -#endif + #ifdef AXOM_USE_MFEM + #include "mfem.hpp" + #endif -#include "axom/fmt.hpp" + #include "axom/fmt.hpp" // clang-format off using seq_exec = axom::SEQ_EXEC; @@ -2070,6 +2070,21 @@ class IntersectionShaper : public Shaper } else { + /* + If the computational mesh is an external conduit::Node, + it must have all necessary fields. We will not create + any field because we don't want overrid the user's memory + management. We will only generate fields for meshes in + sidre::Group, where the user can set the allocator id. + */ + SLIC_ERROR_IF(m_bpNodeExt != nullptr, + "For a computational mesh in an external conduit::Node," + " all output fields must be preallocated before shaping." + " IntersectionShaper will NOT override the user's memory" + " management. The field '" + fieldPath + "' is missing." + " To have IntersectionShaper allocate output memory, you can " + " put the mesh in a sidre::Group and set its allocator id."); + constexpr axom::IndexType componentCount = 1; axom::IndexType shape[2] = {m_cellCount, componentCount}; auto* fieldGrp = m_bpGrp->createGroup(fieldPath); @@ -2086,6 +2101,11 @@ class IntersectionShaper : public Shaper valuesView->allocate(); if(fieldName.rfind("vol_frac_", 0) == 0) { + // TODO: I think this arrangement of matsets is wrong. + // It passes the conduit blueprint verification but maybe + // because conduit doesn't check matsets. + // Needs more verification. + // // This is a material volume fraction field. // Shallow-copy valuesView to (uni-buffer) matsets. const std::string matlName = fieldName.substr(9); @@ -2232,11 +2252,11 @@ class IntersectionShaper : public Shaper constexpr int NUM_COMPS_PER_VERT = 3; // Put mesh in Node so we can use conduit::blueprint utilities. - conduit::Node meshNode; - m_bpGrp->createNativeLayout(meshNode); + // conduit::Node meshNode; + // m_bpGrp->createNativeLayout(m_bpNodeInt); const conduit::Node& topoNode = - meshNode.fetch_existing("topologies").fetch_existing(m_bpTopo); + m_bpNodeInt.fetch_existing("topologies").fetch_existing(m_bpTopo); const std::string coordsetName = topoNode.fetch_existing("coordset").as_string(); @@ -2256,7 +2276,7 @@ class IntersectionShaper : public Shaper XS::usesAllocId(conn.getAllocatorID()), std::string(XS::name()) + " cannot use the connectivity allocator id"); - const conduit::Node& coordNode = meshNode["coordsets"][coordsetName]; + const conduit::Node& coordNode = m_bpNodeInt["coordsets"][coordsetName]; const conduit::Node& coordValues = coordNode.fetch_existing("values"); axom::IndexType vertexCount = coordValues["x"].dtype().number_of_elements(); bool isInterleaved = conduit::blueprint::mcarray::is_interleaved(coordValues); diff --git a/src/axom/quest/Shaper.cpp b/src/axom/quest/Shaper.cpp index 362c08c962..86d360e029 100644 --- a/src/axom/quest/Shaper.cpp +++ b/src/axom/quest/Shaper.cpp @@ -11,7 +11,9 @@ #include "axom/quest/interface/internal/QuestHelpers.hpp" #include "axom/quest/Shaper.hpp" #include "axom/quest/DiscreteShape.hpp" +#include "axom/quest/util/mesh_helpers.hpp" #include "conduit_blueprint_mesh.hpp" +#include "axom/core/WhereMacro.hpp" #include "axom/fmt.hpp" @@ -30,6 +32,12 @@ constexpr double Shaper::DEFAULT_VERTEX_WELD_THRESHOLD; Shaper::Shaper(const klee::ShapeSet& shapeSet, sidre::MFEMSidreDataCollection* dc) : m_shapeSet(shapeSet) , m_dc(dc) +#if defined(AXOM_USE_CONDUIT) + , m_bpGrp(nullptr) + , m_bpTopo() + , m_bpNodeExt(nullptr) + , m_bpNodeInt() +#endif { #if defined(AXOM_USE_MPI) && defined(MFEM_USE_MPI) m_comm = m_dc->GetComm(); @@ -42,44 +50,68 @@ Shaper::Shaper(const klee::ShapeSet& shapeSet, sidre::Group* bpGrp, const std::string& topo) : m_shapeSet(shapeSet) +#if defined(AXOM_USE_CONDUIT) , m_bpGrp(bpGrp) , m_bpTopo(topo.empty() ? bpGrp->getGroup("topologies")->getGroupName(0) : topo) - , m_bpNode(nullptr) + , m_bpNodeExt(nullptr) + , m_bpNodeInt() +#endif , m_comm(MPI_COMM_WORLD) { SLIC_ASSERT(m_bpTopo != sidre::InvalidName); - conduit::Node meshNode; - bpGrp->createNativeLayout(meshNode); + m_bpGrp->createNativeLayout(m_bpNodeInt); + #if defined(AXOM_DEBUG) - conduit::Node info; - SLIC_ASSERT(conduit::blueprint::mesh::verify(meshNode, info)); + std::string whyBad; + bool goodMesh = verifyInputMesh(whyBad); + SLIC_ASSERT_MSG(goodMesh, whyBad); #endif m_cellCount = conduit::blueprint::mesh::topology::length( - meshNode.fetch_existing("topologies").fetch_existing(m_bpTopo)); + m_bpNodeInt.fetch_existing("topologies").fetch_existing(m_bpTopo)); } Shaper::Shaper(const klee::ShapeSet& shapeSet, conduit::Node* bpNode, const std::string& topo) : m_shapeSet(shapeSet) +#if defined(AXOM_USE_CONDUIT) , m_bpGrp(nullptr) , m_bpTopo(topo.empty() ? bpNode->fetch_existing("topologies").child(0).name() : topo) - , m_bpNode(bpNode) + , m_bpNodeExt(bpNode) + , m_bpNodeInt() +#endif , m_comm(MPI_COMM_WORLD) { -#if defined(AXOM_DEBUG) - conduit::Node info; - SLIC_ASSERT(conduit::blueprint::mesh::verify(*bpNode, info)); -#endif - m_bpGrp = m_ds.getRoot()->createGroup("internalGrp"); + m_bpGrp->importConduitTreeExternal(*bpNode); + // We want unstructured topo but can accomodate structured. + const std::string topoType = + bpNode->fetch_existing("topologies").fetch_existing(m_bpTopo).fetch_existing("type").as_string(); + if(topoType == "structured") + { + axom::quest::util::convert_blueprint_structured_explicit_to_unstructured( + m_bpGrp, m_bpTopo); + } + + m_bpGrp->createNativeLayout(m_bpNodeInt); + +#if defined(AXOM_DEBUG) + std::string whyBad; + bool goodMesh = verifyInputMesh(whyBad); + SLIC_ASSERT_MSG(goodMesh, whyBad); +#endif + m_cellCount = conduit::blueprint::mesh::topology::length( - m_bpNode->fetch_existing("topologies").fetch_existing(m_bpTopo)); + bpNode->fetch_existing("topologies").fetch_existing(m_bpTopo)); +} + +Shaper::~Shaper() +{ } void Shaper::setSamplesPerKnotSpan(int nSamples) @@ -176,6 +208,41 @@ void Shaper::loadShapeInternal(const klee::Shape& shape, revolvedVolume = discreteShape.getRevolvedVolume(); } +bool Shaper::verifyInputMesh(std::string& whyBad) const +{ + bool rval = true; + +#if defined(AXOM_USE_CONDUIT) + if(m_bpGrp != nullptr) + { + conduit::Node info; + rval = conduit::blueprint::mesh::verify(m_bpNodeInt, info); + if(rval) + { + std::string topoType = m_bpNodeInt.fetch("topologies")[m_bpTopo]["type"].as_string(); + rval = topoType == "unstructured"; + info[0].set_string("Topology is not unstructured."); + } + if(rval) + { + std::string elemShape = m_bpNodeInt.fetch("topologies")[m_bpTopo]["elements"]["shape"].as_string(); + rval = elemShape == "hex"; + info[0].set_string("Topology elements are not hex."); + } + whyBad = info.to_summary_string(); + } +#endif + +#if defined(AXOM_USE_MFEM) + if(m_dc != nullptr) + { + // No specific requirements for MFEM mesh. + } +#endif + + return rval; +} + // ---------------------------------------------------------------------------- int Shaper::getRank() const diff --git a/src/axom/quest/Shaper.hpp b/src/axom/quest/Shaper.hpp index 0faee5379a..804ef8b22b 100644 --- a/src/axom/quest/Shaper.hpp +++ b/src/axom/quest/Shaper.hpp @@ -41,6 +41,9 @@ namespace quest { /** * Abstract base class for shaping material volume fractions + * + * Shaper requires Axom to be configured with Conduit or MFEM + * or both. */ class Shaper { @@ -63,12 +66,17 @@ class Shaper /*! @brief Construct Shaper to operate on a blueprint-formatted mesh stored in a conduit Node. + + Because \c conduit::Node doesn't support application-specified + allocator id for (only) arrays, the incoming \c bpNode must have + all arrays pre-allocated in a space accessible by the runtime + policy. Any missing space would lead to an exception. */ Shaper(const klee::ShapeSet& shapeSet, conduit::Node* bpNode, const std::string& topo = ""); - virtual ~Shaper() = default; + virtual ~Shaper(); public: // Some default values. @@ -80,6 +88,9 @@ class Shaper /// Refinement type. using RefinementType = DiscreteShape::RefinementType; + //! @brief Verify the input mesh is okay for this class to work with. + bool verifyInputMesh(std::string& whyBad) const; + //@{ //! @name Functions to get and set shaping parameters @@ -197,9 +208,13 @@ class Shaper #if defined(AXOM_USE_CONDUIT) // For mesh represented in Conduit or sidre sidre::DataStore m_ds; - axom::sidre::Group* m_bpGrp {nullptr}; + //! @brief Version of the mesh for computations. + axom::sidre::Group* m_bpGrp; const std::string m_bpTopo; - conduit::Node* m_bpNode {nullptr}; + //! @brief Mesh in an external Node, when provided as a Node. + conduit::Node* m_bpNodeExt; + //! @brief Initial copy of mesh in an internal Node storage. + conduit::Node m_bpNodeInt; #endif axom::IndexType m_cellCount; diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index 5aa22968c1..d1a1ef88b2 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -1601,6 +1601,11 @@ int main(int argc, char** argv) using axom::utilities::string::startsWith; if(params.useBlueprint()) { +#if defined(AXOM_DEBUG) + conduit::Node meshNode, info; + compMeshGrp->createNativeLayout(meshNode); + SLIC_ASSERT(conduit::blueprint::mesh::verify(meshNode, info)); +#endif std::vector materialNames = shaper->getMaterialNames(); for(const auto& materialName : materialNames) { diff --git a/src/axom/quest/util/mesh_helpers.cpp b/src/axom/quest/util/mesh_helpers.cpp index 06d46bd908..da73ca8991 100644 --- a/src/axom/quest/util/mesh_helpers.cpp +++ b/src/axom/quest/util/mesh_helpers.cpp @@ -275,7 +275,7 @@ void convert_blueprint_structured_explicit_to_unstructured( auto* valuesGrp = coordsetGrp->getGroup("values"); curDim = valuesGrp->getView("x")->getShape(2, curShape); assert(curDim == 1); - axom::IndexType vertsShape[2] = {curShape[0], 1}; + const axom::IndexType vertsShape[2] = {curShape[0], 1}; valuesGrp->getView("x")->reshapeArray(2, vertsShape); valuesGrp->getView("y")->reshapeArray(2, vertsShape); valuesGrp->getView("z")->reshapeArray(2, vertsShape); diff --git a/src/axom/quest/util/mesh_helpers.hpp b/src/axom/quest/util/mesh_helpers.hpp index c61bb714eb..45889784c9 100644 --- a/src/axom/quest/util/mesh_helpers.hpp +++ b/src/axom/quest/util/mesh_helpers.hpp @@ -96,8 +96,8 @@ axom::sidre::Group* make_unstructured_blueprint_box_mesh( All input mesh data are expected to have the allocator id of meshGrp->getDefaultAllocatorID(). On output, they will also have - the same allocator id, even though some transfers to and from host - memory are used in intermediate steps. + the same allocator id, even if the intermediate steps have to + transfers memory to and from another space. */ void convert_blueprint_structured_explicit_to_unstructured( axom::sidre::Group* meshGrp, From 512591cd81dd6ae889e53c45707172784758611f Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Fri, 15 Nov 2024 08:00:41 -0800 Subject: [PATCH 056/100] Don't create vol frac data unless/until it's needed in applyReplacementRules. --- src/axom/quest/IntersectionShaper.hpp | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index bf495cacf9..5a7aa4dc3e 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -698,12 +698,6 @@ class IntersectionShaper : public Shaper SLIC_INFO(axom::fmt::format("{:-^80}", " Querying the BVH tree ")); - // Create and register a scalar field for this shape's volume fractions - // The Degrees of Freedom will be in correspondence with the elements - std::string volFracName = - axom::fmt::format("shape_vol_frac_{}", shape.getName()); - auto volFrac = getScalarCellData(volFracName); - populateHexesFromMesh(); axom::ArrayView hexes_device_view = m_hexes.view(); @@ -719,19 +713,6 @@ class IntersectionShaper : public Shaper primal::compute_bounding_box(hexes_device_view[i]); }); // end of loop to initialize hexahedral elements and bounding boxes - // Set each shape volume fraction to 1 - // volFrac may be on the host (MFEM) or device (Sidre). - auto fillVolFrac = AXOM_LAMBDA(axom::IndexType i) { volFrac[i] = 1.0; }; - if(axom::detail::getAllocatorSpace(volFrac.getAllocatorID()) == - MemorySpace::Host) - { - axom::for_all(m_cellCount, fillVolFrac); - } - else - { - axom::for_all(m_cellCount, fillVolFrac); - } - // Set shape components to zero if within threshold snapShapeVerticesToZero(shapes, shape_count, From 6528a5792d0a5ca1885c786d9a5fb69a4cb3d13a Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Fri, 15 Nov 2024 08:48:43 -0800 Subject: [PATCH 057/100] More informative handling of missing fields in conduit mesh. --- src/axom/quest/IntersectionShaper.hpp | 123 +++++++++++++++----------- src/axom/quest/Shaper.hpp | 2 +- 2 files changed, 71 insertions(+), 54 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 5a7aa4dc3e..e7d3e007e5 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -994,6 +994,7 @@ class IntersectionShaper : public Shaper bool newData = !hasData(fieldName); axom::ArrayView cfgf = getScalarCellData(fieldName); + SLIC_ASSERT(!cfgf.empty()); if(newData) { @@ -1092,7 +1093,10 @@ class IntersectionShaper : public Shaper axom::fmt::format("shape_vol_frac_{}", shape.getName()); // auto* shapeVolFrac = this->getDC()->GetField(shapeVolFracName); auto shapeVolFrac = getScalarCellData(shapeVolFracName); - // SLIC_ASSERT(shapeVolFrac != nullptr); + SLIC_ERROR_IF(shapeVolFrac.empty(), + "Field '" + shapeVolFracName + "' must be pre-allocated" + " in the Conduit Node computational mesh before using" + " IntersectionShaper::applyReplacementRules." ); // Allocate some memory for the replacement rule data arrays. int execSpaceAllocatorID = axom::execution_space::allocatorID(); @@ -1486,6 +1490,7 @@ class IntersectionShaper : public Shaper bool newData = !hasData(materialVolFracName); auto matVolFrac = getScalarCellData(materialVolFracName); + SLIC_ASSERT(!matVolFrac.empty()); if(newData) { // Zero out the volume fractions (on host). @@ -2011,6 +2016,11 @@ class IntersectionShaper : public Shaper Also add the corresponding entry in the blueprint field "matsets/fieldName/volume_fractions". + + If mesh is in an external Conduit Node, and the field + doesn't exist, emit a warning and return an empty ArrayView. + We don't add fields to the external Conduit Node. + \see Shaper::Shaper(). */ axom::ArrayView getScalarCellData(const std::string& fieldName, bool volumeDependent = false) @@ -2051,63 +2061,70 @@ class IntersectionShaper : public Shaper } else { - /* - If the computational mesh is an external conduit::Node, - it must have all necessary fields. We will not create - any field because we don't want overrid the user's memory - management. We will only generate fields for meshes in - sidre::Group, where the user can set the allocator id. - */ - SLIC_ERROR_IF(m_bpNodeExt != nullptr, - "For a computational mesh in an external conduit::Node," - " all output fields must be preallocated before shaping." - " IntersectionShaper will NOT override the user's memory" - " management. The field '" + fieldPath + "' is missing." - " To have IntersectionShaper allocate output memory, you can " - " put the mesh in a sidre::Group and set its allocator id."); - - constexpr axom::IndexType componentCount = 1; - axom::IndexType shape[2] = {m_cellCount, componentCount}; - auto* fieldGrp = m_bpGrp->createGroup(fieldPath); - // valuesView = fieldGrp->createView("values"); - valuesView = - fieldGrp->createViewWithShape("values", - axom::sidre::DataTypeId::FLOAT64_ID, - 2, - shape); - fieldGrp->createView("association")->setString("element"); - fieldGrp->createView("topology")->setString(m_bpTopo); - fieldGrp->createView("volume_dependent") - ->setString(std::string(volumeDependent ? "true" : "false")); - valuesView->allocate(); - if(fieldName.rfind("vol_frac_", 0) == 0) + if(m_bpNodeExt != nullptr) { - // TODO: I think this arrangement of matsets is wrong. - // It passes the conduit blueprint verification but maybe - // because conduit doesn't check matsets. - // Needs more verification. - // - // This is a material volume fraction field. - // Shallow-copy valuesView to (uni-buffer) matsets. - const std::string matlName = fieldName.substr(9); - axom::sidre::Group* volFracGrp = nullptr; - if(m_bpGrp->hasGroup("matsets/material/volume_fractions")) - { - volFracGrp = m_bpGrp->getGroup("matsets/material/volume_fractions"); - } - else + /* + If the computational mesh is an external conduit::Node, it + must have all necessary fields. We will only generate + fields for meshes in sidre::Group, where the user can set + the allocator id for only array data. conduit::Node doesn't + have this capability. + */ + SLIC_WARNING_IF(m_bpNodeExt != nullptr, + "For a computational mesh in a conduit::Node, all" + " output fields must be preallocated before shaping." + " IntersectionShaper will NOT contravene the user's" + " memory management. The cell-centered field '" + + fieldPath + "' is missing. Please pre-allocate" + " this output memory, or to have IntersectionShaper" + " allocate it, pass in the mesh as a sidre::Group" + " with your specific allocator id."); + } + else + { + constexpr axom::IndexType componentCount = 1; + axom::IndexType shape[2] = {m_cellCount, componentCount}; + auto* fieldGrp = m_bpGrp->createGroup(fieldPath); + // valuesView = fieldGrp->createView("values"); + valuesView = + fieldGrp->createViewWithShape("values", + axom::sidre::DataTypeId::FLOAT64_ID, + 2, + shape); + fieldGrp->createView("association")->setString("element"); + fieldGrp->createView("topology")->setString(m_bpTopo); + fieldGrp->createView("volume_dependent") + ->setString(std::string(volumeDependent ? "true" : "false")); + valuesView->allocate(); + if(fieldName.rfind("vol_frac_", 0) == 0) { - volFracGrp = - m_bpGrp->createGroup("matsets/material/volume_fractions"); - m_bpGrp->createViewString("matsets/material/topology", m_bpTopo); + // TODO: I think this arrangement of matsets is wrong. + // It passes the conduit blueprint verification but maybe + // because conduit doesn't check matsets. + // Needs more verification. + // + // This is a material volume fraction field. + // Shallow-copy valuesView to (uni-buffer) matsets. + const std::string matlName = fieldName.substr(9); + axom::sidre::Group* volFracGrp = nullptr; + if(m_bpGrp->hasGroup("matsets/material/volume_fractions")) + { + volFracGrp = m_bpGrp->getGroup("matsets/material/volume_fractions"); + } + else + { + volFracGrp = + m_bpGrp->createGroup("matsets/material/volume_fractions"); + m_bpGrp->createViewString("matsets/material/topology", m_bpTopo); + } + auto* valuesViewInMatsets = volFracGrp->copyView(valuesView); + valuesViewInMatsets->rename(matlName); } - auto* valuesViewInMatsets = volFracGrp->copyView(valuesView); - valuesViewInMatsets->rename(matlName); } + rval = + axom::ArrayView(static_cast(valuesView->getVoidPtr()), + m_cellCount); } - rval = - axom::ArrayView(static_cast(valuesView->getVoidPtr()), - m_cellCount); } #endif return rval; diff --git a/src/axom/quest/Shaper.hpp b/src/axom/quest/Shaper.hpp index 804ef8b22b..d29ffe730a 100644 --- a/src/axom/quest/Shaper.hpp +++ b/src/axom/quest/Shaper.hpp @@ -70,7 +70,7 @@ class Shaper Because \c conduit::Node doesn't support application-specified allocator id for (only) arrays, the incoming \c bpNode must have all arrays pre-allocated in a space accessible by the runtime - policy. Any missing space would lead to an exception. + policy. Any needed-but-missing space would lead to an exception. */ Shaper(const klee::ShapeSet& shapeSet, conduit::Node* bpNode, From 7ee5a7e2ac31866a053872a0daa66a813c41420a Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Fri, 15 Nov 2024 11:19:02 -0800 Subject: [PATCH 058/100] Allow access to individual shape results for apps that have their own replacement step. --- src/axom/quest/IntersectionShaper.hpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index e7d3e007e5..f1e47d303c 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -1366,6 +1366,10 @@ class IntersectionShaper : public Shaper // Runs the shaping query, based on the policy member and shape format set // (default is sequential) + // Fills m_overlap_volumes and m_hex_volumes, whose data + // will be in the default memory space for m_execPolicy. + // The data will be used in applyReplacementRules and can + // also be accessed by getOverlapVolumes() and getHexVolumes(). void runShapeQuery(const klee::Shape& shape) override { AXOM_ANNOTATE_SCOPE("runShapeQuery"); @@ -1427,6 +1431,16 @@ class IntersectionShaper : public Shaper } } + axom::ArrayView getOverlapVolumes() const + { + return m_overlap_volumes.view(); + } + + axom::ArrayView getHexVolumes() const + { + return m_hex_volumes.view(); + } + void adjustVolumeFractions() override { // Implementation here -- not sure if this will require anything for intersection-based shaping From 5d13366d490e7357a674eb4d6d3a4ce6cc541f85 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Fri, 15 Nov 2024 11:23:09 -0800 Subject: [PATCH 059/100] Add notes for future code clean-up. --- src/axom/quest/IntersectionShaper.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index f1e47d303c..79847bb900 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -677,6 +677,7 @@ class IntersectionShaper : public Shaper // Generate the BVH tree over the shapes // Access-aligned bounding boxes + // Does m_aabbs need to be a member? It's only used here. BTNG. m_aabbs = axom::Array(shape_count, shape_count, device_allocator); @@ -698,9 +699,11 @@ class IntersectionShaper : public Shaper SLIC_INFO(axom::fmt::format("{:-^80}", " Querying the BVH tree ")); + // Does m_hexes have to be a member? It can be passed into populateHexesFromMesh. BTNG. populateHexesFromMesh(); axom::ArrayView hexes_device_view = m_hexes.view(); + // Does m_hex_bbs have to be a member? I's only used here. BTNG. m_hex_bbs = axom::Array(m_cellCount, m_cellCount, device_allocator); axom::ArrayView hex_bbs_device_view = m_hex_bbs.view(); @@ -2176,6 +2179,7 @@ class IntersectionShaper : public Shaper auto vertCoords_device_view = vertCoords.view(); + // Does m_hexes have to be a member? It can be a function argument. BTNG. m_hexes = axom::Array(m_cellCount, m_cellCount, allocId); axom::ArrayView hexes_device_view = m_hexes.view(); constexpr double ZERO_THRESHOLD = 1.e-10; From 81363e6bb15d703a27efef4663a8836c595c921a Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Thu, 21 Nov 2024 11:44:21 -0800 Subject: [PATCH 060/100] Set the directory path from the ShapeSet when constructing Shaper. This makes it clear that we need the path, not the entire ShapeSet. Get the directory from the path just once, instead of every time we refer to a relative path. --- src/axom/quest/DiscreteShape.cpp | 17 +++++++++-------- src/axom/quest/DiscreteShape.hpp | 10 ++++------ src/axom/quest/IntersectionShaper.hpp | 19 +++++++++++++++++-- src/axom/quest/Shaper.cpp | 21 ++++++++++++++++++++- src/axom/quest/Shaper.hpp | 12 ++++++++++++ 5 files changed, 62 insertions(+), 17 deletions(-) diff --git a/src/axom/quest/DiscreteShape.cpp b/src/axom/quest/DiscreteShape.cpp index 49ea2b2906..27c7ffc3f1 100644 --- a/src/axom/quest/DiscreteShape.cpp +++ b/src/axom/quest/DiscreteShape.cpp @@ -751,20 +751,21 @@ void DiscreteShape::setPercentError(double percent) clampVal(percent, MINIMUM_PERCENT_ERROR, MAXIMUM_PERCENT_ERROR); } +void DiscreteShape::setPrefixPath(const std::string& prefixPath) +{ + SLIC_ERROR_IF(!prefixPath.empty() && !axom::utilities::filesystem::pathExists(prefixPath), + "Path '" + prefixPath + "' does not exist."); + m_prefixPath = prefixPath; +} + std::string DiscreteShape::resolvePath() const { const std::string& geomPath = m_shape.getGeometry().getPath(); - if(geomPath[0] == '/') + if(geomPath[0] == '/' || m_prefixPath.empty()) { return geomPath; } - if(m_prefixPath.empty()) - { - throw std::logic_error("Relative geometry path requires a parent path."); - } - std::string dir; - utilities::filesystem::getDirName(dir, m_prefixPath); - return utilities::filesystem::joinPath(dir, geomPath); + return utilities::filesystem::joinPath(m_prefixPath, geomPath); } void DiscreteShape::setParentGroup(axom::sidre::Group* parentGroup) diff --git a/src/axom/quest/DiscreteShape.hpp b/src/axom/quest/DiscreteShape.hpp index d64ae81f39..f7544d603e 100644 --- a/src/axom/quest/DiscreteShape.hpp +++ b/src/axom/quest/DiscreteShape.hpp @@ -54,11 +54,11 @@ class DiscreteShape @param parentGroup Group under which to put the discrete mesh and support blueprint-tets shapes. If null, don't use sidre and don't support blueprint-tets. - @param prefixPath Path prefix for shape files specified + @param prefixPath Path prefix for shape from a file specified with a relative path. Refinement type is set to DiscreteShape::RefinementUniformSegments - and percent erro is set to 0. See setPercentError() and + and percent error is set to 0. See setPercentError() and setRefinementType(). */ DiscreteShape(const axom::klee::Shape& shape, @@ -70,10 +70,8 @@ class DiscreteShape //@{ //! @name Functions to get and set shaping parameters - void setPrefixPath(const std::string& prefixPath) - { - m_prefixPath = prefixPath; - } + //! @brief Set prefix for shape files specified as relative path. + void setPrefixPath(const std::string& prefixPath); /*! @brief Set the refinement type. diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 79847bb900..aa9fd308e4 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -665,6 +665,10 @@ class IntersectionShaper : public Shaper int shape_count) { + // No need for shape, because it has been converted into m_tets or m_octs, + // which is what the parameter shapes is. + AXOM_UNUSED_VAR(shape); + const int host_allocator = axom::execution_space::allocatorID(); const int device_allocator = axom::execution_space::allocatorID(); @@ -704,6 +708,8 @@ class IntersectionShaper : public Shaper axom::ArrayView hexes_device_view = m_hexes.view(); // Does m_hex_bbs have to be a member? I's only used here. BTNG. + // Or if it's saved, it needs not be recomputed if the computational + // mesh doesn't change. m_hex_bbs = axom::Array(m_cellCount, m_cellCount, device_allocator); axom::ArrayView hex_bbs_device_view = m_hex_bbs.view(); @@ -1319,8 +1325,13 @@ class IntersectionShaper : public Shaper //@} public: - // Prepares the shaping query, based on the policy member set - // (default is sequential) + /*! + \brief Prepares the shaping query, based on the policy member set + (default is sequential) + + \internal This method populates m_tets or m_octs from the given \c + shape. These arrays are used in runShapeQuery. + */ void prepareShapeQuery(klee::Dimensions shapeDimension, const klee::Shape& shape) override { @@ -2428,8 +2439,12 @@ class IntersectionShaper : public Shaper double m_revolvedVolume {DEFAULT_REVOLVED_VOLUME}; std::string m_free_mat_name; + //! @brief Volumes of cells in the computational mesh. axom::Array m_hex_volumes; + + //! @brief Overlap volumes of cells in the computational mesh and the last shape. axom::Array m_overlap_volumes; + double m_vertexWeldThreshold {1.e-10}; // Guard these to prevent warnings. int m_octcount {0}; diff --git a/src/axom/quest/Shaper.cpp b/src/axom/quest/Shaper.cpp index 86d360e029..6d98f87267 100644 --- a/src/axom/quest/Shaper.cpp +++ b/src/axom/quest/Shaper.cpp @@ -43,6 +43,8 @@ Shaper::Shaper(const klee::ShapeSet& shapeSet, sidre::MFEMSidreDataCollection* d m_comm = m_dc->GetComm(); #endif m_cellCount = m_dc->GetMesh()->GetNE(); + + setFilePath(shapeSet.getPath()); } #endif @@ -60,6 +62,7 @@ Shaper::Shaper(const klee::ShapeSet& shapeSet, { SLIC_ASSERT(m_bpTopo != sidre::InvalidName); + // This may take too long if there are repeated construction. m_bpGrp->createNativeLayout(m_bpNodeInt); #if defined(AXOM_DEBUG) @@ -70,6 +73,8 @@ Shaper::Shaper(const klee::ShapeSet& shapeSet, m_cellCount = conduit::blueprint::mesh::topology::length( m_bpNodeInt.fetch_existing("topologies").fetch_existing(m_bpTopo)); + + setFilePath(shapeSet.getPath()); } Shaper::Shaper(const klee::ShapeSet& shapeSet, @@ -108,12 +113,26 @@ Shaper::Shaper(const klee::ShapeSet& shapeSet, m_cellCount = conduit::blueprint::mesh::topology::length( bpNode->fetch_existing("topologies").fetch_existing(m_bpTopo)); + + setFilePath(shapeSet.getPath()); } Shaper::~Shaper() { } +void Shaper::setFilePath(const std::string& filePath) +{ + if (filePath.empty()) + { + m_prefixPath.clear(); + } + else + { + utilities::filesystem::getDirName(m_prefixPath, filePath); + } +} + void Shaper::setSamplesPerKnotSpan(int nSamples) { using axom::utilities::clampLower; @@ -197,7 +216,7 @@ void Shaper::loadShapeInternal(const klee::Shape& shape, shape.getGeometry().getFormat())); // Code for discretizing shapes has been factored into DiscreteShape class. - DiscreteShape discreteShape(shape, m_dataStore.getRoot(), m_shapeSet.getPath()); + DiscreteShape discreteShape(shape, m_dataStore.getRoot(), m_prefixPath); discreteShape.setVertexWeldThreshold(m_vertexWeldThreshold); discreteShape.setRefinementType(m_refinementType); if(percentError > 0) diff --git a/src/axom/quest/Shaper.hpp b/src/axom/quest/Shaper.hpp index d29ffe730a..6b3b66ff91 100644 --- a/src/axom/quest/Shaper.hpp +++ b/src/axom/quest/Shaper.hpp @@ -102,6 +102,14 @@ class Shaper //@} + /*! + @brief Set path of shape input file. + + The path is used to resolve relative paths that may have been + specified in the file. + */ + void setFilePath(const std::string& filePath); + mint::Mesh* getSurfaceMesh() const { return m_surfaceMesh.get(); } bool isVerbose() const { return m_verboseOutput; } @@ -200,6 +208,9 @@ class Shaper const klee::ShapeSet& m_shapeSet; + //! \brief Prefix path for shape file names with relative path. + std::string m_prefixPath; + #if defined(AXOM_USE_MFEM) // For mesh represented as MFEMSidreDataCollection sidre::MFEMSidreDataCollection* m_dc {nullptr}; @@ -217,6 +228,7 @@ class Shaper conduit::Node m_bpNodeInt; #endif + //! @brief Number of cells in computational mesh (m_dc or m_bpGrp). axom::IndexType m_cellCount; std::shared_ptr m_surfaceMesh; From 1eb49adfa4fd39d38f96ce6377e4f0688714b36c Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Thu, 21 Nov 2024 11:52:30 -0800 Subject: [PATCH 061/100] Remove ShapeSet's obsolete path resolution code. --- src/axom/klee/ShapeSet.cpp | 15 ----------- src/axom/klee/ShapeSet.hpp | 10 ------- src/axom/klee/tests/klee_shape_set.cpp | 37 -------------------------- 3 files changed, 62 deletions(-) diff --git a/src/axom/klee/ShapeSet.cpp b/src/axom/klee/ShapeSet.cpp index 326000bce9..6a5d384f1c 100644 --- a/src/axom/klee/ShapeSet.cpp +++ b/src/axom/klee/ShapeSet.cpp @@ -21,21 +21,6 @@ void ShapeSet::setShapes(std::vector shapes) void ShapeSet::setPath(const std::string &path) { m_path = path; } -std::string ShapeSet::resolvePath(const std::string &filePath) const -{ - if(m_path.empty()) - { - throw std::logic_error("The ShapeSet's path has not been set"); - } - if(filePath[0] == '/') - { - return filePath; - } - std::string dir; - utilities::filesystem::getDirName(dir, m_path); - return utilities::filesystem::joinPath(dir, filePath); -} - void ShapeSet::setDimensions(Dimensions dimensions) { m_dimensions = dimensions; diff --git a/src/axom/klee/ShapeSet.hpp b/src/axom/klee/ShapeSet.hpp index f8c197b31a..602ada06b2 100644 --- a/src/axom/klee/ShapeSet.hpp +++ b/src/axom/klee/ShapeSet.hpp @@ -51,16 +51,6 @@ class ShapeSet */ const std::string &getPath() const { return m_path; } - /** - * Resolves a path relative to the path of this ShapeSet. - * - * \param filePath the path to resolve - * \return if the given path is absolute, then the given path. Otherwise, - * the path is interpreted as being relative to the directory containing - * this ShapeSet, and that is is returned. - */ - std::string resolvePath(const std::string &filePath) const; - /** * Sets the dimensions for all shapes in the ShapeSet. * diff --git a/src/axom/klee/tests/klee_shape_set.cpp b/src/axom/klee/tests/klee_shape_set.cpp index 40be40a136..9c7f4f3888 100644 --- a/src/axom/klee/tests/klee_shape_set.cpp +++ b/src/axom/klee/tests/klee_shape_set.cpp @@ -15,43 +15,6 @@ namespace klee { namespace { -TEST(ShapeSetTest, resolvePath_noPathSet) -{ - ShapeSet shapeSet; - EXPECT_THROW(shapeSet.resolvePath("anyPath"), std::logic_error); -} - -TEST(ShapeSetTest, resolvePath_startWithSimpleFileName) -{ - ShapeSet shapeSet; - shapeSet.setPath("file.yaml"); - EXPECT_EQ("newPath.txt", shapeSet.resolvePath("newPath.txt")); - EXPECT_EQ("d1/d2/newPath.txt", shapeSet.resolvePath("d1/d2/newPath.txt")); - EXPECT_EQ("/abs/path/newPath.txt", - shapeSet.resolvePath("/abs/path/newPath.txt")); -} - -TEST(ShapeSetTest, resolvePath_startWithRelativeFileName) -{ - ShapeSet shapeSet; - shapeSet.setPath("path/to/file.yaml"); - EXPECT_EQ("path/to/newPath.txt", shapeSet.resolvePath("newPath.txt")); - EXPECT_EQ("path/to/d1/d2/newPath.txt", - shapeSet.resolvePath("d1/d2/newPath.txt")); - EXPECT_EQ("/abs/path/newPath.txt", - shapeSet.resolvePath("/abs/path/newPath.txt")); -} - -TEST(ShapeSetTest, resolvePath_startWithAbsoluteFileName) -{ - ShapeSet shapeSet; - shapeSet.setPath("/path/to/file.yaml"); - EXPECT_EQ("/path/to/newPath.txt", shapeSet.resolvePath("newPath.txt")); - EXPECT_EQ("/path/to/d1/d2/newPath.txt", - shapeSet.resolvePath("d1/d2/newPath.txt")); - EXPECT_EQ("/other/abs/path/newPath.txt", - shapeSet.resolvePath("/other/abs/path/newPath.txt")); -} TEST(ShapeSetTest, dimensions_getAndSet) { From eb6b86cf4eb68be56fdad96ea01f18b0ca47afc2 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Thu, 21 Nov 2024 12:57:49 -0800 Subject: [PATCH 062/100] Remove resolvePath in favor of using the file utilities. --- src/axom/core/utilities/FileUtilities.cpp | 10 ++++++++++ src/axom/core/utilities/FileUtilities.hpp | 12 ++++++++++++ src/axom/quest/DiscreteShape.cpp | 13 ++----------- src/axom/quest/DiscreteShape.hpp | 3 --- 4 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/axom/core/utilities/FileUtilities.cpp b/src/axom/core/utilities/FileUtilities.cpp index c359ed9dd6..d1fd616ec6 100644 --- a/src/axom/core/utilities/FileUtilities.cpp +++ b/src/axom/core/utilities/FileUtilities.cpp @@ -106,6 +106,16 @@ int makeDirsForPath(const std::string& path) return err; } +//----------------------------------------------------------------------------- +std::string prefixRelativePath(const std::string& path, const std::string& prefix) +{ + if(path[0] == '/' || prefix.empty()) + { + return path; + } + return utilities::filesystem::joinPath(prefix, path); +} + //----------------------------------------------------------------------------- void getDirName(std::string& dir, const std::string& path) { diff --git a/src/axom/core/utilities/FileUtilities.hpp b/src/axom/core/utilities/FileUtilities.hpp index 9f70620ab5..1e0dcab931 100644 --- a/src/axom/core/utilities/FileUtilities.hpp +++ b/src/axom/core/utilities/FileUtilities.hpp @@ -66,6 +66,18 @@ std::string joinPath(const std::string& fileDir, */ int makeDirsForPath(const std::string& path); + +/*! + * \brief Add a prefix to a path if it is relative. + * + * \param [in] path string representing an absolute or relative path + * \param [in] prefix string representing a directory path + * + * \return \c prefix + \c path (with delimiter) if the path is + * relative or \c prefix is empty, or just \c path otherwise. + */ +std::string prefixRelativePath(const std::string& path, const std::string& prefix); + /*! * \brief Get directory name from a path that contains a file name * diff --git a/src/axom/quest/DiscreteShape.cpp b/src/axom/quest/DiscreteShape.cpp index 27c7ffc3f1..ae4727e22f 100644 --- a/src/axom/quest/DiscreteShape.cpp +++ b/src/axom/quest/DiscreteShape.cpp @@ -157,7 +157,8 @@ std::shared_ptr DiscreteShape::createMeshRepresentation() // We handled all the non-file formats. The rest are file formats. const std::string& file_format = geometryFormat; - std::string shapePath = resolvePath(); + std::string shapePath = axom::utilities::filesystem::prefixRelativePath( + m_shape.getGeometry().getPath(), m_prefixPath); SLIC_INFO("Reading file: " << shapePath << "..."); // Initialize revolved volume. @@ -758,16 +759,6 @@ void DiscreteShape::setPrefixPath(const std::string& prefixPath) m_prefixPath = prefixPath; } -std::string DiscreteShape::resolvePath() const -{ - const std::string& geomPath = m_shape.getGeometry().getPath(); - if(geomPath[0] == '/' || m_prefixPath.empty()) - { - return geomPath; - } - return utilities::filesystem::joinPath(m_prefixPath, geomPath); -} - void DiscreteShape::setParentGroup(axom::sidre::Group* parentGroup) { if(parentGroup) diff --git a/src/axom/quest/DiscreteShape.hpp b/src/axom/quest/DiscreteShape.hpp index f7544d603e..d0790553f9 100644 --- a/src/axom/quest/DiscreteShape.hpp +++ b/src/axom/quest/DiscreteShape.hpp @@ -167,9 +167,6 @@ class DiscreteShape void applyTransforms(); numerics::Matrix getTransforms() const; - //! @brief Returns the full geometry path. - std::string resolvePath() const; - /*! @brief Set the parent group for this object to store data. */ From 1c723baa4d23437b43eff0a9850bc942e3638a9e Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Thu, 21 Nov 2024 12:58:56 -0800 Subject: [PATCH 063/100] Add utility to return the parent path from a filesystem path. --- src/axom/core/utilities/FileUtilities.cpp | 16 ++++++++++++++++ src/axom/core/utilities/FileUtilities.hpp | 10 ++++++++++ src/axom/quest/Shaper.cpp | 2 +- 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/axom/core/utilities/FileUtilities.cpp b/src/axom/core/utilities/FileUtilities.cpp index d1fd616ec6..81c1811468 100644 --- a/src/axom/core/utilities/FileUtilities.cpp +++ b/src/axom/core/utilities/FileUtilities.cpp @@ -116,6 +116,22 @@ std::string prefixRelativePath(const std::string& path, const std::string& prefi return utilities::filesystem::joinPath(prefix, path); } +//----------------------------------------------------------------------------- +std::string getParentPath(const std::string& path) +{ + char separator = '/'; + std::size_t found = path.rfind(separator); + + std::string parent; + + if(found != std::string::npos) + { + parent = path.substr(0, found); + } + + return parent; +} + //----------------------------------------------------------------------------- void getDirName(std::string& dir, const std::string& path) { diff --git a/src/axom/core/utilities/FileUtilities.hpp b/src/axom/core/utilities/FileUtilities.hpp index 1e0dcab931..0d6c31b9a5 100644 --- a/src/axom/core/utilities/FileUtilities.hpp +++ b/src/axom/core/utilities/FileUtilities.hpp @@ -78,6 +78,16 @@ int makeDirsForPath(const std::string& path); */ std::string prefixRelativePath(const std::string& path, const std::string& prefix); +/*! + * \brief Get parent path name from a filesystem path. + * + * \param [in] path an absolute or relative filesystem path + * + * \return a directory path formed by removing the last part of the + * input path + */ +std::string getParentPath(const std::string& path); + /*! * \brief Get directory name from a path that contains a file name * diff --git a/src/axom/quest/Shaper.cpp b/src/axom/quest/Shaper.cpp index 6d98f87267..06e2f53db8 100644 --- a/src/axom/quest/Shaper.cpp +++ b/src/axom/quest/Shaper.cpp @@ -129,7 +129,7 @@ void Shaper::setFilePath(const std::string& filePath) } else { - utilities::filesystem::getDirName(m_prefixPath, filePath); + m_prefixPath = utilities::filesystem::getParentPath(filePath); } } From 2f577e38a41b88aff248daa93e98d87d8d2b49dc Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Fri, 22 Nov 2024 15:25:35 -0800 Subject: [PATCH 064/100] Fix missing and unneeded #includes. --- src/axom/quest/IntersectionShaper.hpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index aa9fd308e4..e0aecc3ffc 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -19,10 +19,13 @@ #include "axom/slic.hpp" #include "axom/slam.hpp" #include "axom/primal.hpp" - #include "axom/mint.hpp" - #include "axom/spin.hpp" + #include "axom/sidre/core/Group.hpp" + #include "axom/sidre/core/View.hpp" + #include "axom/mint/mesh/UnstructuredMesh.hpp" + #include "axom/mint/utils/vtk_utils.hpp" #include "axom/klee.hpp" #include "axom/quest/Shaper.hpp" + #include "axom/quest/Discretize.hpp" #include "axom/spin/BVH.hpp" #include "axom/quest/interface/internal/mpicomm_wrapper.hpp" #include "axom/quest/interface/internal/QuestHelpers.hpp" @@ -34,6 +37,12 @@ #include "axom/fmt.hpp" + #if defined(AXOM_USE_CONDUIT) + #include "conduit_node.hpp" + #include "conduit_blueprint_mesh.hpp" + #include "conduit_blueprint_mcarray.hpp" + #endif + // clang-format off using seq_exec = axom::SEQ_EXEC; From e0e09e6c39093aa7fd74b06125e754f669573de1 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Fri, 22 Nov 2024 18:21:54 -0800 Subject: [PATCH 065/100] Replace kernel to set array values with calls to fill. Presumably, the specialized fill method can do no worse than the kernels, and they are cleaner. --- src/axom/quest/IntersectionShaper.hpp | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index e0aecc3ffc..4ffb6648e7 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -850,23 +850,17 @@ class IntersectionShaper : public Shaper // or clip(tet,tet) for Pro/E meshes m_overlap_volumes = axom::Array(m_cellCount, m_cellCount, device_allocator); + m_overlap_volumes.fill(0.0); // Hex volume is the volume of the hexahedron element m_hex_volumes = axom::Array(m_cellCount, m_cellCount, device_allocator); + m_hex_volumes.fill(0.0); axom::ArrayView overlap_volumes_device_view = m_overlap_volumes.view(); axom::ArrayView hex_volumes_device_view = m_hex_volumes.view(); - // Set initial values to 0 - axom::for_all( - m_cellCount, - AXOM_LAMBDA(axom::IndexType i) { - overlap_volumes_device_view[i] = 0; - hex_volumes_device_view[i] = 0; - }); - SLIC_INFO( axom::fmt::format("{:-^80}", " Calculating hexahedron element volume ")); From 6c660f649bbc627007573c9e992a60e0c07c0043 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Fri, 22 Nov 2024 19:29:07 -0800 Subject: [PATCH 066/100] Add logic to avoid unnecessarily recomputing some arrays when shaping multiple shapes. We are not testing multiple shapes yet, but we should. --- src/axom/quest/IntersectionShaper.hpp | 98 ++++++++++++++++----------- 1 file changed, 60 insertions(+), 38 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 4ffb6648e7..a97739ffc7 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -712,24 +712,31 @@ class IntersectionShaper : public Shaper SLIC_INFO(axom::fmt::format("{:-^80}", " Querying the BVH tree ")); - // Does m_hexes have to be a member? It can be passed into populateHexesFromMesh. BTNG. - populateHexesFromMesh(); - axom::ArrayView hexes_device_view = m_hexes.view(); + if (m_hexes.empty()) + { + // m_hexes depend only on mesh so should not change once computed. + populateHexesFromMesh(); + } + axom::ArrayView hexes_device_view = m_hexes.view(); - // Does m_hex_bbs have to be a member? I's only used here. BTNG. - // Or if it's saved, it needs not be recomputed if the computational - // mesh doesn't change. - m_hex_bbs = - axom::Array(m_cellCount, m_cellCount, device_allocator); - axom::ArrayView hex_bbs_device_view = m_hex_bbs.view(); + if (m_hex_bbs.empty()) + { + // Does m_hex_bbs have to be a member? I's only used here. BTNG. + // Or if it's saved, it needs not be recomputed if the computational + // mesh doesn't change. + m_hex_bbs = + axom::Array(m_cellCount, m_cellCount, device_allocator); - // Get bounding boxes for hexahedral elements - axom::for_all( - m_cellCount, - AXOM_LAMBDA(axom::IndexType i) { - hex_bbs_device_view[i] = - primal::compute_bounding_box(hexes_device_view[i]); - }); // end of loop to initialize hexahedral elements and bounding boxes + + // Get bounding boxes for hexahedral elements + axom::ArrayView hex_bbs_device_view = m_hex_bbs.view(); + axom::for_all( + m_cellCount, + AXOM_LAMBDA(axom::IndexType i) { + hex_bbs_device_view[i] = + primal::compute_bounding_box(hexes_device_view[i]); + }); // end of loop to initialize hexahedral elements and bounding boxes + } // Set shape components to zero if within threshold snapShapeVerticesToZero(shapes, @@ -741,6 +748,8 @@ class IntersectionShaper : public Shaper "{:-^80}", " Finding shape candidates for each hexahedral element ")); + axom::ArrayView hex_bbs_device_view = m_hex_bbs.view(); + axom::Array offsets(m_cellCount, m_cellCount, device_allocator); axom::Array counts(m_cellCount, m_cellCount, device_allocator); axom::Array candidates; @@ -776,12 +785,21 @@ class IntersectionShaper : public Shaper auto shape_candidates_device_view = shape_candidates_device.view(); // Tetrahedrons from hexes (24 for each hex) - axom::Array tets_from_hexes_device( - m_cellCount * NUM_TETS_PER_HEX, - m_cellCount * NUM_TETS_PER_HEX, - device_allocator); + bool doInitializeTetsFromHexes = false; + if (m_tets_from_hexes_device.empty()) + { + m_tets_from_hexes_device = axom::Array( + m_cellCount * NUM_TETS_PER_HEX, + m_cellCount * NUM_TETS_PER_HEX, + device_allocator); + doInitializeTetsFromHexes = true; + } + else + { + SLIC_ASSERT( m_tets_from_hexes_device.size() == m_cellCount * NUM_TETS_PER_HEX ); + } axom::ArrayView tets_from_hexes_device_view = - tets_from_hexes_device.view(); + m_tets_from_hexes_device.view(); // Index into 'tets' axom::Array tet_indices_device( @@ -797,25 +815,28 @@ class IntersectionShaper : public Shaper axom::Array(newTotalCandidates_host, device_allocator); auto newTotalCandidates_device_view = newTotalCandidates_device.view(); - SLIC_INFO(axom::fmt::format( - "{:-^80}", - " Decomposing each hexahedron element into 24 tetrahedrons ")); + if (doInitializeTetsFromHexes) + { + SLIC_INFO(axom::fmt::format( + "{:-^80}", + " Decomposing each hexahedron element into 24 tetrahedrons ")); - using TetHexArray = axom::StackArray; + using TetHexArray = axom::StackArray; - { - AXOM_ANNOTATE_SCOPE("init_tets"); - axom::for_all( - m_cellCount, - AXOM_LAMBDA(axom::IndexType i) { - TetHexArray cur_tets; - hexes_device_view[i].triangulate(cur_tets); + { + AXOM_ANNOTATE_SCOPE("init_tets"); + axom::for_all( + m_cellCount, + AXOM_LAMBDA(axom::IndexType i) { + TetHexArray cur_tets; + hexes_device_view[i].triangulate(cur_tets); - for(int j = 0; j < NUM_TETS_PER_HEX; j++) - { - tets_from_hexes_device_view[i * NUM_TETS_PER_HEX + j] = cur_tets[j]; - } - }); + for(int j = 0; j < NUM_TETS_PER_HEX; j++) + { + tets_from_hexes_device_view[i * NUM_TETS_PER_HEX + j] = cur_tets[j]; + } + }); + } } SLIC_INFO( @@ -2193,7 +2214,7 @@ class IntersectionShaper : public Shaper auto vertCoords_device_view = vertCoords.view(); - // Does m_hexes have to be a member? It can be a function argument. BTNG. + SLIC_ASSERT(m_hexes.empty()); // No need to repeat this work. m_hexes = axom::Array(m_cellCount, m_cellCount, allocId); axom::ArrayView hexes_device_view = m_hexes.view(); constexpr double ZERO_THRESHOLD = 1.e-10; @@ -2459,6 +2480,7 @@ class IntersectionShaper : public Shaper axom::Array m_aabbs; axom::Array m_hexes; axom::Array m_hex_bbs; + axom::Array m_tets_from_hexes_device; // Views of volume-fraction data owned by grid. std::vector> m_vf_grid_functions; From 92b33ed0a687d3dd9d61afb3b41b18e35372dafd Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Tue, 26 Nov 2024 10:45:04 -0800 Subject: [PATCH 067/100] Fix bug accessing existing field in blueprint mesh. --- src/axom/quest/IntersectionShaper.hpp | 16 +++++++++------- .../quest/examples/quest_shape_in_memory.cpp | 6 +++--- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index a97739ffc7..0ea1e4397b 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -1024,12 +1024,12 @@ class IntersectionShaper : public Shaper // consider the free material something it needs to write as a matset. const std::string fieldName(materialNameToFieldName(m_free_mat_name)); - bool newData = !hasData(fieldName); + bool makeNewData = !hasData(fieldName); axom::ArrayView cfgf = getScalarCellData(fieldName); SLIC_ASSERT(!cfgf.empty()); - if(newData) + if(makeNewData) { AXOM_ANNOTATE_SCOPE("compute_free"); @@ -1539,11 +1539,11 @@ class IntersectionShaper : public Shaper // Get or create the volume fraction field for this shape's material auto materialVolFracName = materialNameToFieldName(materialName); - bool newData = !hasData(materialVolFracName); + bool makeNewData = !hasData(materialVolFracName); auto matVolFrac = getScalarCellData(materialVolFracName); SLIC_ASSERT(!matVolFrac.empty()); - if(newData) + if(makeNewData) { // Zero out the volume fractions (on host). #ifdef AXOM_USE_UMPIRE @@ -2095,6 +2095,7 @@ class IntersectionShaper : public Shaper rval = axom::ArrayView(gridFunc->GetData(), gridFunc->Size()); } #endif + #if defined(AXOM_USE_CONDUIT) if(m_bpGrp != nullptr) { @@ -2173,10 +2174,11 @@ class IntersectionShaper : public Shaper valuesViewInMatsets->rename(matlName); } } - rval = - axom::ArrayView(static_cast(valuesView->getVoidPtr()), - m_cellCount); } + + rval = + axom::ArrayView(static_cast(valuesView->getVoidPtr()), + m_cellCount); } #endif return rval; diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index 647312093b..4d4395f376 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -74,9 +74,6 @@ struct Input // Shape transformation parameters std::vector scaleFactors; - // Mesh format: mfem or blueprint - std::string meshFormat = "mfem"; - // Inline mesh parameters std::vector boxMins {-2, -2, -2}; std::vector boxMaxs {2, 2, 2}; @@ -480,6 +477,9 @@ axom::sidre::Group* createBoxMesh(axom::sidre::Group* meshGrp) SLIC_ASSERT(conduit::blueprint::mesh::verify(meshNode, info)); #endif + // State group is optional to blueprint, and we don't use it, but mint checks for it. + meshGrp->createGroup("state"); + return meshGrp; } From 703b0bec6b917d092cbe1a471681cdad466567cf Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Tue, 26 Nov 2024 11:42:35 -0800 Subject: [PATCH 068/100] Autoformat and remove unused code. --- src/axom/core/utilities/FileUtilities.hpp | 4 +- src/axom/quest/DiscreteShape.cpp | 8 ++-- src/axom/quest/IntersectionShaper.hpp | 47 ++++++++++++----------- src/axom/quest/Shaper.cpp | 26 +++++++------ src/axom/sidre/core/Group.hpp | 2 +- 5 files changed, 47 insertions(+), 40 deletions(-) diff --git a/src/axom/core/utilities/FileUtilities.hpp b/src/axom/core/utilities/FileUtilities.hpp index 0d6c31b9a5..bfe1406f96 100644 --- a/src/axom/core/utilities/FileUtilities.hpp +++ b/src/axom/core/utilities/FileUtilities.hpp @@ -66,7 +66,6 @@ std::string joinPath(const std::string& fileDir, */ int makeDirsForPath(const std::string& path); - /*! * \brief Add a prefix to a path if it is relative. * @@ -76,7 +75,8 @@ int makeDirsForPath(const std::string& path); * \return \c prefix + \c path (with delimiter) if the path is * relative or \c prefix is empty, or just \c path otherwise. */ -std::string prefixRelativePath(const std::string& path, const std::string& prefix); +std::string prefixRelativePath(const std::string& path, + const std::string& prefix); /*! * \brief Get parent path name from a filesystem path. diff --git a/src/axom/quest/DiscreteShape.cpp b/src/axom/quest/DiscreteShape.cpp index 7c8daa904c..3b352cc277 100644 --- a/src/axom/quest/DiscreteShape.cpp +++ b/src/axom/quest/DiscreteShape.cpp @@ -158,7 +158,8 @@ std::shared_ptr DiscreteShape::createMeshRepresentation() const std::string& file_format = geometryFormat; std::string shapePath = axom::utilities::filesystem::prefixRelativePath( - m_shape.getGeometry().getPath(), m_prefixPath); + m_shape.getGeometry().getPath(), + m_prefixPath); SLIC_INFO("Reading file: " << shapePath << "..."); // Initialize revolved volume. @@ -768,8 +769,9 @@ void DiscreteShape::setPercentError(double percent) void DiscreteShape::setPrefixPath(const std::string& prefixPath) { - SLIC_ERROR_IF(!prefixPath.empty() && !axom::utilities::filesystem::pathExists(prefixPath), - "Path '" + prefixPath + "' does not exist."); + SLIC_ERROR_IF( + !prefixPath.empty() && !axom::utilities::filesystem::pathExists(prefixPath), + "Path '" + prefixPath + "' does not exist."); m_prefixPath = prefixPath; } diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 0ea1e4397b..71f2fb3224 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -712,14 +712,14 @@ class IntersectionShaper : public Shaper SLIC_INFO(axom::fmt::format("{:-^80}", " Querying the BVH tree ")); - if (m_hexes.empty()) + if(m_hexes.empty()) { // m_hexes depend only on mesh so should not change once computed. populateHexesFromMesh(); } axom::ArrayView hexes_device_view = m_hexes.view(); - if (m_hex_bbs.empty()) + if(m_hex_bbs.empty()) { // Does m_hex_bbs have to be a member? I's only used here. BTNG. // Or if it's saved, it needs not be recomputed if the computational @@ -727,7 +727,6 @@ class IntersectionShaper : public Shaper m_hex_bbs = axom::Array(m_cellCount, m_cellCount, device_allocator); - // Get bounding boxes for hexahedral elements axom::ArrayView hex_bbs_device_view = m_hex_bbs.view(); axom::for_all( @@ -786,17 +785,18 @@ class IntersectionShaper : public Shaper // Tetrahedrons from hexes (24 for each hex) bool doInitializeTetsFromHexes = false; - if (m_tets_from_hexes_device.empty()) + if(m_tets_from_hexes_device.empty()) { - m_tets_from_hexes_device = axom::Array( - m_cellCount * NUM_TETS_PER_HEX, - m_cellCount * NUM_TETS_PER_HEX, - device_allocator); + m_tets_from_hexes_device = + axom::Array(m_cellCount * NUM_TETS_PER_HEX, + m_cellCount * NUM_TETS_PER_HEX, + device_allocator); doInitializeTetsFromHexes = true; } else { - SLIC_ASSERT( m_tets_from_hexes_device.size() == m_cellCount * NUM_TETS_PER_HEX ); + SLIC_ASSERT(m_tets_from_hexes_device.size() == + m_cellCount * NUM_TETS_PER_HEX); } axom::ArrayView tets_from_hexes_device_view = m_tets_from_hexes_device.view(); @@ -815,11 +815,11 @@ class IntersectionShaper : public Shaper axom::Array(newTotalCandidates_host, device_allocator); auto newTotalCandidates_device_view = newTotalCandidates_device.view(); - if (doInitializeTetsFromHexes) + if(doInitializeTetsFromHexes) { SLIC_INFO(axom::fmt::format( - "{:-^80}", - " Decomposing each hexahedron element into 24 tetrahedrons ")); + "{:-^80}", + " Decomposing each hexahedron element into 24 tetrahedrons ")); using TetHexArray = axom::StackArray; @@ -1127,9 +1127,10 @@ class IntersectionShaper : public Shaper // auto* shapeVolFrac = this->getDC()->GetField(shapeVolFracName); auto shapeVolFrac = getScalarCellData(shapeVolFracName); SLIC_ERROR_IF(shapeVolFrac.empty(), - "Field '" + shapeVolFracName + "' must be pre-allocated" - " in the Conduit Node computational mesh before using" - " IntersectionShaper::applyReplacementRules." ); + "Field '" + shapeVolFracName + + "' must be pre-allocated" + " in the Conduit Node computational mesh before using" + " IntersectionShaper::applyReplacementRules."); // Allocate some memory for the replacement rule data arrays. int execSpaceAllocatorID = axom::execution_space::allocatorID(); @@ -2127,11 +2128,12 @@ class IntersectionShaper : public Shaper "For a computational mesh in a conduit::Node, all" " output fields must be preallocated before shaping." " IntersectionShaper will NOT contravene the user's" - " memory management. The cell-centered field '" - + fieldPath + "' is missing. Please pre-allocate" - " this output memory, or to have IntersectionShaper" - " allocate it, pass in the mesh as a sidre::Group" - " with your specific allocator id."); + " memory management. The cell-centered field '" + + fieldPath + + "' is missing. Please pre-allocate" + " this output memory, or to have IntersectionShaper" + " allocate it, pass in the mesh as a sidre::Group" + " with your specific allocator id."); } else { @@ -2162,7 +2164,8 @@ class IntersectionShaper : public Shaper axom::sidre::Group* volFracGrp = nullptr; if(m_bpGrp->hasGroup("matsets/material/volume_fractions")) { - volFracGrp = m_bpGrp->getGroup("matsets/material/volume_fractions"); + volFracGrp = + m_bpGrp->getGroup("matsets/material/volume_fractions"); } else { @@ -2216,7 +2219,7 @@ class IntersectionShaper : public Shaper auto vertCoords_device_view = vertCoords.view(); - SLIC_ASSERT(m_hexes.empty()); // No need to repeat this work. + SLIC_ASSERT(m_hexes.empty()); // No need to repeat this work. m_hexes = axom::Array(m_cellCount, m_cellCount, allocId); axom::ArrayView hexes_device_view = m_hexes.view(); constexpr double ZERO_THRESHOLD = 1.e-10; diff --git a/src/axom/quest/Shaper.cpp b/src/axom/quest/Shaper.cpp index 06e2f53db8..e2f8505936 100644 --- a/src/axom/quest/Shaper.cpp +++ b/src/axom/quest/Shaper.cpp @@ -13,7 +13,6 @@ #include "axom/quest/DiscreteShape.hpp" #include "axom/quest/util/mesh_helpers.hpp" #include "conduit_blueprint_mesh.hpp" -#include "axom/core/WhereMacro.hpp" #include "axom/fmt.hpp" @@ -32,12 +31,12 @@ constexpr double Shaper::DEFAULT_VERTEX_WELD_THRESHOLD; Shaper::Shaper(const klee::ShapeSet& shapeSet, sidre::MFEMSidreDataCollection* dc) : m_shapeSet(shapeSet) , m_dc(dc) -#if defined(AXOM_USE_CONDUIT) + #if defined(AXOM_USE_CONDUIT) , m_bpGrp(nullptr) , m_bpTopo() , m_bpNodeExt(nullptr) , m_bpNodeInt() -#endif + #endif { #if defined(AXOM_USE_MPI) && defined(MFEM_USE_MPI) m_comm = m_dc->GetComm(); @@ -95,12 +94,15 @@ Shaper::Shaper(const klee::ShapeSet& shapeSet, m_bpGrp->importConduitTreeExternal(*bpNode); // We want unstructured topo but can accomodate structured. - const std::string topoType = - bpNode->fetch_existing("topologies").fetch_existing(m_bpTopo).fetch_existing("type").as_string(); + const std::string topoType = bpNode->fetch_existing("topologies") + .fetch_existing(m_bpTopo) + .fetch_existing("type") + .as_string(); if(topoType == "structured") { axom::quest::util::convert_blueprint_structured_explicit_to_unstructured( - m_bpGrp, m_bpTopo); + m_bpGrp, + m_bpTopo); } m_bpGrp->createNativeLayout(m_bpNodeInt); @@ -117,13 +119,11 @@ Shaper::Shaper(const klee::ShapeSet& shapeSet, setFilePath(shapeSet.getPath()); } -Shaper::~Shaper() -{ -} +Shaper::~Shaper() { } void Shaper::setFilePath(const std::string& filePath) { - if (filePath.empty()) + if(filePath.empty()) { m_prefixPath.clear(); } @@ -238,13 +238,15 @@ bool Shaper::verifyInputMesh(std::string& whyBad) const rval = conduit::blueprint::mesh::verify(m_bpNodeInt, info); if(rval) { - std::string topoType = m_bpNodeInt.fetch("topologies")[m_bpTopo]["type"].as_string(); + std::string topoType = + m_bpNodeInt.fetch("topologies")[m_bpTopo]["type"].as_string(); rval = topoType == "unstructured"; info[0].set_string("Topology is not unstructured."); } if(rval) { - std::string elemShape = m_bpNodeInt.fetch("topologies")[m_bpTopo]["elements"]["shape"].as_string(); + std::string elemShape = + m_bpNodeInt.fetch("topologies")[m_bpTopo]["elements"]["shape"].as_string(); rval = elemShape == "hex"; info[0].set_string("Topology elements are not hex."); } diff --git a/src/axom/sidre/core/Group.hpp b/src/axom/sidre/core/Group.hpp index 144cd097b7..da64965dea 100644 --- a/src/axom/sidre/core/Group.hpp +++ b/src/axom/sidre/core/Group.hpp @@ -1346,7 +1346,7 @@ class Group * \brief Print given number of levels of Group sub-tree * starting at this Group object to an output stream. */ - void printTree(const int nlevels, std::ostream& os) const; + void printTree(const int nlevels, std::ostream& os = std::cout) const; //@} From f8c7d7d6e9fb6baceef69990b8ec077570803420 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Fri, 6 Dec 2024 13:41:18 -0800 Subject: [PATCH 069/100] Add timers and make other changes. --- src/axom/quest/IntersectionShaper.hpp | 65 +++++++++++++++++-- .../quest/examples/quest_shape_in_memory.cpp | 4 +- 2 files changed, 63 insertions(+), 6 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 71f2fb3224..5060dffb3b 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -674,6 +674,7 @@ class IntersectionShaper : public Shaper int shape_count) { + AXOM_ANNOTATE_SCOPE("IntersectionShaper::runShapeQueryImpl"); // No need for shape, because it has been converted into m_tets or m_octs, // which is what the parameter shapes is. AXOM_UNUSED_VAR(shape); @@ -689,7 +690,7 @@ class IntersectionShaper : public Shaper " Inserting shapes' bounding boxes into BVH ")); // Generate the BVH tree over the shapes - // Access-aligned bounding boxes + // Axis-aligned bounding boxes // Does m_aabbs need to be a member? It's only used here. BTNG. m_aabbs = axom::Array(shape_count, shape_count, device_allocator); @@ -784,6 +785,10 @@ class IntersectionShaper : public Shaper auto shape_candidates_device_view = shape_candidates_device.view(); // Tetrahedrons from hexes (24 for each hex) + // TODO: Why break hexes into tets if we can hex-tet clip (clip_impl.hpp:502). + // Possibly because ShapeType can also be oct, and we don't do hex-oct clip. + // At least not yet. We could break the oct into 6 tets and do 6 hex-tet clips. + // Isn't that better than breaking the hex into 24 tets and doing 24 oct-tet clips? bool doInitializeTetsFromHexes = false; if(m_tets_from_hexes_device.empty()) { @@ -901,13 +906,13 @@ class IntersectionShaper : public Shaper constexpr bool tryFixOrientation = true; { + AXOM_ANNOTATE_SCOPE("IntersectionShaper::clipLoop"); // Copy calculated total back to host axom::Array newTotalCandidates_calc_host = axom::Array(newTotalCandidates_device, host_allocator); - AXOM_ANNOTATE_SCOPE("tet_shape_volume"); axom::for_all( - newTotalCandidates_calc_host[0], + newTotalCandidates_calc_host[0], // Number of candidates found. AXOM_LAMBDA(axom::IndexType i) { const int index = hex_indices_device_view[i]; const int shapeIndex = shape_candidates_device_view[i]; @@ -1480,6 +1485,58 @@ class IntersectionShaper : public Shaper return m_hex_volumes.view(); } + double sumOverlapVolumes(bool global = true) const + { + double overlapVol = 0.0; + switch(m_execPolicy) + { + #if defined(AXOM_USE_OPENMP) + case RuntimePolicy::omp: + overlapVol = sumArray(m_overlap_volumes.data(), + m_overlap_volumes.size()); + break; + #endif // AXOM_USE_OPENMP + #if defined(AXOM_USE_CUDA) && defined(AXOM_USE_UMPIRE) + case RuntimePolicy::cuda: + overlapVol = sumArray(m_overlap_volumes.data(), + m_overlap_volumes.size()); + break; + #endif // AXOM_USE_CUDA + #if defined(AXOM_USE_HIP) && defined(AXOM_USE_UMPIRE) + case RuntimePolicy::hip: + overlapVol = sumArray(m_overlap_volumes.data(), + m_overlap_volumes.size()); + break; + #endif // AXOM_USE_HIP + case RuntimePolicy::seq: + default: + overlapVol = sumArray(m_overlap_volumes.data(), + m_overlap_volumes.size()); + break; + } + + if (global) + { + overlapVol = this->allReduceSum(overlapVol); + } + return overlapVol; + } + + template + Summable sumArray(const Summable* a, axom::IndexType count) const + { + using LoopPolicy = typename axom::execution_space::loop_policy; + using ReducePolicy = typename axom::execution_space::reduce_policy; + RAJA::ReduceSum vsum{0}; + RAJA::forall( + RAJA::RangeSegment(0, count), + AXOM_LAMBDA(RAJA::Index_type i) { + vsum += a[i]; + }); + Summable sum = static_cast(vsum.get()); + return sum; + } + void adjustVolumeFractions() override { // Implementation here -- not sure if this will require anything for intersection-based shaping @@ -2471,7 +2528,7 @@ class IntersectionShaper : public Shaper //! @brief Volumes of cells in the computational mesh. axom::Array m_hex_volumes; - //! @brief Overlap volumes of cells in the computational mesh and the last shape. + //! @brief Overlap volumes of cells in the computational mesh for the last shape. axom::Array m_overlap_volumes; double m_vertexWeldThreshold {1.e-10}; diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index 4d4395f376..0a27b9a132 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -256,7 +256,7 @@ struct Input ->check(axom::CLI::PositiveNumber); app.add_option("--length", length) - ->description("Length of cone/cyl/VOR shape") + ->description("Length of cone/cyl/VOR shape, avg length of hex.") ->check(axom::CLI::PositiveNumber); app.add_option("--dir", direction) @@ -804,7 +804,7 @@ axom::klee::Shape createShape_Hex() SLIC_ASSERT(params.scaleFactors.empty() || params.scaleFactors.size() == 3); - const double md = params.length < 0 ? 0.6 : params.length; + const double md = params.length < 0 ? 0.6 : params.length/2; const double lg = 1.2 * md; const double sm = 0.8 * md; const Point3D p {-lg, -md, -sm}; From b0998b4fa4bd06f2d3fc73fdb7fc93c0bc708170 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Wed, 11 Dec 2024 15:17:24 -0800 Subject: [PATCH 070/100] Fix new code that work on Conduit mesh. This broke the MFEM tests. --- src/axom/quest/IntersectionShaper.hpp | 20 +- src/axom/quest/examples/CMakeLists.txt | 24 +- .../quest/examples/quest_shape_in_memory.cpp | 246 +++++++++++------- 3 files changed, 187 insertions(+), 103 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 5060dffb3b..89cfeb4dc6 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -30,6 +30,7 @@ #include "axom/quest/interface/internal/mpicomm_wrapper.hpp" #include "axom/quest/interface/internal/QuestHelpers.hpp" #include "axom/fmt.hpp" + #include "axom/core/WhereMacro.hpp" #ifdef AXOM_USE_MFEM #include "mfem.hpp" @@ -354,7 +355,11 @@ class IntersectionShaper : public Shaper const std::string& topo = "") : Shaper(shapeSet, bpNode, topo) , m_free_mat_name("free") - { } + { + // We cannot always create Conduit Nodes, so catch missing nodes early if possible. + SLIC_ERROR_IF(!bpNode->has_child("fields"), + "Input blueprint mesh lacks the 'fields' Node."); + } #endif //@{ @@ -874,11 +879,13 @@ class IntersectionShaper : public Shaper // Overlap volume is the volume of clip(oct,tet) for c2c // or clip(tet,tet) for Pro/E meshes + // TODO: don't reallocate. m_cellCount shouldn't change. m_overlap_volumes = axom::Array(m_cellCount, m_cellCount, device_allocator); m_overlap_volumes.fill(0.0); // Hex volume is the volume of the hexahedron element + // TODO: Make reallocate and recompute explicit (with a resetMesh or something). m_hex_volumes = axom::Array(m_cellCount, m_cellCount, device_allocator); m_hex_volumes.fill(0.0); @@ -1030,7 +1037,6 @@ class IntersectionShaper : public Shaper const std::string fieldName(materialNameToFieldName(m_free_mat_name)); bool makeNewData = !hasData(fieldName); - axom::ArrayView cfgf = getScalarCellData(fieldName); SLIC_ASSERT(!cfgf.empty()); @@ -1562,6 +1568,8 @@ class IntersectionShaper : public Shaper if(m_bpGrp) { auto fieldsGrp = m_bpGrp->getGroup("fields"); + SLIC_ERROR_IF(fieldsGrp == nullptr, + "Input blueprint mesh lacks the 'fields' Group/Node."); for(auto& group : fieldsGrp->groups()) { std::string materialName = fieldNameToMaterialName(group.getName()); @@ -1598,7 +1606,6 @@ class IntersectionShaper : public Shaper auto materialVolFracName = materialNameToFieldName(materialName); bool makeNewData = !hasData(materialVolFracName); - auto matVolFrac = getScalarCellData(materialVolFracName); SLIC_ASSERT(!matVolFrac.empty()); if(makeNewData) @@ -2157,7 +2164,7 @@ class IntersectionShaper : public Shaper #if defined(AXOM_USE_CONDUIT) if(m_bpGrp != nullptr) { - std::string fieldPath = axom::fmt::format("fields/{}", fieldName); + std::string fieldPath = "fields/" + fieldName; auto dtype = conduit::DataType::float64(m_cellCount); axom::sidre::View* valuesView = nullptr; if(m_bpGrp->hasGroup(fieldPath)) @@ -2189,8 +2196,9 @@ class IntersectionShaper : public Shaper fieldPath + "' is missing. Please pre-allocate" " this output memory, or to have IntersectionShaper" - " allocate it, pass in the mesh as a sidre::Group" - " with your specific allocator id."); + " allocate it, construct the IntersectionShaper" + " with the mesh as a sidre::Group with your" + " specific allocator id."); } else { diff --git a/src/axom/quest/examples/CMakeLists.txt b/src/axom/quest/examples/CMakeLists.txt index 026aee4bf1..5aaa789b3c 100644 --- a/src/axom/quest/examples/CMakeLists.txt +++ b/src/axom/quest/examples/CMakeLists.txt @@ -294,7 +294,6 @@ if((CONDUIT_FOUND OR endif() set(_testshapes "tetmesh" "tet" "hex" "sphere" "cyl" "cone" "vor" "all" "plane") - foreach(_policy ${_policies}) foreach(_testshape ${_testshapes}) set(_testname "quest_shape_in_memory_${_policy}_${_testshape}") @@ -306,10 +305,33 @@ if((CONDUIT_FOUND OR --refinements 2 --scale 0.75 0.75 0.75 --dir 0.2 0.4 0.8 + --meshType bpSidre inline_mesh --min -2 -2 -2 --max 2 2 2 --res 30 30 30 NUM_MPI_TASKS ${_nranks}) endforeach() endforeach() + + # Test Conduit and MFEM mesh types with a subset of shapes (so it doesn't take too long) + set(_testshapes "tetmesh" "sphere" "cyl") + set(_testMeshTypes "bpConduit" "mfem") + foreach(_policy ${_policies}) + foreach(_testMeshType ${_testMeshTypes}) + foreach(_testshape ${_testshapes}) + set(_testname "quest_shape_in_memory_${_policy}_${_testMeshType}_${_testshape}") + axom_add_test( + NAME ${_testname} + COMMAND quest_shape_in_memory_ex + --policy ${_policy} + --testShape ${_testshape} + --refinements 2 + --scale 0.75 0.75 0.75 + --dir 0.2 0.4 0.8 + --meshType ${_testMeshType} + inline_mesh --min -2 -2 -2 --max 2 2 2 --res 8 8 8 + NUM_MPI_TASKS ${_nranks}) + endforeach() + endforeach() + endforeach() endif() endif() diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index 0a27b9a132..ca3a64778a 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -85,6 +85,10 @@ struct Input SLIC_ASSERT(boxMaxs.size() == d); return int(d); } + int getBoxCellCount() const + { + return boxResolution[0]*boxResolution[1]*boxResolution[2]; + } // The shape to run. std::string testShape {"tetmesh"}; @@ -109,13 +113,14 @@ struct Input std::string backgroundMaterial; // clang-format off - enum class MeshType { bp = 0, mfem = 1 }; + enum class MeshType { bpSidre = 0, bpConduit = 1, mfem = 2 }; const std::map meshTypeChoices - { {"bp", MeshType::bp} , {"mfem", MeshType::mfem} }; + { {"bpSidre", MeshType::bpSidre} , {"bpConduit", MeshType::bpConduit}, {"mfem", MeshType::mfem} }; // clang-format on - MeshType meshType {MeshType::bp}; + MeshType meshType {MeshType::bpSidre}; bool useMfem() { return meshType == MeshType::mfem; } - bool useBlueprint() { return meshType == MeshType::bp; } + bool useBlueprintSidre() { return meshType == MeshType::bpSidre; } + bool useBlueprintConduit() { return meshType == MeshType::bpConduit; } private: bool m_verboseOutput {false}; @@ -459,6 +464,13 @@ const std::string topoName = "mesh"; const std::string coordsetName = "coords"; int cellCount = -1; +// Computational mesh in different forms, initialized in main +#if defined(AXOM_USE_MFEM) +std::shared_ptr shapingDC; +#endif +axom::sidre::Group* compMeshGrp = nullptr; +std::shared_ptr compMeshNode; + axom::sidre::Group* createBoxMesh(axom::sidre::Group* meshGrp) { using BBox3D = primal::BoundingBox; @@ -1034,7 +1046,6 @@ axom::sidre::View* getElementVolumes( Get vertex coordinates. We use UnstructuredMesh for this, so get it on host first then transfer to device if needed. */ -#if 1 auto* connData = meshGrp->getGroup("topologies") ->getGroup(topoName) ->getGroup("elements") @@ -1087,37 +1098,6 @@ axom::sidre::View* getElementVolumes( } } }); -#else - axom::Array vertCoords(cellCount * NUM_VERTS_PER_HEX, - cellCount * NUM_VERTS_PER_HEX); - auto vertCoordsView = vertCoords.view(); - - // DEBUG to here: This code doesn't work when the mesh is on device. - // Fix this error-checking code. Or copy the mesh to host for checking. - // NOTE: This method uses a mix of host and device loops. Make more consistent. - for(axom::IndexType cellIdx = 0; cellIdx < cellCount; ++cellIdx) - { - // Get the indices of this element's vertices - axom::IndexType* verts = mesh.getCellNodeIDs(cellIdx); - mesh.getCellNodeIDs(cellIdx, verts); - - // Get the coordinates for the vertices - for(int j = 0; j < NUM_VERTS_PER_HEX; ++j) - { - int vertIdx = cellIdx * NUM_VERTS_PER_HEX + j; - for(int k = 0; k < NUM_COMPS_PER_VERT; k++) - { - vertCoordsView[vertIdx][k] = mesh.getNodeCoordinate(verts[j], k); - } - } - } - if(vertCoords.getAllocatorID() != meshGrp->getDefaultAllocatorID()) - { - vertCoords = - axom::Array(vertCoords, meshGrp->getDefaultAllocatorID()); - vertCoordsView = vertCoords.view(); - } -#endif // Set vertex coords to zero if within threshold. // (I don't know why we do this. I'm following examples.) @@ -1230,7 +1210,7 @@ double sumMaterialVolumesImpl(sidre::Group* meshGrp, const std::string& material SLIC_ASSERT(conduit::blueprint::mesh::verify(meshNode, info)); } #endif - std::string topoPath = axom::fmt::format("topologies/{}", topoName); + std::string topoPath = "topologies/" + topoName; conduit::Node& topoNode = meshNode.fetch_existing(topoPath); const int cellCount = conduit::blueprint::mesh::topology::length(topoNode); @@ -1242,9 +1222,8 @@ double sumMaterialVolumesImpl(sidre::Group* meshGrp, const std::string& material elementVols->getNumElements()); // Get material volume fractions - const auto vfFieldName = axom::fmt::format("vol_frac_{}", material); - const auto vfFieldValuesPath = - axom::fmt::format("fields/{}/values", vfFieldName); + const auto vfFieldName = "vol_frac_" + material; + const auto vfFieldValuesPath = "fields/" + vfFieldName + "/values"; axom::sidre::View* volFrac = meshGrp->getView(vfFieldValuesPath); axom::ArrayView volFracView(volFrac->getArray(), cellCount); @@ -1339,6 +1318,73 @@ void saveMesh(const sidre::Group& mesh, const std::string& filename) saveMesh(tmpMesh, filename); } +axom::ArrayView getFieldAsArrayView(const std::string& fieldName) +{ + axom::ArrayView fieldDataView; + if(params.useBlueprintSidre()) + { + std::string valuesPath = "fields/" + fieldName + "/values"; + axom::sidre::View* fieldValues = compMeshGrp->getView(valuesPath); + double* fieldData = fieldValues->getArray(); + fieldDataView = axom::ArrayView(fieldData, + fieldValues->getNumElements()); + } + if(params.useBlueprintConduit()) + { + std::string valuesPath = "fields/" + fieldName + "/values"; + conduit::Node& fieldValues = compMeshNode->fetch_existing(valuesPath); + double* fieldData = fieldValues.as_double_ptr(); + fieldDataView = axom::ArrayView(fieldData, + fieldValues.dtype().number_of_elements()); + } +#if defined (AXOM_USE_MFEM) + if(params.useMfem()) + { + // auto* mfemMesh = shapingDC->GetMesh(); + mfem::GridFunction* gridFunc = shapingDC->GetField(fieldName); + fieldDataView = axom::ArrayView(gridFunc->GetData(), gridFunc->Size()); + } +#endif + return fieldDataView; +} + + +//!@brief Fill a sidre array View with a value. +// No error checking. +template void fillSidreViewData(axom::sidre::View* view, const T& value) +{ + double* valuesPtr = view->getData(); + switch (params.policy) { +#if defined(AXOM_USE_CUDA) + case RuntimePolicy::cuda: + axom::for_all>( + view->getNumElements(), + AXOM_LAMBDA(axom::IndexType i) { valuesPtr[i] = value; }); + break; +#endif +#if defined(AXOM_USE_HIP) + case RuntimePolicy::hip: + axom::for_all>( + view->getNumElements(), + AXOM_LAMBDA(axom::IndexType i) { valuesPtr[i] = value; }); + break; +#endif +#if defined(AXOM_USE_OMP) + case RuntimePolicy::omp: + axom::for_all( + view->getNumElements(), + AXOM_LAMBDA(axom::IndexType i) { valuesPtr[i] = value; }); + break; +#endif + case RuntimePolicy::seq: + default: + axom::for_all( + view->getNumElements(), + AXOM_LAMBDA(axom::IndexType i) { valuesPtr[i] = value; }); + break; + } +} + //------------------------------------------------------------------------------ int main(int argc, char** argv) { @@ -1390,16 +1436,6 @@ int main(int argc, char** argv) } #endif -#ifdef AXOM_USE_UMPIRE - const axom::MemorySpace memorySpace = - axom::detail::getAllocatorSpace(defaultAllocId); - const bool onHost = memorySpace == axom::MemorySpace::Host || - memorySpace == axom::MemorySpace::Dynamic || - memorySpace == axom::MemorySpace::Unified; -#else - const bool onHost = true; -#endif - AXOM_ANNOTATE_BEGIN("quest example for shaping primals"); AXOM_ANNOTATE_BEGIN("init"); @@ -1491,8 +1527,6 @@ int main(int argc, char** argv) } } - const klee::Dimensions shapeDim = shapeSet.getDimensions(); - // Apply error checking #ifndef AXOM_USE_C2C SLIC_ERROR_IF(shapeDim == klee::Dimensions::Two, @@ -1501,7 +1535,6 @@ int main(int argc, char** argv) #endif #if defined(AXOM_USE_MFEM) - std::shared_ptr shapingDC; if(params.useMfem()) { AXOM_ANNOTATE_BEGIN("load mesh"); @@ -1546,37 +1579,59 @@ int main(int argc, char** argv) "-DAXOM_ENABLE_MFEM_SIDRE_DATACOLLECTION."); #endif - axom::sidre::Group* compMeshGrp = nullptr; - if(params.useBlueprint()) + if(params.useBlueprintSidre() || params.useBlueprintConduit()) { compMeshGrp = ds.getRoot()->createGroup("compMesh"); compMeshGrp->setDefaultAllocator(defaultAllocId); createBoxMesh(compMeshGrp); - conduit::Node meshNode; - compMeshGrp->createNativeLayout(meshNode); - SLIC_INFO(axom::fmt::format("{:-^80}", "Generated Blueprint mesh")); - if(onHost) + + if (params.useBlueprintConduit()) { - meshNode.print(); + // Intersection requires conduit mesh to have array data pre-allocated. + auto makeField = [&](const std::string& fieldName, double initValue) { + auto fieldGrp = compMeshGrp->createGroup("fields/" + fieldName); + axom::IndexType shape[] = {params.getBoxCellCount(), 1}; + fieldGrp->createViewString("association", "element"); + fieldGrp->createViewString("topology", topoName); + fieldGrp->createViewString("volume_dependent", "true"); + axom::sidre::View* valuesView = fieldGrp->createViewWithShapeAndAllocate( + "values", + axom::sidre::detail::SidreTT::id, + 2, shape); + fillSidreViewData(valuesView, initValue); + }; + makeField("vol_frac_free", 1.0); + const auto &shapes = shapeSet.getShapes(); + for( const auto& shape : shapes ) { + makeField("vol_frac_" + shape.getMaterial(), 0.0); // Used in volume fraction computation + makeField("shape_vol_frac_" + shape.getName(), 0.0); // Used in applyReplacementRules + } } - const conduit::Node& topoNode = meshNode["topologies"][topoName]; - cellCount = conduit::blueprint::mesh::topology::length(topoNode); - } - // TODO Port to GPUs. Shaper should be data-parallel, but data may not be on devices yet. - using ExecSpace = typename axom::SEQ_EXEC; - using ReducePolicy = typename axom::execution_space::reduce_policy; + /* + Shallow-copy compMeshGrp into compMeshNode, + so that any change in one is reflected in the other. + */ + compMeshNode = std::make_shared(); + compMeshGrp->createNativeLayout(*compMeshNode); + SLIC_INFO(axom::fmt::format("{:-^80}", "Generated Blueprint mesh")); + cellCount = params.getBoxCellCount(); + } //--------------------------------------------------------------------------- // Initialize the shaping query object //--------------------------------------------------------------------------- AXOM_ANNOTATE_BEGIN("setup shaping problem"); std::shared_ptr shaper = nullptr; - if(params.useBlueprint()) + if(params.useBlueprintSidre()) { shaper = std::make_shared(shapeSet, compMeshGrp); } + if(params.useBlueprintConduit()) + { + shaper = std::make_shared(shapeSet, compMeshNode.get()); + } #if defined(AXOM_USE_MFEM) if(params.useMfem()) { @@ -1640,7 +1695,7 @@ int main(int argc, char** argv) slic::flushStreams(); // Generate a spatial index over the shape - shaper->prepareShapeQuery(shapeDim, shape); + shaper->prepareShapeQuery(shapeSet.getDimensions(), shape); slic::flushStreams(); // Query the mesh against this shape @@ -1671,12 +1726,11 @@ int main(int argc, char** argv) // Compute and print volumes of each material's volume fraction //--------------------------------------------------------------------------- using axom::utilities::string::startsWith; - if(params.useBlueprint()) + if(params.useBlueprintSidre() || params.useBlueprintConduit()) { #if defined(AXOM_DEBUG) - conduit::Node meshNode, info; - compMeshGrp->createNativeLayout(meshNode); - SLIC_ASSERT(conduit::blueprint::mesh::verify(meshNode, info)); + conduit::Node info; + SLIC_ASSERT(conduit::blueprint::mesh::verify(*compMeshNode, info)); #endif std::vector materialNames = shaper->getMaterialNames(); for(const auto& materialName : materialNames) @@ -1718,32 +1772,30 @@ int main(int argc, char** argv) int failCounts = 0; - axom::sidre::Group* volFracGroups = nullptr; - if(params.useBlueprint()) - { - volFracGroups = compMeshGrp->getGroup("matsets/material/volume_fractions"); - } -#if defined(AXOM_USE_MFEM) - if(params.useMfem()) + std::vector allVfNames(1, "free"); // All volume fraction names plus "free". + for(const auto& shape : shapeSet.getShapes()) { - volFracGroups = - shapingDC->GetBPGroup()->getGroup("matsets/material/volume_fractions"); + allVfNames.push_back(shape.getMaterial()); } -#endif + + // For error checking, work on host. + using ExecSpace = typename axom::SEQ_EXEC; + using ReducePolicy = typename axom::execution_space::reduce_policy; //--------------------------------------------------------------------------- // Correctness test: volume fractions should be in [0,1]. //--------------------------------------------------------------------------- RAJA::ReduceSum rangeViolationCount(0); - for(axom::sidre::Group& materialGroup : volFracGroups->groups()) + for(const auto& vfName : allVfNames) { - axom::sidre::View* values = materialGroup.getView("value"); - double* volFracData = values->getArray(); - axom::ArrayView volFracDataView(volFracData, cellCount); + std::string fieldName = "vol_frac_" + vfName; + axom::ArrayView vfView = getFieldAsArrayView(fieldName); + axom::Array vfHostArray(vfView, axom::execution_space::allocatorID()); + vfView = vfHostArray.view(); axom::for_all( cellCount, AXOM_LAMBDA(axom::IndexType i) { - bool bad = volFracDataView[i] < 0.0 || volFracDataView[i] > 1.0; + bool bad = vfView[i] < 0.0 || vfView[i] > 1.0; rangeViolationCount += bad; }); } @@ -1762,20 +1814,21 @@ int main(int argc, char** argv) axom::Array volSums(cellCount); volSums.fill(0.0); axom::ArrayView volSumsView = volSums.view(); - for(axom::sidre::Group& materialGroup : volFracGroups->groups()) + RAJA::ReduceSum nonUnitSums(0); + for(const auto& vfName : allVfNames) { - axom::sidre::View* values = materialGroup.getView("value"); - double* volFracData = values->getArray(); - axom::ArrayView volFracDataView(volFracData, cellCount); + std::string fieldName = "vol_frac_" + vfName; + axom::ArrayView vfView = getFieldAsArrayView(fieldName); + axom::Array vfHostArray(vfView, axom::execution_space::allocatorID()); + vfView = vfHostArray.view(); axom::for_all( cellCount, - AXOM_LAMBDA(axom::IndexType i) { volSumsView[i] += volFracDataView[i]; }); + AXOM_LAMBDA(axom::IndexType i) { volSumsView[i] += vfView[i]; }); } - RAJA::ReduceSum nonUnitSums(0); axom::for_all( cellCount, AXOM_LAMBDA(axom::IndexType i) { - bool bad = !axom::utilities::isNearlyEqual(volSums[i], 0.0); + bool bad = !axom::utilities::isNearlyEqual(volSumsView[i], 1.0); nonUnitSums += bad; }); @@ -1808,7 +1861,7 @@ int main(int argc, char** argv) const std::string& materialName = shape.getMaterial(); double shapeVol = -1; - if(params.useBlueprint()) + if(params.useBlueprintSidre() || params.useBlueprintConduit()) { shapeVol = sumMaterialVolumes(compMeshGrp, materialName); } @@ -1851,7 +1904,7 @@ int main(int argc, char** argv) if(!params.outputFile.empty()) { std::string fileName = params.outputFile + ".volfracs"; - if(params.useBlueprint()) + if(params.useBlueprintSidre()) { saveMesh(*compMeshGrp, fileName); SLIC_INFO(axom::fmt::format("{:-^80}", "Wrote output mesh " + fileName)); @@ -1864,6 +1917,7 @@ int main(int argc, char** argv) #endif } + shapingDC.reset(); shaper.reset(); //--------------------------------------------------------------------------- From 5218ddd0b14a12a714f420af97fb10b829b7e5d6 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Thu, 12 Dec 2024 15:43:28 -0800 Subject: [PATCH 071/100] remove obsolete code. --- src/axom/quest/IntersectionShaper.hpp | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 89cfeb4dc6..dfc57e8631 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -2216,31 +2216,6 @@ class IntersectionShaper : public Shaper fieldGrp->createView("volume_dependent") ->setString(std::string(volumeDependent ? "true" : "false")); valuesView->allocate(); - if(fieldName.rfind("vol_frac_", 0) == 0) - { - // TODO: I think this arrangement of matsets is wrong. - // It passes the conduit blueprint verification but maybe - // because conduit doesn't check matsets. - // Needs more verification. - // - // This is a material volume fraction field. - // Shallow-copy valuesView to (uni-buffer) matsets. - const std::string matlName = fieldName.substr(9); - axom::sidre::Group* volFracGrp = nullptr; - if(m_bpGrp->hasGroup("matsets/material/volume_fractions")) - { - volFracGrp = - m_bpGrp->getGroup("matsets/material/volume_fractions"); - } - else - { - volFracGrp = - m_bpGrp->createGroup("matsets/material/volume_fractions"); - m_bpGrp->createViewString("matsets/material/topology", m_bpTopo); - } - auto* valuesViewInMatsets = volFracGrp->copyView(valuesView); - valuesViewInMatsets->rename(matlName); - } } } From 0e1c249b611a38ce92eaa36c79078de587a64c62 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Fri, 13 Dec 2024 10:08:23 -0800 Subject: [PATCH 072/100] Don't test MFEM mesh if not configured with MFEM. --- src/axom/quest/examples/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/axom/quest/examples/CMakeLists.txt b/src/axom/quest/examples/CMakeLists.txt index 5aaa789b3c..d7edfce2f2 100644 --- a/src/axom/quest/examples/CMakeLists.txt +++ b/src/axom/quest/examples/CMakeLists.txt @@ -313,7 +313,8 @@ if((CONDUIT_FOUND OR # Test Conduit and MFEM mesh types with a subset of shapes (so it doesn't take too long) set(_testshapes "tetmesh" "sphere" "cyl") - set(_testMeshTypes "bpConduit" "mfem") + set(_testMeshTypes "bpConduit") + blt_list_append(TO _testMeshTypes ELEMENTS "mfem" IF MFEM_FOUND) foreach(_policy ${_policies}) foreach(_testMeshType ${_testMeshTypes}) foreach(_testshape ${_testshapes}) From 383ea19f90344565ce28c1076793805809105d4c Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Fri, 13 Dec 2024 12:09:37 -0800 Subject: [PATCH 073/100] Guess allocator directly from external data's pointer. Taking the allocator from Groups with external data isn't reliable, due to the way we import data. External data may correspond to multiple allocators. --- src/axom/quest/util/mesh_helpers.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/axom/quest/util/mesh_helpers.cpp b/src/axom/quest/util/mesh_helpers.cpp index da73ca8991..d621097c1f 100644 --- a/src/axom/quest/util/mesh_helpers.cpp +++ b/src/axom/quest/util/mesh_helpers.cpp @@ -215,7 +215,7 @@ void convert_blueprint_structured_explicit_to_unstructured( back to meshGrp memory space. */ coordsetGrp = meshGrp->getGroup("coordsets")->getGroup(coordsetName); - int coordsetAllocId = coordsetGrp->getDefaultAllocatorID(); + int coordsetAllocId = axom::getAllocatorIDFromPointer(coordsetGrp->getView("values/x")->getVoidPtr()); MemorySpace memSpace = detail::getAllocatorSpace(coordsetAllocId); sidre::Group* stashGrp = nullptr; sidre::Group* stashedValuesGrp = nullptr; From fe6e75819dede7d3b165fbceacfc97941f513723 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Fri, 13 Dec 2024 19:03:09 -0800 Subject: [PATCH 074/100] Fix bug that put some blueprint mesh data in the wrong memspace. --- src/axom/quest/IntersectionShaper.hpp | 31 +++++++++++++++------------ src/axom/quest/util/mesh_helpers.cpp | 12 +++++------ 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index dfc57e8631..cfb14c8771 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -30,7 +30,6 @@ #include "axom/quest/interface/internal/mpicomm_wrapper.hpp" #include "axom/quest/interface/internal/QuestHelpers.hpp" #include "axom/fmt.hpp" - #include "axom/core/WhereMacro.hpp" #ifdef AXOM_USE_MFEM #include "mfem.hpp" @@ -355,11 +354,7 @@ class IntersectionShaper : public Shaper const std::string& topo = "") : Shaper(shapeSet, bpNode, topo) , m_free_mat_name("free") - { - // We cannot always create Conduit Nodes, so catch missing nodes early if possible. - SLIC_ERROR_IF(!bpNode->has_child("fields"), - "Input blueprint mesh lacks the 'fields' Node."); - } + { } #endif //@{ @@ -2357,20 +2352,19 @@ class IntersectionShaper : public Shaper topoNode.fetch_existing("coordset").as_string(); // Assume unstructured and hexahedral - SLIC_ASSERT(topoNode["type"].as_string() == "unstructured"); - SLIC_ASSERT(topoNode["elements/shape"].as_string() == "hex"); + SLIC_ERROR_IF(topoNode["type"].as_string() != "unstructured", + "topology type must be 'unstructured'"); + SLIC_ERROR_IF(topoNode["elements/shape"].as_string() != "hex", + "element shape must be 'hex'"); const auto& connNode = topoNode["elements/connectivity"]; - SLIC_ASSERT_MSG(connNode.dtype().is_int32(), - "IntersectionShaper internal error: missing logic to " - "handle multiple types."); + SLIC_ERROR_IF(!connNode.dtype().is_int32(), + "IntersectionShaper internal error: missing logic to " + "handle multiple types."); const std::int32_t* connPtr = connNode.as_int32_ptr(); axom::ArrayView conn(connPtr, m_cellCount, NUM_VERTS_PER_HEX); - SLIC_ASSERT_MSG( - XS::usesAllocId(conn.getAllocatorID()), - std::string(XS::name()) + " cannot use the connectivity allocator id"); const conduit::Node& coordNode = m_bpNodeInt["coordsets"][coordsetName]; const conduit::Node& coordValues = coordNode.fetch_existing("values"); @@ -2389,6 +2383,15 @@ class IntersectionShaper : public Shaper {vertexCount}, stride)}; + SLIC_ERROR_IF( + !XS::usesAllocId(axom::getAllocatorIDFromPointer(conn.data())), + std::string(XS::name()) + axom::fmt::format(" execution space cannot use the connectivity allocator id {}", + axom::getAllocatorIDFromPointer(conn.data()))); + SLIC_ERROR_IF( + !XS::usesAllocId(axom::getAllocatorIDFromPointer(coordArrays[0].data())), + std::string(XS::name()) + axom::fmt::format(" execution space cannot use the coordset allocator id {}", + axom::getAllocatorIDFromPointer(coordArrays[0].data()))); + vertCoords = axom::Array(m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT, m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT, diff --git a/src/axom/quest/util/mesh_helpers.cpp b/src/axom/quest/util/mesh_helpers.cpp index d621097c1f..ae89998852 100644 --- a/src/axom/quest/util/mesh_helpers.cpp +++ b/src/axom/quest/util/mesh_helpers.cpp @@ -204,7 +204,7 @@ void convert_blueprint_structured_explicit_to_unstructured( const std::string& topoName) { const std::string& coordsetName = - meshGrp->getView(axom::fmt::format("topologies/{}/coordset", topoName)) + meshGrp->getView("topologies/" + topoName + "/coordset") ->getString(); sidre::Group* coordsetGrp = nullptr; @@ -215,8 +215,8 @@ void convert_blueprint_structured_explicit_to_unstructured( back to meshGrp memory space. */ coordsetGrp = meshGrp->getGroup("coordsets")->getGroup(coordsetName); - int coordsetAllocId = axom::getAllocatorIDFromPointer(coordsetGrp->getView("values/x")->getVoidPtr()); - MemorySpace memSpace = detail::getAllocatorSpace(coordsetAllocId); + int newDataAllocId = axom::getAllocatorIDFromPointer(coordsetGrp->getView("values/x")->getVoidPtr()); + MemorySpace memSpace = detail::getAllocatorSpace(newDataAllocId); sidre::Group* stashGrp = nullptr; sidre::Group* stashedValuesGrp = nullptr; if(memSpace == MemorySpace::Device) @@ -233,7 +233,7 @@ void convert_blueprint_structured_explicit_to_unstructured( conduit::Node curMesh; meshGrp->createNativeLayout(curMesh); const conduit::Node& curTopo = - curMesh.fetch_existing(axom::fmt::format("topologies/{}", topoName)); + curMesh.fetch_existing("topologies/" + topoName); SLIC_ASSERT( conduit::blueprint::mesh::topology::structured::verify(curTopo, info)); @@ -247,13 +247,13 @@ void convert_blueprint_structured_explicit_to_unstructured( // Copy unstructured back into meshGrp. meshGrp->getGroup("topologies")->destroyGroup(topoName); auto* topoGrp = meshGrp->getGroup("topologies")->createGroup(topoName); + topoGrp->setDefaultAllocator(newDataAllocId); topoGrp->importConduitTree(newTopo); topoGrp->getView("coordset")->setString(coordsetName); meshGrp->getGroup("coordsets")->destroyGroup(coordsetName); coordsetGrp = meshGrp->getGroup("coordsets")->createGroup(coordsetName); - SLIC_ASSERT(coordsetGrp->getDefaultAllocatorID() == - meshGrp->getDefaultAllocatorID()); + coordsetGrp->setDefaultAllocator(newDataAllocId); coordsetGrp->importConduitTree(newCoords); const bool addExtraDataForMint = true; From 9ca000a41a8944d4a93a583ab4fc93329935b6a5 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Fri, 13 Dec 2024 19:21:25 -0800 Subject: [PATCH 075/100] Remove vestige of 2D support that was copied from another example. --- .../quest/examples/quest_shape_in_memory.cpp | 103 ++++++++---------- 1 file changed, 45 insertions(+), 58 deletions(-) diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index ca3a64778a..1710ed04da 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -1447,58 +1447,50 @@ int main(int argc, char** argv) //--------------------------------------------------------------------------- axom::klee::ShapeSet shapeSet; SLIC_ERROR_IF(params.getBoxDim() != 3, "This example is only in 3D."); - switch(params.getBoxDim()) + if(params.testShape == "tetmesh") { - case 2: - shapeSet = create2DShapeSet(ds); - break; - case 3: - if(params.testShape == "tetmesh") - { - shapeSet = createShapeSet(createShape_TetMesh(ds)); - } - else if(params.testShape == "tet") - { - shapeSet = createShapeSet(createShape_Tet()); - } - else if(params.testShape == "hex") - { - shapeSet = createShapeSet(createShape_Hex()); - } - else if(params.testShape == "sphere") - { - shapeSet = createShapeSet(createShape_Sphere()); - } - else if(params.testShape == "cyl") - { - shapeSet = createShapeSet(createShape_Cylinder()); - } - else if(params.testShape == "cone") - { - shapeSet = createShapeSet(createShape_Cone()); - } - else if(params.testShape == "vor") - { - shapeSet = createShapeSet(createShape_Vor()); - } - else if(params.testShape == "plane") - { - shapeSet = createShapeSet(createShape_Plane()); - } - else if(params.testShape == "all") - { - std::vector shapesVec; - shapesVec.push_back(createShape_TetMesh(ds)); - shapesVec.push_back(createShape_Tet()); - shapesVec.push_back(createShape_Hex()); - shapesVec.push_back(createShape_Sphere()); - shapesVec.push_back(createShape_Vor()); - shapesVec.push_back(createShape_Cylinder()); - shapesVec.push_back(createShape_Cone()); - shapeSet.setShapes(shapesVec); - shapeSet.setDimensions(axom::klee::Dimensions::Three); - } - break; + shapeSet = createShapeSet(createShape_TetMesh(ds)); + } + else if(params.testShape == "tet") + { + shapeSet = createShapeSet(createShape_Tet()); + } + else if(params.testShape == "hex") + { + shapeSet = createShapeSet(createShape_Hex()); + } + else if(params.testShape == "sphere") + { + shapeSet = createShapeSet(createShape_Sphere()); + } + else if(params.testShape == "cyl") + { + shapeSet = createShapeSet(createShape_Cylinder()); + } + else if(params.testShape == "cone") + { + shapeSet = createShapeSet(createShape_Cone()); + } + else if(params.testShape == "vor") + { + shapeSet = createShapeSet(createShape_Vor()); + } + else if(params.testShape == "plane") + { + shapeSet = createShapeSet(createShape_Plane()); + } + else if(params.testShape == "all") + { + std::vector shapesVec; + shapesVec.push_back(createShape_TetMesh(ds)); + shapesVec.push_back(createShape_Tet()); + shapesVec.push_back(createShape_Hex()); + shapesVec.push_back(createShape_Sphere()); + shapesVec.push_back(createShape_Vor()); + shapesVec.push_back(createShape_Cylinder()); + shapesVec.push_back(createShape_Cone()); + shapeSet.setShapes(shapesVec); + shapeSet.setDimensions(axom::klee::Dimensions::Three); } // Save the discrete shapes for viz and testing. @@ -1527,13 +1519,6 @@ int main(int argc, char** argv) } } - // Apply error checking -#ifndef AXOM_USE_C2C - SLIC_ERROR_IF(shapeDim == klee::Dimensions::Two, - "Shaping with contour files requires an Axom configured with " - "the C2C library"); -#endif - #if defined(AXOM_USE_MFEM) if(params.useMfem()) { @@ -1917,7 +1902,9 @@ int main(int argc, char** argv) #endif } +#if defined(AXOM_USE_MFEM) shapingDC.reset(); +#endif shaper.reset(); //--------------------------------------------------------------------------- From e3b553c4d76ec51dbb01e56076cffbf92ac74f63 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Fri, 3 Jan 2025 13:10:25 -0800 Subject: [PATCH 076/100] Convert structured to unstructured Blueprint mesh in Axom. Using Conduit is too slow, due to the need to copy device data to host and back. This change requires a change in the shaper constructors, because the conversion code needs to be told what execution space to run in. --- src/axom/quest/IntersectionShaper.hpp | 58 +++--- src/axom/quest/SamplingShaper.hpp | 2 +- src/axom/quest/Shaper.cpp | 24 ++- src/axom/quest/Shaper.hpp | 14 +- .../quest/examples/quest_shape_in_memory.cpp | 68 ++++--- src/axom/quest/examples/shaping_driver.cpp | 4 +- .../quest/tests/quest_intersection_shaper.cpp | 6 +- src/axom/quest/util/mesh_helpers.cpp | 181 ++++++++++++------ src/axom/quest/util/mesh_helpers.hpp | 11 ++ 9 files changed, 243 insertions(+), 125 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index cfb14c8771..0be4205928 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -325,9 +325,10 @@ class IntersectionShaper : public Shaper /*! @brief Construct Shaper to operate on an MFEM mesh. */ - IntersectionShaper(const klee::ShapeSet& shapeSet, + IntersectionShaper(RuntimePolicy runtimePolicy, + const klee::ShapeSet& shapeSet, sidre::MFEMSidreDataCollection* dc) - : Shaper(shapeSet, dc) + : Shaper(runtimePolicy, shapeSet, dc) { m_free_mat_name = "free"; } @@ -338,10 +339,11 @@ class IntersectionShaper : public Shaper @brief Construct Shaper to operate on a blueprint-formatted mesh stored in a sidre Group. */ - IntersectionShaper(const klee::ShapeSet& shapeSet, + IntersectionShaper(RuntimePolicy runtimePolicy, + const klee::ShapeSet& shapeSet, sidre::Group* bpGrp, const std::string& topo = "") - : Shaper(shapeSet, bpGrp, topo) + : Shaper(runtimePolicy, shapeSet, bpGrp, topo) , m_free_mat_name("free") { } @@ -349,10 +351,11 @@ class IntersectionShaper : public Shaper @brief Construct Shaper to operate on a blueprint-formatted mesh stored in a Conduit Node. */ - IntersectionShaper(const klee::ShapeSet& shapeSet, + IntersectionShaper(RuntimePolicy runtimePolicy, + const klee::ShapeSet& shapeSet, conduit::Node* bpNode, const std::string& topo = "") - : Shaper(shapeSet, bpNode, topo) + : Shaper(runtimePolicy, shapeSet, bpNode, topo) , m_free_mat_name("free") { } #endif @@ -362,8 +365,6 @@ class IntersectionShaper : public Shaper void setLevel(int level) { m_level = level; } - void setExecPolicy(RuntimePolicy policy) { m_execPolicy = policy; } - /*! * \brief Set the name of the material used to account for free volume fractions. * \param name The new name of the material. This name cannot contain @@ -914,7 +915,7 @@ class IntersectionShaper : public Shaper axom::Array(newTotalCandidates_device, host_allocator); axom::for_all( - newTotalCandidates_calc_host[0], // Number of candidates found. + newTotalCandidates_calc_host[0], // Number of candidates found. AXOM_LAMBDA(axom::IndexType i) { const int index = hex_indices_device_view[i]; const int shapeIndex = shape_candidates_device_view[i]; @@ -1493,47 +1494,45 @@ class IntersectionShaper : public Shaper { #if defined(AXOM_USE_OPENMP) case RuntimePolicy::omp: - overlapVol = sumArray(m_overlap_volumes.data(), - m_overlap_volumes.size()); + overlapVol = + sumArray(m_overlap_volumes.data(), m_overlap_volumes.size()); break; #endif // AXOM_USE_OPENMP #if defined(AXOM_USE_CUDA) && defined(AXOM_USE_UMPIRE) case RuntimePolicy::cuda: - overlapVol = sumArray(m_overlap_volumes.data(), - m_overlap_volumes.size()); + overlapVol = + sumArray(m_overlap_volumes.data(), m_overlap_volumes.size()); break; #endif // AXOM_USE_CUDA #if defined(AXOM_USE_HIP) && defined(AXOM_USE_UMPIRE) case RuntimePolicy::hip: - overlapVol = sumArray(m_overlap_volumes.data(), - m_overlap_volumes.size()); + overlapVol = + sumArray(m_overlap_volumes.data(), m_overlap_volumes.size()); break; #endif // AXOM_USE_HIP case RuntimePolicy::seq: default: - overlapVol = sumArray(m_overlap_volumes.data(), - m_overlap_volumes.size()); + overlapVol = + sumArray(m_overlap_volumes.data(), m_overlap_volumes.size()); break; } - if (global) + if(global) { overlapVol = this->allReduceSum(overlapVol); } return overlapVol; } - template + template Summable sumArray(const Summable* a, axom::IndexType count) const { using LoopPolicy = typename axom::execution_space::loop_policy; using ReducePolicy = typename axom::execution_space::reduce_policy; - RAJA::ReduceSum vsum{0}; + RAJA::ReduceSum vsum {0}; RAJA::forall( RAJA::RangeSegment(0, count), - AXOM_LAMBDA(RAJA::Index_type i) { - vsum += a[i]; - }); + AXOM_LAMBDA(RAJA::Index_type i) { vsum += a[i]; }); Summable sum = static_cast(vsum.get()); return sum; } @@ -2385,12 +2384,16 @@ class IntersectionShaper : public Shaper SLIC_ERROR_IF( !XS::usesAllocId(axom::getAllocatorIDFromPointer(conn.data())), - std::string(XS::name()) + axom::fmt::format(" execution space cannot use the connectivity allocator id {}", - axom::getAllocatorIDFromPointer(conn.data()))); + std::string(XS::name()) + + axom::fmt::format( + " execution space cannot use the connectivity allocator id {}", + axom::getAllocatorIDFromPointer(conn.data()))); SLIC_ERROR_IF( !XS::usesAllocId(axom::getAllocatorIDFromPointer(coordArrays[0].data())), - std::string(XS::name()) + axom::fmt::format(" execution space cannot use the coordset allocator id {}", - axom::getAllocatorIDFromPointer(coordArrays[0].data()))); + std::string(XS::name()) + + axom::fmt::format( + " execution space cannot use the coordset allocator id {}", + axom::getAllocatorIDFromPointer(coordArrays[0].data()))); vertCoords = axom::Array(m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT, @@ -2506,7 +2509,6 @@ class IntersectionShaper : public Shaper } private: - RuntimePolicy m_execPolicy {RuntimePolicy::seq}; int m_level {DEFAULT_CIRCLE_REFINEMENT_LEVEL}; double m_revolvedVolume {DEFAULT_REVOLVED_VOLUME}; std::string m_free_mat_name; diff --git a/src/axom/quest/SamplingShaper.hpp b/src/axom/quest/SamplingShaper.hpp index 360f1f0d79..bef2680e72 100644 --- a/src/axom/quest/SamplingShaper.hpp +++ b/src/axom/quest/SamplingShaper.hpp @@ -318,7 +318,7 @@ class SamplingShaper : public Shaper public: SamplingShaper(const klee::ShapeSet& shapeSet, sidre::MFEMSidreDataCollection* dc) - : Shaper(shapeSet, dc) + : Shaper(Shaper::RuntimePolicy::seq, shapeSet, dc) { } ~SamplingShaper() diff --git a/src/axom/quest/Shaper.cpp b/src/axom/quest/Shaper.cpp index e2f8505936..d5ac1068fd 100644 --- a/src/axom/quest/Shaper.cpp +++ b/src/axom/quest/Shaper.cpp @@ -28,8 +28,11 @@ constexpr double Shaper::MAXIMUM_PERCENT_ERROR; constexpr double Shaper::DEFAULT_VERTEX_WELD_THRESHOLD; #if defined(AXOM_USE_MFEM) -Shaper::Shaper(const klee::ShapeSet& shapeSet, sidre::MFEMSidreDataCollection* dc) - : m_shapeSet(shapeSet) +Shaper::Shaper(RuntimePolicy execPolicy, + const klee::ShapeSet& shapeSet, + sidre::MFEMSidreDataCollection* dc) + : m_execPolicy(execPolicy) + , m_shapeSet(shapeSet) , m_dc(dc) #if defined(AXOM_USE_CONDUIT) , m_bpGrp(nullptr) @@ -47,10 +50,12 @@ Shaper::Shaper(const klee::ShapeSet& shapeSet, sidre::MFEMSidreDataCollection* d } #endif -Shaper::Shaper(const klee::ShapeSet& shapeSet, +Shaper::Shaper(RuntimePolicy execPolicy, + const klee::ShapeSet& shapeSet, sidre::Group* bpGrp, const std::string& topo) - : m_shapeSet(shapeSet) + : m_execPolicy(execPolicy) + , m_shapeSet(shapeSet) #if defined(AXOM_USE_CONDUIT) , m_bpGrp(bpGrp) , m_bpTopo(topo.empty() ? bpGrp->getGroup("topologies")->getGroupName(0) : topo) @@ -76,10 +81,12 @@ Shaper::Shaper(const klee::ShapeSet& shapeSet, setFilePath(shapeSet.getPath()); } -Shaper::Shaper(const klee::ShapeSet& shapeSet, +Shaper::Shaper(RuntimePolicy execPolicy, + const klee::ShapeSet& shapeSet, conduit::Node* bpNode, const std::string& topo) - : m_shapeSet(shapeSet) + : m_execPolicy(execPolicy) + , m_shapeSet(shapeSet) #if defined(AXOM_USE_CONDUIT) , m_bpGrp(nullptr) , m_bpTopo(topo.empty() ? bpNode->fetch_existing("topologies").child(0).name() @@ -89,6 +96,7 @@ Shaper::Shaper(const klee::ShapeSet& shapeSet, #endif , m_comm(MPI_COMM_WORLD) { + AXOM_ANNOTATE_SCOPE("Shaper::Shaper_Node"); m_bpGrp = m_ds.getRoot()->createGroup("internalGrp"); m_bpGrp->importConduitTreeExternal(*bpNode); @@ -100,9 +108,11 @@ Shaper::Shaper(const klee::ShapeSet& shapeSet, .as_string(); if(topoType == "structured") { + AXOM_ANNOTATE_SCOPE("Shaper::convertStructured"); axom::quest::util::convert_blueprint_structured_explicit_to_unstructured( m_bpGrp, - m_bpTopo); + m_bpTopo, + m_execPolicy); } m_bpGrp->createNativeLayout(m_bpNodeInt); diff --git a/src/axom/quest/Shaper.hpp b/src/axom/quest/Shaper.hpp index 6b3b66ff91..7afc200a4a 100644 --- a/src/axom/quest/Shaper.hpp +++ b/src/axom/quest/Shaper.hpp @@ -25,6 +25,7 @@ #include "axom/klee.hpp" #include "axom/mint.hpp" #include "axom/quest/DiscreteShape.hpp" +#include "axom/core/execution/runtime_policy.hpp" #if defined(AXOM_USE_MFEM) #include "mfem.hpp" @@ -48,18 +49,23 @@ namespace quest class Shaper { public: + using RuntimePolicy = axom::runtime_policy::Policy; + #if defined(AXOM_USE_MFEM) /*! @brief Construct Shaper to operate on an MFEM mesh. */ - Shaper(const klee::ShapeSet& shapeSet, sidre::MFEMSidreDataCollection* dc); + Shaper(RuntimePolicy execPolicy, + const klee::ShapeSet& shapeSet, + sidre::MFEMSidreDataCollection* dc); #endif /*! @brief Construct Shaper to operate on a blueprint-formatted mesh stored in a sidre Group. */ - Shaper(const klee::ShapeSet& shapeSet, + Shaper(RuntimePolicy execPolicy, + const klee::ShapeSet& shapeSet, sidre::Group* bpMesh, const std::string& topo = ""); @@ -72,7 +78,8 @@ class Shaper all arrays pre-allocated in a space accessible by the runtime policy. Any needed-but-missing space would lead to an exception. */ - Shaper(const klee::ShapeSet& shapeSet, + Shaper(RuntimePolicy execPolicy, + const klee::ShapeSet& shapeSet, conduit::Node* bpNode, const std::string& topo = ""); @@ -204,6 +211,7 @@ class Shaper int getRank() const; protected: + RuntimePolicy m_execPolicy; sidre::DataStore m_dataStore; const klee::ShapeSet& m_shapeSet; diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index 1710ed04da..f853a884ef 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -87,7 +87,7 @@ struct Input } int getBoxCellCount() const { - return boxResolution[0]*boxResolution[1]*boxResolution[2]; + return boxResolution[0] * boxResolution[1] * boxResolution[2]; } // The shape to run. @@ -816,7 +816,7 @@ axom::klee::Shape createShape_Hex() SLIC_ASSERT(params.scaleFactors.empty() || params.scaleFactors.size() == 3); - const double md = params.length < 0 ? 0.6 : params.length/2; + const double md = params.length < 0 ? 0.6 : params.length / 2; const double lg = 1.2 * md; const double sm = 0.8 * md; const Point3D p {-lg, -md, -sm}; @@ -1326,35 +1326,38 @@ axom::ArrayView getFieldAsArrayView(const std::string& fieldName) std::string valuesPath = "fields/" + fieldName + "/values"; axom::sidre::View* fieldValues = compMeshGrp->getView(valuesPath); double* fieldData = fieldValues->getArray(); - fieldDataView = axom::ArrayView(fieldData, - fieldValues->getNumElements()); + fieldDataView = + axom::ArrayView(fieldData, fieldValues->getNumElements()); } if(params.useBlueprintConduit()) { std::string valuesPath = "fields/" + fieldName + "/values"; conduit::Node& fieldValues = compMeshNode->fetch_existing(valuesPath); double* fieldData = fieldValues.as_double_ptr(); - fieldDataView = axom::ArrayView(fieldData, - fieldValues.dtype().number_of_elements()); + fieldDataView = + axom::ArrayView(fieldData, + fieldValues.dtype().number_of_elements()); } -#if defined (AXOM_USE_MFEM) +#if defined(AXOM_USE_MFEM) if(params.useMfem()) { // auto* mfemMesh = shapingDC->GetMesh(); mfem::GridFunction* gridFunc = shapingDC->GetField(fieldName); - fieldDataView = axom::ArrayView(gridFunc->GetData(), gridFunc->Size()); + fieldDataView = + axom::ArrayView(gridFunc->GetData(), gridFunc->Size()); } #endif return fieldDataView; } - //!@brief Fill a sidre array View with a value. // No error checking. -template void fillSidreViewData(axom::sidre::View* view, const T& value) +template +void fillSidreViewData(axom::sidre::View* view, const T& value) { double* valuesPtr = view->getData(); - switch (params.policy) { + switch(params.policy) + { #if defined(AXOM_USE_CUDA) case RuntimePolicy::cuda: axom::for_all>( @@ -1571,7 +1574,7 @@ int main(int argc, char** argv) createBoxMesh(compMeshGrp); - if (params.useBlueprintConduit()) + if(params.useBlueprintConduit()) { // Intersection requires conduit mesh to have array data pre-allocated. auto makeField = [&](const std::string& fieldName, double initValue) { @@ -1583,14 +1586,18 @@ int main(int argc, char** argv) axom::sidre::View* valuesView = fieldGrp->createViewWithShapeAndAllocate( "values", axom::sidre::detail::SidreTT::id, - 2, shape); + 2, + shape); fillSidreViewData(valuesView, initValue); }; makeField("vol_frac_free", 1.0); - const auto &shapes = shapeSet.getShapes(); - for( const auto& shape : shapes ) { - makeField("vol_frac_" + shape.getMaterial(), 0.0); // Used in volume fraction computation - makeField("shape_vol_frac_" + shape.getName(), 0.0); // Used in applyReplacementRules + const auto& shapes = shapeSet.getShapes(); + for(const auto& shape : shapes) + { + makeField("vol_frac_" + shape.getMaterial(), + 0.0); // Used in volume fraction computation + makeField("shape_vol_frac_" + shape.getName(), + 0.0); // Used in applyReplacementRules } } @@ -1611,21 +1618,26 @@ int main(int argc, char** argv) std::shared_ptr shaper = nullptr; if(params.useBlueprintSidre()) { - shaper = std::make_shared(shapeSet, compMeshGrp); + shaper = std::make_shared(params.policy, + shapeSet, + compMeshGrp); } if(params.useBlueprintConduit()) { - shaper = std::make_shared(shapeSet, compMeshNode.get()); + shaper = std::make_shared(params.policy, + shapeSet, + compMeshNode.get()); } #if defined(AXOM_USE_MFEM) if(params.useMfem()) { - shaper = - std::make_shared(shapeSet, shapingDC.get()); + shaper = std::make_shared(params.policy, + shapeSet, + shapingDC.get()); } #endif SLIC_ASSERT(shaper != nullptr); - shaper->setExecPolicy(params.policy); + // shaper->setExecPolicy(params.policy); // Set generic parameters for the base Shaper instance shaper->setVertexWeldThreshold(params.weldThresh); @@ -1757,7 +1769,9 @@ int main(int argc, char** argv) int failCounts = 0; - std::vector allVfNames(1, "free"); // All volume fraction names plus "free". + std::vector allVfNames( + 1, + "free"); // All volume fraction names plus "free". for(const auto& shape : shapeSet.getShapes()) { allVfNames.push_back(shape.getMaterial()); @@ -1775,7 +1789,9 @@ int main(int argc, char** argv) { std::string fieldName = "vol_frac_" + vfName; axom::ArrayView vfView = getFieldAsArrayView(fieldName); - axom::Array vfHostArray(vfView, axom::execution_space::allocatorID()); + axom::Array vfHostArray( + vfView, + axom::execution_space::allocatorID()); vfView = vfHostArray.view(); axom::for_all( cellCount, @@ -1804,7 +1820,9 @@ int main(int argc, char** argv) { std::string fieldName = "vol_frac_" + vfName; axom::ArrayView vfView = getFieldAsArrayView(fieldName); - axom::Array vfHostArray(vfView, axom::execution_space::allocatorID()); + axom::Array vfHostArray( + vfView, + axom::execution_space::allocatorID()); vfView = vfHostArray.view(); axom::for_all( cellCount, diff --git a/src/axom/quest/examples/shaping_driver.cpp b/src/axom/quest/examples/shaping_driver.cpp index f20ef0be46..766b0c4bc9 100644 --- a/src/axom/quest/examples/shaping_driver.cpp +++ b/src/axom/quest/examples/shaping_driver.cpp @@ -567,7 +567,8 @@ int main(int argc, char** argv) break; case ShapingMethod::Intersection: #if defined(AXOM_USE_RAJA) && defined(AXOM_USE_UMPIRE) - shaper = new quest::IntersectionShaper(params.shapeSet, &shapingDC); + shaper = + new quest::IntersectionShaper(params.policy, params.shapeSet, &shapingDC); #else SLIC_ERROR( "IntersectionShaper requires Axom to be configured with Umpire."); @@ -614,7 +615,6 @@ int main(int argc, char** argv) if(auto* intersectionShaper = dynamic_cast(shaper)) { intersectionShaper->setLevel(params.refinementLevel); - intersectionShaper->setExecPolicy(params.policy); if(!params.backgroundMaterial.empty()) { diff --git a/src/axom/quest/tests/quest_intersection_shaper.cpp b/src/axom/quest/tests/quest_intersection_shaper.cpp index 03fe202283..f6cf35b63d 100644 --- a/src/axom/quest/tests/quest_intersection_shaper.cpp +++ b/src/axom/quest/tests/quest_intersection_shaper.cpp @@ -324,9 +324,8 @@ void replacementRuleTest(const std::string &shapeFile, // the C2C reader. dc.SetComm(MPI_COMM_WORLD); #endif - quest::IntersectionShaper shaper(shapeSet, &dc); + quest::IntersectionShaper shaper(policy, shapeSet, &dc); shaper.setLevel(refinementLevel); - shaper.setExecPolicy(policy); // Borrowed from shaping_driver. const klee::Dimensions shapeDim = shapeSet.getDimensions(); @@ -460,11 +459,10 @@ void IntersectionWithErrorTolerances(const std::string &filebase, // the C2C reader. dc.SetComm(MPI_COMM_WORLD); #endif - quest::IntersectionShaper shaper(shapeSet, &dc); + quest::IntersectionShaper shaper(policy, shapeSet, &dc); shaper.setLevel(refinementLevel); shaper.setPercentError(targetPercentError); shaper.setRefinementType(quest::DiscreteShape::RefinementDynamic); - shaper.setExecPolicy(policy); // Borrowed from shaping_driver (there should just be one shape) const klee::Dimensions shapeDim = shapeSet.getDimensions(); diff --git a/src/axom/quest/util/mesh_helpers.cpp b/src/axom/quest/util/mesh_helpers.cpp index ae89998852..3ce0d07d43 100644 --- a/src/axom/quest/util/mesh_helpers.cpp +++ b/src/axom/quest/util/mesh_helpers.cpp @@ -195,70 +195,134 @@ axom::sidre::Group* make_unstructured_blueprint_box_mesh( topologyName, coordsetName, runtimePolicy); - convert_blueprint_structured_explicit_to_unstructured(meshGrp, topologyName); + convert_blueprint_structured_explicit_to_unstructured(meshGrp, + topologyName, + runtimePolicy); return meshGrp; } void convert_blueprint_structured_explicit_to_unstructured( axom::sidre::Group* meshGrp, - const std::string& topoName) + const std::string& topoName, + axom::runtime_policy::Policy runtimePolicy) { - const std::string& coordsetName = - meshGrp->getView("topologies/" + topoName + "/coordset") - ->getString(); - - sidre::Group* coordsetGrp = nullptr; - #if defined(AXOM_USE_UMPIRE) - /* If using device memory, temporarily copy coords to host - so we can use conduit's blueprint::mesh utilities. - When we re-import the newCoords, we will get the data - back to meshGrp memory space. - */ - coordsetGrp = meshGrp->getGroup("coordsets")->getGroup(coordsetName); - int newDataAllocId = axom::getAllocatorIDFromPointer(coordsetGrp->getView("values/x")->getVoidPtr()); - MemorySpace memSpace = detail::getAllocatorSpace(newDataAllocId); - sidre::Group* stashGrp = nullptr; - sidre::Group* stashedValuesGrp = nullptr; - if(memSpace == MemorySpace::Device) + if(runtimePolicy == axom::runtime_policy::Policy::seq) + { + convert_blueprint_structured_explicit_to_unstructured_impl( + meshGrp, + topoName); + } + #if defined(AXOM_RUNTIME_POLICY_USE_OPENMP) + if(runtimePolicy == axom::runtime_policy::Policy::omp) + { + convert_blueprint_structured_explicit_to_unstructured_impl( + meshGrp, + topoName); + } + #endif + #if defined(AXOM_RUNTIME_POLICY_USE_CUDA) + if(runtimePolicy == axom::runtime_policy::Policy::cuda) + { + convert_blueprint_structured_explicit_to_unstructured_impl>( + meshGrp, + topoName); + } + #endif + #if defined(AXOM_RUNTIME_POLICY_USE_HIP) + if(runtimePolicy == axom::runtime_policy::Policy::hip) { - int hostAllocId = execution_space::allocatorID(); - stashGrp = meshGrp->createGroup("tempStash"); - stashedValuesGrp = stashGrp->moveGroup(coordsetGrp->getGroup("values")); - coordsetGrp->deepCopyGroup(stashedValuesGrp, hostAllocId); + convert_blueprint_structured_explicit_to_unstructured_impl>( + meshGrp, + topoName); } #endif +} - // Convert mesh to conduit::Node to use conduit's blueprint support. - conduit::Node info; - conduit::Node curMesh; - meshGrp->createNativeLayout(curMesh); - const conduit::Node& curTopo = - curMesh.fetch_existing("topologies/" + topoName); - SLIC_ASSERT( - conduit::blueprint::mesh::topology::structured::verify(curTopo, info)); - - // Use conduit to convert to unstructured. - conduit::Node newTopo; - conduit::Node newCoords; - conduit::blueprint::mesh::topology::structured::to_unstructured(curTopo, - newTopo, - newCoords); - - // Copy unstructured back into meshGrp. - meshGrp->getGroup("topologies")->destroyGroup(topoName); - auto* topoGrp = meshGrp->getGroup("topologies")->createGroup(topoName); - topoGrp->setDefaultAllocator(newDataAllocId); - topoGrp->importConduitTree(newTopo); - topoGrp->getView("coordset")->setString(coordsetName); - - meshGrp->getGroup("coordsets")->destroyGroup(coordsetName); - coordsetGrp = meshGrp->getGroup("coordsets")->createGroup(coordsetName); - coordsetGrp->setDefaultAllocator(newDataAllocId); - coordsetGrp->importConduitTree(newCoords); +template +void convert_blueprint_structured_explicit_to_unstructured_impl( + axom::sidre::Group* meshGrp, + const std::string& topoName) +{ + AXOM_ANNOTATE_SCOPE("convert_to_unstructured"); + constexpr int DIM = 3; + + const std::string& coordsetName = + meshGrp->getView("topologies/" + topoName + "/coordset")->getString(); + + sidre::Group* coordsetGrp = + meshGrp->getGroup("coordsets")->getGroup(coordsetName); + int newDataAllocId = axom::getAllocatorIDFromPointer( + coordsetGrp->getView("values/x")->getVoidPtr()); + + axom::sidre::Group* topoGrp = + meshGrp->getGroup("topologies")->getGroup(topoName); + axom::sidre::View* topoTypeView = topoGrp->getView("type"); + SLIC_ASSERT(std::string(topoTypeView->getString()) == "structured"); + topoTypeView->setString("unstructured"); + topoGrp->createView("elements/shape")->setString("hex"); + + axom::sidre::Group* topoElemGrp = topoGrp->getGroup("elements"); + axom::sidre::Group* topoDimsGrp = topoElemGrp->getGroup("dims"); + + // Assuming no ghost, but we should eventually support ghosts. + SLIC_ASSERT(!topoGrp->hasGroup("elements/offsets")); + SLIC_ASSERT(!topoGrp->hasGroup("elements/strides")); + + const axom::StackArray cShape { + topoDimsGrp->getView("i")->getData(), + topoDimsGrp->getView("j")->getData(), + topoDimsGrp->getView("k")->getData()}; + const axom::StackArray vShape {cShape[0] + 1, + cShape[1] + 1, + cShape[2] + 1}; + + const axom::IndexType cCount = cShape[0] * cShape[1] * cShape[2]; + + const axom::StackArray connShape {cCount, 8}; + axom::sidre::View* connView = topoGrp->createViewWithShapeAndAllocate( + "elements/connectivity", + axom::sidre::detail::SidreTT::id, + 2, + connShape.begin(), + newDataAllocId); + axom::ArrayView connArrayView( + static_cast(connView->getVoidPtr()), + connShape); + + axom::MDMapping cIdMapping(cShape, axom::ArrayStrideOrder::COLUMN); + axom::MDMapping vIdMapping(vShape, axom::ArrayStrideOrder::COLUMN); + + const axom::StackArray, 8> cornerOffsets { + {{0, 0, 0}, + {1, 0, 0}, + {1, 1, 0}, + {0, 1, 0}, + {0, 0, 1}, + {1, 0, 1}, + {1, 1, 1}, + {0, 1, 1}}}; + axom::for_all( + cCount, + AXOM_LAMBDA(axom::IndexType iCell) { + axom::StackArray cIdx = + cIdMapping.toMultiIndex(iCell); + for(int n = 0; n < 8; ++n) + { + const auto& cornerOffset = cornerOffsets[n]; + axom::StackArray vIdx; + for(int d = 0; d < DIM; ++d) + { + vIdx[d] = cIdx[d] + cornerOffset[d]; + } + axom::IndexType iVert = vIdMapping.toFlatIndex(vIdx); + connArrayView(iCell, n) = iVert; + } + }); const bool addExtraDataForMint = true; if(addExtraDataForMint) { + AXOM_ANNOTATE_BEGIN("add_extra"); /* Constructing a mint mesh from meshGrp fails unless we add some extra data. Blueprint doesn't require this extra data. (The mesh @@ -285,22 +349,29 @@ void convert_blueprint_structured_explicit_to_unstructured( auto* connView = elementsGrp->getView("connectivity"); curDim = connView->getShape(2, curShape); constexpr axom::IndexType NUM_VERTS_PER_HEX = 8; - SLIC_ASSERT(curDim == 1); - SLIC_ASSERT(curShape[0] % NUM_VERTS_PER_HEX == 0); - axom::IndexType connShape[2] = {curShape[0] / NUM_VERTS_PER_HEX, - NUM_VERTS_PER_HEX}; - connView->reshapeArray(2, connShape); + SLIC_ASSERT(curDim == 1 || curDim == 2); + if(curDim == 1) + { + SLIC_ASSERT(curShape[0] % NUM_VERTS_PER_HEX == 0); + axom::IndexType connShape[2] = {curShape[0] / NUM_VERTS_PER_HEX, + NUM_VERTS_PER_HEX}; + connView->reshapeArray(2, connShape); + } // mint::Mesh requires connectivity strides, even though Blueprint doesn't. elementsGrp->createViewScalar("stride", NUM_VERTS_PER_HEX); // mint::Mesh requires field group, even though Blueprint doesn't. meshGrp->createGroup("fields"); + AXOM_ANNOTATE_END("add_extra"); } #if defined(AXOM_DEBUG) + AXOM_ANNOTATE_BEGIN("validate_post"); + conduit::Node info; bool isValid = verifyBlueprintMesh(meshGrp, info); SLIC_ASSERT_MSG(isValid, "Internal error: Generated mesh is invalid."); + AXOM_ANNOTATE_END("validate_post"); #endif return; diff --git a/src/axom/quest/util/mesh_helpers.hpp b/src/axom/quest/util/mesh_helpers.hpp index 45889784c9..92aa5157bd 100644 --- a/src/axom/quest/util/mesh_helpers.hpp +++ b/src/axom/quest/util/mesh_helpers.hpp @@ -93,6 +93,11 @@ axom::sidre::Group* make_unstructured_blueprint_box_mesh( /*! \brief Convert a structured explicit blueprint mesh to unstructured. + \param meshGrp Put the mesh in this Group + \param topologyName Name of the blueprint topoloyy to use. + \param runtimePolicy Runtime policy, see axom::runtime_policy. + Memory in \c meshGrp must be compatible with the + specified policy. All input mesh data are expected to have the allocator id of meshGrp->getDefaultAllocatorID(). On output, they will also have @@ -100,6 +105,12 @@ axom::sidre::Group* make_unstructured_blueprint_box_mesh( transfers memory to and from another space. */ void convert_blueprint_structured_explicit_to_unstructured( + axom::sidre::Group* meshGrp, + const std::string& topoName, + axom::runtime_policy::Policy runtimePolicy); + +template +void convert_blueprint_structured_explicit_to_unstructured_impl( axom::sidre::Group* meshGrp, const std::string& topoName); From 0c77c090f8c0615886914f2972824616fa4d6b9f Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Sat, 4 Jan 2025 03:53:41 -0800 Subject: [PATCH 077/100] Fix non-C2C build. --- src/axom/quest/interface/inout.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/axom/quest/interface/inout.cpp b/src/axom/quest/interface/inout.cpp index 9566bf821d..c3e0a992b1 100644 --- a/src/axom/quest/interface/inout.cpp +++ b/src/axom/quest/interface/inout.cpp @@ -112,7 +112,6 @@ struct InOutHelper revolvedVolume, comm); #else - AXOM_UNUSED_VAR(revolvedVolume); SLIC_WARNING(fmt::format( "Cannot read contour file: C2C not enabled in this configuration.", file)); From 9d6eacc5cfd7ebdcfef5a56befdd42565e87f112 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Sat, 4 Jan 2025 05:02:08 -0800 Subject: [PATCH 078/100] Fix non-Conduit build. --- src/axom/quest/util/mesh_helpers.cpp | 26 +++++++++++++++----------- src/axom/quest/util/mesh_helpers.hpp | 3 +++ 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/axom/quest/util/mesh_helpers.cpp b/src/axom/quest/util/mesh_helpers.cpp index 3ce0d07d43..4f9108f563 100644 --- a/src/axom/quest/util/mesh_helpers.cpp +++ b/src/axom/quest/util/mesh_helpers.cpp @@ -8,9 +8,11 @@ #include #endif #include -#include -#include #include "axom/mint/mesh/UnstructuredMesh.hpp" +#if defined(AXOM_USE_CONDUIT) + #include +#endif +#include namespace axom { @@ -180,7 +182,6 @@ axom::sidre::Group* make_structured_blueprint_box_mesh( return meshGrp; } - #if defined(AXOM_USE_CONDUIT) axom::sidre::Group* make_unstructured_blueprint_box_mesh( axom::sidre::Group* meshGrp, const primal::BoundingBox& bbox, @@ -212,30 +213,30 @@ void convert_blueprint_structured_explicit_to_unstructured( meshGrp, topoName); } - #if defined(AXOM_RUNTIME_POLICY_USE_OPENMP) + #if defined(AXOM_RUNTIME_POLICY_USE_OPENMP) if(runtimePolicy == axom::runtime_policy::Policy::omp) { convert_blueprint_structured_explicit_to_unstructured_impl( meshGrp, topoName); } - #endif - #if defined(AXOM_RUNTIME_POLICY_USE_CUDA) + #endif + #if defined(AXOM_RUNTIME_POLICY_USE_CUDA) if(runtimePolicy == axom::runtime_policy::Policy::cuda) { convert_blueprint_structured_explicit_to_unstructured_impl>( meshGrp, topoName); } - #endif - #if defined(AXOM_RUNTIME_POLICY_USE_HIP) + #endif + #if defined(AXOM_RUNTIME_POLICY_USE_HIP) if(runtimePolicy == axom::runtime_policy::Policy::hip) { convert_blueprint_structured_explicit_to_unstructured_impl>( meshGrp, topoName); } - #endif + #endif } template @@ -251,6 +252,8 @@ void convert_blueprint_structured_explicit_to_unstructured_impl( sidre::Group* coordsetGrp = meshGrp->getGroup("coordsets")->getGroup(coordsetName); + SLIC_ASSERT(std::string(coordsetGrp->getView("type")->getString()) == + "explicit"); int newDataAllocId = axom::getAllocatorIDFromPointer( coordsetGrp->getView("values/x")->getVoidPtr()); @@ -366,17 +369,18 @@ void convert_blueprint_structured_explicit_to_unstructured_impl( AXOM_ANNOTATE_END("add_extra"); } - #if defined(AXOM_DEBUG) + #if defined(AXOM_DEBUG) && defined(AXOM_USE_CONDUIT) AXOM_ANNOTATE_BEGIN("validate_post"); conduit::Node info; bool isValid = verifyBlueprintMesh(meshGrp, info); SLIC_ASSERT_MSG(isValid, "Internal error: Generated mesh is invalid."); AXOM_ANNOTATE_END("validate_post"); - #endif + #endif return; } + #if defined(AXOM_USE_CONDUIT) bool verifyBlueprintMesh(const axom::sidre::Group* meshGrp, conduit::Node info) { conduit::Node meshNode; diff --git a/src/axom/quest/util/mesh_helpers.hpp b/src/axom/quest/util/mesh_helpers.hpp index 92aa5157bd..aeccb68bd5 100644 --- a/src/axom/quest/util/mesh_helpers.hpp +++ b/src/axom/quest/util/mesh_helpers.hpp @@ -11,6 +11,9 @@ #if defined(AXOM_USE_SIDRE) #include "axom/sidre.hpp" #endif +#if defined(AXOM_USE_CONDUIT) + #include +#endif #ifdef AXOM_USE_MFEM #include "mfem.hpp" From 1d2abe3629d54a6de15f809052c90a375ce7db61 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Mon, 6 Jan 2025 05:05:18 -0800 Subject: [PATCH 079/100] Always use axom::IndexType for Blueprint connectivity data. --- src/axom/core/CMakeLists.txt | 1 + src/axom/core/Types.hpp | 10 ++++++++ src/axom/quest/IntersectionShaper.hpp | 25 +++++++++---------- .../quest/examples/quest_shape_in_memory.cpp | 10 ++++---- src/axom/quest/util/mesh_helpers.cpp | 12 ++++----- 5 files changed, 34 insertions(+), 24 deletions(-) diff --git a/src/axom/core/CMakeLists.txt b/src/axom/core/CMakeLists.txt index f1dbf22b26..1a87c367dd 100644 --- a/src/axom/core/CMakeLists.txt +++ b/src/axom/core/CMakeLists.txt @@ -110,6 +110,7 @@ blt_list_append( TO core_depends ELEMENTS caliper IF CALIPER_FOUND ) blt_list_append( TO core_depends ELEMENTS camp IF CAMP_FOUND ) blt_list_append( TO core_depends ELEMENTS umpire IF UMPIRE_FOUND ) blt_list_append( TO core_depends ELEMENTS RAJA IF RAJA_FOUND ) +blt_list_append( TO core_depends ELEMENTS conduit::conduit IF CONDUIT_FOUND ) blt_list_append( TO core_depends ELEMENTS mpi IF AXOM_ENABLE_MPI ) # HACK: RAJA's dependencies are not getting added to core due to a bug in diff --git a/src/axom/core/Types.hpp b/src/axom/core/Types.hpp index 535c852570..8556d81cf9 100644 --- a/src/axom/core/Types.hpp +++ b/src/axom/core/Types.hpp @@ -15,6 +15,10 @@ // Axom includes #include "axom/config.hpp" +#if defined(AXOM_USE_CONDUIT) +#include "conduit/conduit_data_type.hpp" +#endif + // C/C++ includes #include @@ -62,8 +66,14 @@ using uint64 = std::uint64_t; /*!< 64-bit unsigned integer type */ #if defined(AXOM_USE_64BIT_INDEXTYPE) && !defined(AXOM_NO_INT64_T) using IndexType = std::int64_t; +#if defined(AXOM_USE_CONDUIT) +static constexpr conduit::DataType::TypeID conduitDataIdOfIndexType = conduit::DataType::INT64_ID; +#endif #else using IndexType = std::int32_t; +#if defined(AXOM_USE_CONDUIT) +static constexpr conduit::DataType::TypeID conduitDataIdOfIndexType = conduit::DataType::INT32_ID; +#endif #endif static constexpr IndexType InvalidIndex = -1; diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 0be4205928..43e8deb501 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -2357,13 +2357,18 @@ class IntersectionShaper : public Shaper "element shape must be 'hex'"); const auto& connNode = topoNode["elements/connectivity"]; - SLIC_ERROR_IF(!connNode.dtype().is_int32(), - "IntersectionShaper internal error: missing logic to " - "handle multiple types."); - const std::int32_t* connPtr = connNode.as_int32_ptr(); - axom::ArrayView conn(connPtr, - m_cellCount, - NUM_VERTS_PER_HEX); + SLIC_ERROR_IF( + !XS::usesAllocId(axom::getAllocatorIDFromPointer(connNode.data_ptr())), + std::string(XS::name()) + + axom::fmt::format( + " execution space cannot use the connectivity allocator id {}", + axom::getAllocatorIDFromPointer(connNode.data_ptr()))); + SLIC_ERROR_IF(connNode.dtype().id() != conduitDataIdOfIndexType, + "IntersectionShaper error: connectivity data type must be axom::IndexType."); + const auto* connPtr = static_cast(connNode.data_ptr()); + axom::ArrayView conn(connPtr, + m_cellCount, + NUM_VERTS_PER_HEX); const conduit::Node& coordNode = m_bpNodeInt["coordsets"][coordsetName]; const conduit::Node& coordValues = coordNode.fetch_existing("values"); @@ -2382,12 +2387,6 @@ class IntersectionShaper : public Shaper {vertexCount}, stride)}; - SLIC_ERROR_IF( - !XS::usesAllocId(axom::getAllocatorIDFromPointer(conn.data())), - std::string(XS::name()) + - axom::fmt::format( - " execution space cannot use the connectivity allocator id {}", - axom::getAllocatorIDFromPointer(conn.data()))); SLIC_ERROR_IF( !XS::usesAllocId(axom::getAllocatorIDFromPointer(coordArrays[0].data())), std::string(XS::name()) + diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index 107661d0da..ccf3013669 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -1050,7 +1050,7 @@ axom::sidre::View* getElementVolumes( ->getGroup(topoName) ->getGroup("elements") ->getView("connectivity"); - SLIC_ASSERT(connData->getNode().dtype().is_int32()); + SLIC_ASSERT(connData->getNode().dtype().id() == axom::conduitDataIdOfIndexType); conduit::Node coordNode; meshGrp->getGroup("coordsets") @@ -1071,11 +1071,11 @@ axom::sidre::View* getElementVolumes( {vertexCount}, stride)}; - const std::int32_t* connPtr = connData->getArray(); + const axom::IndexType* connPtr = connData->getArray(); SLIC_ASSERT(connPtr != nullptr); - axom::ArrayView conn(connPtr, - cellCount, - NUM_VERTS_PER_HEX); + axom::ArrayView conn(connPtr, + cellCount, + NUM_VERTS_PER_HEX); axom::Array vertCoords(cellCount * NUM_VERTS_PER_HEX, cellCount * NUM_VERTS_PER_HEX, XS::allocatorID()); diff --git a/src/axom/quest/util/mesh_helpers.cpp b/src/axom/quest/util/mesh_helpers.cpp index 4f9108f563..d045fd30e9 100644 --- a/src/axom/quest/util/mesh_helpers.cpp +++ b/src/axom/quest/util/mesh_helpers.cpp @@ -272,9 +272,9 @@ void convert_blueprint_structured_explicit_to_unstructured_impl( SLIC_ASSERT(!topoGrp->hasGroup("elements/strides")); const axom::StackArray cShape { - topoDimsGrp->getView("i")->getData(), - topoDimsGrp->getView("j")->getData(), - topoDimsGrp->getView("k")->getData()}; + axom::IndexType(topoDimsGrp->getView("i")->getNode().to_value()), + axom::IndexType(topoDimsGrp->getView("j")->getNode().to_value()), + axom::IndexType(topoDimsGrp->getView("k")->getNode().to_value())}; const axom::StackArray vShape {cShape[0] + 1, cShape[1] + 1, cShape[2] + 1}; @@ -456,9 +456,9 @@ void fill_cartesian_coords_3d_impl(const primal::BoundingBox& domainB SLIC_ASSERT(mapping == zView.mapping()); // Mesh resolution - const axom::primal::NumericArray res {shape[0] - 1, - shape[1] - 1, - shape[2] - 1}; + const axom::primal::NumericArray res {shape[0] - 1, + shape[1] - 1, + shape[2] - 1}; // Mesh spacings. double dx = (domainBox.getMax()[0] - domainBox.getMin()[0]) / res[0]; From 310f1ad7741357b78f509e5080e8831044c1f41c Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Mon, 6 Jan 2025 05:23:44 -0800 Subject: [PATCH 080/100] Autoformat. --- src/axom/core/Types.hpp | 16 +++++++++------- src/axom/quest/IntersectionShaper.hpp | 6 ++++-- .../quest/examples/quest_shape_in_memory.cpp | 3 ++- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/axom/core/Types.hpp b/src/axom/core/Types.hpp index 8556d81cf9..171b674e51 100644 --- a/src/axom/core/Types.hpp +++ b/src/axom/core/Types.hpp @@ -16,7 +16,7 @@ #include "axom/config.hpp" #if defined(AXOM_USE_CONDUIT) -#include "conduit/conduit_data_type.hpp" + #include "conduit/conduit_data_type.hpp" #endif // C/C++ includes @@ -66,14 +66,16 @@ using uint64 = std::uint64_t; /*!< 64-bit unsigned integer type */ #if defined(AXOM_USE_64BIT_INDEXTYPE) && !defined(AXOM_NO_INT64_T) using IndexType = std::int64_t; -#if defined(AXOM_USE_CONDUIT) -static constexpr conduit::DataType::TypeID conduitDataIdOfIndexType = conduit::DataType::INT64_ID; -#endif + #if defined(AXOM_USE_CONDUIT) +static constexpr conduit::DataType::TypeID conduitDataIdOfIndexType = + conduit::DataType::INT64_ID; + #endif #else using IndexType = std::int32_t; -#if defined(AXOM_USE_CONDUIT) -static constexpr conduit::DataType::TypeID conduitDataIdOfIndexType = conduit::DataType::INT32_ID; -#endif + #if defined(AXOM_USE_CONDUIT) +static constexpr conduit::DataType::TypeID conduitDataIdOfIndexType = + conduit::DataType::INT32_ID; + #endif #endif static constexpr IndexType InvalidIndex = -1; diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 43e8deb501..c050feccc0 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -2364,8 +2364,10 @@ class IntersectionShaper : public Shaper " execution space cannot use the connectivity allocator id {}", axom::getAllocatorIDFromPointer(connNode.data_ptr()))); SLIC_ERROR_IF(connNode.dtype().id() != conduitDataIdOfIndexType, - "IntersectionShaper error: connectivity data type must be axom::IndexType."); - const auto* connPtr = static_cast(connNode.data_ptr()); + "IntersectionShaper error: connectivity data type must be " + "axom::IndexType."); + const auto* connPtr = + static_cast(connNode.data_ptr()); axom::ArrayView conn(connPtr, m_cellCount, NUM_VERTS_PER_HEX); diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index ccf3013669..bbf835ad26 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -1050,7 +1050,8 @@ axom::sidre::View* getElementVolumes( ->getGroup(topoName) ->getGroup("elements") ->getView("connectivity"); - SLIC_ASSERT(connData->getNode().dtype().id() == axom::conduitDataIdOfIndexType); + SLIC_ASSERT(connData->getNode().dtype().id() == + axom::conduitDataIdOfIndexType); conduit::Node coordNode; meshGrp->getGroup("coordsets") From e4ab7ae33b315568d297a95498bc261a6d215b4a Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Mon, 6 Jan 2025 17:46:18 -0800 Subject: [PATCH 081/100] Timer for Shaper::loadShapeInternal. --- src/axom/quest/Shaper.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/axom/quest/Shaper.cpp b/src/axom/quest/Shaper.cpp index d5ac1068fd..7a160f62c4 100644 --- a/src/axom/quest/Shaper.cpp +++ b/src/axom/quest/Shaper.cpp @@ -214,6 +214,7 @@ void Shaper::loadShapeInternal(const klee::Shape& shape, double percentError, double& revolvedVolume) { + AXOM_ANNOTATE_SCOPE("Shaper::loadShapeInternal"); internal::ScopedLogLevelChanger logLevelChanger( this->isVerbose() ? slic::message::Debug : slic::message::Warning); From 9921031a43f165c97b4a6b1f5d607a36da3a849a Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Tue, 7 Jan 2025 20:22:25 -0800 Subject: [PATCH 082/100] Remove a check that unintentionally requires user data to be Umpire-allocated. --- src/axom/quest/IntersectionShaper.hpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index c050feccc0..7513d77df2 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -2389,13 +2389,6 @@ class IntersectionShaper : public Shaper {vertexCount}, stride)}; - SLIC_ERROR_IF( - !XS::usesAllocId(axom::getAllocatorIDFromPointer(coordArrays[0].data())), - std::string(XS::name()) + - axom::fmt::format( - " execution space cannot use the coordset allocator id {}", - axom::getAllocatorIDFromPointer(coordArrays[0].data()))); - vertCoords = axom::Array(m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT, m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT, From 5d2372e2a7e9338c2c9aea2046c2eb4800842ec1 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Tue, 7 Jan 2025 20:23:45 -0800 Subject: [PATCH 083/100] Fix breakage that occurs when user data isn't Umpire-allocated. --- src/axom/core/execution/runtime_policy.hpp | 1 + src/axom/quest/Shaper.cpp | 29 ++++++++++++++++++++++ src/axom/quest/util/mesh_helpers.cpp | 5 +--- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/axom/core/execution/runtime_policy.hpp b/src/axom/core/execution/runtime_policy.hpp index 445bb538ff..a894146b6b 100644 --- a/src/axom/core/execution/runtime_policy.hpp +++ b/src/axom/core/execution/runtime_policy.hpp @@ -7,6 +7,7 @@ #define AXOM_CORE_EXECUTION_RUNTIME_POLICY_HPP_ #include "axom/config.hpp" /* for compile time defs. */ +#include #include "axom/fmt/format.h" /* for axom::fmt */ #include diff --git a/src/axom/quest/Shaper.cpp b/src/axom/quest/Shaper.cpp index 7a160f62c4..5169c1152d 100644 --- a/src/axom/quest/Shaper.cpp +++ b/src/axom/quest/Shaper.cpp @@ -16,6 +16,34 @@ #include "axom/fmt.hpp" +/// \brief Return default allocator id for a runtime policy. +static int policyToDefaultAllocatorID(axom::runtime_policy::Policy policy) +{ + if(policy == axom::runtime_policy::Policy::seq) + { + return axom::execution_space::allocatorID(); + } + #if defined(AXOM_RUNTIME_POLICY_USE_OPENMP) + if(policy == axom::runtime_policy::Policy::omp) + { + return axom::execution_space::allocatorID(); + } + #endif + #if defined(AXOM_RUNTIME_POLICY_USE_CUDA) + if(policy == axom::runtime_policy::Policy::cuda) + { + return axom::execution_space>::allocatorID(); + } + #endif + #if defined(AXOM_RUNTIME_POLICY_USE_HIP) + if(policy == axom::runtime_policy::Policy::hip) + { + return axom::execution_space>::allocatorID(); + } + #endif + return axom::INVALID_ALLOCATOR_ID; +} + namespace axom { namespace quest @@ -98,6 +126,7 @@ Shaper::Shaper(RuntimePolicy execPolicy, { AXOM_ANNOTATE_SCOPE("Shaper::Shaper_Node"); m_bpGrp = m_ds.getRoot()->createGroup("internalGrp"); + m_bpGrp->setDefaultAllocator(policyToDefaultAllocatorID(m_execPolicy)); m_bpGrp->importConduitTreeExternal(*bpNode); diff --git a/src/axom/quest/util/mesh_helpers.cpp b/src/axom/quest/util/mesh_helpers.cpp index d045fd30e9..aea68cd79c 100644 --- a/src/axom/quest/util/mesh_helpers.cpp +++ b/src/axom/quest/util/mesh_helpers.cpp @@ -254,8 +254,6 @@ void convert_blueprint_structured_explicit_to_unstructured_impl( meshGrp->getGroup("coordsets")->getGroup(coordsetName); SLIC_ASSERT(std::string(coordsetGrp->getView("type")->getString()) == "explicit"); - int newDataAllocId = axom::getAllocatorIDFromPointer( - coordsetGrp->getView("values/x")->getVoidPtr()); axom::sidre::Group* topoGrp = meshGrp->getGroup("topologies")->getGroup(topoName); @@ -286,8 +284,7 @@ void convert_blueprint_structured_explicit_to_unstructured_impl( "elements/connectivity", axom::sidre::detail::SidreTT::id, 2, - connShape.begin(), - newDataAllocId); + connShape.begin()); axom::ArrayView connArrayView( static_cast(connView->getVoidPtr()), connShape); From 1fe19a2796560ca95fd417be189136d0324be193 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Tue, 14 Jan 2025 16:07:57 -0800 Subject: [PATCH 084/100] Autoformat. --- src/axom/core/execution/runtime_policy.hpp | 2 +- src/axom/quest/IntersectionShaper.hpp | 10 +++++----- src/axom/quest/Shaper.cpp | 12 ++++++------ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/axom/core/execution/runtime_policy.hpp b/src/axom/core/execution/runtime_policy.hpp index a894146b6b..246488ea67 100644 --- a/src/axom/core/execution/runtime_policy.hpp +++ b/src/axom/core/execution/runtime_policy.hpp @@ -6,7 +6,7 @@ #ifndef AXOM_CORE_EXECUTION_RUNTIME_POLICY_HPP_ #define AXOM_CORE_EXECUTION_RUNTIME_POLICY_HPP_ -#include "axom/config.hpp" /* for compile time defs. */ +#include "axom/config.hpp" /* for compile time defs. */ #include #include "axom/fmt/format.h" /* for axom::fmt */ diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 1e850dc031..52691c8cdf 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -416,13 +416,13 @@ class IntersectionShaper : public Shaper } } - // The following private methods are made public due to CUDA compilers - // requirements for methods that call device functions. -#if defined(__CUDACC__) + // The following private methods are made public due to CUDA compilers + // requirements for methods that call device functions. + #if defined(__CUDACC__) public: -#else + #else private: -#endif + #endif //@{ //! @name Private functions related to the stages for a given shape diff --git a/src/axom/quest/Shaper.cpp b/src/axom/quest/Shaper.cpp index 5169c1152d..4a09173eec 100644 --- a/src/axom/quest/Shaper.cpp +++ b/src/axom/quest/Shaper.cpp @@ -23,24 +23,24 @@ static int policyToDefaultAllocatorID(axom::runtime_policy::Policy policy) { return axom::execution_space::allocatorID(); } - #if defined(AXOM_RUNTIME_POLICY_USE_OPENMP) +#if defined(AXOM_RUNTIME_POLICY_USE_OPENMP) if(policy == axom::runtime_policy::Policy::omp) { return axom::execution_space::allocatorID(); } - #endif - #if defined(AXOM_RUNTIME_POLICY_USE_CUDA) +#endif +#if defined(AXOM_RUNTIME_POLICY_USE_CUDA) if(policy == axom::runtime_policy::Policy::cuda) { return axom::execution_space>::allocatorID(); } - #endif - #if defined(AXOM_RUNTIME_POLICY_USE_HIP) +#endif +#if defined(AXOM_RUNTIME_POLICY_USE_HIP) if(policy == axom::runtime_policy::Policy::hip) { return axom::execution_space>::allocatorID(); } - #endif +#endif return axom::INVALID_ALLOCATOR_ID; } From 91fceda9216fe35b8264136a5aef62764060c54e Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Tue, 14 Jan 2025 16:16:35 -0800 Subject: [PATCH 085/100] Less confusing variable name. --- src/axom/core/Types.hpp | 4 ++-- src/axom/quest/IntersectionShaper.hpp | 2 +- src/axom/quest/examples/quest_shape_in_memory.cpp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/axom/core/Types.hpp b/src/axom/core/Types.hpp index 171b674e51..d39789691f 100644 --- a/src/axom/core/Types.hpp +++ b/src/axom/core/Types.hpp @@ -67,13 +67,13 @@ using uint64 = std::uint64_t; /*!< 64-bit unsigned integer type */ #if defined(AXOM_USE_64BIT_INDEXTYPE) && !defined(AXOM_NO_INT64_T) using IndexType = std::int64_t; #if defined(AXOM_USE_CONDUIT) -static constexpr conduit::DataType::TypeID conduitDataIdOfIndexType = +static constexpr conduit::DataType::TypeID conduitDataIdOfAxomIndexType = conduit::DataType::INT64_ID; #endif #else using IndexType = std::int32_t; #if defined(AXOM_USE_CONDUIT) -static constexpr conduit::DataType::TypeID conduitDataIdOfIndexType = +static constexpr conduit::DataType::TypeID conduitDataIdOfAxomIndexType = conduit::DataType::INT32_ID; #endif #endif diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 52691c8cdf..46b2d90be7 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -2370,7 +2370,7 @@ class IntersectionShaper : public Shaper axom::fmt::format( " execution space cannot use the connectivity allocator id {}", axom::getAllocatorIDFromPointer(connNode.data_ptr()))); - SLIC_ERROR_IF(connNode.dtype().id() != conduitDataIdOfIndexType, + SLIC_ERROR_IF(connNode.dtype().id() != conduitDataIdOfAxomIndexType, "IntersectionShaper error: connectivity data type must be " "axom::IndexType."); const auto* connPtr = diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index 890bddbb2d..5679f05f9a 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -1057,7 +1057,7 @@ axom::sidre::View* getElementVolumes( ->getGroup("elements") ->getView("connectivity"); SLIC_ASSERT(connData->getNode().dtype().id() == - axom::conduitDataIdOfIndexType); + axom::conduitDataIdOfAxomIndexType); conduit::Node coordNode; meshGrp->getGroup("coordsets") From ff0236c524a4423f3e1fc4c89ea3823e870cc84f Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Thu, 16 Jan 2025 09:34:38 -0800 Subject: [PATCH 086/100] Silence warning about unreachable instruction. --- src/axom/primal/geometry/KnotVector.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/axom/primal/geometry/KnotVector.hpp b/src/axom/primal/geometry/KnotVector.hpp index bfc3ca2f3a..a82d230021 100644 --- a/src/axom/primal/geometry/KnotVector.hpp +++ b/src/axom/primal/geometry/KnotVector.hpp @@ -157,8 +157,8 @@ class KnotVector */ T& operator[](axom::IndexType i) { - return m_knots[i]; SLIC_ASSERT(isValid()); + return m_knots[i]; } /// \brief Return the degree of the knot vector From 92fb7af0061fa5099e9972f8d893ddf09a4a9622 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Thu, 16 Jan 2025 09:35:51 -0800 Subject: [PATCH 087/100] Fix crash in usesAllocId when the id is not from Umpire. --- src/axom/core/execution/execution_space.hpp | 2 +- src/axom/core/execution/internal/cuda_exec.hpp | 2 +- src/axom/core/execution/internal/hip_exec.hpp | 2 +- src/axom/core/execution/internal/omp_exec.hpp | 2 +- src/axom/core/execution/internal/seq_exec.hpp | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/axom/core/execution/execution_space.hpp b/src/axom/core/execution/execution_space.hpp index 2dbab4777e..3fb3408f2d 100644 --- a/src/axom/core/execution/execution_space.hpp +++ b/src/axom/core/execution/execution_space.hpp @@ -99,7 +99,7 @@ struct execution_space } static bool usesAllocId(int allocId) noexcept { - return usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); + return allocId == 0 || usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); } }; diff --git a/src/axom/core/execution/internal/cuda_exec.hpp b/src/axom/core/execution/internal/cuda_exec.hpp index 7fbe66f50f..8369464bab 100644 --- a/src/axom/core/execution/internal/cuda_exec.hpp +++ b/src/axom/core/execution/internal/cuda_exec.hpp @@ -75,7 +75,7 @@ struct execution_space> } static bool usesAllocId(int allocId) noexcept { - return usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); + return allocId == 0 || usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); } }; diff --git a/src/axom/core/execution/internal/hip_exec.hpp b/src/axom/core/execution/internal/hip_exec.hpp index c03e434fda..ab245d2f2c 100644 --- a/src/axom/core/execution/internal/hip_exec.hpp +++ b/src/axom/core/execution/internal/hip_exec.hpp @@ -73,7 +73,7 @@ struct execution_space> } static bool usesAllocId(int allocId) noexcept { - return usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); + return allocId == 0 || usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); } }; diff --git a/src/axom/core/execution/internal/omp_exec.hpp b/src/axom/core/execution/internal/omp_exec.hpp index bbc0fec051..f38d23ad9e 100644 --- a/src/axom/core/execution/internal/omp_exec.hpp +++ b/src/axom/core/execution/internal/omp_exec.hpp @@ -74,7 +74,7 @@ struct execution_space } static bool usesAllocId(int allocId) noexcept { - return usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); + return allocId == 0 || usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); } }; diff --git a/src/axom/core/execution/internal/seq_exec.hpp b/src/axom/core/execution/internal/seq_exec.hpp index 2049416ae1..a15c61e097 100644 --- a/src/axom/core/execution/internal/seq_exec.hpp +++ b/src/axom/core/execution/internal/seq_exec.hpp @@ -83,7 +83,7 @@ struct execution_space } static bool usesAllocId(int allocId) noexcept { - return usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); + return allocId == 0 || usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); } }; From 6b84ee7893a08604352add2060ab4cbceff93a57 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Thu, 16 Jan 2025 09:37:04 -0800 Subject: [PATCH 088/100] Add missing methods in ASYNC versions of device execution spaces. --- src/axom/core/execution/internal/cuda_exec.hpp | 12 ++++++++++++ src/axom/core/execution/internal/hip_exec.hpp | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/axom/core/execution/internal/cuda_exec.hpp b/src/axom/core/execution/internal/cuda_exec.hpp index 8369464bab..410151c8dc 100644 --- a/src/axom/core/execution/internal/cuda_exec.hpp +++ b/src/axom/core/execution/internal/cuda_exec.hpp @@ -107,6 +107,18 @@ struct execution_space> { return axom::getUmpireResourceAllocatorID(umpire::resource::Device); } + static constexpr runtime_policy::Policy runtimePolicy() noexcept + { + return runtime_policy::Policy::cuda; + } + static bool usesMemorySpace(axom::MemorySpace m) noexcept + { + return m == memory_space || m == MemorySpace::Unified; + } + static bool usesAllocId(int allocId) noexcept + { + return allocId == 0 || usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); + } }; } // namespace axom diff --git a/src/axom/core/execution/internal/hip_exec.hpp b/src/axom/core/execution/internal/hip_exec.hpp index ab245d2f2c..bba77b9041 100644 --- a/src/axom/core/execution/internal/hip_exec.hpp +++ b/src/axom/core/execution/internal/hip_exec.hpp @@ -102,6 +102,18 @@ struct execution_space> { return axom::getUmpireResourceAllocatorID(umpire::resource::Device); } + static constexpr runtime_policy::Policy runtimePolicy() noexcept + { + return runtime_policy::Policy::hip; + } + static bool usesMemorySpace(axom::MemorySpace m) noexcept + { + return m == memory_space || m == MemorySpace::Unified; + } + static bool usesAllocId(int allocId) noexcept + { + return allocId == 0 || usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); + } }; } // namespace axom From 5fce87db8909a5a5d3354a2c53bdd7ba3de63d1f Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Thu, 16 Jan 2025 09:37:46 -0800 Subject: [PATCH 089/100] Update RELEASE-NOTES for interface changes. --- RELEASE-NOTES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index be388661e1..1b19578f7a 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -20,6 +20,8 @@ The Axom project release numbers follow [Semantic Versioning](http://semver.org/ ## [Unreleased] - Release date yyyy-mm-dd ### Added +- Support in `quest::IntersectionShaper` for Blueprint mesh stored in a `conduit::Node` + or `sidre::Group`. - A number of new `klee::Geometry` constructors are added, for the different shapes now supported. This is a temporary change. The class will be subclassed in the future to support a diversity of geometries. - Support some analytical shapes in `IntersectionShaper`. @@ -32,6 +34,8 @@ to use Open Cascade's file I/O capabilities in support of Quest applications. - Adds a Quest example that reads in a STEP file using Open Cascade and processes its geometry ### Changed +- `quest::Shaper` and `quest::IntersectionShaper` constructors require a runtime policy. + Changing the policy after construction is no longer supported. - Importing Conduit array data into `sidre::View` now allocates destination data using the `View`'s parent's allocator ID, instead of always using host memory. This is consistent with the behavior of deep-copying data From d810dd61f5ea35ce56508ee514a0eb69891aabe3 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Thu, 16 Jan 2025 09:40:42 -0800 Subject: [PATCH 090/100] Changes to comments. --- src/axom/quest/IntersectionShaper.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 46b2d90be7..0901be2dc1 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -730,9 +730,8 @@ class IntersectionShaper : public Shaper if(m_hex_bbs.empty()) { - // Does m_hex_bbs have to be a member? I's only used here. BTNG. - // Or if it's saved, it needs not be recomputed if the computational - // mesh doesn't change. + // Compute m_hex_bbs only once. We assume that the mesh doesn't change + // once set. m_hex_bbs = axom::Array(m_cellCount, m_cellCount, device_allocator); @@ -1589,8 +1588,9 @@ class IntersectionShaper : public Shaper * * \param materialName The name of the material. * - * \return A pair containing the associated grid function and material - * number (its order in the list). + * \return A pair containing the associated grid function (an element + * in the m_vf_grid_functions array) and material + * number (its index in the array). */ std::pair, int> getMaterial(const std::string& materialName) { From d14f583563102eaa132e9efc7a5ffd043b587861 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Thu, 16 Jan 2025 12:46:47 -0800 Subject: [PATCH 091/100] Update IntersectionShaper dox comments. --- src/axom/quest/IntersectionShaper.hpp | 30 ++++++++++++++++++++------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 0901be2dc1..e2fc4d76e1 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -268,8 +268,8 @@ AXOM_HOST_DEVICE inline void TempArrayView::finalize() //--------------------------------------------------------------------------- /** * \class - * \brief Intersects a solid, created by revolving a c2c contour or by - * loading a Pro/E mesh, with an input mesh to produce volume fractions. + * \brief Intersects a solid with an input mesh to produce volume fractions. + * The solid may be any supported by the \c klee::Geometry class. * * The IntersectionShaper generates material volume fractions: * @@ -281,16 +281,30 @@ AXOM_HOST_DEVICE inline void TempArrayView::finalize() * intersected with the mesh. The octahedra are intersected with the input * mesh to produce volume fractions. * - * - For Pro/E, an input mesh of 3D tetrahedra are loaded in. + * - For tetrahedral mesh (including Pro/E), an input mesh of 3D tetrahedra is loaded in. * Each tetrahedron has its own respective volume. The tetrahedra are * intersected with the input mesh to produce volume fractions. * - * Volume fractions are represented as a GridFunction with a special prefix, + * - For analytical geometries, the shapes are discretized into a tetrahedral + * mesh first. Sphere and surfaces-of-revolution discretization uses + * the refinement level specified in the \c Geometry. + * + * The input mesh can be an MFEM mesh stored as a \c + * sidre::MFEMSidreDataCollection or be a Blueprint mesh stored as a + * \c conduit::Node or a \c sidre::Group. + * + * IntersectionShaper requires Axom configured with RAJA and Umpire. + * + * Support for replacement rules exists for MFEM input meshes. + * Replacement rules for Blueprint meshes is not yet supported. + * The following comments apply to replacement rules. + * + * Volume fractions are represented in the input mesh as a GridFunction with a special prefix, * currently "vol_frac_", followed by a material name. Volume fractions * can be present in the input data collection prior to shaping and the * IntersectionShaper will augment them when changes are needed such as when - * a material overwrites them. If a new material is not yet represented by - * a grid function, one will be added. + * a material overwrites them. If a new material is not yet represented in + * the mesh, one will be added. * * In addition to user-specified materials, the IntersectionShaper creates * a "free" material that is used to account for volume fractions that are @@ -299,8 +313,6 @@ AXOM_HOST_DEVICE inline void TempArrayView::finalize() * starts out as all 1's indicating that it contains 100% of all possible * material in a zone. Volume fractions for other materials are then * subtracted from the free material so no zone exceeds 100% of material. - * - * IntersectionShaper requires Axom configured with RAJA and Umpire. */ class IntersectionShaper : public Shaper { @@ -2228,8 +2240,10 @@ class IntersectionShaper : public Shaper return rval; } +#if defined(__CUDACC__) public: // These methods should be private, but NVCC complains unless its public. +#endif template void populateHexesFromMesh() From 7e7c141312bb63da427eeeadbd4eee78e6f94048 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Thu, 16 Jan 2025 13:49:47 -0800 Subject: [PATCH 092/100] Autoformat. --- src/axom/core/execution/execution_space.hpp | 3 ++- src/axom/core/execution/internal/cuda_exec.hpp | 6 ++++-- src/axom/core/execution/internal/hip_exec.hpp | 6 ++++-- src/axom/core/execution/internal/omp_exec.hpp | 3 ++- src/axom/core/execution/internal/seq_exec.hpp | 3 ++- src/axom/quest/IntersectionShaper.hpp | 6 +++--- 6 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/axom/core/execution/execution_space.hpp b/src/axom/core/execution/execution_space.hpp index 3fb3408f2d..e65f63fe1c 100644 --- a/src/axom/core/execution/execution_space.hpp +++ b/src/axom/core/execution/execution_space.hpp @@ -99,7 +99,8 @@ struct execution_space } static bool usesAllocId(int allocId) noexcept { - return allocId == 0 || usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); + return allocId == 0 || + usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); } }; diff --git a/src/axom/core/execution/internal/cuda_exec.hpp b/src/axom/core/execution/internal/cuda_exec.hpp index 410151c8dc..36ff1901fe 100644 --- a/src/axom/core/execution/internal/cuda_exec.hpp +++ b/src/axom/core/execution/internal/cuda_exec.hpp @@ -75,7 +75,8 @@ struct execution_space> } static bool usesAllocId(int allocId) noexcept { - return allocId == 0 || usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); + return allocId == 0 || + usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); } }; @@ -117,7 +118,8 @@ struct execution_space> } static bool usesAllocId(int allocId) noexcept { - return allocId == 0 || usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); + return allocId == 0 || + usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); } }; } // namespace axom diff --git a/src/axom/core/execution/internal/hip_exec.hpp b/src/axom/core/execution/internal/hip_exec.hpp index bba77b9041..8d292f0fb0 100644 --- a/src/axom/core/execution/internal/hip_exec.hpp +++ b/src/axom/core/execution/internal/hip_exec.hpp @@ -73,7 +73,8 @@ struct execution_space> } static bool usesAllocId(int allocId) noexcept { - return allocId == 0 || usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); + return allocId == 0 || + usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); } }; @@ -112,7 +113,8 @@ struct execution_space> } static bool usesAllocId(int allocId) noexcept { - return allocId == 0 || usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); + return allocId == 0 || + usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); } }; } // namespace axom diff --git a/src/axom/core/execution/internal/omp_exec.hpp b/src/axom/core/execution/internal/omp_exec.hpp index f38d23ad9e..860bc7d677 100644 --- a/src/axom/core/execution/internal/omp_exec.hpp +++ b/src/axom/core/execution/internal/omp_exec.hpp @@ -74,7 +74,8 @@ struct execution_space } static bool usesAllocId(int allocId) noexcept { - return allocId == 0 || usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); + return allocId == 0 || + usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); } }; diff --git a/src/axom/core/execution/internal/seq_exec.hpp b/src/axom/core/execution/internal/seq_exec.hpp index a15c61e097..10833cb584 100644 --- a/src/axom/core/execution/internal/seq_exec.hpp +++ b/src/axom/core/execution/internal/seq_exec.hpp @@ -83,7 +83,8 @@ struct execution_space } static bool usesAllocId(int allocId) noexcept { - return allocId == 0 || usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); + return allocId == 0 || + usesMemorySpace(axom::detail::getAllocatorSpace(allocId)); } }; diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index e2fc4d76e1..642f25598c 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -2240,10 +2240,10 @@ class IntersectionShaper : public Shaper return rval; } -#if defined(__CUDACC__) + #if defined(__CUDACC__) public: - // These methods should be private, but NVCC complains unless its public. -#endif + // These methods should be private, but NVCC complains unless its public. + #endif template void populateHexesFromMesh() From 68b0808d97c5d77d3ef891f43dff7f1be192c0c9 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Thu, 16 Jan 2025 15:21:40 -0800 Subject: [PATCH 093/100] Change "volume of revolution" to "surface of revolution." To be in line with other codes' convention. --- src/axom/klee/Geometry.cpp | 10 +-- src/axom/klee/Geometry.hpp | 34 +++++----- src/axom/quest/DiscreteShape.cpp | 20 +++--- src/axom/quest/DiscreteShape.hpp | 6 +- src/axom/quest/Shaper.cpp | 2 +- src/axom/quest/examples/CMakeLists.txt | 2 +- .../quest/examples/quest_shape_in_memory.cpp | 66 +++++++++---------- 7 files changed, 70 insertions(+), 70 deletions(-) diff --git a/src/axom/klee/Geometry.cpp b/src/axom/klee/Geometry.cpp index 939e542926..8ddcf51aca 100644 --- a/src/axom/klee/Geometry.cpp +++ b/src/axom/klee/Geometry.cpp @@ -86,19 +86,19 @@ Geometry::Geometry(const TransformableGeometryProperties& startProperties, Geometry::Geometry(const TransformableGeometryProperties& startProperties, const axom::Array& discreteFunction, - const Point3D& vorBase, - const Vector3D& vorDirection, + const Point3D& sorBase, + const Vector3D& sorDirection, axom::IndexType levelOfRefinement, std::shared_ptr operator_) : m_startProperties(startProperties) - , m_format("vor3D") + , m_format("sor3D") , m_path() , m_meshGroup(nullptr) , m_topology() , m_sphere() , m_discreteFunction(discreteFunction) - , m_vorBase(vorBase) - , m_vorDirection(vorDirection) + , m_sorBase(sorBase) + , m_sorDirection(sorDirection) , m_levelOfRefinement(levelOfRefinement) , m_operator(std::move(operator_)) { } diff --git a/src/axom/klee/Geometry.hpp b/src/axom/klee/Geometry.hpp index e23d095a99..c08812200b 100644 --- a/src/axom/klee/Geometry.hpp +++ b/src/axom/klee/Geometry.hpp @@ -132,27 +132,27 @@ class Geometry std::shared_ptr operator_); /** - * Create a volume-of-revolution (VOR) Geometry object. + * Create a volume-of-revolution (SOR) Geometry object. * * \param startProperties the transformable properties before any * operators are applied * \param discreteFunction Discrete function describing the surface * of revolution. - * \param vorBase Coordinates of the base of the VOR. - * \param vorDirection VOR axis, in the direction of increasing z. + * \param sorBase Coordinates of the base of the SOR. + * \param sorDirection SOR axis, in the direction of increasing z. * \param levelOfRefinement Number of refinement levels to use for - * discretizing the VOR. + * discretizing the SOR. * \param operator_ a possibly null operator to apply to the geometry. * * The \c discreteFunction should be an Nx2 array, interpreted as * (z,r) pairs, where z is the axial distance and r is the radius. - * The \c vorBase coordinates corresponds to z=0. - * \c vorAxis should point in the direction of increasing z. + * The \c sorBase coordinates corresponds to z=0. + * \c sorAxis should point in the direction of increasing z. */ Geometry(const TransformableGeometryProperties &startProperties, const axom::Array &discreteFunction, - const Point3D &vorBase, - const Vector3D &vorDirection, + const Point3D &sorBase, + const Vector3D &sorDirection, axom::IndexType levelOfRefinement, std::shared_ptr operator_); @@ -181,7 +181,7 @@ class Geometry * - "blueprint-tets" = Blueprint tetrahedral mesh in memory * - "tet3D" = 3D tetrahedron (4 points) * - "sphere3D" = 3D sphere, as \c primal::Sphere - * - "vor3D" = 3D volume of revolution. + * - "sor3D" = 3D volume of revolution. * - "cone3D" = 3D cone, as \c primal::Cone * - "cylinder3D" = 3D cylinder, as \c primal::Cylinder * - "hex3D" = 3D hexahedron (8 points) @@ -216,14 +216,14 @@ class Geometry const std::string &getBlueprintTopology() const; /** - @brief Return the VOR axis direction. + @brief Return the SOR axis direction. */ - const Vector3D getVorDirection() const { return m_vorDirection; } + const Vector3D getSorDirection() const { return m_sorDirection; } /** - @brief Return the 3D coordinates of the VOR base. + @brief Return the 3D coordinates of the SOR base. */ - const Point3D getVorBaseCoords() const { return m_vorBase; } + const Point3D getSorBaseCoords() const { return m_sorBase; } /*! @brief Predicate that returns true when the shape has an associated geometry @@ -336,11 +336,11 @@ class Geometry //! @brief The discrete 2D function, as an Nx2 array, if used. axom::Array m_discreteFunction; - //!@brief The point corresponding to z=0 on the VOR axis. - Point3D m_vorBase; + //!@brief The point corresponding to z=0 on the SOR axis. + Point3D m_sorBase; - //!@brief VOR axis in the direction of increasing z. - Vector3D m_vorDirection; + //!@brief SOR axis in the direction of increasing z. + Vector3D m_sorDirection; //!@brief Level of refinement for discretizing curved // analytical shapes and surfaces of revolutions. diff --git a/src/axom/quest/DiscreteShape.cpp b/src/axom/quest/DiscreteShape.cpp index 328b188de9..8ee86cd2b4 100644 --- a/src/axom/quest/DiscreteShape.cpp +++ b/src/axom/quest/DiscreteShape.cpp @@ -134,9 +134,9 @@ std::shared_ptr DiscreteShape::createMeshRepresentation() { createRepresentationOfSphere(); } - if(geometryFormat == "vor3D") + if(geometryFormat == "sor3D") { - createRepresentationOfVOR(); + createRepresentationOfSOR(); } if(m_meshRep) @@ -408,7 +408,7 @@ void DiscreteShape::createRepresentationOfPlane() // clang-format on // Rotate and translate boundingHex to align with the plane. - numerics::Matrix rotate = vorAxisRotMatrix(plane.getNormal()); + numerics::Matrix rotate = sorAxisRotMatrix(plane.getNormal()); const auto translate = plane.getNormal() * plane.getOffset(); for(int i = 0; i < 8; ++i) { @@ -531,11 +531,11 @@ void DiscreteShape::createRepresentationOfSphere() applyTransforms(); } -void DiscreteShape::createRepresentationOfVOR() +void DiscreteShape::createRepresentationOfSOR() { // Construct the tet m_meshRep from the volume-of-revolution. - auto& vorGeom = m_shape.getGeometry(); - const auto& discreteFcn = vorGeom.getDiscreteFunction(); + auto& sorGeom = m_shape.getGeometry(); + const auto& discreteFcn = sorGeom.getDiscreteFunction(); // Generate the Octahedra axom::Array octs; @@ -551,9 +551,9 @@ void DiscreteShape::createRepresentationOfVOR() AXOM_UNUSED_VAR(good); SLIC_ASSERT(good); - // Rotate to the VOR axis direction and translate to the base location. - numerics::Matrix rotate = vorAxisRotMatrix(vorGeom.getVorDirection()); - const auto& translate = vorGeom.getVorBaseCoords(); + // Rotate to the SOR axis direction and translate to the base location. + numerics::Matrix rotate = sorAxisRotMatrix(sorGeom.getSorDirection()); + const auto& translate = sorGeom.getSorBaseCoords(); auto octsView = octs.view(); axom::for_all( octCount, @@ -683,7 +683,7 @@ numerics::Matrix DiscreteShape::getTransforms() const } // Return a 3x3 matrix that rotates coordinates from the x-axis to the given direction. -numerics::Matrix DiscreteShape::vorAxisRotMatrix(const Vector3D& dir) +numerics::Matrix DiscreteShape::sorAxisRotMatrix(const Vector3D& dir) { // Note that the rotation matrix is not unique. static const Vector3D x {1.0, 0.0, 0.0}; diff --git a/src/axom/quest/DiscreteShape.hpp b/src/axom/quest/DiscreteShape.hpp index 3529850397..5a3d23768d 100644 --- a/src/axom/quest/DiscreteShape.hpp +++ b/src/axom/quest/DiscreteShape.hpp @@ -174,7 +174,7 @@ class DiscreteShape void setParentGroup(axom::sidre::Group* parentGroup); //!@brief Return a 3x3 matrix that rotate coordinates from the x-axis to the given direction. - numerics::Matrix vorAxisRotMatrix(const Vector3D& dir); + numerics::Matrix sorAxisRotMatrix(const Vector3D& dir); void clearInternalData(); @@ -190,8 +190,8 @@ class DiscreteShape void createRepresentationOfPlane(); //!@brief Create the internal mesh representation of the analytical sphere. void createRepresentationOfSphere(); - //!@brief Create the internal mesh representation of the analytical VOR. - void createRepresentationOfVOR(); + //!@brief Create the internal mesh representation of the analytical SOR. + void createRepresentationOfSOR(); }; } // namespace quest diff --git a/src/axom/quest/Shaper.cpp b/src/axom/quest/Shaper.cpp index 4a09173eec..e955ed1eb0 100644 --- a/src/axom/quest/Shaper.cpp +++ b/src/axom/quest/Shaper.cpp @@ -227,7 +227,7 @@ bool Shaper::isValidFormat(const std::string& format) const return (format == "stl" || format == "proe" || format == "c2c" || format == "blueprint-tets" || format == "tet3D" || format == "hex3D" || format == "plane3D" || format == "sphere3D" || - format == "vor3D" || format == "none"); + format == "sor3D" || format == "none"); } void Shaper::loadShape(const klee::Shape& shape) diff --git a/src/axom/quest/examples/CMakeLists.txt b/src/axom/quest/examples/CMakeLists.txt index 9fd03b720f..22b0a52dd6 100644 --- a/src/axom/quest/examples/CMakeLists.txt +++ b/src/axom/quest/examples/CMakeLists.txt @@ -293,7 +293,7 @@ if((CONDUIT_FOUND OR blt_list_append(TO _policies ELEMENTS "hip" IF AXOM_ENABLE_HIP) endif() - set(_testshapes "tetmesh" "tet" "hex" "sphere" "cyl" "cone" "vor" "all" "plane") + set(_testshapes "tetmesh" "tet" "hex" "sphere" "cyl" "cone" "sor" "all" "plane") foreach(_policy ${_policies}) foreach(_testshape ${_testshapes}) set(_testname "quest_shape_in_memory_${_policy}_${_testshape}") diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index 5679f05f9a..027309b504 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -97,7 +97,7 @@ struct Input "sphere", "cyl", "cone", - "vor", + "sor", "tet", "hex", "plane", @@ -253,7 +253,7 @@ struct Input #endif app.add_option("--center", center) - ->description("Center of sphere or base of cone/cyl/VOR (x,y[,z]) shape") + ->description("Center of sphere or base of cone/cyl/SOR (x,y[,z]) shape") ->expected(2, 3); app.add_option("--radius", radius) @@ -261,12 +261,12 @@ struct Input ->check(axom::CLI::PositiveNumber); app.add_option("--length", length) - ->description("Length of cone/cyl/VOR shape, avg length of hex.") + ->description("Length of cone/cyl/SOR shape, avg length of hex.") ->check(axom::CLI::PositiveNumber); app.add_option("--dir", direction) ->description( - "Direction of axis of rotation (cone/cyl/VOR (x,y[,z])), or rotated " + "Direction of axis of rotation (cone/cyl/SOR (x,y[,z])), or rotated " "x-axis (hex, tet, tetmesh, and sphere), or positive normal direction " "(plane).") ->expected(2, 3); @@ -665,9 +665,9 @@ axom::klee::Shape createShape_TetMesh(sidre::DataStore& ds) return tetShape; } -axom::klee::Geometry createGeometry_Vor( - axom::primal::Point& vorBase, - axom::primal::Vector& vorDirection, +axom::klee::Geometry createGeometry_Sor( + axom::primal::Point& sorBase, + axom::primal::Vector& sorDirection, axom::Array& discreteFunction, std::shared_ptr& compositeOp) { @@ -678,20 +678,20 @@ axom::klee::Geometry createGeometry_Vor( SLIC_ASSERT(params.scaleFactors.empty() || params.scaleFactors.size() == 3); const axom::IndexType levelOfRefinement = params.refinementLevel; - axom::klee::Geometry vorGeometry(prop, + axom::klee::Geometry sorGeometry(prop, discreteFunction, - vorBase, - vorDirection, + sorBase, + sorDirection, levelOfRefinement, compositeOp); - return vorGeometry; + return sorGeometry; } -axom::klee::Shape createShape_Vor() +axom::klee::Shape createShape_Sor() { - Point3D vorBase = params.center.empty() ? Point3D {0.0, 0.0, 0.0} + Point3D sorBase = params.center.empty() ? Point3D {0.0, 0.0, 0.0} : Point3D {params.center.data()}; - axom::primal::Vector vorDirection = params.direction.empty() + axom::primal::Vector sorDirection = params.direction.empty() ? primal::Vector3D {0.1, 0.2, 0.4} : primal::Vector3D {params.direction.data()}; const int numIntervals = 5; @@ -720,19 +720,19 @@ axom::klee::Shape createShape_Vor() addScaleOperator(*compositeOp); addTranslateOperator(*compositeOp, -1, -1, 1); - axom::klee::Geometry vorGeometry = - createGeometry_Vor(vorBase, vorDirection, discreteFunction, compositeOp); + axom::klee::Geometry sorGeometry = + createGeometry_Sor(sorBase, sorDirection, discreteFunction, compositeOp); - axom::klee::Shape vorShape("vor", "VOR", {}, {}, vorGeometry); + axom::klee::Shape sorShape("sor", "SOR", {}, {}, sorGeometry); - return vorShape; + return sorShape; } axom::klee::Shape createShape_Cylinder() { - Point3D vorBase = params.center.empty() ? Point3D {0.0, 0.0, 0.0} + Point3D sorBase = params.center.empty() ? Point3D {0.0, 0.0, 0.0} : Point3D {params.center.data()}; - axom::primal::Vector vorDirection = params.direction.empty() + axom::primal::Vector sorDirection = params.direction.empty() ? primal::Vector3D {0.1, 0.2, 0.4} : primal::Vector3D {params.direction.data()}; // discreteFunction are discrete z-r pairs describing the function @@ -749,19 +749,19 @@ axom::klee::Shape createShape_Cylinder() addScaleOperator(*compositeOp); addTranslateOperator(*compositeOp, 1, -1, 1); - axom::klee::Geometry vorGeometry = - createGeometry_Vor(vorBase, vorDirection, discreteFunction, compositeOp); + axom::klee::Geometry sorGeometry = + createGeometry_Sor(sorBase, sorDirection, discreteFunction, compositeOp); - axom::klee::Shape vorShape("cyl", "CYL", {}, {}, vorGeometry); + axom::klee::Shape sorShape("cyl", "CYL", {}, {}, sorGeometry); - return vorShape; + return sorShape; } axom::klee::Shape createShape_Cone() { - Point3D vorBase = params.center.empty() ? Point3D {0.0, 0.0, 0.0} + Point3D sorBase = params.center.empty() ? Point3D {0.0, 0.0, 0.0} : Point3D {params.center.data()}; - axom::primal::Vector vorDirection = params.direction.empty() + axom::primal::Vector sorDirection = params.direction.empty() ? primal::Vector3D {0.1, 0.2, 0.4} : primal::Vector3D {params.direction.data()}; // discreteFunction are discrete z-r pairs describing the function @@ -779,12 +779,12 @@ axom::klee::Shape createShape_Cone() addScaleOperator(*compositeOp); addTranslateOperator(*compositeOp, 1, 1, -1); - axom::klee::Geometry vorGeometry = - createGeometry_Vor(vorBase, vorDirection, discreteFunction, compositeOp); + axom::klee::Geometry sorGeometry = + createGeometry_Sor(sorBase, sorDirection, discreteFunction, compositeOp); - axom::klee::Shape vorShape("cone", "CONE", {}, {}, vorGeometry); + axom::klee::Shape sorShape("cone", "CONE", {}, {}, sorGeometry); - return vorShape; + return sorShape; } axom::klee::Shape createShape_Tet() @@ -1481,9 +1481,9 @@ int main(int argc, char** argv) { shapeSet = createShapeSet(createShape_Cone()); } - else if(params.testShape == "vor") + else if(params.testShape == "sor") { - shapeSet = createShapeSet(createShape_Vor()); + shapeSet = createShapeSet(createShape_Sor()); } else if(params.testShape == "plane") { @@ -1496,7 +1496,7 @@ int main(int argc, char** argv) shapesVec.push_back(createShape_Tet()); shapesVec.push_back(createShape_Hex()); shapesVec.push_back(createShape_Sphere()); - shapesVec.push_back(createShape_Vor()); + shapesVec.push_back(createShape_Sor()); shapesVec.push_back(createShape_Cylinder()); shapesVec.push_back(createShape_Cone()); shapeSet.setShapes(shapesVec); From a765335d13fba6c4c2311f16bd6f59a22a917d52 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Fri, 17 Jan 2025 10:45:27 -0800 Subject: [PATCH 094/100] Dox comment changes and remove an unused method. --- src/axom/quest/DiscreteShape.cpp | 7 +++++++ src/axom/quest/IntersectionShaper.hpp | 27 ++++++++++++--------------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/axom/quest/DiscreteShape.cpp b/src/axom/quest/DiscreteShape.cpp index 8ee86cd2b4..805d60ddb4 100644 --- a/src/axom/quest/DiscreteShape.cpp +++ b/src/axom/quest/DiscreteShape.cpp @@ -571,6 +571,13 @@ void DiscreteShape::createRepresentationOfSOR() }); // Dump discretized octs as a tet mesh + // + // NOTE: mesh_from_discretized_polyline returns a tet mesh. + // IntersectionShaper can handle octahedra, which may be faster + // because it has 8x fewer elements. This method returns the + // discretization as a mesh. But mint doesn't support octs and + // Blueprint support octs only as polyhedra. We can always return + // an array of primal::Octahedron instead of a mesh. axom::mint::Mesh* mesh; axom::quest::mesh_from_discretized_polyline(octs.view(), octCount, diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 642f25598c..0f80dd9881 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -86,24 +86,24 @@ template class TempArrayView { public: - #if defined(AXOM_USE_MFEM) /*! - * \brief Host constructor that accepts the grid function. + * \brief Host constructor that accepts data in an Array. * - * \param gf The grid function that will be accessed/modified by the view. + * \param gf The data that will be accessed/modified by this view. * \param _needResult Whether the data needs to be brought back to the host * from the device. */ - AXOM_HOST TempArrayView(mfem::GridFunction* gf, bool _needResult = true) - { - initialize(gf->GetData(), gf->Size(), _needResult); - } - #endif - AXOM_HOST TempArrayView(axom::Array& gf, bool _needResult = true) { initialize(gf.data(), gf.size(), _needResult); } + /*! + * \brief Host constructor that accepts data in an ArrayView. + * + * \param gf The data that will be accessed/modified by this view. + * \param _needResult Whether the data needs to be brought back to the host + * from the device. + */ AXOM_HOST TempArrayView(axom::ArrayView& gf, bool _needResult = true) { initialize(gf.data(), gf.size(), _needResult); @@ -804,10 +804,8 @@ class IntersectionShaper : public Shaper auto shape_candidates_device_view = shape_candidates_device.view(); // Tetrahedrons from hexes (24 for each hex) - // TODO: Why break hexes into tets if we can hex-tet clip (clip_impl.hpp:502). - // Possibly because ShapeType can also be oct, and we don't do hex-oct clip. - // At least not yet. We could break the oct into 6 tets and do 6 hex-tet clips. - // Isn't that better than breaking the hex into 24 tets and doing 24 oct-tet clips? + // NOTE: Why break hexes into tets if we can hex-tet clip (clip_impl.hpp:502). + // Possibly because ShapeType can also be oct, and we don't currently do hex-oct clip. bool doInitializeTetsFromHexes = false; if(m_tets_from_hexes_device.empty()) { @@ -2242,7 +2240,7 @@ class IntersectionShaper : public Shaper #if defined(__CUDACC__) public: - // These methods should be private, but NVCC complains unless its public. + // These methods should be private, but NVCC complains unless they're public. #endif template @@ -2497,7 +2495,6 @@ class IntersectionShaper : public Shaper private: /// Create and return a new volume fraction grid function for the current mesh - // TODO: change to generic name. Nothing in here is about volume fractions. BTNG. mfem::GridFunction* newVolFracGridFunction() { mfem::Mesh* mesh = getDC()->GetMesh(); From 2c2a370d3c0089d8c2187cd3900ec457c28e4ff1 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Fri, 17 Jan 2025 10:45:52 -0800 Subject: [PATCH 095/100] Avoid unneccesarily re-allocating memory. --- src/axom/quest/IntersectionShaper.hpp | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 0f80dd9881..6a80544075 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -891,16 +891,20 @@ class IntersectionShaper : public Shaper // Overlap volume is the volume of clip(oct,tet) for c2c // or clip(tet,tet) for Pro/E meshes - // TODO: don't reallocate. m_cellCount shouldn't change. - m_overlap_volumes = - axom::Array(m_cellCount, m_cellCount, device_allocator); + if (m_overlap_volumes.empty()) + { + m_overlap_volumes = + axom::Array(m_cellCount, m_cellCount, device_allocator); + } m_overlap_volumes.fill(0.0); // Hex volume is the volume of the hexahedron element - // TODO: Make reallocate and recompute explicit (with a resetMesh or something). - m_hex_volumes = - axom::Array(m_cellCount, m_cellCount, device_allocator); - m_hex_volumes.fill(0.0); + if (m_hex_volumes.empty()) + { + m_hex_volumes = + axom::Array(m_cellCount, m_cellCount, device_allocator); + m_hex_volumes.fill(0.0); + } axom::ArrayView overlap_volumes_device_view = m_overlap_volumes.view(); From 2a9d6f965ae2f65df96149753e8939c222865e91 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Sat, 18 Jan 2025 04:05:09 -0800 Subject: [PATCH 096/100] Add convenience method to get default allocator id for a runtime execution policy. --- src/axom/core/execution/execution_space.hpp | 32 +++++++++++++++++++ src/axom/quest/Shaper.cpp | 28 ---------------- .../quest/examples/quest_shape_in_memory.cpp | 17 ++-------- 3 files changed, 34 insertions(+), 43 deletions(-) diff --git a/src/axom/core/execution/execution_space.hpp b/src/axom/core/execution/execution_space.hpp index e65f63fe1c..f619fc66df 100644 --- a/src/axom/core/execution/execution_space.hpp +++ b/src/axom/core/execution/execution_space.hpp @@ -123,4 +123,36 @@ struct execution_space #include "axom/core/execution/internal/hip_exec.hpp" #endif +namespace axom +{ + +/// \brief Return default allocator id for a runtime policy. +static const std::map s_policyToDefaultAllocatorID { + {axom::runtime_policy::Policy::seq, + axom::execution_space::allocatorID()} +#if defined(AXOM_RUNTIME_POLICY_USE_OPENMP) + , + {axom::runtime_policy::Policy::omp, + axom::execution_space::allocatorID()} +#endif +#if defined(AXOM_RUNTIME_POLICY_USE_CUDA) + , + {axom::runtime_policy::Policy::cuda, + axom::execution_space>::allocatorID()} +#endif +#if defined(AXOM_RUNTIME_POLICY_USE_HIP) + , + {axom::runtime_policy::Policy::hip, + axom::execution_space>::allocatorID()} +#endif +}; + +/// \brief Return default allocator id for a runtime policy. +inline int policyToDefaultAllocatorID(axom::runtime_policy::Policy policy) +{ + return s_policyToDefaultAllocatorID.find(policy)->second; +} + +} // namespace axom + #endif // AXOM_EXECUTIONSPACE_HPP_ diff --git a/src/axom/quest/Shaper.cpp b/src/axom/quest/Shaper.cpp index e955ed1eb0..87d4551c08 100644 --- a/src/axom/quest/Shaper.cpp +++ b/src/axom/quest/Shaper.cpp @@ -16,34 +16,6 @@ #include "axom/fmt.hpp" -/// \brief Return default allocator id for a runtime policy. -static int policyToDefaultAllocatorID(axom::runtime_policy::Policy policy) -{ - if(policy == axom::runtime_policy::Policy::seq) - { - return axom::execution_space::allocatorID(); - } -#if defined(AXOM_RUNTIME_POLICY_USE_OPENMP) - if(policy == axom::runtime_policy::Policy::omp) - { - return axom::execution_space::allocatorID(); - } -#endif -#if defined(AXOM_RUNTIME_POLICY_USE_CUDA) - if(policy == axom::runtime_policy::Policy::cuda) - { - return axom::execution_space>::allocatorID(); - } -#endif -#if defined(AXOM_RUNTIME_POLICY_USE_HIP) - if(policy == axom::runtime_policy::Policy::hip) - { - return axom::execution_space>::allocatorID(); - } -#endif - return axom::INVALID_ALLOCATOR_ID; -} - namespace axom { namespace quest diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index 027309b504..5ae45949bb 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -1431,20 +1431,7 @@ int main(int argc, char** argv) axom::utilities::raii::AnnotationsWrapper annotations_raii_wrapper( params.annotationMode); - // We will use array memory compatible with the specified runtime policy. - int defaultAllocId = axom::execution_space::allocatorID(); -#if defined(AXOM_USE_CUDA) - if(params.policy == RuntimePolicy::cuda) - { - defaultAllocId = axom::execution_space>::allocatorID(); - } -#endif -#if defined(AXOM_USE_HIP) - if(params.policy == RuntimePolicy::hip) - { - defaultAllocId = axom::execution_space>::allocatorID(); - } -#endif + const int allocatorId = axom::policyToDefaultAllocatorID(params.policy); AXOM_ANNOTATE_BEGIN("quest example for shaping primals"); AXOM_ANNOTATE_BEGIN("init"); @@ -1577,7 +1564,7 @@ int main(int argc, char** argv) if(params.useBlueprintSidre() || params.useBlueprintConduit()) { compMeshGrp = ds.getRoot()->createGroup("compMesh"); - compMeshGrp->setDefaultAllocator(defaultAllocId); + compMeshGrp->setDefaultAllocator(allocatorId); createBoxMesh(compMeshGrp); From 8ab007a91e14ce791e6c14c0af0a9d556a8f377d Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Sat, 18 Jan 2025 04:07:06 -0800 Subject: [PATCH 097/100] Let user specify allocator ID for shapers. Users can provide a memory pool to bypass slow CUDA memory allocations/deallocations. --- src/axom/quest/IntersectionShaper.hpp | 32 ++++++++++++------- src/axom/quest/SamplingShaper.hpp | 5 ++- src/axom/quest/Shaper.cpp | 6 ++++ src/axom/quest/Shaper.hpp | 5 +++ .../quest/examples/quest_shape_in_memory.cpp | 3 ++ src/axom/quest/examples/shaping_driver.cpp | 7 ++-- .../quest/tests/quest_intersection_shaper.cpp | 12 +++++-- 7 files changed, 53 insertions(+), 17 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 6a80544075..3a942d5e4b 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -335,12 +335,13 @@ class IntersectionShaper : public Shaper public: #if defined(AXOM_USE_MFEM) /*! - @brief Construct Shaper to operate on an MFEM mesh. + \brief Construct Shaper to operate on an MFEM mesh. */ IntersectionShaper(RuntimePolicy runtimePolicy, + int allocatorId, const klee::ShapeSet& shapeSet, sidre::MFEMSidreDataCollection* dc) - : Shaper(runtimePolicy, shapeSet, dc) + : Shaper(runtimePolicy, allocatorId, shapeSet, dc) { m_free_mat_name = "free"; } @@ -348,26 +349,33 @@ class IntersectionShaper : public Shaper #if defined(AXOM_USE_CONDUIT) /*! - @brief Construct Shaper to operate on a blueprint-formatted mesh + \brief Construct Shaper to operate on a blueprint-formatted mesh stored in a sidre Group. + \param [in] runtimePolicy A value from RuntimePolicy. + The simplest policy is RuntimePolicy::seq, which specifies + running sequentially on the CPU. + \param [in] allocatorID Data allocator ID. Choose something compatible + with \c runtimePolicy. See \c execution_space. */ IntersectionShaper(RuntimePolicy runtimePolicy, + int allocatorId, const klee::ShapeSet& shapeSet, sidre::Group* bpGrp, const std::string& topo = "") - : Shaper(runtimePolicy, shapeSet, bpGrp, topo) + : Shaper(runtimePolicy, allocatorId, shapeSet, bpGrp, topo) , m_free_mat_name("free") { } /*! - @brief Construct Shaper to operate on a blueprint-formatted mesh + \brief Construct Shaper to operate on a blueprint-formatted mesh stored in a Conduit Node. */ IntersectionShaper(RuntimePolicy runtimePolicy, + int allocatorId, const klee::ShapeSet& shapeSet, conduit::Node* bpNode, const std::string& topo = "") - : Shaper(runtimePolicy, shapeSet, bpNode, topo) + : Shaper(runtimePolicy, allocatorId, shapeSet, bpNode, topo) , m_free_mat_name("free") { } #endif @@ -891,7 +899,7 @@ class IntersectionShaper : public Shaper // Overlap volume is the volume of clip(oct,tet) for c2c // or clip(tet,tet) for Pro/E meshes - if (m_overlap_volumes.empty()) + if(m_overlap_volumes.empty()) { m_overlap_volumes = axom::Array(m_cellCount, m_cellCount, device_allocator); @@ -899,7 +907,7 @@ class IntersectionShaper : public Shaper m_overlap_volumes.fill(0.0); // Hex volume is the volume of the hexahedron element - if (m_hex_volumes.empty()) + if(m_hex_volumes.empty()) { m_hex_volumes = axom::Array(m_cellCount, m_cellCount, device_allocator); @@ -2143,7 +2151,7 @@ class IntersectionShaper : public Shaper return has; } - /*! @brief Get a scalar double-type field data from the mesh, + /*! \brief Get a scalar double-type field data from the mesh, "fields/fieldName/values", creating it if it doesn't exist. Also add the corresponding entry in the blueprint field @@ -2319,7 +2327,7 @@ class IntersectionShaper : public Shaper }); // end of loop to initialize hexahedral elements and bounding boxes } - //!@brief Set shape vertices to zero of within threshold. + //!\brief Set shape vertices to zero of within threshold. template void snapShapeVerticesToZero(axom::Array& shapes, axom::IndexType shapeCount, @@ -2529,10 +2537,10 @@ class IntersectionShaper : public Shaper double m_revolvedVolume {DEFAULT_REVOLVED_VOLUME}; std::string m_free_mat_name; - //! @brief Volumes of cells in the computational mesh. + //! \brief Volumes of cells in the computational mesh. axom::Array m_hex_volumes; - //! @brief Overlap volumes of cells in the computational mesh for the last shape. + //! \brief Overlap volumes of cells in the computational mesh for the last shape. axom::Array m_overlap_volumes; double m_vertexWeldThreshold {1.e-10}; diff --git a/src/axom/quest/SamplingShaper.hpp b/src/axom/quest/SamplingShaper.hpp index bef2680e72..63e2c0bdad 100644 --- a/src/axom/quest/SamplingShaper.hpp +++ b/src/axom/quest/SamplingShaper.hpp @@ -318,7 +318,10 @@ class SamplingShaper : public Shaper public: SamplingShaper(const klee::ShapeSet& shapeSet, sidre::MFEMSidreDataCollection* dc) - : Shaper(Shaper::RuntimePolicy::seq, shapeSet, dc) + : Shaper(Shaper::RuntimePolicy::seq, + axom::policyToDefaultAllocatorID(axom::runtime_policy::Policy::seq), + shapeSet, + dc) { } ~SamplingShaper() diff --git a/src/axom/quest/Shaper.cpp b/src/axom/quest/Shaper.cpp index 87d4551c08..b3de404cd4 100644 --- a/src/axom/quest/Shaper.cpp +++ b/src/axom/quest/Shaper.cpp @@ -29,9 +29,11 @@ constexpr double Shaper::DEFAULT_VERTEX_WELD_THRESHOLD; #if defined(AXOM_USE_MFEM) Shaper::Shaper(RuntimePolicy execPolicy, + int allocatorId, const klee::ShapeSet& shapeSet, sidre::MFEMSidreDataCollection* dc) : m_execPolicy(execPolicy) + , m_allocatorId(allocatorId) , m_shapeSet(shapeSet) , m_dc(dc) #if defined(AXOM_USE_CONDUIT) @@ -51,10 +53,12 @@ Shaper::Shaper(RuntimePolicy execPolicy, #endif Shaper::Shaper(RuntimePolicy execPolicy, + int allocatorId, const klee::ShapeSet& shapeSet, sidre::Group* bpGrp, const std::string& topo) : m_execPolicy(execPolicy) + , m_allocatorId(allocatorId) , m_shapeSet(shapeSet) #if defined(AXOM_USE_CONDUIT) , m_bpGrp(bpGrp) @@ -82,10 +86,12 @@ Shaper::Shaper(RuntimePolicy execPolicy, } Shaper::Shaper(RuntimePolicy execPolicy, + int allocatorId, const klee::ShapeSet& shapeSet, conduit::Node* bpNode, const std::string& topo) : m_execPolicy(execPolicy) + , m_allocatorId(allocatorId) , m_shapeSet(shapeSet) #if defined(AXOM_USE_CONDUIT) , m_bpGrp(nullptr) diff --git a/src/axom/quest/Shaper.hpp b/src/axom/quest/Shaper.hpp index 6ac6fa4237..88ed87cb79 100644 --- a/src/axom/quest/Shaper.hpp +++ b/src/axom/quest/Shaper.hpp @@ -56,6 +56,7 @@ class Shaper @brief Construct Shaper to operate on an MFEM mesh. */ Shaper(RuntimePolicy execPolicy, + int allocatorId, const klee::ShapeSet& shapeSet, sidre::MFEMSidreDataCollection* dc); #endif @@ -65,6 +66,7 @@ class Shaper stored in a sidre Group. */ Shaper(RuntimePolicy execPolicy, + int allocatorId, const klee::ShapeSet& shapeSet, sidre::Group* bpMesh, const std::string& topo = ""); @@ -79,6 +81,7 @@ class Shaper policy. Any needed-but-missing space would lead to an exception. */ Shaper(RuntimePolicy execPolicy, + int allocatorId, const klee::ShapeSet& shapeSet, conduit::Node* bpNode, const std::string& topo = ""); @@ -212,6 +215,8 @@ class Shaper protected: RuntimePolicy m_execPolicy; + int m_allocatorId; + sidre::DataStore m_dataStore; const klee::ShapeSet& m_shapeSet; diff --git a/src/axom/quest/examples/quest_shape_in_memory.cpp b/src/axom/quest/examples/quest_shape_in_memory.cpp index 5ae45949bb..4fda13f9f7 100644 --- a/src/axom/quest/examples/quest_shape_in_memory.cpp +++ b/src/axom/quest/examples/quest_shape_in_memory.cpp @@ -1613,12 +1613,14 @@ int main(int argc, char** argv) if(params.useBlueprintSidre()) { shaper = std::make_shared(params.policy, + allocatorId, shapeSet, compMeshGrp); } if(params.useBlueprintConduit()) { shaper = std::make_shared(params.policy, + allocatorId, shapeSet, compMeshNode.get()); } @@ -1626,6 +1628,7 @@ int main(int argc, char** argv) if(params.useMfem()) { shaper = std::make_shared(params.policy, + allocatorId, shapeSet, shapingDC.get()); } diff --git a/src/axom/quest/examples/shaping_driver.cpp b/src/axom/quest/examples/shaping_driver.cpp index 766b0c4bc9..85ff4a7d23 100644 --- a/src/axom/quest/examples/shaping_driver.cpp +++ b/src/axom/quest/examples/shaping_driver.cpp @@ -567,8 +567,11 @@ int main(int argc, char** argv) break; case ShapingMethod::Intersection: #if defined(AXOM_USE_RAJA) && defined(AXOM_USE_UMPIRE) - shaper = - new quest::IntersectionShaper(params.policy, params.shapeSet, &shapingDC); + shaper = new quest::IntersectionShaper( + params.policy, + axom::policyToDefaultAllocatorID(axom::runtime_policy::Policy::seq), + params.shapeSet, + &shapingDC); #else SLIC_ERROR( "IntersectionShaper requires Axom to be configured with Umpire."); diff --git a/src/axom/quest/tests/quest_intersection_shaper.cpp b/src/axom/quest/tests/quest_intersection_shaper.cpp index f6cf35b63d..f7d474d962 100644 --- a/src/axom/quest/tests/quest_intersection_shaper.cpp +++ b/src/axom/quest/tests/quest_intersection_shaper.cpp @@ -324,7 +324,11 @@ void replacementRuleTest(const std::string &shapeFile, // the C2C reader. dc.SetComm(MPI_COMM_WORLD); #endif - quest::IntersectionShaper shaper(policy, shapeSet, &dc); + quest::IntersectionShaper shaper( + policy, + axom::policyToDefaultAllocatorID(axom::runtime_policy::Policy::seq), + shapeSet, + &dc); shaper.setLevel(refinementLevel); // Borrowed from shaping_driver. @@ -459,7 +463,11 @@ void IntersectionWithErrorTolerances(const std::string &filebase, // the C2C reader. dc.SetComm(MPI_COMM_WORLD); #endif - quest::IntersectionShaper shaper(policy, shapeSet, &dc); + quest::IntersectionShaper shaper( + policy, + axom::policyToDefaultAllocatorID(axom::runtime_policy::Policy::seq), + shapeSet, + &dc); shaper.setLevel(refinementLevel); shaper.setPercentError(targetPercentError); shaper.setRefinementType(quest::DiscreteShape::RefinementDynamic); From 5abc0af3c5f57a7a855a7ba1adeb58ea97a5a040 Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Wed, 22 Jan 2025 09:13:49 -0800 Subject: [PATCH 098/100] Move one-time initialization into setMeshDependentData(). --- src/axom/quest/IntersectionShaper.hpp | 277 +++++++++++++++----------- src/axom/quest/Shaper.cpp | 14 +- 2 files changed, 170 insertions(+), 121 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index 3a942d5e4b..bcb9865ddc 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -380,6 +380,124 @@ class IntersectionShaper : public Shaper { } #endif + void setMeshDependentData() + { + AXOM_ANNOTATE_SCOPE("IntersectionShaper::setMeshDependentData"); + switch(m_execPolicy) + { + case RuntimePolicy::seq: + setMeshDependentDataImpl(); + break; + #if defined(AXOM_RUNTIME_POLICY_USE_OPENMP) + case RuntimePolicy::omp: + setMeshDependentDataImpl(); + break; + #endif + #if defined(AXOM_RUNTIME_POLICY_USE_CUDA) + case RuntimePolicy::cuda: + setMeshDependentDataImpl(); + break; + #endif + #if defined(AXOM_RUNTIME_POLICY_USE_HIP) + case RuntimePolicy::hip: + setMeshDependentDataImpl(); + break; + #endif + default: + SLIC_ERROR("Axom Internal error: Unhandled execution policy."); + } + } + + template + void setMeshDependentDataImpl() + { + constexpr int NUM_TETS_PER_HEX = 24; + + AXOM_ANNOTATE_BEGIN("allocate m_tets_from_hexes_device"); + m_tets_from_hexes_device = + axom::Array(ArrayOptions::Uninitialized(), + m_cellCount * NUM_TETS_PER_HEX, + m_cellCount * NUM_TETS_PER_HEX, + m_allocatorId); + AXOM_ANNOTATE_END("allocate m_tets_from_hexes_device"); + + populateHexesFromMesh(); + auto hexesView = m_hexes.view(); + + AXOM_ANNOTATE_BEGIN("allocate m_hex_volumes"); + m_hex_volumes = axom::Array(m_cellCount, m_cellCount, m_allocatorId); + AXOM_ANNOTATE_END("allocate m_hex_volumes"); + m_hex_volumes.fill(0.0); + + SLIC_INFO( + axom::fmt::format("{:-^80}", " Calculating hexahedron element volume ")); + auto hexVolumesView = m_hex_volumes.view(); + AXOM_ANNOTATE_BEGIN("hex_volume"); + axom::for_all( + m_cellCount, + AXOM_LAMBDA(axom::IndexType i) { + hexVolumesView[i] = hexesView[i].volume(); + }); + AXOM_ANNOTATE_END("hex_volume"); + + SLIC_INFO(axom::fmt::format( + "{:-^80}", + " Decomposing each hexahedron element into 24 tetrahedrons ")); + + AXOM_ANNOTATE_BEGIN("populate m_hex_bbs"); + m_hex_bbs = + axom::Array(m_cellCount, m_cellCount, m_allocatorId); + + // Get bounding boxes for hexahedral elements + axom::ArrayView hexBbsView = m_hex_bbs.view(); + axom::for_all( + m_cellCount, + AXOM_LAMBDA(axom::IndexType i) { + hexBbsView[i] = primal::compute_bounding_box(hexesView[i]); + }); // end of loop to initialize hexahedral elements and bounding boxes + AXOM_ANNOTATE_END("populate m_hex_bbs"); + + using TetHexArray = axom::StackArray; + + auto tetsFromHexesView = m_tets_from_hexes_device.view(); + AXOM_ANNOTATE_BEGIN("init_tets"); + axom::for_all( + m_cellCount, + AXOM_LAMBDA(axom::IndexType i) { + TetHexArray cur_tets; + hexesView[i].triangulate(cur_tets); + + for(int j = 0; j < NUM_TETS_PER_HEX; j++) + { + tetsFromHexesView[i * NUM_TETS_PER_HEX + j] = cur_tets[j]; + } + }); + AXOM_ANNOTATE_END("init_tets"); + + AXOM_ANNOTATE_BEGIN("allocate m_overlap_volumes"); + m_overlap_volumes = + axom::Array(m_cellCount, m_cellCount, m_allocatorId); + AXOM_ANNOTATE_END("allocate m_overlap_volumes"); + + // Non-essential output + auto overlapVolumesView = m_overlap_volumes.view(); + using REDUCE_POL = typename axom::execution_space::reduce_policy; + RAJA::ReduceSum totalOverlap(0); + RAJA::ReduceSum totalHex(0); + axom::for_all( + m_cellCount, + AXOM_LAMBDA(axom::IndexType i) { + totalOverlap += overlapVolumesView[i]; + totalHex += hexVolumesView[i]; + }); + SLIC_INFO(axom::fmt::format(axom::utilities::locale(), + "Total overlap volume with shape is {:.3Lf}", + this->allReduceSum(totalOverlap))); + SLIC_INFO(axom::fmt::format(axom::utilities::locale(), + "Total mesh volume is {:.3Lf}", + this->allReduceSum(totalHex))); + } + //@{ //! @name Functions to get and set shaping parameters related to intersection; supplements parameters in base class @@ -453,7 +571,7 @@ class IntersectionShaper : public Shaper { const int host_allocator = axom::execution_space::allocatorID(); - const int device_allocator = axom::execution_space::allocatorID(); + const int device_allocator = m_allocatorId; // Number of tets in mesh m_tetcount = m_surfaceMesh->getNumberOfCells(); @@ -703,13 +821,18 @@ class IntersectionShaper : public Shaper { AXOM_ANNOTATE_SCOPE("IntersectionShaper::runShapeQueryImpl"); - // No need for shape, because it has been converted into m_tets or m_octs, - // which is what the parameter shapes is. + // No need for parameter shape, because it has been converted into + // m_tets or m_octs, which is what the parameter shapes is. AXOM_UNUSED_VAR(shape); + if(m_hexes.empty()) + { + setMeshDependentData(); + } + const int host_allocator = axom::execution_space::allocatorID(); - const int device_allocator = axom::execution_space::allocatorID(); + const int device_allocator = m_allocatorId; constexpr int NUM_TETS_PER_HEX = 24; constexpr double ZERO_THRESHOLD = 1.e-10; @@ -741,29 +864,21 @@ class IntersectionShaper : public Shaper SLIC_INFO(axom::fmt::format("{:-^80}", " Querying the BVH tree ")); - if(m_hexes.empty()) - { - // m_hexes depend only on mesh so should not change once computed. - populateHexesFromMesh(); - } axom::ArrayView hexes_device_view = m_hexes.view(); - if(m_hex_bbs.empty()) - { - // Compute m_hex_bbs only once. We assume that the mesh doesn't change - // once set. - m_hex_bbs = - axom::Array(m_cellCount, m_cellCount, device_allocator); + // Compute m_hex_bbs only once. We assume that the mesh doesn't change + // once set. + m_hex_bbs = + axom::Array(m_cellCount, m_cellCount, device_allocator); - // Get bounding boxes for hexahedral elements - axom::ArrayView hex_bbs_device_view = m_hex_bbs.view(); - axom::for_all( - m_cellCount, - AXOM_LAMBDA(axom::IndexType i) { - hex_bbs_device_view[i] = - primal::compute_bounding_box(hexes_device_view[i]); - }); // end of loop to initialize hexahedral elements and bounding boxes - } + // Get bounding boxes for hexahedral elements + axom::ArrayView hex_bbs_device_view = m_hex_bbs.view(); + axom::for_all( + m_cellCount, + AXOM_LAMBDA(axom::IndexType i) { + hex_bbs_device_view[i] = + primal::compute_bounding_box(hexes_device_view[i]); + }); // end of loop to initialize hexahedral elements and bounding boxes // Set shape components to zero if within threshold snapShapeVerticesToZero(shapes, @@ -775,99 +890,71 @@ class IntersectionShaper : public Shaper "{:-^80}", " Finding shape candidates for each hexahedral element ")); - axom::ArrayView hex_bbs_device_view = m_hex_bbs.view(); - axom::Array offsets(m_cellCount, m_cellCount, device_allocator); axom::Array counts(m_cellCount, m_cellCount, device_allocator); axom::Array candidates; + AXOM_ANNOTATE_BEGIN("bvh.findBoundingBoxes"); bvh.findBoundingBoxes(offsets, counts, candidates, m_cellCount, hex_bbs_device_view); + AXOM_ANNOTATE_END("bvh.findBoundingBoxes"); // Get the total number of candidates using REDUCE_POL = typename axom::execution_space::reduce_policy; using ATOMIC_POL = typename axom::execution_space::atomic_policy; const auto counts_device_view = counts.view(); + AXOM_ANNOTATE_BEGIN("populate totalCandidates"); RAJA::ReduceSum totalCandidates(0); axom::for_all( m_cellCount, AXOM_LAMBDA(axom::IndexType i) { totalCandidates += counts_device_view[i]; }); + AXOM_ANNOTATE_END("populate totalCandidates"); + AXOM_ANNOTATE_BEGIN("allocate scratch space"); // Initialize hexahedron indices and shape candidates + AXOM_ANNOTATE_BEGIN("allocate hex_indices"); axom::Array hex_indices_device( totalCandidates.get() * NUM_TETS_PER_HEX, totalCandidates.get() * NUM_TETS_PER_HEX, device_allocator); + AXOM_ANNOTATE_END("allocate hex_indices"); auto hex_indices_device_view = hex_indices_device.view(); + AXOM_ANNOTATE_BEGIN("allocate shape_candidates"); axom::Array shape_candidates_device( totalCandidates.get() * NUM_TETS_PER_HEX, totalCandidates.get() * NUM_TETS_PER_HEX, device_allocator); + AXOM_ANNOTATE_END("allocate shape_candidates"); auto shape_candidates_device_view = shape_candidates_device.view(); // Tetrahedrons from hexes (24 for each hex) - // NOTE: Why break hexes into tets if we can hex-tet clip (clip_impl.hpp:502). - // Possibly because ShapeType can also be oct, and we don't currently do hex-oct clip. - bool doInitializeTetsFromHexes = false; - if(m_tets_from_hexes_device.empty()) - { - m_tets_from_hexes_device = - axom::Array(m_cellCount * NUM_TETS_PER_HEX, - m_cellCount * NUM_TETS_PER_HEX, - device_allocator); - doInitializeTetsFromHexes = true; - } - else - { - SLIC_ASSERT(m_tets_from_hexes_device.size() == - m_cellCount * NUM_TETS_PER_HEX); - } axom::ArrayView tets_from_hexes_device_view = m_tets_from_hexes_device.view(); // Index into 'tets' + AXOM_ANNOTATE_BEGIN("allocate tet_indices_device"); axom::Array tet_indices_device( totalCandidates.get() * NUM_TETS_PER_HEX, totalCandidates.get() * NUM_TETS_PER_HEX, device_allocator); + AXOM_ANNOTATE_END("allocate tet_indices_device"); auto tet_indices_device_view = tet_indices_device.view(); + AXOM_ANNOTATE_END("allocate scratch space"); // New total number of candidates after omitting degenerate shapes + AXOM_ANNOTATE_BEGIN("newTotalCandidates memory"); axom::Array newTotalCandidates_host(1, 1, host_allocator); newTotalCandidates_host[0] = 0; axom::Array newTotalCandidates_device = axom::Array(newTotalCandidates_host, device_allocator); auto newTotalCandidates_device_view = newTotalCandidates_device.view(); - - if(doInitializeTetsFromHexes) - { - SLIC_INFO(axom::fmt::format( - "{:-^80}", - " Decomposing each hexahedron element into 24 tetrahedrons ")); - - using TetHexArray = axom::StackArray; - - { - AXOM_ANNOTATE_SCOPE("init_tets"); - axom::for_all( - m_cellCount, - AXOM_LAMBDA(axom::IndexType i) { - TetHexArray cur_tets; - hexes_device_view[i].triangulate(cur_tets); - - for(int j = 0; j < NUM_TETS_PER_HEX; j++) - { - tets_from_hexes_device_view[i * NUM_TETS_PER_HEX + j] = cur_tets[j]; - } - }); - } - } + AXOM_ANNOTATE_END("newTotalCandidates memory"); SLIC_INFO( axom::fmt::format("{:-^80}", @@ -899,36 +986,11 @@ class IntersectionShaper : public Shaper // Overlap volume is the volume of clip(oct,tet) for c2c // or clip(tet,tet) for Pro/E meshes - if(m_overlap_volumes.empty()) - { - m_overlap_volumes = - axom::Array(m_cellCount, m_cellCount, device_allocator); - } m_overlap_volumes.fill(0.0); - // Hex volume is the volume of the hexahedron element - if(m_hex_volumes.empty()) - { - m_hex_volumes = - axom::Array(m_cellCount, m_cellCount, device_allocator); - m_hex_volumes.fill(0.0); - } - axom::ArrayView overlap_volumes_device_view = m_overlap_volumes.view(); - axom::ArrayView hex_volumes_device_view = m_hex_volumes.view(); - - SLIC_INFO( - axom::fmt::format("{:-^80}", " Calculating hexahedron element volume ")); - { - AXOM_ANNOTATE_SCOPE("hex_volume"); - axom::for_all( - m_cellCount, - AXOM_LAMBDA(axom::IndexType i) { - hex_volumes_device_view[i] = hexes_device_view[i].volume(); - }); - } SLIC_INFO(axom::fmt::format( "{:-^80}", " Calculating element overlap volume from each tet-shape pair ")); @@ -937,7 +999,7 @@ class IntersectionShaper : public Shaper constexpr bool tryFixOrientation = true; { - AXOM_ANNOTATE_SCOPE("IntersectionShaper::clipLoop"); + AXOM_ANNOTATE_SCOPE("clipLoop"); // Copy calculated total back to host axom::Array newTotalCandidates_calc_host = axom::Array(newTotalCandidates_device, host_allocator); @@ -966,23 +1028,6 @@ class IntersectionShaper : public Shaper } }); } - - RAJA::ReduceSum totalOverlap(0); - RAJA::ReduceSum totalHex(0); - - axom::for_all( - m_cellCount, - AXOM_LAMBDA(axom::IndexType i) { - totalOverlap += overlap_volumes_device_view[i]; - totalHex += hex_volumes_device_view[i]; - }); - - SLIC_INFO(axom::fmt::format(axom::utilities::locale(), - "Total overlap volume with shape is {:.3Lf}", - this->allReduceSum(totalOverlap))); - SLIC_INFO(axom::fmt::format(axom::utilities::locale(), - "Total mesh volume is {:.3Lf}", - this->allReduceSum(totalHex))); } // end of runShapeQueryImpl() function // These methods are private in support of replacement rules. @@ -1168,10 +1213,8 @@ class IntersectionShaper : public Shaper " IntersectionShaper::applyReplacementRules."); // Allocate some memory for the replacement rule data arrays. - int execSpaceAllocatorID = axom::execution_space::allocatorID(); - - Array vf_subtract_array(dataSize, dataSize, execSpaceAllocatorID); - Array vf_writable_array(dataSize, dataSize, execSpaceAllocatorID); + Array vf_subtract_array(dataSize, dataSize, m_allocatorId); + Array vf_writable_array(dataSize, dataSize, m_allocatorId); ArrayView vf_subtract(vf_subtract_array); ArrayView vf_writable(vf_writable_array); @@ -2258,11 +2301,11 @@ class IntersectionShaper : public Shaper template void populateHexesFromMesh() { - using XS = axom::execution_space; + AXOM_ANNOTATE_SCOPE("populateHexesFromMesh"); constexpr int NUM_VERTS_PER_HEX = 8; constexpr int NUM_COMPS_PER_VERT = 3; - const int allocId = XS::allocatorID(); + const int allocId = m_allocatorId; axom::Array vertCoords( m_cellCount * NUM_VERTS_PER_HEX * NUM_COMPS_PER_VERT, @@ -2284,7 +2327,6 @@ class IntersectionShaper : public Shaper auto vertCoords_device_view = vertCoords.view(); - SLIC_ASSERT(m_hexes.empty()); // No need to repeat this work. m_hexes = axom::Array(m_cellCount, m_cellCount, allocId); axom::ArrayView hexes_device_view = m_hexes.view(); constexpr double ZERO_THRESHOLD = 1.e-10; @@ -2333,6 +2375,7 @@ class IntersectionShaper : public Shaper axom::IndexType shapeCount, double zeroThreshold) { + AXOM_ANNOTATE_SCOPE("snapShapeVerticesToZero"); axom::ArrayView shapesView = shapes.view(); axom::for_all( shapeCount, @@ -2363,7 +2406,7 @@ class IntersectionShaper : public Shaper { using XS = axom::execution_space; - const int allocId = XS::allocatorID(); + const int allocId = m_allocatorId; // Initialize vertices from blueprint mesh and // set each shape volume fraction to 1 diff --git a/src/axom/quest/Shaper.cpp b/src/axom/quest/Shaper.cpp index b3de404cd4..af5e190d75 100644 --- a/src/axom/quest/Shaper.cpp +++ b/src/axom/quest/Shaper.cpp @@ -33,7 +33,9 @@ Shaper::Shaper(RuntimePolicy execPolicy, const klee::ShapeSet& shapeSet, sidre::MFEMSidreDataCollection* dc) : m_execPolicy(execPolicy) - , m_allocatorId(allocatorId) + , m_allocatorId(allocatorId != axom::INVALID_ALLOCATOR_ID + ? allocatorId + : axom::policyToDefaultAllocatorID(execPolicy)) , m_shapeSet(shapeSet) , m_dc(dc) #if defined(AXOM_USE_CONDUIT) @@ -58,7 +60,9 @@ Shaper::Shaper(RuntimePolicy execPolicy, sidre::Group* bpGrp, const std::string& topo) : m_execPolicy(execPolicy) - , m_allocatorId(allocatorId) + , m_allocatorId(allocatorId != axom::INVALID_ALLOCATOR_ID + ? allocatorId + : axom::policyToDefaultAllocatorID(execPolicy)) , m_shapeSet(shapeSet) #if defined(AXOM_USE_CONDUIT) , m_bpGrp(bpGrp) @@ -91,7 +95,9 @@ Shaper::Shaper(RuntimePolicy execPolicy, conduit::Node* bpNode, const std::string& topo) : m_execPolicy(execPolicy) - , m_allocatorId(allocatorId) + , m_allocatorId(allocatorId != axom::INVALID_ALLOCATOR_ID + ? allocatorId + : axom::policyToDefaultAllocatorID(execPolicy)) , m_shapeSet(shapeSet) #if defined(AXOM_USE_CONDUIT) , m_bpGrp(nullptr) @@ -104,7 +110,7 @@ Shaper::Shaper(RuntimePolicy execPolicy, { AXOM_ANNOTATE_SCOPE("Shaper::Shaper_Node"); m_bpGrp = m_ds.getRoot()->createGroup("internalGrp"); - m_bpGrp->setDefaultAllocator(policyToDefaultAllocatorID(m_execPolicy)); + m_bpGrp->setDefaultAllocator(m_allocatorId); m_bpGrp->importConduitTreeExternal(*bpNode); From 5b53a7bb7a8787d37a36dfba0839a0022afe2d6d Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Wed, 22 Jan 2025 10:41:33 -0800 Subject: [PATCH 099/100] Make a member variable local. --- src/axom/quest/IntersectionShaper.hpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/axom/quest/IntersectionShaper.hpp b/src/axom/quest/IntersectionShaper.hpp index bcb9865ddc..510c788a7e 100644 --- a/src/axom/quest/IntersectionShaper.hpp +++ b/src/axom/quest/IntersectionShaper.hpp @@ -821,6 +821,7 @@ class IntersectionShaper : public Shaper { AXOM_ANNOTATE_SCOPE("IntersectionShaper::runShapeQueryImpl"); + // No need for parameter shape, because it has been converted into // m_tets or m_octs, which is what the parameter shapes is. AXOM_UNUSED_VAR(shape); @@ -842,13 +843,11 @@ class IntersectionShaper : public Shaper // Generate the BVH tree over the shapes // Axis-aligned bounding boxes - // Does m_aabbs need to be a member? It's only used here. BTNG. - m_aabbs = - axom::Array(shape_count, shape_count, device_allocator); + axom::Array aabbs(shape_count, shape_count, device_allocator); axom::ArrayView shapes_device_view = shapes.view(); - axom::ArrayView aabbs_device_view = m_aabbs.view(); + axom::ArrayView aabbs_device_view = aabbs.view(); // Get the bounding boxes for the shapes axom::for_all( @@ -2594,7 +2593,6 @@ class IntersectionShaper : public Shaper axom::Array m_octs; axom::Array m_tets; - axom::Array m_aabbs; axom::Array m_hexes; axom::Array m_hex_bbs; axom::Array m_tets_from_hexes_device; From 6bd51f85363b53f71dd5a70c8189c37e84c219ab Mon Sep 17 00:00:00 2001 From: "Brian T.N. Gunney" Date: Wed, 22 Jan 2025 12:34:35 -0800 Subject: [PATCH 100/100] Replace bad allocator id with auto. --- src/axom/quest/tests/quest_intersection_shaper.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/axom/quest/tests/quest_intersection_shaper.cpp b/src/axom/quest/tests/quest_intersection_shaper.cpp index 670e152249..5bd7687727 100644 --- a/src/axom/quest/tests/quest_intersection_shaper.cpp +++ b/src/axom/quest/tests/quest_intersection_shaper.cpp @@ -326,7 +326,7 @@ void replacementRuleTest(const std::string &shapeFile, #endif quest::IntersectionShaper shaper( policy, - axom::policyToDefaultAllocatorID(axom::runtime_policy::Policy::seq), + axom::INVALID_ALLOCATOR_ID, shapeSet, &dc); shaper.setLevel(refinementLevel); @@ -465,7 +465,7 @@ void IntersectionWithErrorTolerances(const std::string &filebase, #endif quest::IntersectionShaper shaper( policy, - axom::policyToDefaultAllocatorID(axom::runtime_policy::Policy::seq), + axom::INVALID_ALLOCATOR_ID, shapeSet, &dc); shaper.setLevel(refinementLevel);