From 40e409fb42d96aaacd79e0b12d6fcaf92117ccb9 Mon Sep 17 00:00:00 2001 From: psakiev Date: Wed, 3 May 2023 16:55:41 -0600 Subject: [PATCH 1/7] Example usage of FieldManager This PR demonstrates how a Kernel can be converted to use a FieldManager. The main goal is to move the field registration away from higher level abstractions like realm and equationsystem, to lower ones (algs and kernels). This will essentially eliminate the need for `RegisterNodalFields` type calls, and will eliminate a range of bugs that can come in from not registering fields in the right place. Having kernels and algs register the fields they need will also reduce the amount of fixturing needed for unit tests since constructing the object will register the fields for you. You can then finalize the mesh and populate the fields with initial values. (Not demonstrated in this PR). --- include/FieldManager.h | 25 ++++++++++---- include/node_kernels/TKESSTNodeKernel.h | 12 +++---- src/FieldManager.C | 4 +-- src/TurbKineticEnergyEquationSystem.C | 3 +- src/node_kernels/TKESSTNodeKernel.C | 34 ++++++++++--------- .../node_kernels/UnitTestSSTNodeKernel.C | 8 +++-- 6 files changed, 50 insertions(+), 36 deletions(-) diff --git a/include/FieldManager.h b/include/FieldManager.h index d84572185..8777d21cf 100644 --- a/include/FieldManager.h +++ b/include/FieldManager.h @@ -14,7 +14,6 @@ #include #include "stk_mesh/base/GetNgpField.hpp" #include -#include namespace stk { namespace mesh { @@ -62,12 +61,12 @@ class FieldManager T* register_field( const std::string& name, const stk::mesh::PartVector& parts, - const void* init_val = nullptr, + const void* initVal = nullptr, stk::mesh::FieldState state = stk::mesh::FieldState::StateNone) { const int numStates = 0; const int numComponents = 0; - register_field(name, parts, numStates, numComponents, init_val); + register_field(name, parts, numStates, numComponents, initVal); return get_field_ptr(name, state); } @@ -84,10 +83,10 @@ class FieldManager const stk::mesh::PartVector& parts, const int numStates, const int numComponents, - const void* init_val = nullptr, + const void* initVal = nullptr, stk::mesh::FieldState state = stk::mesh::FieldState::StateNone) { - register_field(name, parts, numStates, numComponents, init_val); + register_field(name, parts, numStates, numComponents, initVal); return get_field_ptr(name, state); } @@ -116,7 +115,7 @@ class FieldManager /// would otherwise be defined in the field Registry class. /// /// If numStates = 0 then the number of states comes from the - /// field Registry. Same for numComponents = 0 and init_val = nullptr. + /// field Registry. Same for numComponents = 0 and initVal = nullptr. /// /// This is useful for dynamic fields that depend on the input /// options to define the number of states or number of components since the @@ -128,7 +127,19 @@ class FieldManager const stk::mesh::PartVector& parts, const int numStates = 0, const int numComponents = 0, - const void* init_val = nullptr); + const void* initVal = nullptr) const; + + FieldPointerTypes register_field( + const std::string& name, + stk::mesh::Part* part, + const int numStates = 0, + const int numComponents = 0, + const void* initVal = nullptr) const + { + stk::mesh::PartVector parts; + parts.push_back(part); + register_field(name, parts, numStates, numComponents, initVal); + } /// Given the named field that has already been registered on the CPU /// return the GPU version of the same field. diff --git a/include/node_kernels/TKESSTNodeKernel.h b/include/node_kernels/TKESSTNodeKernel.h index c85f20e04..8ce5dc9d6 100644 --- a/include/node_kernels/TKESSTNodeKernel.h +++ b/include/node_kernels/TKESSTNodeKernel.h @@ -10,12 +10,14 @@ #ifndef TKESSTNODEKERNEL_H #define TKESSTNODEKERNEL_H +#include "LinearSystem.h" #include "node_kernels/NodeKernel.h" #include "FieldTypeDef.h" #include "stk_mesh/base/BulkData.hpp" #include "stk_mesh/base/Ngp.hpp" #include "stk_mesh/base/NgpField.hpp" #include "stk_mesh/base/Types.hpp" +#include namespace sierra { namespace nalu { @@ -25,7 +27,8 @@ class Realm; class TKESSTNodeKernel : public NGPNodeKernel { public: - TKESSTNodeKernel(const stk::mesh::MetaData&); + TKESSTNodeKernel( + const stk::mesh::MetaData&, const FieldManager&, stk::mesh::Part* part); TKESSTNodeKernel() = delete; @@ -48,13 +51,6 @@ class TKESSTNodeKernel : public NGPNodeKernel stk::mesh::NgpField dudx_; stk::mesh::NgpField dualNodalVolume_; - unsigned tkeID_{stk::mesh::InvalidOrdinal}; - unsigned sdrID_{stk::mesh::InvalidOrdinal}; - unsigned densityID_{stk::mesh::InvalidOrdinal}; - unsigned tviscID_{stk::mesh::InvalidOrdinal}; - unsigned dudxID_{stk::mesh::InvalidOrdinal}; - unsigned dualNodalVolumeID_{stk::mesh::InvalidOrdinal}; - NodeKernelTraits::DblType betaStar_; NodeKernelTraits::DblType tkeProdLimitRatio_; NodeKernelTraits::DblType tkeAmb_; diff --git a/src/FieldManager.C b/src/FieldManager.C index 1e30e7263..b778a2554 100644 --- a/src/FieldManager.C +++ b/src/FieldManager.C @@ -36,7 +36,7 @@ FieldManager::register_field( const stk::mesh::PartVector& parts, const int numStates, const int numComponents, - const void* init_val) + const void* initVal) const { auto definition = FieldRegistry::query(numDimensions_, numStates_, name); @@ -48,7 +48,7 @@ FieldManager::register_field( const int num_components = numComponents ? numComponents : def.num_components; - const val_type* init = static_cast(init_val); + const val_type* init = static_cast(initVal); auto* id = &(meta_.declare_field(def.rank, name, num_states)); for (auto&& part : parts) { stk::mesh::put_field_on_mesh(*id, *part, num_components, init); diff --git a/src/TurbKineticEnergyEquationSystem.C b/src/TurbKineticEnergyEquationSystem.C index 0f7d988af..54cde3239 100644 --- a/src/TurbKineticEnergyEquationSystem.C +++ b/src/TurbKineticEnergyEquationSystem.C @@ -303,7 +303,8 @@ TurbKineticEnergyEquationSystem::register_interior_algorithm( nodeAlg.add_kernel(realm_.meta_data()); break; case TurbulenceModel::SST: - nodeAlg.add_kernel(realm_.meta_data()); + nodeAlg.add_kernel( + realm_.meta_data(), *(realm_.fieldManager_.get()), part); break; case TurbulenceModel::SSTLR: nodeAlg.add_kernel(realm_.meta_data()); diff --git a/src/node_kernels/TKESSTNodeKernel.C b/src/node_kernels/TKESSTNodeKernel.C index 6fe7202f1..5fbe0d22e 100644 --- a/src/node_kernels/TKESSTNodeKernel.C +++ b/src/node_kernels/TKESSTNodeKernel.C @@ -19,29 +19,31 @@ namespace sierra { namespace nalu { -TKESSTNodeKernel::TKESSTNodeKernel(const stk::mesh::MetaData& meta) - : NGPNodeKernel(), - tkeID_(get_field_ordinal(meta, "turbulent_ke")), - sdrID_(get_field_ordinal(meta, "specific_dissipation_rate")), - densityID_(get_field_ordinal(meta, "density")), - tviscID_(get_field_ordinal(meta, "turbulent_viscosity")), - dudxID_(get_field_ordinal(meta, "dudx")), - dualNodalVolumeID_(get_field_ordinal(meta, "dual_nodal_volume")), - nDim_(meta.spatial_dimension()) +TKESSTNodeKernel::TKESSTNodeKernel( + const stk::mesh::MetaData& meta, + const FieldManager& manager, + stk::mesh::Part* part) + : NGPNodeKernel(), nDim_(meta.spatial_dimension()) { + manager.register_field("turbulent_ke", part); + manager.register_field("specific_dissipation_rate", part); + manager.register_field("density", part); + manager.register_field("turbulent_viscosity", part); + manager.register_field("dudx", part); + manager.register_field("dual_nodal_volume", part); } void TKESSTNodeKernel::setup(Realm& realm) { - const auto& fieldMgr = realm.ngp_field_manager(); + const auto& fieldMgr = *(realm.fieldManager_.get()); - tke_ = fieldMgr.get_field(tkeID_); - sdr_ = fieldMgr.get_field(sdrID_); - density_ = fieldMgr.get_field(densityID_); - tvisc_ = fieldMgr.get_field(tviscID_); - dudx_ = fieldMgr.get_field(dudxID_); - dualNodalVolume_ = fieldMgr.get_field(dualNodalVolumeID_); + tke_ = fieldMgr.get_ngp_field_ptr("turbulent_ke"); + sdr_ = fieldMgr.get_ngp_field_ptr("specific_dissipation_rate"); + density_ = fieldMgr.get_ngp_field_ptr("density"); + tvisc_ = fieldMgr.get_ngp_field_ptr("turbulent_viscosity"); + dudx_ = fieldMgr.get_ngp_field_ptr("dudx"); + dualNodalVolume_ = fieldMgr.get_ngp_field_ptr("dual_nodal_volume"); // Update turbulence model constants betaStar_ = realm.get_turb_model_constant(TM_betaStar); diff --git a/unit_tests/node_kernels/UnitTestSSTNodeKernel.C b/unit_tests/node_kernels/UnitTestSSTNodeKernel.C index d41791f03..acf24511c 100644 --- a/unit_tests/node_kernels/UnitTestSSTNodeKernel.C +++ b/unit_tests/node_kernels/UnitTestSSTNodeKernel.C @@ -636,6 +636,8 @@ TEST_F(SSTKernelHex8Mesh, NGP_tke_sst_node) if (bulk_->parallel_size() > 1) return; + // TDODO we can eliminate all the excess fields if we decide to do our field + // init after we add the kernels fill_mesh_and_init_fields(); // Setup solution options @@ -646,7 +648,8 @@ TEST_F(SSTKernelHex8Mesh, NGP_tke_sst_node) unit_test_utils::NodeHelperObjects helperObjs( bulk_, stk::topology::HEX_8, 1, partVec_[0]); - helperObjs.nodeAlg->add_kernel(*meta_); + helperObjs.nodeAlg->add_kernel( + *meta_, *(helperObjs.realm.fieldManager_.get()), partVec_[0]); helperObjs.execute(); @@ -685,7 +688,8 @@ TEST_F(SSTKernelHex8Mesh, NGP_tke_sst_sust_node) realm.solutionOptions_->turbModelConstantMap_[sierra::nalu::TM_tkeAmb] = 5.0; realm.solutionOptions_->turbModelConstantMap_[sierra::nalu::TM_sdrAmb] = 50.0; - helperObjs.nodeAlg->add_kernel(*meta_); + helperObjs.nodeAlg->add_kernel( + *meta_, *(realm.fieldManager_.get()), partVec_[0]); helperObjs.execute(); From 8160ddd101b94ed09e175a7fbaf48d54cc5fe8cb Mon Sep 17 00:00:00 2001 From: psakiev Date: Wed, 3 May 2023 21:35:27 -0600 Subject: [PATCH 2/7] Hack in FieldManager initialzation --- unit_tests/UnitTestHelperObjects.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/unit_tests/UnitTestHelperObjects.h b/unit_tests/UnitTestHelperObjects.h index bdf065a1f..9fc463a68 100644 --- a/unit_tests/UnitTestHelperObjects.h +++ b/unit_tests/UnitTestHelperObjects.h @@ -16,6 +16,7 @@ #include #include +#include namespace unit_test_utils { @@ -33,6 +34,10 @@ struct HelperObjectsBase eqSystem(eqSystems) { realm.bulkData_ = bulk; + // hack + // realm.setup_field_manager(); + const int numStates = 3; + realm.fieldManager_ = std::make_unique(realm.meta_data(), numStates); } virtual ~HelperObjectsBase() { delete naluObj; } From c4a57c15d3b0e2c2eddb8c1079ecd7160371f69c Mon Sep 17 00:00:00 2001 From: psakiev Date: Thu, 4 May 2023 10:57:35 -0600 Subject: [PATCH 3/7] Change order --- unit_tests/UnitTestHelperObjects.h | 3 ++- unit_tests/node_kernels/UnitTestSSTNodeKernel.C | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/unit_tests/UnitTestHelperObjects.h b/unit_tests/UnitTestHelperObjects.h index 9fc463a68..dbc4acc55 100644 --- a/unit_tests/UnitTestHelperObjects.h +++ b/unit_tests/UnitTestHelperObjects.h @@ -37,7 +37,8 @@ struct HelperObjectsBase // hack // realm.setup_field_manager(); const int numStates = 3; - realm.fieldManager_ = std::make_unique(realm.meta_data(), numStates); + realm.fieldManager_ = std::make_unique( + realm.meta_data(), numStates); } virtual ~HelperObjectsBase() { delete naluObj; } diff --git a/unit_tests/node_kernels/UnitTestSSTNodeKernel.C b/unit_tests/node_kernels/UnitTestSSTNodeKernel.C index acf24511c..cbbd14351 100644 --- a/unit_tests/node_kernels/UnitTestSSTNodeKernel.C +++ b/unit_tests/node_kernels/UnitTestSSTNodeKernel.C @@ -636,10 +636,6 @@ TEST_F(SSTKernelHex8Mesh, NGP_tke_sst_node) if (bulk_->parallel_size() > 1) return; - // TDODO we can eliminate all the excess fields if we decide to do our field - // init after we add the kernels - fill_mesh_and_init_fields(); - // Setup solution options solnOpts_.meshMotion_ = false; solnOpts_.externalMeshDeformation_ = false; @@ -651,6 +647,10 @@ TEST_F(SSTKernelHex8Mesh, NGP_tke_sst_node) helperObjs.nodeAlg->add_kernel( *meta_, *(helperObjs.realm.fieldManager_.get()), partVec_[0]); + // TDODO we can eliminate all the excess fields if we decide to do our field + // init after we add the kernels + fill_mesh_and_init_fields(); + helperObjs.execute(); Kokkos::deep_copy( From 858d7e1a4016c3a698b0c694bca597222e5baa92 Mon Sep 17 00:00:00 2001 From: psakiev Date: Wed, 17 Apr 2024 21:23:25 -0600 Subject: [PATCH 4/7] Get changes building again --- include/FieldManager.h | 7 +---- include/SmartField.h | 6 ++--- include/node_kernels/TKESSTNodeKernel.h | 3 ++- src/FieldManager.C | 2 +- src/TurbKineticEnergyEquationSystem.C | 5 +++- src/node_kernels/TKESSTNodeKernel.C | 26 +++++++++---------- unit_tests/UnitTestHelperObjects.h | 9 +++---- .../node_kernels/UnitTestSSTNodeKernel.C | 6 +++-- 8 files changed, 32 insertions(+), 32 deletions(-) diff --git a/include/FieldManager.h b/include/FieldManager.h index 030bbda69..a0f1e5813 100644 --- a/include/FieldManager.h +++ b/include/FieldManager.h @@ -123,12 +123,7 @@ class FieldManager const stk::mesh::PartVector& parts, const int numStates = 0, const int numComponents = 0, - const void* initVal = nullptr) const - { - stk::mesh::PartVector parts; - parts.push_back(part); - register_field(name, parts, numStates, numComponents, initVal); - } + const void* initVal = nullptr) const; /// Given the named field that has already been registered on the CPU diff --git a/include/SmartField.h b/include/SmartField.h index af4f58005..99be1e24e 100644 --- a/include/SmartField.h +++ b/include/SmartField.h @@ -445,11 +445,11 @@ struct MakeSmartField }; template -using SmartDeviceField = SmartField; +using SmartDeviceField = SmartField, DEVICE, ACCESS>; template -using SmartHostField = SmartField; +using SmartHostField = SmartField, HOST, ACCESS>; template -using SmartLegacyField = SmartField; +using SmartLegacyField = SmartField, LEGACY, ACCESS>; } // namespace sierra::nalu #endif diff --git a/include/node_kernels/TKESSTNodeKernel.h b/include/node_kernels/TKESSTNodeKernel.h index 8ce5dc9d6..14b3af607 100644 --- a/include/node_kernels/TKESSTNodeKernel.h +++ b/include/node_kernels/TKESSTNodeKernel.h @@ -28,7 +28,7 @@ class TKESSTNodeKernel : public NGPNodeKernel { public: TKESSTNodeKernel( - const stk::mesh::MetaData&, const FieldManager&, stk::mesh::Part* part); + const stk::mesh::MetaData&, const FieldManager&, stk::mesh::PartVector& parts); TKESSTNodeKernel() = delete; @@ -44,6 +44,7 @@ class TKESSTNodeKernel : public NGPNodeKernel const stk::mesh::FastMeshIndex&) override; private: + // TODO can we make these SmartFields? Do we need to? review how kernels manage syncs stk::mesh::NgpField tke_; stk::mesh::NgpField sdr_; stk::mesh::NgpField density_; diff --git a/src/FieldManager.C b/src/FieldManager.C index 007666f53..0ccced6e8 100644 --- a/src/FieldManager.C +++ b/src/FieldManager.C @@ -49,7 +49,7 @@ FieldManager::register_field( numComponents ? numComponents : def.num_components; const FieldLayout layout = def.layout; - const val_type* init = static_cast(init_val); + const val_type* init = static_cast(initVal); auto* id = &(meta_.declare_field(def.rank, name, num_states)); for (auto&& part : parts) { diff --git a/src/TurbKineticEnergyEquationSystem.C b/src/TurbKineticEnergyEquationSystem.C index 92aa82dc0..97f30a7f8 100644 --- a/src/TurbKineticEnergyEquationSystem.C +++ b/src/TurbKineticEnergyEquationSystem.C @@ -298,13 +298,16 @@ TurbKineticEnergyEquationSystem::register_interior_algorithm( if (!elementMassAlg) nodeAlg.add_kernel(realm_.bulk_data(), tke_); + // hacky solution since updating everything to part vecs at once would be painful and error prone + stk::mesh::PartVector tempVec({part}); + switch (turbulenceModel_) { case TurbulenceModel::KSGS: nodeAlg.add_kernel(realm_.meta_data()); break; case TurbulenceModel::SST: nodeAlg.add_kernel( - realm_.meta_data(), *(realm_.fieldManager_.get()), part); + realm_.meta_data(), *(realm_.fieldManager_.get()), tempVec); break; case TurbulenceModel::SSTLR: nodeAlg.add_kernel(realm_.meta_data()); diff --git a/src/node_kernels/TKESSTNodeKernel.C b/src/node_kernels/TKESSTNodeKernel.C index 5fbe0d22e..b39d39a29 100644 --- a/src/node_kernels/TKESSTNodeKernel.C +++ b/src/node_kernels/TKESSTNodeKernel.C @@ -22,15 +22,15 @@ namespace nalu { TKESSTNodeKernel::TKESSTNodeKernel( const stk::mesh::MetaData& meta, const FieldManager& manager, - stk::mesh::Part* part) + stk::mesh::PartVector& parts) : NGPNodeKernel(), nDim_(meta.spatial_dimension()) { - manager.register_field("turbulent_ke", part); - manager.register_field("specific_dissipation_rate", part); - manager.register_field("density", part); - manager.register_field("turbulent_viscosity", part); - manager.register_field("dudx", part); - manager.register_field("dual_nodal_volume", part); + manager.register_field("turbulent_ke", parts); + manager.register_field("specific_dissipation_rate", parts); + manager.register_field("density", parts); + manager.register_field("turbulent_viscosity", parts); + manager.register_field("dudx", parts); + manager.register_field("dual_nodal_volume", parts); } void @@ -38,12 +38,12 @@ TKESSTNodeKernel::setup(Realm& realm) { const auto& fieldMgr = *(realm.fieldManager_.get()); - tke_ = fieldMgr.get_ngp_field_ptr("turbulent_ke"); - sdr_ = fieldMgr.get_ngp_field_ptr("specific_dissipation_rate"); - density_ = fieldMgr.get_ngp_field_ptr("density"); - tvisc_ = fieldMgr.get_ngp_field_ptr("turbulent_viscosity"); - dudx_ = fieldMgr.get_ngp_field_ptr("dudx"); - dualNodalVolume_ = fieldMgr.get_ngp_field_ptr("dual_nodal_volume"); + tke_ = fieldMgr.get_ngp_field_ptr("turbulent_ke"); + sdr_ = fieldMgr.get_ngp_field_ptr("specific_dissipation_rate"); + density_ = fieldMgr.get_ngp_field_ptr("density"); + tvisc_ = fieldMgr.get_ngp_field_ptr("turbulent_viscosity"); + dudx_ = fieldMgr.get_ngp_field_ptr("dudx"); + dualNodalVolume_ = fieldMgr.get_ngp_field_ptr("dual_nodal_volume"); // Update turbulence model constants betaStar_ = realm.get_turb_model_constant(TM_betaStar); diff --git a/unit_tests/UnitTestHelperObjects.h b/unit_tests/UnitTestHelperObjects.h index 5c6f35c60..1034d020a 100644 --- a/unit_tests/UnitTestHelperObjects.h +++ b/unit_tests/UnitTestHelperObjects.h @@ -34,11 +34,10 @@ struct HelperObjectsBase eqSystem(eqSystems) { realm.bulkData_ = bulk; - // hack - // realm.setup_field_manager(); - const int numStates = 3; - realm.fieldManager_ = std::make_unique( - realm.meta_data(), numStates); + realm.setup_field_manager(); + // const int numStates = 3; + // realm.fieldManager_ = std::make_unique( + // realm.meta_data(), numStates); } virtual ~HelperObjectsBase() { delete naluObj; } diff --git a/unit_tests/node_kernels/UnitTestSSTNodeKernel.C b/unit_tests/node_kernels/UnitTestSSTNodeKernel.C index 632103564..b772242e1 100644 --- a/unit_tests/node_kernels/UnitTestSSTNodeKernel.C +++ b/unit_tests/node_kernels/UnitTestSSTNodeKernel.C @@ -734,8 +734,9 @@ TEST_F(SSTKernelHex8Mesh, NGP_tke_sst_node) unit_test_utils::NodeHelperObjects helperObjs( bulk_, stk::topology::HEX_8, 1, partVec_[0]); + /* helperObjs.nodeAlg->add_kernel( - *meta_, *(helperObjs.realm.fieldManager_.get()), partVec_[0]); + *meta_, *(helperObjs.realm.fieldManager_.get()), partVec_); // TDODO we can eliminate all the excess fields if we decide to do our field // init after we add the kernels @@ -756,6 +757,7 @@ TEST_F(SSTKernelHex8Mesh, NGP_tke_sst_node) helperObjs.linsys->rhs_, hex8_golds::rhs, 1.0e-12); unit_test_kernel_utils::expect_all_near<8>( helperObjs.linsys->lhs_, hex8_golds::lhs, 1.0e-12); + */ } TEST_F(SSTKernelHex8Mesh, NGP_tke_sst_sust_node) @@ -779,7 +781,7 @@ TEST_F(SSTKernelHex8Mesh, NGP_tke_sst_sust_node) realm.solutionOptions_->turbModelConstantMap_[sierra::nalu::TM_sdrAmb] = 50.0; helperObjs.nodeAlg->add_kernel( - *meta_, *(realm.fieldManager_.get()), partVec_[0]); + *meta_, *(realm.fieldManager_.get()), partVec_); helperObjs.execute(); From 547dfe66f73b2443603b4216e5cc7b8347160304 Mon Sep 17 00:00:00 2001 From: psakiev Date: Thu, 18 Apr 2024 00:27:47 -0600 Subject: [PATCH 5/7] SST unit tests are passing --- src/FieldRegistry.C | 6 +- src/Simulation.C | 9 +- src/TurbKineticEnergyEquationSystem.C | 2 + unit_tests/UnitTestHelperObjects.h | 8 +- unit_tests/kernels/UnitTestKernelUtils.h | 94 ++++++++++++------- .../node_kernels/UnitTestSSTNodeKernel.C | 5 +- 6 files changed, 80 insertions(+), 44 deletions(-) diff --git a/src/FieldRegistry.C b/src/FieldRegistry.C index e89efb7f9..262d56141 100644 --- a/src/FieldRegistry.C +++ b/src/FieldRegistry.C @@ -53,9 +53,10 @@ Registry() {"div_mesh_velocity", SingleStateNodalScalar}, {"dkdx", SingleStateNodalVector}, {"dpdx", SingleStateNodalVector}, - {"dual_nodal_volume", SingleStateNodalScalar}, + {"dual_nodal_volume", MultiStateNodalScalar}, {"dudx", SingleStateNodalTensor}, {"dwdx", SingleStateNodalVector}, + {"dynamic_pressure", SingleStateNodalScalar}, // Used in testing {"edge_area_vector", SingleStateEdgeVector}, {"effective_viscosity" , SingleStateNodalScalar}, {"elemCentroid", SingleStateElemVector}, @@ -83,7 +84,7 @@ Registry() {"pressure", SingleStateNodalScalar}, {"rans_time_scale" , SingleStateNodalScalar}, {"scalarQ", SingleStateNodalScalar}, - {"specific_dissipation_rate", MultiStateNodalScalar}, + {"specific_dissipation_rate", SingleStateNodalScalar}, {"thermal_conductivity", SingleStateNodalScalar}, {"specific_heat" , SingleStateNodalScalar}, {"sst_f_one_blending" , SingleStateNodalScalar}, @@ -94,6 +95,7 @@ Registry() {"turbulent_viscosity" , SingleStateNodalScalar}, {"velocity", MultiStateNodalVector}, {"velocity_rtm", SingleStateNodalVector}, + {"velocity_bc", MultiStateNodalVector}, // Used in testing {"viscosity", SingleStateNodalScalar} }; // clang-format on diff --git a/src/Simulation.C b/src/Simulation.C index f2f6461e9..4ec5cd8f4 100644 --- a/src/Simulation.C +++ b/src/Simulation.C @@ -173,9 +173,12 @@ Simulation::setSerializedIOGroupSize(int siogs) void Simulation::breadboard() { - realms_->breadboard(); - timeIntegrator_->breadboard(); - transfers_->breadboard(); + if (realms_) + realms_->breadboard(); + if (timeIntegrator_) + timeIntegrator_->breadboard(); + if (transfers_) + transfers_->breadboard(); } void diff --git a/src/TurbKineticEnergyEquationSystem.C b/src/TurbKineticEnergyEquationSystem.C index 97f30a7f8..6d6faa362 100644 --- a/src/TurbKineticEnergyEquationSystem.C +++ b/src/TurbKineticEnergyEquationSystem.C @@ -306,6 +306,8 @@ TurbKineticEnergyEquationSystem::register_interior_algorithm( nodeAlg.add_kernel(realm_.meta_data()); break; case TurbulenceModel::SST: + // should be able to keep creating actually since stk fields can be registered over and over as no-ops + // registration should still happen nodeAlg.add_kernel( realm_.meta_data(), *(realm_.fieldManager_.get()), tempVec); break; diff --git a/unit_tests/UnitTestHelperObjects.h b/unit_tests/UnitTestHelperObjects.h index 1034d020a..f7bc6c7e1 100644 --- a/unit_tests/UnitTestHelperObjects.h +++ b/unit_tests/UnitTestHelperObjects.h @@ -34,10 +34,12 @@ struct HelperObjectsBase eqSystem(eqSystems) { realm.bulkData_ = bulk; + // TODO fix this! realm should not be getting time integrator this way!! + naluObj->sim_.breadboard(); + assert(naluObj->sim_.timeIntegrator_ != NULL); + // bread board didn't work so set integrator manually + realm.timeIntegrator_ = naluObj->sim_.timeIntegrator_; realm.setup_field_manager(); - // const int numStates = 3; - // realm.fieldManager_ = std::make_unique( - // realm.meta_data(), numStates); } virtual ~HelperObjectsBase() { delete naluObj; } diff --git a/unit_tests/kernels/UnitTestKernelUtils.h b/unit_tests/kernels/UnitTestKernelUtils.h index 2f134e893..287a82b01 100644 --- a/unit_tests/kernels/UnitTestKernelUtils.h +++ b/unit_tests/kernels/UnitTestKernelUtils.h @@ -256,12 +256,33 @@ class TestKernelHex8Mesh : public ::testing::Test TestKernelHex8Mesh() : comm_(MPI_COMM_WORLD), spatialDim_(3), solnOpts_(), coordinates_(nullptr) { + const int numStates = 3; stk::mesh::MeshBuilder meshBuilder(MPI_COMM_WORLD); meshBuilder.set_spatial_dimension(spatialDim_); bulk_ = meshBuilder.create(); meta_ = &bulk_->mesh_meta_data(); meta_->use_simple_fields(); + fieldManager = + std::make_shared(*meta_, numStates); + + const stk::mesh::PartVector parts(1, &meta_->universal_part()); + + naluGlobalId_ = fieldManager->register_field("nalu_global_id", parts); + tpetGlobalId_ = fieldManager->register_field("tpet_global_id", parts); + fieldManager->register_field("dual_nodal_volume", parts, 3); + dnvField_ = fieldManager->get_field_ptr("dual_nodal_volume"); + divMeshVelField_ = fieldManager->register_field("div_mesh_velocity", parts); + edgeAreaVec_ = fieldManager->register_field("edge_area_vector", parts); + elementVolume_ = fieldManager->register_field("element_volume", parts); + // need to check on this one + exposedAreaVec_ = + &meta_->declare_field(meta_->side_rank(), "exposed_area_vector"); + stk::mesh::put_field_on_mesh( + *exposedAreaVec_, meta_->universal_part(), + spatialDim_ * sierra::nalu::AlgTraitsQuad4::numScsIp_, nullptr); + + /* naluGlobalId_ = &meta_->declare_field( stk::topology::NODE_RANK, "nalu_global_id", 1); tpetGlobalId_ = &meta_->declare_field( @@ -274,8 +295,6 @@ class TestKernelHex8Mesh : public ::testing::Test stk::topology::EDGE_RANK, "edge_area_vector"); elementVolume_ = &meta_->declare_field(stk::topology::ELEM_RANK, "element_volume"); - exposedAreaVec_ = - &meta_->declare_field(meta_->side_rank(), "exposed_area_vector"); stk::mesh::put_field_on_mesh( *naluGlobalId_, meta_->universal_part(), nullptr); @@ -290,9 +309,7 @@ class TestKernelHex8Mesh : public ::testing::Test *edgeAreaVec_, stk::io::FieldOutputType::VECTOR_3D); stk::mesh::put_field_on_mesh( *elementVolume_, meta_->universal_part(), nullptr); - stk::mesh::put_field_on_mesh( - *exposedAreaVec_, meta_->universal_part(), - spatialDim_ * sierra::nalu::AlgTraitsQuad4::numScsIp_, nullptr); + */ } virtual ~TestKernelHex8Mesh() {} @@ -329,6 +346,7 @@ class TestKernelHex8Mesh : public ::testing::Test unsigned spatialDim_; stk::mesh::MetaData* meta_; std::shared_ptr bulk_; + std::shared_ptr fieldManager; stk::mesh::PartVector partVec_; sierra::nalu::SolutionOptions solnOpts_; @@ -361,35 +379,45 @@ class LowMachKernelHex8Mesh : public TestKernelHex8Mesh { public: LowMachKernelHex8Mesh() - : TestKernelHex8Mesh(), - velocity_( - &meta_->declare_field(stk::topology::NODE_RANK, "velocity", 2)), - dpdx_(&meta_->declare_field(stk::topology::NODE_RANK, "dpdx", 2)), - density_( - &meta_->declare_field(stk::topology::NODE_RANK, "density", 2)), - pressure_( - &meta_->declare_field(stk::topology::NODE_RANK, "pressure", 2)), - Udiag_(&meta_->declare_field( - stk::topology::NODE_RANK, "momentum_diag", 2)), - velocityBC_( - &meta_->declare_field(stk::topology::NODE_RANK, "velocity_bc")), - dynP_( - &meta_->declare_field(meta_->side_rank(), "dynamic_pressure")) + : TestKernelHex8Mesh() { - stk::mesh::put_field_on_mesh( - *velocity_, meta_->universal_part(), spatialDim_, nullptr); - stk::io::set_field_output_type( - *velocity_, stk::io::FieldOutputType::VECTOR_3D); - stk::mesh::put_field_on_mesh( - *dpdx_, meta_->universal_part(), spatialDim_, nullptr); - stk::io::set_field_output_type(*dpdx_, stk::io::FieldOutputType::VECTOR_3D); - stk::mesh::put_field_on_mesh(*density_, meta_->universal_part(), nullptr); - stk::mesh::put_field_on_mesh(*pressure_, meta_->universal_part(), nullptr); - stk::mesh::put_field_on_mesh(*Udiag_, meta_->universal_part(), nullptr); - stk::mesh::put_field_on_mesh( - *velocityBC_, meta_->universal_part(), spatialDim_, nullptr); - stk::io::set_field_output_type( - *velocityBC_, stk::io::FieldOutputType::VECTOR_3D); + const stk::mesh::PartVector parts(1, &meta_->universal_part()); + velocity_ = fieldManager->register_field("velocity", parts); + dpdx_ = fieldManager->register_field("dpdx", parts); + density_ = fieldManager->register_field("density", parts); + pressure_ = fieldManager->register_field("pressure", parts); + Udiag_ = fieldManager->register_field("momentum_diag", parts); + velocityBC_ = fieldManager->register_field("velocity_bc", parts); + // dynP_ = fieldManager->register_field("dynamic_pressure", parts); + + + // velocity_( + // &meta_->declare_field(stk::topology::NODE_RANK, "velocity", 2)), + // dpdx_(&meta_->declare_field(stk::topology::NODE_RANK, "dpdx", 2)), + // density_( + // &meta_->declare_field(stk::topology::NODE_RANK, "density", 2)), + // pressure_( + // &meta_->declare_field(stk::topology::NODE_RANK, "pressure", 2)), + // Udiag_(&meta_->declare_field( + // stk::topology::NODE_RANK, "momentum_diag", 2)), + // velocityBC_( + // &meta_->declare_field(stk::topology::NODE_RANK, "velocity_bc")), + dynP_ = + &meta_->declare_field(meta_->side_rank(), "dynamic_pressure"); + // stk::mesh::put_field_on_mesh( + // *velocity_, meta_->universal_part(), spatialDim_, nullptr); + // stk::io::set_field_output_type( + // *velocity_, stk::io::FieldOutputType::VECTOR_3D); + // stk::mesh::put_field_on_mesh( + // *dpdx_, meta_->universal_part(), spatialDim_, nullptr); + // stk::io::set_field_output_type(*dpdx_, stk::io::FieldOutputType::VECTOR_3D); + // stk::mesh::put_field_on_mesh(*density_, meta_->universal_part(), nullptr); + // stk::mesh::put_field_on_mesh(*pressure_, meta_->universal_part(), nullptr); + // stk::mesh::put_field_on_mesh(*Udiag_, meta_->universal_part(), nullptr); + // stk::mesh::put_field_on_mesh( + // *velocityBC_, meta_->universal_part(), spatialDim_, nullptr); + // stk::io::set_field_output_type( + // *velocityBC_, stk::io::FieldOutputType::VECTOR_3D); stk::mesh::put_field_on_mesh( *dynP_, meta_->universal_part(), sierra::nalu::AlgTraitsQuad4::numScsIp_, nullptr); diff --git a/unit_tests/node_kernels/UnitTestSSTNodeKernel.C b/unit_tests/node_kernels/UnitTestSSTNodeKernel.C index b772242e1..a1328eef3 100644 --- a/unit_tests/node_kernels/UnitTestSSTNodeKernel.C +++ b/unit_tests/node_kernels/UnitTestSSTNodeKernel.C @@ -731,16 +731,16 @@ TEST_F(SSTKernelHex8Mesh, NGP_tke_sst_node) solnOpts_.externalMeshDeformation_ = false; solnOpts_.initialize_turbulence_constants(); + fill_mesh_and_init_fields(); + unit_test_utils::NodeHelperObjects helperObjs( bulk_, stk::topology::HEX_8, 1, partVec_[0]); - /* helperObjs.nodeAlg->add_kernel( *meta_, *(helperObjs.realm.fieldManager_.get()), partVec_); // TDODO we can eliminate all the excess fields if we decide to do our field // init after we add the kernels - fill_mesh_and_init_fields(); helperObjs.execute(); @@ -757,7 +757,6 @@ TEST_F(SSTKernelHex8Mesh, NGP_tke_sst_node) helperObjs.linsys->rhs_, hex8_golds::rhs, 1.0e-12); unit_test_kernel_utils::expect_all_near<8>( helperObjs.linsys->lhs_, hex8_golds::lhs, 1.0e-12); - */ } TEST_F(SSTKernelHex8Mesh, NGP_tke_sst_sust_node) From 877a49d5eda5e1b2f943053a4e569a94016d5fce Mon Sep 17 00:00:00 2001 From: psakiev Date: Thu, 18 Apr 2024 00:32:31 -0600 Subject: [PATCH 6/7] Remove crufty comments --- unit_tests/kernels/UnitTestKernelUtils.h | 70 ++---------------------- 1 file changed, 6 insertions(+), 64 deletions(-) diff --git a/unit_tests/kernels/UnitTestKernelUtils.h b/unit_tests/kernels/UnitTestKernelUtils.h index 287a82b01..559703021 100644 --- a/unit_tests/kernels/UnitTestKernelUtils.h +++ b/unit_tests/kernels/UnitTestKernelUtils.h @@ -269,47 +269,16 @@ class TestKernelHex8Mesh : public ::testing::Test naluGlobalId_ = fieldManager->register_field("nalu_global_id", parts); tpetGlobalId_ = fieldManager->register_field("tpet_global_id", parts); - fieldManager->register_field("dual_nodal_volume", parts, 3); - dnvField_ = fieldManager->get_field_ptr("dual_nodal_volume"); + dnvField_ = fieldManager->register_field("dual_nodal_volume", parts); divMeshVelField_ = fieldManager->register_field("div_mesh_velocity", parts); edgeAreaVec_ = fieldManager->register_field("edge_area_vector", parts); elementVolume_ = fieldManager->register_field("element_volume", parts); - // need to check on this one + // currently don't handle side ranks exposedAreaVec_ = &meta_->declare_field(meta_->side_rank(), "exposed_area_vector"); stk::mesh::put_field_on_mesh( *exposedAreaVec_, meta_->universal_part(), spatialDim_ * sierra::nalu::AlgTraitsQuad4::numScsIp_, nullptr); - - - /* - naluGlobalId_ = &meta_->declare_field( - stk::topology::NODE_RANK, "nalu_global_id", 1); - tpetGlobalId_ = &meta_->declare_field( - stk::topology::NODE_RANK, "tpet_global_id", 1); - dnvField_ = &meta_->declare_field( - stk::topology::NODE_RANK, "dual_nodal_volume", 3); - divMeshVelField_ = &meta_->declare_field( - stk::topology::NODE_RANK, "div_mesh_velocity"); - edgeAreaVec_ = &meta_->declare_field( - stk::topology::EDGE_RANK, "edge_area_vector"); - elementVolume_ = - &meta_->declare_field(stk::topology::ELEM_RANK, "element_volume"); - - stk::mesh::put_field_on_mesh( - *naluGlobalId_, meta_->universal_part(), nullptr); - stk::mesh::put_field_on_mesh( - *tpetGlobalId_, meta_->universal_part(), nullptr); - stk::mesh::put_field_on_mesh(*dnvField_, meta_->universal_part(), nullptr); - stk::mesh::put_field_on_mesh( - *divMeshVelField_, meta_->universal_part(), nullptr); - stk::mesh::put_field_on_mesh( - *edgeAreaVec_, meta_->universal_part(), spatialDim_, nullptr); - stk::io::set_field_output_type( - *edgeAreaVec_, stk::io::FieldOutputType::VECTOR_3D); - stk::mesh::put_field_on_mesh( - *elementVolume_, meta_->universal_part(), nullptr); - */ } virtual ~TestKernelHex8Mesh() {} @@ -388,39 +357,12 @@ class LowMachKernelHex8Mesh : public TestKernelHex8Mesh pressure_ = fieldManager->register_field("pressure", parts); Udiag_ = fieldManager->register_field("momentum_diag", parts); velocityBC_ = fieldManager->register_field("velocity_bc", parts); - // dynP_ = fieldManager->register_field("dynamic_pressure", parts); - - - // velocity_( - // &meta_->declare_field(stk::topology::NODE_RANK, "velocity", 2)), - // dpdx_(&meta_->declare_field(stk::topology::NODE_RANK, "dpdx", 2)), - // density_( - // &meta_->declare_field(stk::topology::NODE_RANK, "density", 2)), - // pressure_( - // &meta_->declare_field(stk::topology::NODE_RANK, "pressure", 2)), - // Udiag_(&meta_->declare_field( - // stk::topology::NODE_RANK, "momentum_diag", 2)), - // velocityBC_( - // &meta_->declare_field(stk::topology::NODE_RANK, "velocity_bc")), + + // currently don't handle side ranks dynP_ = &meta_->declare_field(meta_->side_rank(), "dynamic_pressure"); - // stk::mesh::put_field_on_mesh( - // *velocity_, meta_->universal_part(), spatialDim_, nullptr); - // stk::io::set_field_output_type( - // *velocity_, stk::io::FieldOutputType::VECTOR_3D); - // stk::mesh::put_field_on_mesh( - // *dpdx_, meta_->universal_part(), spatialDim_, nullptr); - // stk::io::set_field_output_type(*dpdx_, stk::io::FieldOutputType::VECTOR_3D); - // stk::mesh::put_field_on_mesh(*density_, meta_->universal_part(), nullptr); - // stk::mesh::put_field_on_mesh(*pressure_, meta_->universal_part(), nullptr); - // stk::mesh::put_field_on_mesh(*Udiag_, meta_->universal_part(), nullptr); - // stk::mesh::put_field_on_mesh( - // *velocityBC_, meta_->universal_part(), spatialDim_, nullptr); - // stk::io::set_field_output_type( - // *velocityBC_, stk::io::FieldOutputType::VECTOR_3D); - stk::mesh::put_field_on_mesh( - *dynP_, meta_->universal_part(), sierra::nalu::AlgTraitsQuad4::numScsIp_, - nullptr); + stk::mesh::put_field_on_mesh( + *dynP_, meta_->universal_part(), sierra::nalu::AlgTraitsQuad4::numScsIp_, nullptr); } virtual ~LowMachKernelHex8Mesh() {} From 0a08cb820d7945a9c7ff0a6cd53d01a4fd5cc56f Mon Sep 17 00:00:00 2001 From: psakiev Date: Thu, 18 Apr 2024 13:10:03 -0600 Subject: [PATCH 7/7] Fix error in registry --- src/FieldRegistry.C | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FieldRegistry.C b/src/FieldRegistry.C index 262d56141..033e3c2f8 100644 --- a/src/FieldRegistry.C +++ b/src/FieldRegistry.C @@ -46,7 +46,7 @@ Registry() {"coordinates", SingleStateNodalVector}, {"coordinates_copy", SingleStateNodalVector}, // Used in testing {"current_coordinates", SingleStateNodalVector}, - {"density", SingleStateNodalScalar}, + {"density", MultiStateNodalScalar}, {"dhdx", SingleStateNodalVector}, {"diffFluxCoeff", SingleStateNodalScalar}, {"discreteLaplacian", SingleStateNodalScalar},