From b9c1d789de8c35d21c1093b78a954e7cad35b946 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 9 Sep 2024 12:02:17 -0700 Subject: [PATCH 001/314] Add: `MultiFabRegister` Add a central location to store all `MultiFab`s in. --- Source/WarpX.H | 4 + Source/ablastr/fields/CMakeLists.txt | 6 + Source/ablastr/fields/Make.package | 1 + Source/ablastr/fields/MultiFabRegister.H | 126 ++++++++++++++++++++ Source/ablastr/fields/MultiFabRegister.cpp | 129 +++++++++++++++++++++ 5 files changed, 266 insertions(+) create mode 100644 Source/ablastr/fields/MultiFabRegister.H create mode 100644 Source/ablastr/fields/MultiFabRegister.cpp diff --git a/Source/WarpX.H b/Source/WarpX.H index 903e97549dd..871e6efbbd0 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -49,6 +49,7 @@ #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/export.H" +#include #include #include @@ -1500,6 +1501,9 @@ private: amrex::Vector t_old; amrex::Vector dt; + // Field container + ablastr::fields::MultiFabRegister m_multifab; + // Particle container std::unique_ptr mypc; std::unique_ptr multi_diags; diff --git a/Source/ablastr/fields/CMakeLists.txt b/Source/ablastr/fields/CMakeLists.txt index 56acc678217..011d765a6bb 100644 --- a/Source/ablastr/fields/CMakeLists.txt +++ b/Source/ablastr/fields/CMakeLists.txt @@ -1,5 +1,11 @@ foreach(D IN LISTS WarpX_DIMS) warpx_set_suffix_dims(SD ${D}) + + target_sources(ablastr_${SD} + PRIVATE + MultiFabRegister.cpp + ) + if(ABLASTR_FFT AND D EQUAL 3) target_sources(ablastr_${SD} PRIVATE diff --git a/Source/ablastr/fields/Make.package b/Source/ablastr/fields/Make.package index 01392991559..727a17b6de8 100644 --- a/Source/ablastr/fields/Make.package +++ b/Source/ablastr/fields/Make.package @@ -1,4 +1,5 @@ ifeq ($(USE_FFT),TRUE) + CEXE_sources += MultiFabRegister.cpp ifeq ($(DIM),3) CEXE_sources += IntegratedGreenFunctionSolver.cpp endif diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H new file mode 100644 index 00000000000..d69298aa042 --- /dev/null +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -0,0 +1,126 @@ +/* Copyright 2024 The ABLAST Community + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + * Authors: Axel Huebl, ... + */ +#ifndef ABLASTR_FIELDS_MF_REGISTER_H +#define ABLASTR_FIELDS_MF_REGISTER_H + + +#include + +#include +#include +#include +#include +#include +#include + + +namespace ablastr::fields +{ + struct MultiFabOwner + { + // TODO: also add iMultiFab via std::variant + + /** owned (i)MultiFab data */ + amrex::MultiFab m_mf; + + /** redistribute */ + bool redistribute = true; + }; + + /** This is a register of fields aka amrex::(i)MultiFabs. + * + * This is owned by a simulation instance. All used fields should be registered here. + */ + struct MultiFabRegister + { + /** title + * + * body body + * body + * body + * + * @param name ... + * @param ba ... + * @param dm ... + * @param ncomp ... + * @param ngrow ... + * @param level ... + * @param redistribute ... + * @param initial_value ... + * @return pointer to newly allocated MultiFab + */ + amrex::MultiFab* + alloc_init ( + std::string name, + const amrex::BoxArray& ba, + const amrex::DistributionMapping& dm, + const int ncomp, + const amrex::IntVect& ngrow, + const int level, + bool redistribute = true, + std::optional initial_value = std::nullopt + ); + + /** title + * + * body body + * body + * + * @param name ... + */ + void + alloc_like ( + std::string other_key + ); + + /** title + * + * body body + * body + * + * @param name ... + * @return ... + */ + amrex::MultiFab* + get ( + std::string name + ); + + /** title + * + * body body + * body + * + * @return ... + */ + std::vector + list (); + + /** title + * + * body body + * body + * + * @param name ... + * @return ... + */ + void + erase ( + std::string name + ); + + private: + std::map< + std::string, + MultiFabOwner + > m_mf_register; + }; + +} // namespace ablastr::fields + +#endif // ABLASTR_FIELDS_MF_REGISTER_H diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp new file mode 100644 index 00000000000..1f10657f8e2 --- /dev/null +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -0,0 +1,129 @@ +/* Copyright 2024 The ABLAST Community + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + * Authors: Axel Huebl, ... + */ + + +#include "MultiFabRegister.H" + + + +namespace ablastr::fields +{ + + amrex::MultiFab* + MultiFabRegister::alloc_init ( + std::string name, + const amrex::BoxArray& ba, + const amrex::DistributionMapping& dm, + const int ncomp, + const amrex::IntVect& ngrow, + const int level, + bool redistribute, + std::optional initial_value + ) + { + // create name + // Add the suffix "[level=level]" + name.append("[level=").append(std::to_string(level)).append("]"); + + // Checks + // TODO: does the key already exist? error + + // allocate + const auto tag = amrex::MFInfo().SetTag(name); + auto [it, success] = m_mf_register.emplace( + std::make_pair( + name, + MultiFabOwner{{ba, dm, ncomp, ngrow, tag}, redistribute} + ) + ); + if (!success) { + throw std::runtime_error("MultiFabRegister::alloc_init failed for " + name); + } + + // a short-hand alias for the code below + amrex::MultiFab & mf = it->second.m_mf; + + // initialize with value + if (initial_value) { + mf.setVal(*initial_value); + } + + return &mf; + } + + void + MultiFabRegister::alloc_like ( + std::string /* other_key */ + ) + { + throw std::runtime_error("MultiFabRegister::alloc_like not yet implemented"); + + // Checks + // TODO: does the key already exist? error + } + + /** title + * + * body body + * body + * + * @param name ... + * @return ... + */ + amrex::MultiFab* + MultiFabRegister::get ( + std::string name + ) + { + if (m_mf_register.count(name) == 0) { + throw std::runtime_error("MultiFabRegister::get name does not exist in register: " + name); + } + amrex::MultiFab & mf = m_mf_register[name].m_mf; + + return &mf; + } + + /** title + * + * body body + * body + * + * @return ... + */ + std::vector + MultiFabRegister::list () + { + std::vector names; + names.reserve(m_mf_register.size()); + for (auto const & str : m_mf_register) { names.push_back(str.first); } + + return names; + } + + + /** title + * + * body body + * body + * + * @param name ... + * @return ... + */ + void + MultiFabRegister::erase ( + std::string name + ) + { + if (m_mf_register.count(name) != 1) { + throw std::runtime_error("MultiFabRegister::remove name does not exist in register: " + name); + } + m_mf_register.erase(name); + } + + +} // namespace ablastr::fields From 6e7f7fb1a29de3683ae65ad8eb5718177b2efaec Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 9 Sep 2024 12:55:56 -0700 Subject: [PATCH 002/314] Add Missing Includes --- Source/ablastr/fields/MultiFabRegister.H | 4 +++- Source/ablastr/fields/MultiFabRegister.cpp | 4 ---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index d69298aa042..08ef7f70a82 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -8,7 +8,9 @@ #ifndef ABLASTR_FIELDS_MF_REGISTER_H #define ABLASTR_FIELDS_MF_REGISTER_H - +#include +#include +#include #include #include diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 1f10657f8e2..573ef3e8a97 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -5,15 +5,11 @@ * License: BSD-3-Clause-LBNL * Authors: Axel Huebl, ... */ - - #include "MultiFabRegister.H" - namespace ablastr::fields { - amrex::MultiFab* MultiFabRegister::alloc_init ( std::string name, From c4643bb2ff9f655a82811aaa9571d824765ba057 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 9 Sep 2024 13:30:29 -0700 Subject: [PATCH 003/314] `contains` and arg order init --- Source/ablastr/fields/MultiFabRegister.H | 29 ++++++++++++++++------ Source/ablastr/fields/MultiFabRegister.cpp | 22 ++++++++++------ 2 files changed, 36 insertions(+), 15 deletions(-) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 08ef7f70a82..6f92719ddbc 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -52,20 +52,20 @@ namespace ablastr::fields * @param ncomp ... * @param ngrow ... * @param level ... - * @param redistribute ... * @param initial_value ... + * @param redistribute follow the default domain decomposition of the simulation * @return pointer to newly allocated MultiFab */ amrex::MultiFab* alloc_init ( std::string name, - const amrex::BoxArray& ba, - const amrex::DistributionMapping& dm, - const int ncomp, - const amrex::IntVect& ngrow, - const int level, - bool redistribute = true, - std::optional initial_value = std::nullopt + amrex::BoxArray const & ba, + amrex::DistributionMapping const & dm, + int ncomp, + amrex::IntVect const & ngrow, + int level, + std::optional initial_value = std::nullopt, + bool redistribute = true ); /** title @@ -80,6 +80,19 @@ namespace ablastr::fields std::string other_key ); + /** title + * + * body body + * body + * + * @param name ... + * @return true if contained, otherwise false + */ + bool + contains ( + std::string name + ); + /** title * * body body diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 573ef3e8a97..b4062cd875c 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -13,13 +13,13 @@ namespace ablastr::fields amrex::MultiFab* MultiFabRegister::alloc_init ( std::string name, - const amrex::BoxArray& ba, - const amrex::DistributionMapping& dm, - const int ncomp, - const amrex::IntVect& ngrow, - const int level, - bool redistribute, - std::optional initial_value + amrex::BoxArray const & ba, + amrex::DistributionMapping const & dm, + int ncomp, + amrex::IntVect const & ngrow, + int level, + std::optional initial_value, + bool redistribute ) { // create name @@ -63,6 +63,14 @@ namespace ablastr::fields // TODO: does the key already exist? error } + bool + MultiFabRegister::contains ( + std::string name + ) + { + return m_mf_register.count(name) > 0; + } + /** title * * body body From 810f0e1972ec425bf8134f1ef2446c0fff74b488 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 9 Sep 2024 13:57:38 -0700 Subject: [PATCH 004/314] Add Level to: Contains, Get, Erase --- Source/ablastr/fields/MultiFabRegister.H | 32 ++++++++++-- Source/ablastr/fields/MultiFabRegister.cpp | 59 ++++++++++------------ 2 files changed, 56 insertions(+), 35 deletions(-) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 6f92719ddbc..c33b0f7aab5 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -74,10 +74,12 @@ namespace ablastr::fields * body * * @param name ... + * @param other_level ... */ void alloc_like ( - std::string other_key + std::string other_key, + int other_level ); /** title @@ -86,11 +88,13 @@ namespace ablastr::fields * body * * @param name ... + * @param level ... * @return true if contained, otherwise false */ bool contains ( - std::string name + std::string name, + int level ); /** title @@ -99,11 +103,13 @@ namespace ablastr::fields * body * * @param name ... + * @param level ... * @return ... */ amrex::MultiFab* get ( - std::string name + std::string name, + int level ); /** title @@ -122,14 +128,32 @@ namespace ablastr::fields * body * * @param name ... + * @param level ... * @return ... */ void erase ( - std::string name + std::string name, + int level + ); + + /** title + * + * body body + * body + * + * @param name ... + * @param level ... + * @return ... + */ + std::string + mf_name ( + std::string name, + int level ); private: + /** data storage: ownership and lifetime control */ std::map< std::string, MultiFabOwner diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index b4062cd875c..d03eaf2481a 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -22,9 +22,7 @@ namespace ablastr::fields bool redistribute ) { - // create name - // Add the suffix "[level=level]" - name.append("[level=").append(std::to_string(level)).append("]"); + name = mf_name(name, level); // Checks // TODO: does the key already exist? error @@ -54,9 +52,12 @@ namespace ablastr::fields void MultiFabRegister::alloc_like ( - std::string /* other_key */ + std::string /* other_key */, + int /* other_level */ ) { + // other_key = mf_name(other_key, other_level); + throw std::runtime_error("MultiFabRegister::alloc_like not yet implemented"); // Checks @@ -65,25 +66,23 @@ namespace ablastr::fields bool MultiFabRegister::contains ( - std::string name + std::string name, + int level ) { + name = mf_name(name, level); + return m_mf_register.count(name) > 0; } - /** title - * - * body body - * body - * - * @param name ... - * @return ... - */ amrex::MultiFab* MultiFabRegister::get ( - std::string name + std::string name, + int level ) { + name = mf_name(name, level); + if (m_mf_register.count(name) == 0) { throw std::runtime_error("MultiFabRegister::get name does not exist in register: " + name); } @@ -92,13 +91,6 @@ namespace ablastr::fields return &mf; } - /** title - * - * body body - * body - * - * @return ... - */ std::vector MultiFabRegister::list () { @@ -109,25 +101,30 @@ namespace ablastr::fields return names; } - - /** title - * - * body body - * body - * - * @param name ... - * @return ... - */ void MultiFabRegister::erase ( - std::string name + std::string name, + int level ) { + name = mf_name(name, level); + if (m_mf_register.count(name) != 1) { throw std::runtime_error("MultiFabRegister::remove name does not exist in register: " + name); } m_mf_register.erase(name); } + std::string + MultiFabRegister::mf_name ( + std::string name, + int level + ) + { + // Add the suffix "[level=level]" + return name.append("[level=") + .append(std::to_string(level)) + .append("]"); + } } // namespace ablastr::fields From 978b3bbbe38306744177447e155ebcc9736dbded Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 9 Sep 2024 14:07:06 -0700 Subject: [PATCH 005/314] Rule-of-Five --- Source/ablastr/fields/MultiFabRegister.H | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index c33b0f7aab5..8a07b399072 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -40,6 +40,14 @@ namespace ablastr::fields */ struct MultiFabRegister { + // Avoid accidental copies when passing to member functions + MultiFabRegister() = default; + MultiFabRegister(MultiFabRegister const &) = delete; + MultiFabRegister(MultiFabRegister&&) = delete; + MultiFabRegister& operator=(MultiFabRegister const &) = delete; + MultiFabRegister& operator=(MultiFabRegister&&) = delete; + ~MultiFabRegister() = default; + /** title * * body body From 47ba2e754872ed7ce38bf27e06542919c6c3c4ac Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 9 Sep 2024 14:07:17 -0700 Subject: [PATCH 006/314] has -> contains --- Source/ablastr/fields/MultiFabRegister.H | 6 +++--- Source/ablastr/fields/MultiFabRegister.cpp | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 8a07b399072..df362969ef8 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -81,12 +81,12 @@ namespace ablastr::fields * body body * body * - * @param name ... + * @param other_name ... * @param other_level ... */ void alloc_like ( - std::string other_key, + std::string other_name, int other_level ); @@ -100,7 +100,7 @@ namespace ablastr::fields * @return true if contained, otherwise false */ bool - contains ( + has ( std::string name, int level ); diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index d03eaf2481a..77a6156cc87 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -52,20 +52,20 @@ namespace ablastr::fields void MultiFabRegister::alloc_like ( - std::string /* other_key */, + std::string /* other_name */, int /* other_level */ ) { - // other_key = mf_name(other_key, other_level); + // other_name = mf_name(other_name, other_level); throw std::runtime_error("MultiFabRegister::alloc_like not yet implemented"); // Checks - // TODO: does the key already exist? error + // TODO: does the other_name already exist? error } bool - MultiFabRegister::contains ( + MultiFabRegister::has ( std::string name, int level ) From 81703a7735f9d20526a87818f9005453e0a8b108 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 9 Sep 2024 14:23:40 -0700 Subject: [PATCH 007/314] Start Python Bindings --- Source/Python/CMakeLists.txt | 1 + Source/Python/MultiFabRegister.cpp | 44 ++++++++++++++++++++++++++++++ Source/Python/WarpX.cpp | 7 +++++ Source/Python/pyWarpX.cpp | 2 ++ 4 files changed, 54 insertions(+) create mode 100644 Source/Python/MultiFabRegister.cpp diff --git a/Source/Python/CMakeLists.txt b/Source/Python/CMakeLists.txt index 17a75301306..1b4ab90aade 100644 --- a/Source/Python/CMakeLists.txt +++ b/Source/Python/CMakeLists.txt @@ -13,6 +13,7 @@ foreach(D IN LISTS WarpX_DIMS) target_sources(pyWarpX_${SD} PRIVATE # pybind11 + MultiFabRegister.cpp WarpX.cpp ) endif() diff --git a/Source/Python/MultiFabRegister.cpp b/Source/Python/MultiFabRegister.cpp new file mode 100644 index 00000000000..fd9d1201771 --- /dev/null +++ b/Source/Python/MultiFabRegister.cpp @@ -0,0 +1,44 @@ +/* Copyright 2024 The WarpX Community + * + * Authors: Axel Huebl + * License: BSD-3-Clause-LBNL + */ +#include "Python/pyWarpX.H" + +#include + + +void init_MultiFabRegister (py::module & m) +{ + using ablastr::fields::MultiFabRegister; + + py::class_(m, "MultiFabRegister") + .def("alloc_init", + &MultiFabRegister::alloc_init + // ... py::arg("name") + // "..." + ) + .def("alloc_like", + &MultiFabRegister::alloc_like, + py::arg("other_name"), + py::arg("other_level") + // "..." + ) + .def("has", + &MultiFabRegister::has, + py::arg("name"), + py::arg("level") + // "..." + ) + .def("list", + &MultiFabRegister::list + // "..." + ) + .def("erase", + &MultiFabRegister::erase, + py::arg("name"), + py::arg("level") + // "..." + ) + ; +} diff --git a/Source/Python/WarpX.cpp b/Source/Python/WarpX.cpp index e583d42c49c..8e7e8f189e0 100644 --- a/Source/Python/WarpX.cpp +++ b/Source/Python/WarpX.cpp @@ -110,6 +110,13 @@ void init_WarpX (py::module& m) //py::overload_cast< int >(&WarpX::boxArray, py::const_), py::arg("lev") ) + .def("field", + [](WarpX const & wx) { + return wx.multifab_map; + }, + py::return_value_policy::reference_internal, + R"doc(Registry to all WarpX MultiFab (fields).)doc" + ) .def("multifab", [](WarpX const & wx, std::string const multifab_name) { if (wx.multifab_map.count(multifab_name) > 0) { diff --git a/Source/Python/pyWarpX.cpp b/Source/Python/pyWarpX.cpp index 26f4c77502d..e128599abd0 100644 --- a/Source/Python/pyWarpX.cpp +++ b/Source/Python/pyWarpX.cpp @@ -32,6 +32,7 @@ // forward declarations of exposed classes void init_BoundaryBufferParIter (py::module&); void init_MultiParticleContainer (py::module&); +void init_MultiFabRegister (py::module&); void init_ParticleBoundaryBuffer (py::module&); void init_PinnedMemoryParticleContainer (py::module&); void init_WarpXParIter (py::module&); @@ -59,6 +60,7 @@ PYBIND11_MODULE(PYWARPX_MODULE_NAME, m) { )pbdoc"; // note: order from parent to child classes + init_MultiFabRegister(m); init_PinnedMemoryParticleContainer(m); init_WarpXParticleContainer(m); init_WarpXParIter(m); From db75efa314545b5e3c38b2914c494ad68224d2f9 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Mon, 9 Sep 2024 14:37:22 -0700 Subject: [PATCH 008/314] Moving G_fp and G_cp to m_multifab_map --- Source/Evolve/WarpXEvolve.cpp | 8 +++- Source/FieldSolver/Fields.H | 1 - .../FiniteDifferenceSolver/EvolveB.cpp | 6 +-- .../FiniteDifferenceSolver/EvolveG.cpp | 4 +- .../FiniteDifferenceSolver.H | 8 ++-- Source/FieldSolver/WarpXPushFieldsEM.cpp | 48 ++++++++++++------- Source/Initialization/WarpXInitData.cpp | 8 ++-- Source/Parallelization/WarpXComm.cpp | 22 +++++---- Source/Utils/WarpXMovingWindow.cpp | 6 +-- Source/WarpX.H | 6 +-- Source/WarpX.cpp | 24 +++++----- 11 files changed, 79 insertions(+), 62 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index bc18d0da75d..1c6d762b3cf 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -1114,7 +1114,9 @@ WarpX::applyMirrors (Real time) // If div(E)/div(B) cleaning are used, set F/G field to zero if (F_fp[lev]) { NullifyMF(*F_fp[lev], lev, z_min, z_max); } - if (G_fp[lev]) { NullifyMF(*G_fp[lev], lev, z_min, z_max); } + if (m_multifab_map.contains("G_fp", lev)) { + NullifyMF(*m_multifab_map.get("G_fp", lev), lev, z_min, z_max); + } if (lev>0) { @@ -1136,7 +1138,9 @@ WarpX::applyMirrors (Real time) // If div(E)/div(B) cleaning are used, set F/G field to zero if (F_cp[lev]) { NullifyMF(*F_cp[lev], lev, z_min, z_max); } - if (G_cp[lev]) { NullifyMF(*G_cp[lev], lev, z_min, z_max); } + if (m_multifab_map.contains("G_cp", lev)) { + NullifyMF(*m_multifab_map.get("G_cp", lev), lev, z_min, z_max); + } } } } diff --git a/Source/FieldSolver/Fields.H b/Source/FieldSolver/Fields.H index 9e4ce5a71a7..2f26224542a 100644 --- a/Source/FieldSolver/Fields.H +++ b/Source/FieldSolver/Fields.H @@ -33,7 +33,6 @@ namespace warpx::fields current_cp, rho_cp, F_cp, - G_cp, edge_lengths, face_areas, Efield_avg_fp, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp index 4fe9fc76e10..8a8d80c27e9 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp @@ -50,7 +50,7 @@ using namespace amrex; void FiniteDifferenceSolver::EvolveB ( std::array< std::unique_ptr, 3 >& Bfield, std::array< std::unique_ptr, 3 > const& Efield, - std::unique_ptr const& Gfield, + amrex::MultiFab * Gfield, std::array< std::unique_ptr, 3 > const& face_areas, std::array< std::unique_ptr, 3 > const& area_mod, std::array< std::unique_ptr, 3 >& ECTRhofield, @@ -103,7 +103,7 @@ template void FiniteDifferenceSolver::EvolveBCartesian ( std::array< std::unique_ptr, 3 >& Bfield, std::array< std::unique_ptr, 3 > const& Efield, - std::unique_ptr const& Gfield, + amrex::MultiFab * Gfield, // FIXME should be const int lev, amrex::Real const dt ) { amrex::LayoutData* cost = WarpX::getCosts(lev); @@ -169,7 +169,7 @@ void FiniteDifferenceSolver::EvolveBCartesian ( if (Gfield) { // Extract field data for this grid/tile - const Array4 G = Gfield->array(mfi); + Array4 G = Gfield->array(mfi); // Loop over cells and update G amrex::ParallelFor(tbx, tby, tbz, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveG.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveG.cpp index b6bc8fdca7f..730b6067bc5 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveG.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveG.cpp @@ -38,7 +38,7 @@ using namespace amrex; void FiniteDifferenceSolver::EvolveG ( - std::unique_ptr& Gfield, + amrex::MultiFab* Gfield, std::array,3> const& Bfield, amrex::Real const dt) { @@ -70,7 +70,7 @@ void FiniteDifferenceSolver::EvolveG ( template void FiniteDifferenceSolver::EvolveGCartesian ( - std::unique_ptr& Gfield, + amrex::MultiFab* Gfield, std::array,3> const& Bfield, amrex::Real const dt) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 6a33c89c184..3b1f61a601b 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -53,7 +53,7 @@ class FiniteDifferenceSolver void EvolveB ( std::array< std::unique_ptr, 3 >& Bfield, std::array< std::unique_ptr, 3 > const& Efield, - std::unique_ptr const& Gfield, + amrex::MultiFab * Gfield, std::array< std::unique_ptr, 3 > const& face_areas, std::array< std::unique_ptr, 3 > const& area_mod, std::array< std::unique_ptr, 3 >& ECTRhofield, @@ -77,7 +77,7 @@ class FiniteDifferenceSolver int rhocomp, amrex::Real dt ); - void EvolveG (std::unique_ptr& Gfield, + void EvolveG (amrex::MultiFab* Gfield, std::array,3> const& Bfield, amrex::Real dt); @@ -260,7 +260,7 @@ class FiniteDifferenceSolver void EvolveBCartesian ( std::array< std::unique_ptr, 3 >& Bfield, std::array< std::unique_ptr, 3 > const& Efield, - std::unique_ptr const& Gfield, + amrex::MultiFab * Gfield, // FIXME: should be const int lev, amrex::Real dt ); template< typename T_Algo > @@ -282,7 +282,7 @@ class FiniteDifferenceSolver template< typename T_Algo > void EvolveGCartesian ( - std::unique_ptr& Gfield, + amrex::MultiFab* Gfield, std::array,3> const& Bfield, amrex::Real dt); diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 602a2666b27..82eadeb3860 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -219,7 +219,7 @@ WarpX::PSATDBackwardTransformF () // Damp the field in the guard cells for (int lev = 0; lev <= finest_level; ++lev) { - DampFieldsInGuards(lev, F_fp[lev]); + DampFieldsInGuards(lev, F_fp[lev].get()); } } @@ -230,11 +230,15 @@ WarpX::PSATDForwardTransformG () for (int lev = 0; lev <= finest_level; ++lev) { - if (G_fp[lev]) { spectral_solver_fp[lev]->ForwardTransform(lev, *G_fp[lev], Idx.G); } + if (m_multifab_map.contains("G_fp", lev)) { + spectral_solver_fp[lev]->ForwardTransform(lev, *m_multifab_map.get("G_fp", lev), Idx.G); + } if (spectral_solver_cp[lev]) { - if (G_cp[lev]) { spectral_solver_cp[lev]->ForwardTransform(lev, *G_cp[lev], Idx.G); } + if (m_multifab_map.contains("G_cp", lev)) { + spectral_solver_fp[lev]->ForwardTransform(lev, *m_multifab_map.get("G_cp", lev), Idx.G); + } } } } @@ -246,27 +250,29 @@ WarpX::PSATDBackwardTransformG () for (int lev = 0; lev <= finest_level; ++lev) { + if (m_multifab_map.contains("G_fp", lev)) { + MultiFab* G_fp = m_multifab_map.get("G_fp", lev); #ifdef WARPX_DIM_RZ - if (G_fp[lev]) { spectral_solver_fp[lev]->BackwardTransform(lev, *G_fp[lev], Idx.G); } + spectral_solver_fp[lev]->BackwardTransform(lev, *G_fp, Idx.G); #else - if (G_fp[lev]) { spectral_solver_fp[lev]->BackwardTransform(lev, *G_fp[lev], Idx.G, m_fill_guards_fields); } + spectral_solver_fp[lev]->BackwardTransform(lev, *G_fp, Idx.G, m_fill_guards_fields); #endif + DampFieldsInGuards(lev, G_fp); + } + if (spectral_solver_cp[lev]) { + if (m_multifab_map.contains("G_cp", lev)) { + MultiFab* G_cp = m_multifab_map.get("G_cp", lev); #ifdef WARPX_DIM_RZ - if (G_cp[lev]) { spectral_solver_cp[lev]->BackwardTransform(lev, *G_cp[lev], Idx.G); } + spectral_solver_fp[lev]->BackwardTransform(lev, *G_cp, Idx.G); #else - if (G_cp[lev]) { spectral_solver_cp[lev]->BackwardTransform(lev, *G_cp[lev], Idx.G, m_fill_guards_fields); } + spectral_solver_fp[lev]->BackwardTransform(lev, *G_cp, Idx.G, m_fill_guards_fields); #endif + } } } - - // Damp the field in the guard cells - for (int lev = 0; lev <= finest_level; ++lev) - { - DampFieldsInGuards(lev, G_fp[lev]); - } } void WarpX::PSATDForwardTransformJ ( @@ -821,11 +827,13 @@ WarpX::EvolveB (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_typ // Evolve B field in regular cells if (patch_type == PatchType::fine) { - m_fdtd_solver_fp[lev]->EvolveB(Bfield_fp[lev], Efield_fp[lev], G_fp[lev], + m_fdtd_solver_fp[lev]->EvolveB(Bfield_fp[lev], Efield_fp[lev], + m_multifab_map.get("G_fp", lev), m_face_areas[lev], m_area_mod[lev], ECTRhofield[lev], Venl[lev], m_flag_info_face[lev], m_borrowing[lev], lev, a_dt); } else { - m_fdtd_solver_cp[lev]->EvolveB(Bfield_cp[lev], Efield_cp[lev], G_cp[lev], + m_fdtd_solver_cp[lev]->EvolveB(Bfield_cp[lev], Efield_cp[lev], + m_multifab_map.get("G_fp", lev), m_face_areas[lev], m_area_mod[lev], ECTRhofield[lev], Venl[lev], m_flag_info_face[lev], m_borrowing[lev], lev, a_dt); } @@ -1005,11 +1013,15 @@ WarpX::EvolveG (int lev, PatchType patch_type, amrex::Real a_dt, DtType /*a_dt_t // Evolve G field in regular cells if (patch_type == PatchType::fine) { - m_fdtd_solver_fp[lev]->EvolveG(G_fp[lev], Bfield_fp[lev], a_dt); + m_fdtd_solver_fp[lev]->EvolveG( + m_multifab_map.get("G_fp", lev), + Bfield_fp[lev], a_dt); } else // coarse patch { - m_fdtd_solver_cp[lev]->EvolveG(G_cp[lev], Bfield_cp[lev], a_dt); + m_fdtd_solver_cp[lev]->EvolveG( + m_multifab_map.get("G_cp", lev), + Bfield_cp[lev], a_dt); } // TODO Evolution in PML cells will go here @@ -1169,7 +1181,7 @@ WarpX::DampFieldsInGuards(const int lev, } } -void WarpX::DampFieldsInGuards(const int lev, std::unique_ptr& mf) +void WarpX::DampFieldsInGuards(const int lev, amrex::MultiFab* mf) { // Loop over dimensions for (int dampdir = 0; dampdir < AMREX_SPACEDIM; dampdir++) diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 5e5ebb19921..156ac15de8d 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -1226,9 +1226,9 @@ void WarpX::CheckGuardCells() ::CheckGuardCells(*F_fp[lev]); } - if (G_fp[lev]) + if (m_multifab_map.contains("G_fp", lev)) { - ::CheckGuardCells(*G_fp[lev]); + ::CheckGuardCells( *m_multifab_map.get("G_fp", lev) ); } // MultiFabs on coarse patch @@ -1257,9 +1257,9 @@ void WarpX::CheckGuardCells() ::CheckGuardCells(*F_cp[lev]); } - if (G_cp[lev]) + if (m_multifab_map.contains("G_cp", lev)) { - ::CheckGuardCells(*G_cp[lev]); + ::CheckGuardCells( *m_multifab_map.get("G_cp", lev) ); } } } diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 6c44df061fd..b95700f2cc0 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -932,30 +932,36 @@ void WarpX::FillBoundaryG (int lev, PatchType patch_type, IntVect ng, std::optio { if (do_pml && pml[lev] && pml[lev]->ok()) { - if (G_fp[lev]) { pml[lev]->ExchangeG(patch_type, G_fp[lev].get(), do_pml_in_domain); } + if (m_multifab_map.contains("G_fp",lev)) { + pml[lev]->ExchangeG(patch_type, m_multifab_map.get("G_fp",lev), do_pml_in_domain); + } pml[lev]->FillBoundaryG(patch_type, nodal_sync); } - if (G_fp[lev]) + if (m_multifab_map.contains("G_fp",lev)) { const amrex::Periodicity& period = Geom(lev).periodicity(); - const amrex::IntVect& nghost = (safe_guard_cells) ? G_fp[lev]->nGrowVect() : ng; - ablastr::utils::communication::FillBoundary(*G_fp[lev], nghost, WarpX::do_single_precision_comms, period, nodal_sync); + MultiFab* G_fp = m_multifab_map.get("G_fp",lev); + const amrex::IntVect& nghost = (safe_guard_cells) ? G_fp->nGrowVect() : ng; + ablastr::utils::communication::FillBoundary(*G_fp, nghost, WarpX::do_single_precision_comms, period, nodal_sync); } } else if (patch_type == PatchType::coarse) { if (do_pml && pml[lev] && pml[lev]->ok()) { - if (G_cp[lev]) { pml[lev]->ExchangeG(patch_type, G_cp[lev].get(), do_pml_in_domain); } + if (m_multifab_map.contains("G_cp",lev)) { + pml[lev]->ExchangeG(patch_type, m_multifab_map.get("G_cp",lev), do_pml_in_domain); + } pml[lev]->FillBoundaryG(patch_type, nodal_sync); } - if (G_cp[lev]) + if (m_multifab_map.contains("G_cp",lev)) { const amrex::Periodicity& period = Geom(lev-1).periodicity(); - const amrex::IntVect& nghost = (safe_guard_cells) ? G_cp[lev]->nGrowVect() : ng; - ablastr::utils::communication::FillBoundary(*G_cp[lev], nghost, WarpX::do_single_precision_comms, period, nodal_sync); + MultiFab* G_cp = m_multifab_map.get("G_cp",lev); + const amrex::IntVect& nghost = (safe_guard_cells) ? G_cp->nGrowVect() : ng; + ablastr::utils::communication::FillBoundary(*G_cp, nghost, WarpX::do_single_precision_comms, period, nodal_sync); } } } diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 73696838cd4..9f0423e679f 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -322,14 +322,14 @@ WarpX::MoveWindow (const int step, bool move_j) // Shift scalar field G with div(B) cleaning in valid domain // TODO: shift G from pml_rz for RZ geometry with PSATD, once implemented - if (G_fp[lev]) + if (m_multifab_map.contains("G_fp", lev)) { // Fine grid - shiftMF(*G_fp[lev], geom[lev], num_shift, dir, lev, do_update_cost); + shiftMF(*m_multifab_map.get("G_fp", lev), geom[lev], num_shift, dir, lev, do_update_cost); if (lev > 0) { // Coarse grid - shiftMF(*G_cp[lev], geom[lev-1], num_shift_crse, dir, lev, do_update_cost); + shiftMF(*m_multifab_map.get("G_cp", lev), geom[lev-1], num_shift_crse, dir, lev, do_update_cost); } } diff --git a/Source/WarpX.H b/Source/WarpX.H index 871e6efbbd0..71e03191a17 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -733,7 +733,7 @@ public: * can appear in parallel simulations. This will be called * when FieldBoundaryType is set to damped. Scalar version. */ - void DampFieldsInGuards (int lev, std::unique_ptr& mf); + void DampFieldsInGuards (int lev, amrex::MultiFab* mf); #ifdef WARPX_DIM_RZ void ApplyInverseVolumeScalingToCurrentDensity(amrex::MultiFab* Jx, @@ -1502,7 +1502,7 @@ private: amrex::Vector dt; // Field container - ablastr::fields::MultiFabRegister m_multifab; + ablastr::fields::MultiFabRegister m_multifab_map; // Particle container std::unique_ptr mypc; @@ -1522,7 +1522,6 @@ private: // Fine patch amrex::Vector< std::unique_ptr > F_fp; - amrex::Vector< std::unique_ptr > G_fp; amrex::Vector< std::unique_ptr > rho_fp; amrex::Vector< std::unique_ptr > phi_fp; amrex::Vector, 3 > > current_fp; @@ -1603,7 +1602,6 @@ private: // Coarse patch amrex::Vector< std::unique_ptr > F_cp; - amrex::Vector< std::unique_ptr > G_cp; amrex::Vector< std::unique_ptr > rho_cp; amrex::Vector, 3 > > current_cp; amrex::Vector, 3 > > Efield_cp; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index c55eedb87a5..e3899b2be46 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -321,7 +321,6 @@ WarpX::WarpX () Bfield_aux.resize(nlevs_max); F_fp.resize(nlevs_max); - G_fp.resize(nlevs_max); rho_fp.resize(nlevs_max); phi_fp.resize(nlevs_max); current_fp.resize(nlevs_max); @@ -383,7 +382,6 @@ WarpX::WarpX () } F_cp.resize(nlevs_max); - G_cp.resize(nlevs_max); rho_cp.resize(nlevs_max); current_cp.resize(nlevs_max); Efield_cp.resize(nlevs_max); @@ -2149,11 +2147,11 @@ WarpX::ClearLevel (int lev) gather_buffer_masks[lev].reset(); F_fp [lev].reset(); - G_fp [lev].reset(); + m_multifab_map.erase( "G_fp", lev ); + rho_fp[lev].reset(); phi_fp[lev].reset(); F_cp [lev].reset(); - G_cp [lev].reset(); rho_cp[lev].reset(); phi_dotMask[lev].reset(); @@ -2555,7 +2553,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (do_divb_cleaning) { - AllocInitMultiFab(G_fp[lev], amrex::convert(ba, G_nodal_flag), dm, ncomps, ngG, lev, "G_fp", 0.0_rt); + m_multifab_map.alloc_init( + "G_fp", amrex::convert(ba, G_nodal_flag), dm, + ncomps, ngG, lev, 0.0_rt); } if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) @@ -2758,11 +2758,15 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm { if (grid_type == GridType::Collocated) { - AllocInitMultiFab(G_cp[lev], amrex::convert(cba, IntVect::TheUnitVector()), dm, ncomps, ngG, lev, "G_cp", 0.0_rt); + m_multifab_map.alloc_init( + "G_cp", amrex::convert(ba, IntVect::TheUnitVector()), dm, + ncomps, ngG, lev, 0.0_rt); } else // grid_type=staggered or grid_type=hybrid { - AllocInitMultiFab(G_cp[lev], amrex::convert(cba, IntVect::TheZeroVector()), dm, ncomps, ngG, lev, "G_cp", 0.0_rt); + m_multifab_map.alloc_init( + "G_cp", amrex::convert(ba, IntVect::TheZeroVector()), dm, + ncomps, ngG, lev, 0.0_rt); } } @@ -3541,9 +3545,6 @@ WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, cons case FieldType::F_fp : field_pointer = F_fp[lev].get(); break; - case FieldType::G_fp : - field_pointer = G_fp[lev].get(); - break; case FieldType::phi_fp : field_pointer = phi_fp[lev].get(); break; @@ -3565,9 +3566,6 @@ WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, cons case FieldType::F_cp : field_pointer = F_cp[lev].get(); break; - case FieldType::G_cp : - field_pointer = G_cp[lev].get(); - break; case FieldType::edge_lengths : field_pointer = m_edge_lengths[lev][direction].get(); break; From ce9e1e5e386b894c4c3ec9b394c15ef8db029d20 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 9 Sep 2024 14:37:27 -0700 Subject: [PATCH 009/314] Constify: `DownwardD*` --- .../CartesianCKCAlgorithm.H | 8 ++++---- .../CartesianNodalAlgorithm.H | 12 ++++++------ .../CartesianYeeAlgorithm.H | 6 +++--- .../CylindricalYeeAlgorithm.H | 12 ++++++------ 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianCKCAlgorithm.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianCKCAlgorithm.H index 737146f24a3..cf27898cb86 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianCKCAlgorithm.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianCKCAlgorithm.H @@ -26,7 +26,7 @@ struct CartesianCKCAlgorithm { static void InitializeStencilCoefficients ( - std::array& cell_size, + std::array& cell_size, amrex::Vector& stencil_coefs_x, amrex::Vector& stencil_coefs_y, amrex::Vector& stencil_coefs_z ) { @@ -129,7 +129,7 @@ struct CartesianCKCAlgorithm { * Perform derivative along x on a cell-centered grid, from a nodal field `F` */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real UpwardDx ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_x, int const /*n_coefs_x*/, int const i, int const j, int const k, int const ncomp=0 ) { @@ -186,7 +186,7 @@ struct CartesianCKCAlgorithm { * Perform derivative along y on a cell-centered grid, from a nodal field `F` */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real UpwardDy ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_y, int const n_coefs_y, int const i, int const j, int const k, int const ncomp=0 ) { @@ -244,7 +244,7 @@ struct CartesianCKCAlgorithm { * Perform derivative along z on a cell-centered grid, from a nodal field `F` */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real UpwardDz ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_z, int const n_coefs_z, int const i, int const j, int const k, int const ncomp=0 ) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianNodalAlgorithm.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianNodalAlgorithm.H index b693ed8785f..46940e7e306 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianNodalAlgorithm.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianNodalAlgorithm.H @@ -69,7 +69,7 @@ struct CartesianNodalAlgorithm { * account the staggering; but for `CartesianNodalAlgorithm`, they are equivalent) */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real UpwardDx ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_x, int const /*n_coefs_x*/, int const i, int const j, int const k, int const ncomp=0 ) { @@ -89,7 +89,7 @@ struct CartesianNodalAlgorithm { * account the staggering; but for `CartesianNodalAlgorithm`, they are equivalent) */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real DownwardDx ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_x, int const n_coefs_x, int const i, int const j, int const k, int const ncomp=0 ) { @@ -109,7 +109,7 @@ struct CartesianNodalAlgorithm { * account the staggering; but for `CartesianNodalAlgorithm`, they are equivalent) */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real UpwardDy ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_y, int const /*n_coefs_y*/, int const i, int const j, int const k, int const ncomp=0 ) { @@ -129,7 +129,7 @@ struct CartesianNodalAlgorithm { * account the staggering; but for `CartesianNodalAlgorithm`, they are equivalent) */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real DownwardDy ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_y, int const n_coefs_y, int const i, int const j, int const k, int const ncomp=0 ) { @@ -143,7 +143,7 @@ struct CartesianNodalAlgorithm { * account the staggering; but for `CartesianNodalAlgorithm`, they are equivalent) */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real UpwardDz ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_z, int const /*n_coefs_z*/, int const i, int const j, int const k, int const ncomp=0 ) { @@ -164,7 +164,7 @@ struct CartesianNodalAlgorithm { * account the staggering; but for `CartesianNodalAlgorithm`, they are equivalent) */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real DownwardDz ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_z, int const n_coefs_z, int const i, int const j, int const k, int const ncomp=0 ) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianYeeAlgorithm.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianYeeAlgorithm.H index c4978287aec..485698802b6 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianYeeAlgorithm.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianYeeAlgorithm.H @@ -67,7 +67,7 @@ struct CartesianYeeAlgorithm { * Perform derivative along x on a cell-centered grid, from a nodal field `F`*/ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real UpwardDx ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_x, int const /*n_coefs_x*/, int const i, int const j, int const k, int const ncomp=0 ) { @@ -123,7 +123,7 @@ struct CartesianYeeAlgorithm { * Perform derivative along y on a cell-centered grid, from a nodal field `F`*/ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real UpwardDy ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_y, int const n_coefs_y, int const i, int const j, int const k, int const ncomp=0 ) { @@ -189,7 +189,7 @@ struct CartesianYeeAlgorithm { * Perform derivative along z on a cell-centered grid, from a nodal field `F`*/ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real UpwardDz ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_z, int const /*n_coefs_z*/, int const i, int const j, int const k, int const ncomp=0 ) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H index 436f0a83f31..d20e1ef829b 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H @@ -74,7 +74,7 @@ struct CylindricalYeeAlgorithm { * The input parameter `r` is given at the cell-centered position */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real UpwardDrr_over_r ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const r, amrex::Real const dr, amrex::Real const * const coefs_r, int const n_coefs_r, int const i, int const j, int const k, int const comp ) { @@ -92,7 +92,7 @@ struct CylindricalYeeAlgorithm { * The input parameter `r` is given at the cell-centered position */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real DownwardDrr_over_r ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const r, amrex::Real const dr, amrex::Real const * const coefs_r, int const n_coefs_r, int const i, int const j, int const k, int const comp ) { @@ -108,7 +108,7 @@ struct CylindricalYeeAlgorithm { * Perform derivative along r on a cell-centered grid, from a nodal field `F` */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real UpwardDr ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_r, int const n_coefs_r, int const i, int const j, int const k, int const comp ) { @@ -123,7 +123,7 @@ struct CylindricalYeeAlgorithm { * Perform derivative along r on a nodal grid, from a cell-centered field `F` */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real DownwardDr ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_r, int const n_coefs_r, int const i, int const j, int const k, int const comp ) { @@ -156,7 +156,7 @@ struct CylindricalYeeAlgorithm { * Perform derivative along z on a cell-centered grid, from a nodal field `F` */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real UpwardDz ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_z, int const n_coefs_z, int const i, int const j, int const k, int const comp ) { @@ -170,7 +170,7 @@ struct CylindricalYeeAlgorithm { * Perform derivative along z on a nodal grid, from a cell-centered field `F` */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static amrex::Real DownwardDz ( - amrex::Array4 const& F, + amrex::Array4 const& F, amrex::Real const * const coefs_z, int const n_coefs_z, int const i, int const j, int const k, int const comp ) { From 7f3005d5bf01e9f402c838b8ed615a616f635c62 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Mon, 9 Sep 2024 14:39:47 -0700 Subject: [PATCH 010/314] Use has instead of contains --- Source/Evolve/WarpXEvolve.cpp | 4 ++-- Source/FieldSolver/WarpXPushFieldsEM.cpp | 8 ++++---- Source/Initialization/WarpXInitData.cpp | 4 ++-- Source/Parallelization/WarpXComm.cpp | 8 ++++---- Source/Utils/WarpXMovingWindow.cpp | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 1c6d762b3cf..f3981dfcb38 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -1114,7 +1114,7 @@ WarpX::applyMirrors (Real time) // If div(E)/div(B) cleaning are used, set F/G field to zero if (F_fp[lev]) { NullifyMF(*F_fp[lev], lev, z_min, z_max); } - if (m_multifab_map.contains("G_fp", lev)) { + if (m_multifab_map.has("G_fp", lev)) { NullifyMF(*m_multifab_map.get("G_fp", lev), lev, z_min, z_max); } @@ -1138,7 +1138,7 @@ WarpX::applyMirrors (Real time) // If div(E)/div(B) cleaning are used, set F/G field to zero if (F_cp[lev]) { NullifyMF(*F_cp[lev], lev, z_min, z_max); } - if (m_multifab_map.contains("G_cp", lev)) { + if (m_multifab_map.has("G_cp", lev)) { NullifyMF(*m_multifab_map.get("G_cp", lev), lev, z_min, z_max); } } diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 82eadeb3860..2b817decdbb 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -230,13 +230,13 @@ WarpX::PSATDForwardTransformG () for (int lev = 0; lev <= finest_level; ++lev) { - if (m_multifab_map.contains("G_fp", lev)) { + if (m_multifab_map.has("G_fp", lev)) { spectral_solver_fp[lev]->ForwardTransform(lev, *m_multifab_map.get("G_fp", lev), Idx.G); } if (spectral_solver_cp[lev]) { - if (m_multifab_map.contains("G_cp", lev)) { + if (m_multifab_map.has("G_cp", lev)) { spectral_solver_fp[lev]->ForwardTransform(lev, *m_multifab_map.get("G_cp", lev), Idx.G); } } @@ -250,7 +250,7 @@ WarpX::PSATDBackwardTransformG () for (int lev = 0; lev <= finest_level; ++lev) { - if (m_multifab_map.contains("G_fp", lev)) { + if (m_multifab_map.has("G_fp", lev)) { MultiFab* G_fp = m_multifab_map.get("G_fp", lev); #ifdef WARPX_DIM_RZ spectral_solver_fp[lev]->BackwardTransform(lev, *G_fp, Idx.G); @@ -263,7 +263,7 @@ WarpX::PSATDBackwardTransformG () if (spectral_solver_cp[lev]) { - if (m_multifab_map.contains("G_cp", lev)) { + if (m_multifab_map.has("G_cp", lev)) { MultiFab* G_cp = m_multifab_map.get("G_cp", lev); #ifdef WARPX_DIM_RZ spectral_solver_fp[lev]->BackwardTransform(lev, *G_cp, Idx.G); diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 156ac15de8d..477e1a044e2 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -1226,7 +1226,7 @@ void WarpX::CheckGuardCells() ::CheckGuardCells(*F_fp[lev]); } - if (m_multifab_map.contains("G_fp", lev)) + if (m_multifab_map.has("G_fp", lev)) { ::CheckGuardCells( *m_multifab_map.get("G_fp", lev) ); } @@ -1257,7 +1257,7 @@ void WarpX::CheckGuardCells() ::CheckGuardCells(*F_cp[lev]); } - if (m_multifab_map.contains("G_cp", lev)) + if (m_multifab_map.has("G_cp", lev)) { ::CheckGuardCells( *m_multifab_map.get("G_cp", lev) ); } diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index b95700f2cc0..538152d5bd0 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -932,13 +932,13 @@ void WarpX::FillBoundaryG (int lev, PatchType patch_type, IntVect ng, std::optio { if (do_pml && pml[lev] && pml[lev]->ok()) { - if (m_multifab_map.contains("G_fp",lev)) { + if (m_multifab_map.has("G_fp",lev)) { pml[lev]->ExchangeG(patch_type, m_multifab_map.get("G_fp",lev), do_pml_in_domain); } pml[lev]->FillBoundaryG(patch_type, nodal_sync); } - if (m_multifab_map.contains("G_fp",lev)) + if (m_multifab_map.has("G_fp",lev)) { const amrex::Periodicity& period = Geom(lev).periodicity(); MultiFab* G_fp = m_multifab_map.get("G_fp",lev); @@ -950,13 +950,13 @@ void WarpX::FillBoundaryG (int lev, PatchType patch_type, IntVect ng, std::optio { if (do_pml && pml[lev] && pml[lev]->ok()) { - if (m_multifab_map.contains("G_cp",lev)) { + if (m_multifab_map.has("G_cp",lev)) { pml[lev]->ExchangeG(patch_type, m_multifab_map.get("G_cp",lev), do_pml_in_domain); } pml[lev]->FillBoundaryG(patch_type, nodal_sync); } - if (m_multifab_map.contains("G_cp",lev)) + if (m_multifab_map.has("G_cp",lev)) { const amrex::Periodicity& period = Geom(lev-1).periodicity(); MultiFab* G_cp = m_multifab_map.get("G_cp",lev); diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 9f0423e679f..d101244e304 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -322,7 +322,7 @@ WarpX::MoveWindow (const int step, bool move_j) // Shift scalar field G with div(B) cleaning in valid domain // TODO: shift G from pml_rz for RZ geometry with PSATD, once implemented - if (m_multifab_map.contains("G_fp", lev)) + if (m_multifab_map.has("G_fp", lev)) { // Fine grid shiftMF(*m_multifab_map.get("G_fp", lev), geom[lev], num_shift, dir, lev, do_update_cost); From 052e54d5888d4c2c8bcaf5c830a59b5838e7da5e Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 9 Sep 2024 14:43:40 -0700 Subject: [PATCH 011/314] WarpX: Not Movable --- Source/WarpX.H | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/WarpX.H b/Source/WarpX.H index 71e03191a17..7cbf71b377e 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -105,9 +105,9 @@ public: WarpX& operator= ( WarpX const & ) = delete; /** Move constructor */ - WarpX ( WarpX && ) = default; + WarpX ( WarpX && ) = delete; /** Move operator */ - WarpX& operator= ( WarpX && ) = default; + WarpX& operator= ( WarpX && ) = delete; static std::string Version (); //!< Version of WarpX executable static std::string PicsarVersion (); //!< Version of PICSAR dependency From 1117250aeb62ea5d6f58d688a5ba8bcd3beceead Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 9 Sep 2024 14:50:40 -0700 Subject: [PATCH 012/314] Constify GField --- Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp | 6 +++--- .../FiniteDifferenceSolver/FiniteDifferenceSolver.H | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp index 8a8d80c27e9..0609f12f5c3 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp @@ -50,7 +50,7 @@ using namespace amrex; void FiniteDifferenceSolver::EvolveB ( std::array< std::unique_ptr, 3 >& Bfield, std::array< std::unique_ptr, 3 > const& Efield, - amrex::MultiFab * Gfield, + amrex::MultiFab const * Gfield, std::array< std::unique_ptr, 3 > const& face_areas, std::array< std::unique_ptr, 3 > const& area_mod, std::array< std::unique_ptr, 3 >& ECTRhofield, @@ -103,7 +103,7 @@ template void FiniteDifferenceSolver::EvolveBCartesian ( std::array< std::unique_ptr, 3 >& Bfield, std::array< std::unique_ptr, 3 > const& Efield, - amrex::MultiFab * Gfield, // FIXME should be const + amrex::MultiFab const * Gfield, int lev, amrex::Real const dt ) { amrex::LayoutData* cost = WarpX::getCosts(lev); @@ -169,7 +169,7 @@ void FiniteDifferenceSolver::EvolveBCartesian ( if (Gfield) { // Extract field data for this grid/tile - Array4 G = Gfield->array(mfi); + Array4 const G = Gfield->array(mfi); // Loop over cells and update G amrex::ParallelFor(tbx, tby, tbz, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 3b1f61a601b..b9b8104ecf2 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -53,7 +53,7 @@ class FiniteDifferenceSolver void EvolveB ( std::array< std::unique_ptr, 3 >& Bfield, std::array< std::unique_ptr, 3 > const& Efield, - amrex::MultiFab * Gfield, + amrex::MultiFab const * Gfield, std::array< std::unique_ptr, 3 > const& face_areas, std::array< std::unique_ptr, 3 > const& area_mod, std::array< std::unique_ptr, 3 >& ECTRhofield, @@ -260,7 +260,7 @@ class FiniteDifferenceSolver void EvolveBCartesian ( std::array< std::unique_ptr, 3 >& Bfield, std::array< std::unique_ptr, 3 > const& Efield, - amrex::MultiFab * Gfield, // FIXME: should be const + amrex::MultiFab const * Gfield, int lev, amrex::Real dt ); template< typename T_Algo > From 665b346c995d82d14da7ced6517035aa7dfe45e9 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Mon, 9 Sep 2024 15:02:18 -0700 Subject: [PATCH 013/314] Fix getFieldPointer --- Source/Diagnostics/FullDiagnostics.cpp | 4 ++-- Source/FieldSolver/Fields.H | 1 - Source/WarpX.H | 6 +++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index bcf613f49b0..d0b3f278548 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -320,7 +320,7 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) AddRZModesToOutputNames(std::string("F"), ncomp); } } else if ( m_varnames_fields[comp] == "G" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::G_fp, lev), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique( warpx.m_multifab_map.get("G_fp", lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("G"), ncomp); @@ -686,7 +686,7 @@ FullDiagnostics::InitializeFieldFunctors (int lev) } else if ( m_varnames[comp] == "F" ){ m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::F_fp, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "G" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::G_fp, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_multifab_map.get("G_fp", lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "phi" ){ m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::phi_fp, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "part_per_cell" ){ diff --git a/Source/FieldSolver/Fields.H b/Source/FieldSolver/Fields.H index 2f26224542a..55b37a2182c 100644 --- a/Source/FieldSolver/Fields.H +++ b/Source/FieldSolver/Fields.H @@ -25,7 +25,6 @@ namespace warpx::fields current_fp_nodal, rho_fp, F_fp, - G_fp, phi_fp, vector_potential_fp, Efield_cp, diff --git a/Source/WarpX.H b/Source/WarpX.H index 71e03191a17..72bc9c49a4a 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1232,6 +1232,9 @@ public: FiniteDifferenceSolver * get_pointer_fdtd_solver_fp (int lev) { return m_fdtd_solver_fp[lev].get(); } + // Field container + ablastr::fields::MultiFabRegister m_multifab_map; + protected: /** @@ -1501,9 +1504,6 @@ private: amrex::Vector t_old; amrex::Vector dt; - // Field container - ablastr::fields::MultiFabRegister m_multifab_map; - // Particle container std::unique_ptr mypc; std::unique_ptr multi_diags; From 4a96b1d0666354bfe91fce68e1677e33d6e142ca Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 9 Sep 2024 15:08:15 -0700 Subject: [PATCH 014/314] `NullifyMF`: Pass by String --- Source/Evolve/WarpXEvolve.cpp | 56 +++++++++++------------------------ Source/Utils/WarpXUtil.H | 16 ++++++++-- Source/Utils/WarpXUtil.cpp | 22 +++++++++----- 3 files changed, 46 insertions(+), 48 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index f3981dfcb38..b2f98266b7b 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -1096,51 +1096,31 @@ WarpX::applyMirrors (Real time) const amrex::Real dz = WarpX::CellSize(lev)[2]; const amrex::Real z_max = std::max(z_max_tmp, z_min+mirror_z_npoints[i_mirror]*dz); - // Get fine patch field MultiFabs - amrex::MultiFab& Ex = *Efield_fp[lev][0].get(); - amrex::MultiFab& Ey = *Efield_fp[lev][1].get(); - amrex::MultiFab& Ez = *Efield_fp[lev][2].get(); - amrex::MultiFab& Bx = *Bfield_fp[lev][0].get(); - amrex::MultiFab& By = *Bfield_fp[lev][1].get(); - amrex::MultiFab& Bz = *Bfield_fp[lev][2].get(); - - // Set each field to zero between z_min and z_max - NullifyMF(Ex, lev, z_min, z_max); - NullifyMF(Ey, lev, z_min, z_max); - NullifyMF(Ez, lev, z_min, z_max); - NullifyMF(Bx, lev, z_min, z_max); - NullifyMF(By, lev, z_min, z_max); - NullifyMF(Bz, lev, z_min, z_max); + // Set each field on the fine patch to zero between z_min and z_max + NullifyMF(m_multifab_map, "Efield_fp[0]", lev, z_min, z_max); + NullifyMF(m_multifab_map, "Efield_fp[1]", lev, z_min, z_max); + NullifyMF(m_multifab_map, "Efield_fp[2]", lev, z_min, z_max); + NullifyMF(m_multifab_map, "Bfield_fp[0]", lev, z_min, z_max); + NullifyMF(m_multifab_map, "Bfield_fp[1]", lev, z_min, z_max); + NullifyMF(m_multifab_map, "Bfield_fp[2]", lev, z_min, z_max); // If div(E)/div(B) cleaning are used, set F/G field to zero - if (F_fp[lev]) { NullifyMF(*F_fp[lev], lev, z_min, z_max); } - if (m_multifab_map.has("G_fp", lev)) { - NullifyMF(*m_multifab_map.get("G_fp", lev), lev, z_min, z_max); - } + NullifyMF(m_multifab_map, "F_fp", lev, z_min, z_max); + NullifyMF(m_multifab_map, "G_fp", lev, z_min, z_max); if (lev>0) { - // Get coarse patch field MultiFabs - amrex::MultiFab& cEx = *Efield_cp[lev][0].get(); - amrex::MultiFab& cEy = *Efield_cp[lev][1].get(); - amrex::MultiFab& cEz = *Efield_cp[lev][2].get(); - amrex::MultiFab& cBx = *Bfield_cp[lev][0].get(); - amrex::MultiFab& cBy = *Bfield_cp[lev][1].get(); - amrex::MultiFab& cBz = *Bfield_cp[lev][2].get(); - - // Set each field to zero between z_min and z_max - NullifyMF(cEx, lev, z_min, z_max); - NullifyMF(cEy, lev, z_min, z_max); - NullifyMF(cEz, lev, z_min, z_max); - NullifyMF(cBx, lev, z_min, z_max); - NullifyMF(cBy, lev, z_min, z_max); - NullifyMF(cBz, lev, z_min, z_max); + // Set each field on the coarse patch to zero between z_min and z_max + NullifyMF(m_multifab_map, "Efield_cp[0]", lev, z_min, z_max); + NullifyMF(m_multifab_map, "Efield_cp[1]", lev, z_min, z_max); + NullifyMF(m_multifab_map, "Efield_cp[2]", lev, z_min, z_max); + NullifyMF(m_multifab_map, "Bfield_cp[0]", lev, z_min, z_max); + NullifyMF(m_multifab_map, "Bfield_cp[1]", lev, z_min, z_max); + NullifyMF(m_multifab_map, "Bfield_cp[2]", lev, z_min, z_max); // If div(E)/div(B) cleaning are used, set F/G field to zero - if (F_cp[lev]) { NullifyMF(*F_cp[lev], lev, z_min, z_max); } - if (m_multifab_map.has("G_cp", lev)) { - NullifyMF(*m_multifab_map.get("G_cp", lev), lev, z_min, z_max); - } + NullifyMF(m_multifab_map, "F_cp", lev, z_min, z_max); + NullifyMF(m_multifab_map, "G_cp", lev, z_min, z_max); } } } diff --git a/Source/Utils/WarpXUtil.H b/Source/Utils/WarpXUtil.H index 1de03eb61f0..fca0b95ba7f 100644 --- a/Source/Utils/WarpXUtil.H +++ b/Source/Utils/WarpXUtil.H @@ -8,6 +8,8 @@ #ifndef WARPX_UTILS_H_ #define WARPX_UTILS_H_ +#include + #include #include #include @@ -53,9 +55,17 @@ void CheckDims (); */ void CheckGriddingForRZSpectral (); -void NullifyMF(amrex::MultiFab& mf, int lev, amrex::Real zmin, - amrex::Real zmax); - +/** Function that sets the value of MultiFab MF to zero. + * + * For z between zmin and zmax. + */ +void NullifyMF ( + ablastr::fields::MultiFabRegister& multifab_map, + std::string mf_name, + int lev, + amrex::Real zmin, + amrex::Real zmax +); namespace WarpXUtilIO{ /** diff --git a/Source/Utils/WarpXUtil.cpp b/Source/Utils/WarpXUtil.cpp index 2ef4ee55d6e..35d0f08be54 100644 --- a/Source/Utils/WarpXUtil.cpp +++ b/Source/Utils/WarpXUtil.cpp @@ -14,6 +14,7 @@ #include "WarpXProfilerWrapper.H" #include "WarpXUtil.H" +#include #include #include @@ -220,16 +221,23 @@ void ConvertLabParamsToBoost() } -/* \brief Function that sets the value of MultiFab MF to zero for z between - * zmin and zmax. - */ -void NullifyMF(amrex::MultiFab& mf, int lev, amrex::Real zmin, amrex::Real zmax){ +void NullifyMF ( + ablastr::fields::MultiFabRegister& multifab_map, + std::string mf_name, + int lev, + amrex::Real zmin, + amrex::Real zmax +) +{ WARPX_PROFILE("WarpXUtil::NullifyMF()"); - int const ncomp = mf.nComp(); + if (!multifab_map.has(mf_name, lev)) { return; } + + auto * mf = multifab_map.get(mf_name, lev); + int const ncomp = mf->nComp(); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for(amrex::MFIter mfi(mf, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi){ + for(amrex::MFIter mfi(*mf, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi){ const amrex::Box& bx = mfi.tilebox(); // Get box lower and upper physical z bound, and dz const amrex::Real zmin_box = WarpX::LowerCorner(bx, lev, 0._rt).z; @@ -245,7 +253,7 @@ void NullifyMF(amrex::MultiFab& mf, int lev, amrex::Real zmin, amrex::Real zmax) #endif // Check if box intersect with [zmin, zmax] if ( (zmax>zmin_box && zmin<=zmax_box) ){ - const Array4 arr = mf[mfi].array(); + const Array4 arr = (*mf)[mfi].array(); // Set field to 0 between zmin and zmax ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept{ From 1e8df508bfb3f9fe774055b5711130226d369a85 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Mon, 9 Sep 2024 15:12:28 -0700 Subject: [PATCH 015/314] Replace m_multifab_map by m_fields --- Source/Diagnostics/FullDiagnostics.cpp | 4 ++-- Source/Evolve/WarpXEvolve.cpp | 8 ++++---- Source/FieldSolver/WarpXPushFieldsEM.cpp | 24 ++++++++++++------------ Source/Initialization/WarpXInitData.cpp | 8 ++++---- Source/Parallelization/WarpXComm.cpp | 16 ++++++++-------- Source/Utils/WarpXMovingWindow.cpp | 6 +++--- Source/WarpX.H | 2 +- Source/WarpX.cpp | 8 ++++---- 8 files changed, 38 insertions(+), 38 deletions(-) diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index d0b3f278548..1ee725e7312 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -320,7 +320,7 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) AddRZModesToOutputNames(std::string("F"), ncomp); } } else if ( m_varnames_fields[comp] == "G" ){ - m_all_field_functors[lev][comp] = std::make_unique( warpx.m_multifab_map.get("G_fp", lev), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique( warpx.m_fields.get("G_fp", lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("G"), ncomp); @@ -686,7 +686,7 @@ FullDiagnostics::InitializeFieldFunctors (int lev) } else if ( m_varnames[comp] == "F" ){ m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::F_fp, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "G" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_multifab_map.get("G_fp", lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("G_fp", lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "phi" ){ m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::phi_fp, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "part_per_cell" ){ diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index f3981dfcb38..f09dc2fdad8 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -1114,8 +1114,8 @@ WarpX::applyMirrors (Real time) // If div(E)/div(B) cleaning are used, set F/G field to zero if (F_fp[lev]) { NullifyMF(*F_fp[lev], lev, z_min, z_max); } - if (m_multifab_map.has("G_fp", lev)) { - NullifyMF(*m_multifab_map.get("G_fp", lev), lev, z_min, z_max); + if (m_fields.has("G_fp", lev)) { + NullifyMF(*m_fields.get("G_fp", lev), lev, z_min, z_max); } if (lev>0) @@ -1138,8 +1138,8 @@ WarpX::applyMirrors (Real time) // If div(E)/div(B) cleaning are used, set F/G field to zero if (F_cp[lev]) { NullifyMF(*F_cp[lev], lev, z_min, z_max); } - if (m_multifab_map.has("G_cp", lev)) { - NullifyMF(*m_multifab_map.get("G_cp", lev), lev, z_min, z_max); + if (m_fields.has("G_cp", lev)) { + NullifyMF(*m_fields.get("G_cp", lev), lev, z_min, z_max); } } } diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 2b817decdbb..33948bb17d3 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -230,14 +230,14 @@ WarpX::PSATDForwardTransformG () for (int lev = 0; lev <= finest_level; ++lev) { - if (m_multifab_map.has("G_fp", lev)) { - spectral_solver_fp[lev]->ForwardTransform(lev, *m_multifab_map.get("G_fp", lev), Idx.G); + if (m_fields.has("G_fp", lev)) { + spectral_solver_fp[lev]->ForwardTransform(lev, *m_fields.get("G_fp", lev), Idx.G); } if (spectral_solver_cp[lev]) { - if (m_multifab_map.has("G_cp", lev)) { - spectral_solver_fp[lev]->ForwardTransform(lev, *m_multifab_map.get("G_cp", lev), Idx.G); + if (m_fields.has("G_cp", lev)) { + spectral_solver_fp[lev]->ForwardTransform(lev, *m_fields.get("G_cp", lev), Idx.G); } } } @@ -250,8 +250,8 @@ WarpX::PSATDBackwardTransformG () for (int lev = 0; lev <= finest_level; ++lev) { - if (m_multifab_map.has("G_fp", lev)) { - MultiFab* G_fp = m_multifab_map.get("G_fp", lev); + if (m_fields.has("G_fp", lev)) { + MultiFab* G_fp = m_fields.get("G_fp", lev); #ifdef WARPX_DIM_RZ spectral_solver_fp[lev]->BackwardTransform(lev, *G_fp, Idx.G); #else @@ -263,8 +263,8 @@ WarpX::PSATDBackwardTransformG () if (spectral_solver_cp[lev]) { - if (m_multifab_map.has("G_cp", lev)) { - MultiFab* G_cp = m_multifab_map.get("G_cp", lev); + if (m_fields.has("G_cp", lev)) { + MultiFab* G_cp = m_fields.get("G_cp", lev); #ifdef WARPX_DIM_RZ spectral_solver_fp[lev]->BackwardTransform(lev, *G_cp, Idx.G); #else @@ -828,12 +828,12 @@ WarpX::EvolveB (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_typ // Evolve B field in regular cells if (patch_type == PatchType::fine) { m_fdtd_solver_fp[lev]->EvolveB(Bfield_fp[lev], Efield_fp[lev], - m_multifab_map.get("G_fp", lev), + m_fields.get("G_fp", lev), m_face_areas[lev], m_area_mod[lev], ECTRhofield[lev], Venl[lev], m_flag_info_face[lev], m_borrowing[lev], lev, a_dt); } else { m_fdtd_solver_cp[lev]->EvolveB(Bfield_cp[lev], Efield_cp[lev], - m_multifab_map.get("G_fp", lev), + m_fields.get("G_fp", lev), m_face_areas[lev], m_area_mod[lev], ECTRhofield[lev], Venl[lev], m_flag_info_face[lev], m_borrowing[lev], lev, a_dt); } @@ -1014,13 +1014,13 @@ WarpX::EvolveG (int lev, PatchType patch_type, amrex::Real a_dt, DtType /*a_dt_t if (patch_type == PatchType::fine) { m_fdtd_solver_fp[lev]->EvolveG( - m_multifab_map.get("G_fp", lev), + m_fields.get("G_fp", lev), Bfield_fp[lev], a_dt); } else // coarse patch { m_fdtd_solver_cp[lev]->EvolveG( - m_multifab_map.get("G_cp", lev), + m_fields.get("G_cp", lev), Bfield_cp[lev], a_dt); } diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 477e1a044e2..2ee0b3d50ba 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -1226,9 +1226,9 @@ void WarpX::CheckGuardCells() ::CheckGuardCells(*F_fp[lev]); } - if (m_multifab_map.has("G_fp", lev)) + if (m_fields.has("G_fp", lev)) { - ::CheckGuardCells( *m_multifab_map.get("G_fp", lev) ); + ::CheckGuardCells( *m_fields.get("G_fp", lev) ); } // MultiFabs on coarse patch @@ -1257,9 +1257,9 @@ void WarpX::CheckGuardCells() ::CheckGuardCells(*F_cp[lev]); } - if (m_multifab_map.has("G_cp", lev)) + if (m_fields.has("G_cp", lev)) { - ::CheckGuardCells( *m_multifab_map.get("G_cp", lev) ); + ::CheckGuardCells( *m_fields.get("G_cp", lev) ); } } } diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 538152d5bd0..d967ca2a40d 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -932,16 +932,16 @@ void WarpX::FillBoundaryG (int lev, PatchType patch_type, IntVect ng, std::optio { if (do_pml && pml[lev] && pml[lev]->ok()) { - if (m_multifab_map.has("G_fp",lev)) { - pml[lev]->ExchangeG(patch_type, m_multifab_map.get("G_fp",lev), do_pml_in_domain); + if (m_fields.has("G_fp",lev)) { + pml[lev]->ExchangeG(patch_type, m_fields.get("G_fp",lev), do_pml_in_domain); } pml[lev]->FillBoundaryG(patch_type, nodal_sync); } - if (m_multifab_map.has("G_fp",lev)) + if (m_fields.has("G_fp",lev)) { const amrex::Periodicity& period = Geom(lev).periodicity(); - MultiFab* G_fp = m_multifab_map.get("G_fp",lev); + MultiFab* G_fp = m_fields.get("G_fp",lev); const amrex::IntVect& nghost = (safe_guard_cells) ? G_fp->nGrowVect() : ng; ablastr::utils::communication::FillBoundary(*G_fp, nghost, WarpX::do_single_precision_comms, period, nodal_sync); } @@ -950,16 +950,16 @@ void WarpX::FillBoundaryG (int lev, PatchType patch_type, IntVect ng, std::optio { if (do_pml && pml[lev] && pml[lev]->ok()) { - if (m_multifab_map.has("G_cp",lev)) { - pml[lev]->ExchangeG(patch_type, m_multifab_map.get("G_cp",lev), do_pml_in_domain); + if (m_fields.has("G_cp",lev)) { + pml[lev]->ExchangeG(patch_type, m_fields.get("G_cp",lev), do_pml_in_domain); } pml[lev]->FillBoundaryG(patch_type, nodal_sync); } - if (m_multifab_map.has("G_cp",lev)) + if (m_fields.has("G_cp",lev)) { const amrex::Periodicity& period = Geom(lev-1).periodicity(); - MultiFab* G_cp = m_multifab_map.get("G_cp",lev); + MultiFab* G_cp = m_fields.get("G_cp",lev); const amrex::IntVect& nghost = (safe_guard_cells) ? G_cp->nGrowVect() : ng; ablastr::utils::communication::FillBoundary(*G_cp, nghost, WarpX::do_single_precision_comms, period, nodal_sync); } diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index d101244e304..3747268f381 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -322,14 +322,14 @@ WarpX::MoveWindow (const int step, bool move_j) // Shift scalar field G with div(B) cleaning in valid domain // TODO: shift G from pml_rz for RZ geometry with PSATD, once implemented - if (m_multifab_map.has("G_fp", lev)) + if (m_fields.has("G_fp", lev)) { // Fine grid - shiftMF(*m_multifab_map.get("G_fp", lev), geom[lev], num_shift, dir, lev, do_update_cost); + shiftMF(*m_fields.get("G_fp", lev), geom[lev], num_shift, dir, lev, do_update_cost); if (lev > 0) { // Coarse grid - shiftMF(*m_multifab_map.get("G_cp", lev), geom[lev-1], num_shift_crse, dir, lev, do_update_cost); + shiftMF(*m_fields.get("G_cp", lev), geom[lev-1], num_shift_crse, dir, lev, do_update_cost); } } diff --git a/Source/WarpX.H b/Source/WarpX.H index b7ab835c667..66d1689e565 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1233,7 +1233,7 @@ public: FiniteDifferenceSolver * get_pointer_fdtd_solver_fp (int lev) { return m_fdtd_solver_fp[lev].get(); } // Field container - ablastr::fields::MultiFabRegister m_multifab_map; + ablastr::fields::MultiFabRegister m_fields; protected: diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index e3899b2be46..c0ab02b7ed9 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2147,7 +2147,7 @@ WarpX::ClearLevel (int lev) gather_buffer_masks[lev].reset(); F_fp [lev].reset(); - m_multifab_map.erase( "G_fp", lev ); + m_fields.erase( "G_fp", lev ); rho_fp[lev].reset(); phi_fp[lev].reset(); @@ -2553,7 +2553,7 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (do_divb_cleaning) { - m_multifab_map.alloc_init( + m_fields.alloc_init( "G_fp", amrex::convert(ba, G_nodal_flag), dm, ncomps, ngG, lev, 0.0_rt); } @@ -2758,13 +2758,13 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm { if (grid_type == GridType::Collocated) { - m_multifab_map.alloc_init( + m_fields.alloc_init( "G_cp", amrex::convert(ba, IntVect::TheUnitVector()), dm, ncomps, ngG, lev, 0.0_rt); } else // grid_type=staggered or grid_type=hybrid { - m_multifab_map.alloc_init( + m_fields.alloc_init( "G_cp", amrex::convert(ba, IntVect::TheZeroVector()), dm, ncomps, ngG, lev, 0.0_rt); } From 747e65643e018b390870de9e5b0db122c6d06570 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Mon, 9 Sep 2024 15:16:39 -0700 Subject: [PATCH 016/314] Fix merge --- Source/Evolve/WarpXEvolve.cpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 2c4ec0b55d4..e1e28c6baef 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -1097,12 +1097,12 @@ WarpX::applyMirrors (Real time) const amrex::Real z_max = std::max(z_max_tmp, z_min+mirror_z_npoints[i_mirror]*dz); // Set each field on the fine patch to zero between z_min and z_max - NullifyMF(m_multifab_map, "Efield_fp[0]", lev, z_min, z_max); - NullifyMF(m_multifab_map, "Efield_fp[1]", lev, z_min, z_max); - NullifyMF(m_multifab_map, "Efield_fp[2]", lev, z_min, z_max); - NullifyMF(m_multifab_map, "Bfield_fp[0]", lev, z_min, z_max); - NullifyMF(m_multifab_map, "Bfield_fp[1]", lev, z_min, z_max); - NullifyMF(m_multifab_map, "Bfield_fp[2]", lev, z_min, z_max); + NullifyMF(m_fields, "Efield_fp[0]", lev, z_min, z_max); + NullifyMF(m_fields, "Efield_fp[1]", lev, z_min, z_max); + NullifyMF(m_fields, "Efield_fp[2]", lev, z_min, z_max); + NullifyMF(m_fields, "Bfield_fp[0]", lev, z_min, z_max); + NullifyMF(m_fields, "Bfield_fp[1]", lev, z_min, z_max); + NullifyMF(m_fields, "Bfield_fp[2]", lev, z_min, z_max); // If div(E)/div(B) cleaning are used, set F/G field to zero NullifyMF(m_fields, "F_fp", lev, z_min, z_max); @@ -1111,12 +1111,12 @@ WarpX::applyMirrors (Real time) if (lev>0) { // Set each field on the coarse patch to zero between z_min and z_max - NullifyMF(m_multifab_map, "Efield_cp[0]", lev, z_min, z_max); - NullifyMF(m_multifab_map, "Efield_cp[1]", lev, z_min, z_max); - NullifyMF(m_multifab_map, "Efield_cp[2]", lev, z_min, z_max); - NullifyMF(m_multifab_map, "Bfield_cp[0]", lev, z_min, z_max); - NullifyMF(m_multifab_map, "Bfield_cp[1]", lev, z_min, z_max); - NullifyMF(m_multifab_map, "Bfield_cp[2]", lev, z_min, z_max); + NullifyMF(m_fields, "Efield_cp[0]", lev, z_min, z_max); + NullifyMF(m_fields, "Efield_cp[1]", lev, z_min, z_max); + NullifyMF(m_fields, "Efield_cp[2]", lev, z_min, z_max); + NullifyMF(m_fields, "Bfield_cp[0]", lev, z_min, z_max); + NullifyMF(m_fields, "Bfield_cp[1]", lev, z_min, z_max); + NullifyMF(m_fields, "Bfield_cp[2]", lev, z_min, z_max); // If div(E)/div(B) cleaning are used, set F/G field to zero NullifyMF(m_fields, "F_cp", lev, z_min, z_max); From b4331b184901d2b6a093e8f628956d15266e8dcb Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Tue, 10 Sep 2024 00:24:33 +0200 Subject: [PATCH 017/314] test push --- Source/WarpX.H | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/WarpX.H b/Source/WarpX.H index 66d1689e565..21109a2d10c 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1602,7 +1602,7 @@ private: // Coarse patch amrex::Vector< std::unique_ptr > F_cp; - amrex::Vector< std::unique_ptr > rho_cp; + amrex::Vector< std::unique_ptr > rho_cp ; amrex::Vector, 3 > > current_cp; amrex::Vector, 3 > > Efield_cp; amrex::Vector, 3 > > Bfield_cp; From dc11c53ad20b5661a6eb875795e012d2ec496628 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 9 Sep 2024 15:27:41 -0700 Subject: [PATCH 018/314] `CheckGuardCells`: Pass by String --- Source/Initialization/WarpXInitData.cpp | 64 ++++++++++--------------- 1 file changed, 25 insertions(+), 39 deletions(-) diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 2ee0b3d50ba..98306689737 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -34,6 +34,7 @@ #include "Utils/WarpXUtil.H" #include "Python/callbacks.H" +#include #include #include #include @@ -91,8 +92,15 @@ namespace * \brief Check that the number of guard cells is smaller than the number of valid cells, * for a given MultiFab, and abort otherwise. */ - void CheckGuardCells(amrex::MultiFab const& mf) + void CheckGuardCells ( + ablastr::fields::MultiFabRegister& fields, + std::string mf_name, + int lev + ) { + if (!fields.has(mf_name, lev)) { return; } + auto & mf = *fields.get(mf_name, lev); + for (amrex::MFIter mfi(mf); mfi.isValid(); ++mfi) { const amrex::IntVect vc = mfi.validbox().enclosedCells().size(); @@ -1205,62 +1213,40 @@ void WarpX::CheckGuardCells() { for (int dim = 0; dim < 3; ++dim) { - ::CheckGuardCells(*Efield_fp[lev][dim]); - ::CheckGuardCells(*Bfield_fp[lev][dim]); - ::CheckGuardCells(*current_fp[lev][dim]); + ::CheckGuardCells(m_fields, "Efield_fp[" + std::to_string(dim) + "]", lev); + ::CheckGuardCells(m_fields, "Bfield_fp[" + std::to_string(dim) + "]", lev); + ::CheckGuardCells(m_fields, "current_fp[" + std::to_string(dim) + "]", lev); if (WarpX::fft_do_time_averaging) { - ::CheckGuardCells(*Efield_avg_fp[lev][dim]); - ::CheckGuardCells(*Bfield_avg_fp[lev][dim]); + ::CheckGuardCells(m_fields, "Efield_avg_fp[" + std::to_string(dim) + "]", lev); + ::CheckGuardCells(m_fields, "Bfield_avg_fp[" + std::to_string(dim) + "]", lev); } } - if (rho_fp[lev]) - { - ::CheckGuardCells(*rho_fp[lev]); - } - - if (F_fp[lev]) - { - ::CheckGuardCells(*F_fp[lev]); - } - - if (m_fields.has("G_fp", lev)) - { - ::CheckGuardCells( *m_fields.get("G_fp", lev) ); - } + ::CheckGuardCells(m_fields, "rho_fp", lev); + ::CheckGuardCells(m_fields, "F_fp", lev); + ::CheckGuardCells(m_fields, "G_fp", lev); // MultiFabs on coarse patch if (lev > 0) { for (int dim = 0; dim < 3; ++dim) { - ::CheckGuardCells(*Efield_cp[lev][dim]); - ::CheckGuardCells(*Bfield_cp[lev][dim]); - ::CheckGuardCells(*current_cp[lev][dim]); + ::CheckGuardCells(m_fields, "Efield_cp[" + std::to_string(dim) + "]", lev); + ::CheckGuardCells(m_fields, "Bfield_cp[" + std::to_string(dim) + "]", lev); + ::CheckGuardCells(m_fields, "current_cp[" + std::to_string(dim) + "]", lev); if (WarpX::fft_do_time_averaging) { - ::CheckGuardCells(*Efield_avg_cp[lev][dim]); - ::CheckGuardCells(*Bfield_avg_cp[lev][dim]); + ::CheckGuardCells(m_fields, "Efield_avg_cp[" + std::to_string(dim) + "]", lev); + ::CheckGuardCells(m_fields, "Bfield_avg_cp[" + std::to_string(dim) + "]", lev); } } - if (rho_cp[lev]) - { - ::CheckGuardCells(*rho_cp[lev]); - } - - if (F_cp[lev]) - { - ::CheckGuardCells(*F_cp[lev]); - } - - if (m_fields.has("G_cp", lev)) - { - ::CheckGuardCells( *m_fields.get("G_cp", lev) ); - } + ::CheckGuardCells(m_fields, "rho_cp", lev); + ::CheckGuardCells(m_fields, "F_cp", lev); + ::CheckGuardCells(m_fields, "G_cp", lev); } } } From 9bec6bfd103e4c055c38fe7032b00274802728a1 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Mon, 9 Sep 2024 15:41:46 -0700 Subject: [PATCH 019/314] Move current_fp to m_fields --- Source/BoundaryConditions/PML.cpp | 6 +++--- Source/BoundaryConditions/WarpXEvolvePML.cpp | 16 ++++++++++------ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index f413831c74d..5a9e73358d1 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -711,9 +711,9 @@ PML::PML (const int lev, const BoxArray& grid_ba, WarpX::AllocInitMultiFab(pml_B_fp[1], ba_By, dm, ncompb, ngb, lev, "pml_B_fp[y]", 0.0_rt); WarpX::AllocInitMultiFab(pml_B_fp[2], ba_Bz, dm, ncompb, ngb, lev, "pml_B_fp[z]", 0.0_rt); - const amrex::BoxArray ba_jx = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::current_fp, 0,0).ixType().toIntVect()); - const amrex::BoxArray ba_jy = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::current_fp, 0,1).ixType().toIntVect()); - const amrex::BoxArray ba_jz = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::current_fp, 0,2).ixType().toIntVect()); + const amrex::BoxArray ba_jx = amrex::convert(ba, WarpX::GetInstance().m_fields.get("current_fp[x]", 0)->ixType().toIntVect()); + const amrex::BoxArray ba_jy = amrex::convert(ba, WarpX::GetInstance().m_fields.get("current_fp[y]", 0)->ixType().toIntVect()); + const amrex::BoxArray ba_jz = amrex::convert(ba, WarpX::GetInstance().m_fields.get("current_fp[z]", 0)->ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_j_fp[0], ba_jx, dm, 1, ngb, lev, "pml_j_fp[x]", 0.0_rt); WarpX::AllocInitMultiFab(pml_j_fp[1], ba_jy, dm, 1, ngb, lev, "pml_j_fp[y]", 0.0_rt); WarpX::AllocInitMultiFab(pml_j_fp[2], ba_jz, dm, 1, ngb, lev, "pml_j_fp[z]", 0.0_rt); diff --git a/Source/BoundaryConditions/WarpXEvolvePML.cpp b/Source/BoundaryConditions/WarpXEvolvePML.cpp index bbe969052a3..f3a3f88ba45 100644 --- a/Source/BoundaryConditions/WarpXEvolvePML.cpp +++ b/Source/BoundaryConditions/WarpXEvolvePML.cpp @@ -339,12 +339,16 @@ WarpX::CopyJPML () for (int lev = 0; lev <= finest_level; ++lev) { if (pml[lev] && pml[lev]->ok()){ - pml[lev]->CopyJtoPMLs({ current_fp[lev][0].get(), - current_fp[lev][1].get(), - current_fp[lev][2].get() }, - { current_cp[lev][0].get(), - current_cp[lev][1].get(), - current_cp[lev][2].get() }); + pml[lev]->CopyJtoPMLs({ + m_fields.get("current_fp[x]", lev), + m_fields.get("current_fp[y]", lev), + m_fields.get("current_fp[z]", lev) + }, { + m_fields.get("current_cp[x]", lev), + m_fields.get("current_cp[y]", lev), + m_fields.get("current_cp[z]", lev) + } + ); } } } From b13cf047dff8e7dd4976ffd0613a923a70e3143c Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 9 Sep 2024 17:01:06 -0700 Subject: [PATCH 020/314] Add `MultiFabRegister::get_mr_levels` --- Source/ablastr/fields/MultiFabRegister.H | 14 ++++++++++++++ Source/ablastr/fields/MultiFabRegister.cpp | 15 +++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index df362969ef8..5c8e984f1ec 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -120,6 +120,20 @@ namespace ablastr::fields int level ); + /** title + * + * Same as get above, but returns all levels for a name. + * + * @param name ... + * @param finest_level ... + * @return ... + */ + std::vector + get_mr_levels ( + std::string name, + int finest_level + ); + /** title * * body body diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 77a6156cc87..c194f651bb0 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -91,6 +91,21 @@ namespace ablastr::fields return &mf; } + std::vector + MultiFabRegister::get_mr_levels ( + std::string name, + int finest_level + ) + { + std::vector field_on_level; + field_on_level.reserve(finest_level+1); + for (int lvl = 0; lvl<= finest_level; lvl++) + { + field_on_level.push_back(get(name, lvl)); + } + return field_on_level; + } + std::vector MultiFabRegister::list () { From 6feb6380a7d0203d2c4347f5aee4357f19ad55d5 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 10 Sep 2024 09:20:42 -0700 Subject: [PATCH 021/314] Overload: Direction --- Source/ablastr/fields/MultiFabRegister.H | 50 ++++++++++++++++++++++ Source/ablastr/fields/MultiFabRegister.cpp | 50 ++++++++++++++++++++++ 2 files changed, 100 insertions(+) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 5c8e984f1ec..cb10a18aac2 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -120,6 +120,23 @@ namespace ablastr::fields int level ); + /** title + * + * body body + * body + * + * @param name ... + * @param dir ... + * @param level ... + * @return ... + */ + amrex::MultiFab* + get ( + std::string name, + int dir, + int level + ); + /** title * * Same as get above, but returns all levels for a name. @@ -134,6 +151,22 @@ namespace ablastr::fields int finest_level ); + /** title + * + * Same as get above, but returns all levels for a name. + * + * @param name ... + * @param dir ... + * @param finest_level ... + * @return ... + */ + std::vector + get_mr_levels ( + std::string name, + int dir, + int finest_level + ); + /** title * * body body @@ -174,6 +207,23 @@ namespace ablastr::fields int level ); + /** title + * + * body body + * body + * + * @param name ... + * @param dir ... + * @param level ... + * @return ... + */ + std::string + mf_name ( + std::string name, + int dir, + int level + ); + private: /** data storage: ownership and lifetime control */ std::map< diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index c194f651bb0..610299861bd 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -91,6 +91,23 @@ namespace ablastr::fields return &mf; } + amrex::MultiFab* + MultiFabRegister::get ( + std::string name, + int dir, + int level + ) + { + name = mf_name(name, dir, level); + + if (m_mf_register.count(name) == 0) { + throw std::runtime_error("MultiFabRegister::get name does not exist in register: " + name); + } + amrex::MultiFab & mf = m_mf_register[name].m_mf; + + return &mf; + } + std::vector MultiFabRegister::get_mr_levels ( std::string name, @@ -106,6 +123,22 @@ namespace ablastr::fields return field_on_level; } + std::vector + MultiFabRegister::get_mr_levels ( + std::string name, + int dir, + int finest_level + ) + { + std::vector field_on_level; + field_on_level.reserve(finest_level+1); + for (int lvl = 0; lvl<= finest_level; lvl++) + { + field_on_level.push_back(get(name, dir, lvl)); + } + return field_on_level; + } + std::vector MultiFabRegister::list () { @@ -142,4 +175,21 @@ namespace ablastr::fields .append("]"); } + std::string + MultiFabRegister::mf_name ( + std::string name, + int dir, + int level + ) + { + // Add the suffix "[level=level]" + return mf_name( + name + .append("[dir=") + .append(std::to_string(dir)) + .append("]"), + level + ); + } + } // namespace ablastr::fields From 86a5247bcf07eac62102ee1604bce2f01f8fcf0d Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 10 Sep 2024 09:26:42 -0700 Subject: [PATCH 022/314] Alloc and erase current_fp --- Source/WarpX.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index c0ab02b7ed9..efa1832ba0f 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -323,7 +323,6 @@ WarpX::WarpX () F_fp.resize(nlevs_max); rho_fp.resize(nlevs_max); phi_fp.resize(nlevs_max); - current_fp.resize(nlevs_max); Efield_fp.resize(nlevs_max); Bfield_fp.resize(nlevs_max); @@ -2096,11 +2095,15 @@ WarpX::MakeNewLevelFromCoarse (int /*lev*/, amrex::Real /*time*/, const amrex::B void WarpX::ClearLevel (int lev) { + + m_fields.erase( "current_fp[x]", lev ); + m_fields.erase( "current_fp[y]", lev ); + m_fields.erase( "current_fp[z]", lev ); + for (int i = 0; i < 3; ++i) { Efield_aux[lev][i].reset(); Bfield_aux[lev][i].reset(); - current_fp[lev][i].reset(); Efield_fp [lev][i].reset(); Bfield_fp [lev][i].reset(); @@ -2358,9 +2361,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm AllocInitMultiFab(Efield_fp[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp[y]", 0.0_rt); AllocInitMultiFab(Efield_fp[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp[z]", 0.0_rt); - AllocInitMultiFab(current_fp[lev][0], amrex::convert(ba, jx_nodal_flag), dm, ncomps, ngJ, lev, "current_fp[x]", 0.0_rt); - AllocInitMultiFab(current_fp[lev][1], amrex::convert(ba, jy_nodal_flag), dm, ncomps, ngJ, lev, "current_fp[y]", 0.0_rt); - AllocInitMultiFab(current_fp[lev][2], amrex::convert(ba, jz_nodal_flag), dm, ncomps, ngJ, lev, "current_fp[z]", 0.0_rt); + m_fields.alloc_init( "current_fp[x]", amrex::convert(ba, jx_nodal_flag), dm, ncomps, ngJ, lev, 0.0_rt); + m_fields.alloc_init( "current_fp[y]", amrex::convert(ba, jy_nodal_flag), dm, ncomps, ngJ, lev, 0.0_rt); + m_fields.alloc_init( "current_fp[z]", amrex::convert(ba, jz_nodal_flag), dm, ncomps, ngJ, lev, 0.0_rt); if (do_current_centering) { From 75081de5aebb3ead935f5594f7c89c58d4e09428 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 10 Sep 2024 09:46:05 -0700 Subject: [PATCH 023/314] Replace getField, getFieldPointer --- Source/Diagnostics/BTDiagnostics.cpp | 12 ++++++------ Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp | 8 ++++---- .../Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp | 2 +- .../FlushFormats/FlushFormatCheckpoint.cpp | 6 +++--- .../Diagnostics/FlushFormats/FlushFormatPlotfile.cpp | 10 +++++----- Source/Diagnostics/FullDiagnostics.cpp | 6 +++--- Source/Diagnostics/ReducedDiags/FieldReduction.H | 6 +++--- 7 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 1cee9909226..8aec1f2a27d 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -579,11 +579,11 @@ BTDiagnostics::InitializeFieldFunctors (int lev) } else if ( m_cellcenter_varnames[comp] == "Bz" ){ m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 2), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "jx" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 0), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", 0, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "jy" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 1), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", 1, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "jz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 2), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", 2, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "rho" ){ m_cell_center_functors[lev][comp] = std::make_unique(lev, m_crse_ratio); } @@ -695,11 +695,11 @@ BTDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) } else if ( m_cellcenter_varnames_fields[comp] == "Bz" ){ m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 2), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "jr" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 0), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", 0, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "jt" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 1), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", 1, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "jz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 2), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", 2, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "rho" ){ m_cell_center_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, false, -1, false, ncomp); } diff --git a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp index ebaec47b2f1..348ee0ead3d 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp @@ -31,7 +31,7 @@ JFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/ { auto& warpx = WarpX::GetInstance(); /** pointer to source multifab (can be multi-component) */ - amrex::MultiFab* m_mf_src = warpx.getFieldPointer(FieldType::current_fp, m_lev, m_dir); + amrex::MultiFab* m_mf_src = warpx.m_fields.get("current_fp", m_dir, m_lev); // Deposit current if no solver or the electrostatic solver is being used if (m_deposit_current) @@ -40,16 +40,16 @@ JFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/ amrex::Vector, 3 > > current_fp_temp; current_fp_temp.resize(1); - const auto& current_fp_x = warpx.getField(FieldType::current_fp, m_lev,0); + const auto& current_fp_x = *warpx.m_fields.get("current_fp", 0, m_lev); current_fp_temp[0][0] = std::make_unique( current_fp_x, amrex::make_alias, 0, current_fp_x.nComp() ); - const auto& current_fp_y = warpx.getField(FieldType::current_fp, m_lev,1); + const auto& current_fp_y = *warpx.m_fields.get("current_fp",1,m_lev); current_fp_temp[0][1] = std::make_unique( current_fp_y, amrex::make_alias, 0, current_fp_y.nComp() ); - const auto& current_fp_z = warpx.getField(FieldType::current_fp, m_lev,2); + const auto& current_fp_z = *warpx.m_fields.get("current_fp",2,m_lev); current_fp_temp[0][2] = std::make_unique( current_fp_z, amrex::make_alias, 0, current_fp_z.nComp() ); diff --git a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp index aac5869da65..871ba4c7782 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp @@ -31,7 +31,7 @@ JdispFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buff auto* hybrid_pic_model = warpx.get_pointer_HybridPICModel(); /** pointer to total simulation current (J) multifab */ - amrex::MultiFab* mf_j = warpx.getFieldPointer(FieldType::current_fp, m_lev, m_dir); + amrex::MultiFab* mf_j = warpx.m_fields.get("current_fp", m_dir, m_lev); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(hybrid_pic_model, "Displacement current diagnostic is only implemented for the HybridPICModel."); diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index 1a3318ae0d8..11aff8fc233 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -96,11 +96,11 @@ FlushFormatCheckpoint::WriteToFile ( if (warpx.getis_synchronized()) { // Need to save j if synchronized because after restart we need j to evolve E by dt/2. - VisMF::Write(warpx.getField(FieldType::current_fp, lev, 0), + VisMF::Write(*warpx.m_fields.get("current_fp", 0, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jx_fp")); - VisMF::Write(warpx.getField(FieldType::current_fp, lev, 1), + VisMF::Write(*warpx.m_fields.get("current_fp", 1, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jy_fp")); - VisMF::Write(warpx.getField(FieldType::current_fp, lev, 2), + VisMF::Write(*warpx.m_fields.get("current_fp", 2, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jz_fp")); } diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index f6c73d9fa7e..988b375ca5a 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -575,9 +575,9 @@ FlushFormatPlotfile::WriteAllRawFields( WriteRawMF( warpx.getField(FieldType::Efield_fp, lev, 0), dm, raw_pltname, default_level_prefix, "Ex_fp", lev, plot_raw_fields_guards); WriteRawMF( warpx.getField(FieldType::Efield_fp, lev, 1), dm, raw_pltname, default_level_prefix, "Ey_fp", lev, plot_raw_fields_guards); WriteRawMF( warpx.getField(FieldType::Efield_fp, lev, 2), dm, raw_pltname, default_level_prefix, "Ez_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getField(FieldType::current_fp, lev, 0), dm, raw_pltname, default_level_prefix, "jx_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getField(FieldType::current_fp, lev, 1), dm, raw_pltname, default_level_prefix, "jy_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getField(FieldType::current_fp, lev, 2), dm, raw_pltname, default_level_prefix, "jz_fp", lev, plot_raw_fields_guards); + WriteRawMF( *warpx.m_fields.get("current_fp", 0, lev), dm, raw_pltname, default_level_prefix, "jx_fp", lev, plot_raw_fields_guards); + WriteRawMF( *warpx.m_fields.get("current_fp", 1, lev), dm, raw_pltname, default_level_prefix, "jy_fp", lev, plot_raw_fields_guards); + WriteRawMF( *warpx.m_fields.get("current_fp", 2, lev), dm, raw_pltname, default_level_prefix, "jz_fp", lev, plot_raw_fields_guards); WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 0), dm, raw_pltname, default_level_prefix, "Bx_fp", lev, plot_raw_fields_guards); WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 1), dm, raw_pltname, default_level_prefix, "By_fp", lev, plot_raw_fields_guards); WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 2), dm, raw_pltname, default_level_prefix, "Bz_fp", lev, plot_raw_fields_guards); @@ -630,8 +630,8 @@ FlushFormatPlotfile::WriteAllRawFields( warpx.getFieldPointer(FieldType::Bfield_fp, lev, 0), warpx.getFieldPointer(FieldType::Bfield_fp, lev, 1), warpx.getFieldPointer(FieldType::Bfield_fp, lev, 2), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards); WriteCoarseVector( "j", - warpx.getFieldPointer(FieldType::current_cp, lev, 0), warpx.getFieldPointer(FieldType::current_cp, lev, 1), warpx.getFieldPointer(FieldType::current_cp, lev, 2), - warpx.getFieldPointer(FieldType::current_fp, lev, 0), warpx.getFieldPointer(FieldType::current_fp, lev, 1), warpx.getFieldPointer(FieldType::current_fp, lev, 2), + warpx.m_fields.get("current_cp", 0, lev), warpx.m_fields.get("current_cp", 1, lev), warpx.m_fields.get("current_cp", 2, lev), + warpx.m_fields.get("current_fp", 0, lev), warpx.m_fields.get("current_fp", 1, lev), warpx.m_fields.get("current_fp", 2, lev), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards); if (warpx.isFieldInitialized(FieldType::F_fp, lev) && warpx.isFieldInitialized(FieldType::F_cp, lev)) { diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index 1ee725e7312..ea90a4d4b3b 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -182,7 +182,7 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) AMREX_ALWAYS_ASSERT( warpx.getFieldPointer(FieldType::Bfield_aux, lev, dim)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( - warpx.getFieldPointer(FieldType::current_fp, lev, dim)->nComp() == ncomp_multimodefab ); + warpx.m_fields("current_fp", dim, lev)->nComp() == ncomp_multimodefab ); } // Species index to loop over species that dump rho per species @@ -405,7 +405,7 @@ FullDiagnostics::AddRZModesToDiags (int lev) AMREX_ALWAYS_ASSERT( warpx.getFieldPointer(FieldType::Bfield_aux, lev, dim)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( - warpx.getFieldPointer(FieldType::current_fp, lev, dim)->nComp() == ncomp_multimodefab ); + warpx.m_fields.get("current_fp", dim, lev)->nComp() == ncomp_multimodefab ); } // Check if divE is requested @@ -461,7 +461,7 @@ FullDiagnostics::AddRZModesToDiags (int lev) dim, lev, m_crse_ratio, false, deposit_current, ncomp_multimodefab)); deposit_current = false; AddRZModesToOutputNames(std::string("J") + coord[dim], - warpx.getFieldPointer(FieldType::current_fp, 0, 0)->nComp()); + warpx.m_fields.get("current_fp", 0, 0)->nComp()); } // divE if (divE_requested) { diff --git a/Source/Diagnostics/ReducedDiags/FieldReduction.H b/Source/Diagnostics/ReducedDiags/FieldReduction.H index f467499ad56..1d451277deb 100644 --- a/Source/Diagnostics/ReducedDiags/FieldReduction.H +++ b/Source/Diagnostics/ReducedDiags/FieldReduction.H @@ -105,9 +105,9 @@ public: const amrex::MultiFab & Bx = warpx.getField(warpx::fields::FieldType::Bfield_aux, lev,0); const amrex::MultiFab & By = warpx.getField(warpx::fields::FieldType::Bfield_aux, lev,1); const amrex::MultiFab & Bz = warpx.getField(warpx::fields::FieldType::Bfield_aux, lev,2); - const amrex::MultiFab & jx = warpx.getField(warpx::fields::FieldType::current_fp, lev,0); - const amrex::MultiFab & jy = warpx.getField(warpx::fields::FieldType::current_fp, lev,1); - const amrex::MultiFab & jz = warpx.getField(warpx::fields::FieldType::current_fp, lev,2); + const amrex::MultiFab & jx = *warpx.m_fields.get("current_fp", 0, lev); + const amrex::MultiFab & jy = *warpx.m_fields.get("current_fp", 1, lev); + const amrex::MultiFab & jz = *warpx.m_fields.get("current_fp", 2, lev); // General preparation of interpolation and reduction operations From 8a39bce3a3064c769b22c9ed58bc195fc7c8fd13 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 10 Sep 2024 09:58:14 -0700 Subject: [PATCH 024/314] ClearLevel --- Source/WarpX.cpp | 55 +--------------------- Source/ablastr/fields/MultiFabRegister.H | 17 ++++++- Source/ablastr/fields/MultiFabRegister.cpp | 19 +++++++- 3 files changed, 34 insertions(+), 57 deletions(-) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index c0ab02b7ed9..21868e3cec0 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2096,66 +2096,13 @@ WarpX::MakeNewLevelFromCoarse (int /*lev*/, amrex::Real /*time*/, const amrex::B void WarpX::ClearLevel (int lev) { - for (int i = 0; i < 3; ++i) { - Efield_aux[lev][i].reset(); - Bfield_aux[lev][i].reset(); - - current_fp[lev][i].reset(); - Efield_fp [lev][i].reset(); - Bfield_fp [lev][i].reset(); - - Efield_dotMask [lev][i].reset(); - Bfield_dotMask [lev][i].reset(); - Afield_dotMask [lev][i].reset(); - - current_store[lev][i].reset(); - - if (do_current_centering) - { - current_fp_nodal[lev][i].reset(); - } - - if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Vay) - { - current_fp_vay[lev][i].reset(); - } - - if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) - { - vector_potential_fp_nodal[lev][i].reset(); - vector_potential_grad_buf_e_stag[lev][i].reset(); - vector_potential_grad_buf_b_stag[lev][i].reset(); - } - - current_cp[lev][i].reset(); - Efield_cp [lev][i].reset(); - Bfield_cp [lev][i].reset(); - - Efield_cax[lev][i].reset(); - Bfield_cax[lev][i].reset(); - current_buf[lev][i].reset(); - } + m_fields.clear_level(lev); if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { m_hybrid_pic_model->ClearLevel(lev); } - charge_buf[lev].reset(); - - current_buffer_masks[lev].reset(); - gather_buffer_masks[lev].reset(); - - F_fp [lev].reset(); - m_fields.erase( "G_fp", lev ); - - rho_fp[lev].reset(); - phi_fp[lev].reset(); - F_cp [lev].reset(); - rho_cp[lev].reset(); - - phi_dotMask[lev].reset(); - #ifdef WARPX_USE_FFT if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { spectral_solver_fp[lev].reset(); diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index cb10a18aac2..8a32765ea5f 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -30,6 +30,9 @@ namespace ablastr::fields /** owned (i)MultiFab data */ amrex::MultiFab m_mf; + /** the MR level of this (i)MultiFab */ + int level = 0; + /** redistribute */ bool redistribute = true; }; @@ -165,7 +168,7 @@ namespace ablastr::fields std::string name, int dir, int finest_level - ); + ); /** title * @@ -184,7 +187,6 @@ namespace ablastr::fields * * @param name ... * @param level ... - * @return ... */ void erase ( @@ -192,6 +194,17 @@ namespace ablastr::fields int level ); + /** Erase all MultiFabs on a specific MR level. + * + * Calls @see erase for all MultiFabs on a specific level. + * + * @param level the MR level to erase all MultiFabs from + */ + void + clear_level ( + int level + ); + /** title * * body body diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 610299861bd..05e292eab31 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -7,6 +7,8 @@ */ #include "MultiFabRegister.H" +#include + namespace ablastr::fields { @@ -32,7 +34,7 @@ namespace ablastr::fields auto [it, success] = m_mf_register.emplace( std::make_pair( name, - MultiFabOwner{{ba, dm, ncomp, ngrow, tag}, redistribute} + MultiFabOwner{{ba, dm, ncomp, ngrow, tag}, level, redistribute} ) ); if (!success) { @@ -163,6 +165,21 @@ namespace ablastr::fields m_mf_register.erase(name); } + void + MultiFabRegister::clear_level ( + int level + ) + { + // C++20: Replace with std::erase_if + for (auto first = m_mf_register.begin(), last = m_mf_register.end(); first != last;) + { + if (first->second.level == level) + first = m_mf_register.erase(first); + else + ++first; + } + } + std::string MultiFabRegister::mf_name ( std::string name, From c4884651b5e268c610bf4b5f1709043509e77c76 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 10 Sep 2024 10:29:34 -0700 Subject: [PATCH 025/314] Explicit Direction Type --- Source/ablastr/fields/MultiFabRegister.H | 23 +++++++++++++++++++--- Source/ablastr/fields/MultiFabRegister.cpp | 8 ++++---- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 8a32765ea5f..d344082b679 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -23,6 +23,23 @@ namespace ablastr::fields { + + /** Components (base vector directions) of vector/tensor fields. + * + * Because of different staggering, the components of vector/tensor fields are stored + * in separate (i)MultiFab. This + */ + struct Direction + { + int dir = 0; + }; + + /** title + * + * body body + * body + * body + */ struct MultiFabOwner { // TODO: also add iMultiFab via std::variant @@ -136,7 +153,7 @@ namespace ablastr::fields amrex::MultiFab* get ( std::string name, - int dir, + Direction dir, int level ); @@ -166,7 +183,7 @@ namespace ablastr::fields std::vector get_mr_levels ( std::string name, - int dir, + Direction dir, int finest_level ); @@ -233,7 +250,7 @@ namespace ablastr::fields std::string mf_name ( std::string name, - int dir, + Direction dir, int level ); diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 05e292eab31..4886d4e22ea 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -96,7 +96,7 @@ namespace ablastr::fields amrex::MultiFab* MultiFabRegister::get ( std::string name, - int dir, + Direction dir, int level ) { @@ -128,7 +128,7 @@ namespace ablastr::fields std::vector MultiFabRegister::get_mr_levels ( std::string name, - int dir, + Direction dir, int finest_level ) { @@ -195,7 +195,7 @@ namespace ablastr::fields std::string MultiFabRegister::mf_name ( std::string name, - int dir, + Direction dir, int level ) { @@ -203,7 +203,7 @@ namespace ablastr::fields return mf_name( name .append("[dir=") - .append(std::to_string(dir)) + .append(std::to_string(dir.dir)) .append("]"), level ); From be97bda04e54df6f619d507227177721378b2519 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 10 Sep 2024 10:41:06 -0700 Subject: [PATCH 026/314] Use new Direction feature --- Source/Diagnostics/BTDiagnostics.cpp | 16 ++++++++++------ .../Diagnostics/ComputeDiagFunctors/JFunctor.cpp | 10 ++++++---- .../ComputeDiagFunctors/JdispFunctor.cpp | 4 +++- .../FlushFormats/FlushFormatCheckpoint.cpp | 14 ++++++++------ .../FlushFormats/FlushFormatPlotfile.cpp | 12 +++++++----- Source/Diagnostics/FullDiagnostics.cpp | 8 +++++--- Source/Diagnostics/ReducedDiags/FieldReduction.H | 7 ++++--- Source/Diagnostics/WarpXIO.cpp | 4 +++- 8 files changed, 46 insertions(+), 29 deletions(-) diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 8aec1f2a27d..5d74bf1e718 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -525,6 +525,8 @@ BTDiagnostics::DefineCellCenteredMultiFab(int lev) void BTDiagnostics::InitializeFieldFunctors (int lev) { + using ablastr::fields::Direction; + // Initialize fields functors only if do_back_transformed_fields is selected if (!m_do_back_transformed_fields) { return; } @@ -579,11 +581,11 @@ BTDiagnostics::InitializeFieldFunctors (int lev) } else if ( m_cellcenter_varnames[comp] == "Bz" ){ m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 2), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "jx" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", 0, lev), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp",Direction{0}, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "jy" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", 1, lev), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp",Direction{1}, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "jz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", 2, lev), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp",Direction{2}, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "rho" ){ m_cell_center_functors[lev][comp] = std::make_unique(lev, m_crse_ratio); } @@ -656,6 +658,8 @@ void BTDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) { #ifdef WARPX_DIM_RZ + using ablastr::fields::Direction; + auto & warpx = WarpX::GetInstance(); const int ncomp_multimodefab = warpx.getFieldPointer(FieldType::Efield_aux, 0,0)->nComp(); const int ncomp = ncomp_multimodefab; @@ -695,11 +699,11 @@ BTDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) } else if ( m_cellcenter_varnames_fields[comp] == "Bz" ){ m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 2), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "jr" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", 0, lev), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", Direction{0}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "jt" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", 1, lev), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", Direction{1}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "jz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", 2, lev), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", Direction{2}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "rho" ){ m_cell_center_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, false, -1, false, ncomp); } diff --git a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp index 348ee0ead3d..2cb1d8f710b 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp @@ -29,9 +29,11 @@ JFunctor::JFunctor (const int dir, int lev, void JFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/) const { + using ablastr::fields::Direction; + auto& warpx = WarpX::GetInstance(); /** pointer to source multifab (can be multi-component) */ - amrex::MultiFab* m_mf_src = warpx.m_fields.get("current_fp", m_dir, m_lev); + amrex::MultiFab* m_mf_src = warpx.m_fields.get("current_fp",Direction{m_dir},m_lev); // Deposit current if no solver or the electrostatic solver is being used if (m_deposit_current) @@ -40,16 +42,16 @@ JFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/ amrex::Vector, 3 > > current_fp_temp; current_fp_temp.resize(1); - const auto& current_fp_x = *warpx.m_fields.get("current_fp", 0, m_lev); + const auto& current_fp_x = *warpx.m_fields.get("current_fp",Direction{0},m_lev); current_fp_temp[0][0] = std::make_unique( current_fp_x, amrex::make_alias, 0, current_fp_x.nComp() ); - const auto& current_fp_y = *warpx.m_fields.get("current_fp",1,m_lev); + const auto& current_fp_y = *warpx.m_fields.get("current_fp",Direction{1},m_lev); current_fp_temp[0][1] = std::make_unique( current_fp_y, amrex::make_alias, 0, current_fp_y.nComp() ); - const auto& current_fp_z = *warpx.m_fields.get("current_fp",2,m_lev); + const auto& current_fp_z = *warpx.m_fields.get("current_fp",Direction{2},m_lev); current_fp_temp[0][2] = std::make_unique( current_fp_z, amrex::make_alias, 0, current_fp_z.nComp() ); diff --git a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp index 871ba4c7782..d7fae637106 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp @@ -27,11 +27,13 @@ JdispFunctor::JdispFunctor (int dir, int lev, void JdispFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/) const { + using ablastr::fields::Direction; + auto& warpx = WarpX::GetInstance(); auto* hybrid_pic_model = warpx.get_pointer_HybridPICModel(); /** pointer to total simulation current (J) multifab */ - amrex::MultiFab* mf_j = warpx.m_fields.get("current_fp", m_dir, m_lev); + amrex::MultiFab* mf_j = warpx.m_fields.get("current_fp",Direction{m_dir},m_lev); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(hybrid_pic_model, "Displacement current diagnostic is only implemented for the HybridPICModel."); diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index 51b61b36673..ac367121970 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -43,6 +43,8 @@ FlushFormatCheckpoint::WriteToFile ( const amrex::Geometry& /*full_BTD_snapshot*/, bool /*isLastBTDFlush*/) const { + using ablastr::fields::Direction; + WARPX_PROFILE("FlushFormatCheckpoint::WriteToFile()"); auto & warpx = WarpX::GetInstance(); @@ -96,11 +98,11 @@ FlushFormatCheckpoint::WriteToFile ( if (warpx.getis_synchronized()) { // Need to save j if synchronized because after restart we need j to evolve E by dt/2. - VisMF::Write(*warpx.m_fields.get("current_fp", 0, lev), + VisMF::Write(*warpx.m_fields.get("current_fp", Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jx_fp")); - VisMF::Write(*warpx.m_fields.get("current_fp", 1, lev), + VisMF::Write(*warpx.m_fields.get("current_fp", Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jy_fp")); - VisMF::Write(*warpx.m_fields.get("current_fp", 2, lev), + VisMF::Write(*warpx.m_fields.get("current_fp", Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jz_fp")); } @@ -138,11 +140,11 @@ FlushFormatCheckpoint::WriteToFile ( if (warpx.getis_synchronized()) { // Need to save j if synchronized because after restart we need j to evolve E by dt/2. - VisMF::Write(*warpx.m_fields.get("current_cp", 0, lev), + VisMF::Write(*warpx.m_fields.get("current_cp", Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jx_cp")); - VisMF::Write(*warpx.m_fields.get("current_cp", 1, lev), + VisMF::Write(*warpx.m_fields.get("current_cp", Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jy_cp")); - VisMF::Write(*warpx.m_fields.get("current_cp", 2, lev), + VisMF::Write(*warpx.m_fields.get("current_cp", Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jz_cp")); } } diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index 988b375ca5a..c29053ce550 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -554,6 +554,8 @@ FlushFormatPlotfile::WriteAllRawFields( const bool plot_raw_fields, const int nlevels, const std::string& plotfilename, const bool plot_raw_fields_guards) const { + using ablastr::fields::Direction; + if (!plot_raw_fields) { return; } auto & warpx = WarpX::GetInstance(); for (int lev = 0; lev < nlevels; ++lev) @@ -575,9 +577,9 @@ FlushFormatPlotfile::WriteAllRawFields( WriteRawMF( warpx.getField(FieldType::Efield_fp, lev, 0), dm, raw_pltname, default_level_prefix, "Ex_fp", lev, plot_raw_fields_guards); WriteRawMF( warpx.getField(FieldType::Efield_fp, lev, 1), dm, raw_pltname, default_level_prefix, "Ey_fp", lev, plot_raw_fields_guards); WriteRawMF( warpx.getField(FieldType::Efield_fp, lev, 2), dm, raw_pltname, default_level_prefix, "Ez_fp", lev, plot_raw_fields_guards); - WriteRawMF( *warpx.m_fields.get("current_fp", 0, lev), dm, raw_pltname, default_level_prefix, "jx_fp", lev, plot_raw_fields_guards); - WriteRawMF( *warpx.m_fields.get("current_fp", 1, lev), dm, raw_pltname, default_level_prefix, "jy_fp", lev, plot_raw_fields_guards); - WriteRawMF( *warpx.m_fields.get("current_fp", 2, lev), dm, raw_pltname, default_level_prefix, "jz_fp", lev, plot_raw_fields_guards); + WriteRawMF( *warpx.m_fields.get("current_fp",Direction{0}, lev), dm, raw_pltname, default_level_prefix, "jx_fp", lev,plot_raw_fields_guards); + WriteRawMF( *warpx.m_fields.get("current_fp",Direction{1}, lev), dm, raw_pltname, default_level_prefix, "jy_fp", lev,plot_raw_fields_guards); + WriteRawMF( *warpx.m_fields.get("current_fp",Direction{2}, lev), dm, raw_pltname, default_level_prefix, "jz_fp", lev,plot_raw_fields_guards); WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 0), dm, raw_pltname, default_level_prefix, "Bx_fp", lev, plot_raw_fields_guards); WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 1), dm, raw_pltname, default_level_prefix, "By_fp", lev, plot_raw_fields_guards); WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 2), dm, raw_pltname, default_level_prefix, "Bz_fp", lev, plot_raw_fields_guards); @@ -630,8 +632,8 @@ FlushFormatPlotfile::WriteAllRawFields( warpx.getFieldPointer(FieldType::Bfield_fp, lev, 0), warpx.getFieldPointer(FieldType::Bfield_fp, lev, 1), warpx.getFieldPointer(FieldType::Bfield_fp, lev, 2), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards); WriteCoarseVector( "j", - warpx.m_fields.get("current_cp", 0, lev), warpx.m_fields.get("current_cp", 1, lev), warpx.m_fields.get("current_cp", 2, lev), - warpx.m_fields.get("current_fp", 0, lev), warpx.m_fields.get("current_fp", 1, lev), warpx.m_fields.get("current_fp", 2, lev), + warpx.m_fields.get("current_cp", Direction{0}, lev), warpx.m_fields.get("current_cp", Direction{1}, lev), warpx.m_fields.get("current_cp", Direction{2}, lev), + warpx.m_fields.get("current_fp", Direction{0}, lev), warpx.m_fields.get("current_fp", Direction{1}, lev), warpx.m_fields.get("current_fp", Direction{2}, lev), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards); if (warpx.isFieldInitialized(FieldType::F_fp, lev) && warpx.isFieldInitialized(FieldType::F_cp, lev)) { diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index ea90a4d4b3b..9a743138526 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -172,6 +172,7 @@ void FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) { #ifdef WARPX_DIM_RZ + using ablastr::fields::Direction; auto & warpx = WarpX::GetInstance(); const int ncomp_multimodefab = warpx.getFieldPointer(FieldType::Efield_aux, 0, 0)->nComp(); @@ -182,7 +183,7 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) AMREX_ALWAYS_ASSERT( warpx.getFieldPointer(FieldType::Bfield_aux, lev, dim)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( - warpx.m_fields("current_fp", dim, lev)->nComp() == ncomp_multimodefab ); + warpx.m_fields.get("current_fp", Direction{dim}, lev)->nComp() == ncomp_multimodefab ); } // Species index to loop over species that dump rho per species @@ -393,6 +394,7 @@ void FullDiagnostics::AddRZModesToDiags (int lev) { #ifdef WARPX_DIM_RZ + using ablastr::fields::Direction; if (!m_dump_rz_modes) { return; } @@ -405,7 +407,7 @@ FullDiagnostics::AddRZModesToDiags (int lev) AMREX_ALWAYS_ASSERT( warpx.getFieldPointer(FieldType::Bfield_aux, lev, dim)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( - warpx.m_fields.get("current_fp", dim, lev)->nComp() == ncomp_multimodefab ); + warpx.m_fields.get("current_fp", Direction{dim}, lev)->nComp() == ncomp_multimodefab ); } // Check if divE is requested @@ -461,7 +463,7 @@ FullDiagnostics::AddRZModesToDiags (int lev) dim, lev, m_crse_ratio, false, deposit_current, ncomp_multimodefab)); deposit_current = false; AddRZModesToOutputNames(std::string("J") + coord[dim], - warpx.m_fields.get("current_fp", 0, 0)->nComp()); + warpx.m_fields.get("current_fp",Direction{0},0)->nComp()); } // divE if (divE_requested) { diff --git a/Source/Diagnostics/ReducedDiags/FieldReduction.H b/Source/Diagnostics/ReducedDiags/FieldReduction.H index 1d451277deb..111393441bc 100644 --- a/Source/Diagnostics/ReducedDiags/FieldReduction.H +++ b/Source/Diagnostics/ReducedDiags/FieldReduction.H @@ -87,6 +87,7 @@ public: template void ComputeFieldReduction() { + using ablastr::fields::Direction; using namespace amrex::literals; // get a reference to WarpX instance @@ -105,9 +106,9 @@ public: const amrex::MultiFab & Bx = warpx.getField(warpx::fields::FieldType::Bfield_aux, lev,0); const amrex::MultiFab & By = warpx.getField(warpx::fields::FieldType::Bfield_aux, lev,1); const amrex::MultiFab & Bz = warpx.getField(warpx::fields::FieldType::Bfield_aux, lev,2); - const amrex::MultiFab & jx = *warpx.m_fields.get("current_fp", 0, lev); - const amrex::MultiFab & jy = *warpx.m_fields.get("current_fp", 1, lev); - const amrex::MultiFab & jz = *warpx.m_fields.get("current_fp", 2, lev); + const amrex::MultiFab & jx = *warpx.m_fields.get("current_fp",Direction{0},lev); + const amrex::MultiFab & jy = *warpx.m_fields.get("current_fp",Direction{1},lev); + const amrex::MultiFab & jz = *warpx.m_fields.get("current_fp",Direction{2},lev); // General preparation of interpolation and reduction operations diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp index ed629202120..4c3e566663f 100644 --- a/Source/Diagnostics/WarpXIO.cpp +++ b/Source/Diagnostics/WarpXIO.cpp @@ -87,6 +87,8 @@ WarpX::GetRestartDMap (const std::string& chkfile, const amrex::BoxArray& ba, in void WarpX::InitFromCheckpoint () { + using ablastr::fields::Direction; + WARPX_PROFILE("WarpX::InitFromCheckpoint()"); amrex::Print()<< Utils::TextMsg::Info( @@ -278,7 +280,7 @@ WarpX::InitFromCheckpoint () for (int lev = 0; lev < nlevs; ++lev) { for (int i = 0; i < 3; ++i) { - m_fields.get("current_fp", i, lev)->setVal(0.0); + m_fields.get("current_fp",Direction{i},lev)->setVal(0.0); current_fp[lev][i]->setVal(0.0); Efield_fp[lev][i]->setVal(0.0); Bfield_fp[lev][i]->setVal(0.0); From 74835226d73d4b468958f8a159f6d92700eeedad Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 10 Sep 2024 11:11:01 -0700 Subject: [PATCH 027/314] Additional fixes --- Source/Diagnostics/WarpXIO.cpp | 1 - Source/WarpX.cpp | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp index 4c3e566663f..1ba211b45bc 100644 --- a/Source/Diagnostics/WarpXIO.cpp +++ b/Source/Diagnostics/WarpXIO.cpp @@ -281,7 +281,6 @@ WarpX::InitFromCheckpoint () { for (int i = 0; i < 3; ++i) { m_fields.get("current_fp",Direction{i},lev)->setVal(0.0); - current_fp[lev][i]->setVal(0.0); Efield_fp[lev][i]->setVal(0.0); Bfield_fp[lev][i]->setVal(0.0); } diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index b51ec991867..d88e9c88aa7 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2687,9 +2687,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm } // Create the MultiFabs for the current - AllocInitMultiFab(current_cp[lev][0], amrex::convert(cba, jx_nodal_flag), dm, ncomps, ngJ, lev, "current_cp[x]", 0.0_rt); - AllocInitMultiFab(current_cp[lev][1], amrex::convert(cba, jy_nodal_flag), dm, ncomps, ngJ, lev, "current_cp[y]", 0.0_rt); - AllocInitMultiFab(current_cp[lev][2], amrex::convert(cba, jz_nodal_flag), dm, ncomps, ngJ, lev, "current_cp[z]", 0.0_rt); + m_fields.alloc_init( "current_cp[x]", amrex::convert(cba, jx_nodal_flag), dm, ncomps, ngJ, lev, 0.0_rt); + m_fields.alloc_init( "current_cp[y]", amrex::convert(cba, jy_nodal_flag), dm, ncomps, ngJ, lev, 0.0_rt); + m_fields.alloc_init( "current_cp[z]", amrex::convert(cba, jz_nodal_flag), dm, ncomps, ngJ, lev, 0.0_rt); if (rho_ncomps > 0) { AllocInitMultiFab(rho_cp[lev], amrex::convert(cba, rho_nodal_flag), dm, rho_ncomps, ngRho, lev, "rho_cp", 0.0_rt); From eb508a0afa891de6afa0225d7206dc17f11d1abe Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 10 Sep 2024 11:43:19 -0700 Subject: [PATCH 028/314] `MultiFabRegister::get_mr_levels_alldirs` --- Source/WarpX.cpp | 12 +-- Source/ablastr/fields/MultiFabRegister.H | 63 ++++++++++++-- Source/ablastr/fields/MultiFabRegister.cpp | 96 ++++++++++++++++++++-- 3 files changed, 148 insertions(+), 23 deletions(-) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 21868e3cec0..147480ebfd4 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2501,8 +2501,8 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (do_divb_cleaning) { m_fields.alloc_init( - "G_fp", amrex::convert(ba, G_nodal_flag), dm, - ncomps, ngG, lev, 0.0_rt); + "G_fp", lev, amrex::convert(ba, G_nodal_flag), dm, + ncomps, ngG, 0.0_rt); } if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) @@ -2706,14 +2706,14 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (grid_type == GridType::Collocated) { m_fields.alloc_init( - "G_cp", amrex::convert(ba, IntVect::TheUnitVector()), dm, - ncomps, ngG, lev, 0.0_rt); + "G_cp", lev, amrex::convert(ba, IntVect::TheUnitVector()), dm, + ncomps, ngG, 0.0_rt); } else // grid_type=staggered or grid_type=hybrid { m_fields.alloc_init( - "G_cp", amrex::convert(ba, IntVect::TheZeroVector()), dm, - ncomps, ngG, lev, 0.0_rt); + "G_cp", lev, amrex::convert(ba, IntVect::TheZeroVector()), dm, + ncomps, ngG, 0.0_rt); } } diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index d344082b679..f303c8ab5d2 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -32,6 +32,11 @@ namespace ablastr::fields struct Direction { int dir = 0; + + bool operator<(const Direction& other) const + { + return other.dir < this->dir; + } }; /** title @@ -47,11 +52,17 @@ namespace ablastr::fields /** owned (i)MultiFab data */ amrex::MultiFab m_mf; + /** Components (base vector directions) of this (i)MultiFab */ + std::optional dir = std::nullopt; + /** the MR level of this (i)MultiFab */ int level = 0; /** redistribute */ bool redistribute = true; + + /** redistribute on @see amrex::AmrCore::RemakeLevel */ + bool redistribute_on_remake = true; }; /** This is a register of fields aka amrex::(i)MultiFabs. @@ -75,25 +86,40 @@ namespace ablastr::fields * body * * @param name ... + * @param level ... * @param ba ... * @param dm ... * @param ncomp ... * @param ngrow ... - * @param level ... * @param initial_value ... * @param redistribute follow the default domain decomposition of the simulation + * @param redistribute_on_remake redistribute on @see amrex::AmrCore::RemakeLevel * @return pointer to newly allocated MultiFab */ amrex::MultiFab* alloc_init ( std::string name, + int level, amrex::BoxArray const & ba, amrex::DistributionMapping const & dm, int ncomp, amrex::IntVect const & ngrow, + std::optional initial_value = std::nullopt, + bool redistribute = true, + bool redistribute_on_remake = true + ); + amrex::MultiFab* + alloc_init ( + std::string name, + Direction dir, int level, + amrex::BoxArray const & ba, + amrex::DistributionMapping const & dm, + int ncomp, + amrex::IntVect const & ngrow, std::optional initial_value = std::nullopt, - bool redistribute = true + bool redistribute = true, + bool redistribute_on_remake = true ); /** title @@ -125,6 +151,23 @@ namespace ablastr::fields int level ); + /** title + * + * body body + * body + * + * @param name ... + * @param dir ... + * @param level ... + * @return true if contained, otherwise false + */ + bool + has ( + std::string name, + Direction dir, + int level + ); + /** title * * body body @@ -171,19 +214,23 @@ namespace ablastr::fields int finest_level ); - /** title + /** Return a vector field on all MR levels. * - * Same as get above, but returns all levels for a name. + * Out loop: MR levels. + * Inner loop: directions (components). * * @param name ... - * @param dir ... * @param finest_level ... * @return ... */ - std::vector - get_mr_levels ( + std::vector< + std::map< + Direction, + amrex::MultiFab* + > + > + get_mr_levels_alldirs ( std::string name, - Direction dir, int finest_level ); diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 4886d4e22ea..da7c73522b4 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -15,13 +15,56 @@ namespace ablastr::fields amrex::MultiFab* MultiFabRegister::alloc_init ( std::string name, + int level, amrex::BoxArray const & ba, amrex::DistributionMapping const & dm, int ncomp, amrex::IntVect const & ngrow, + std::optional initial_value, + bool redistribute, + bool redistribute_on_remake + ) + { + name = mf_name(name, level); + + // Checks + // TODO: does the key already exist? error + + // allocate + const auto tag = amrex::MFInfo().SetTag(name); + auto [it, success] = m_mf_register.emplace( + std::make_pair( + name, + MultiFabOwner{{ba, dm, ncomp, ngrow, tag}, std::nullopt, level, redistribute, redistribute_on_remake} + ) + ); + if (!success) { + throw std::runtime_error("MultiFabRegister::alloc_init failed for " + name); + } + + // a short-hand alias for the code below + amrex::MultiFab & mf = it->second.m_mf; + + // initialize with value + if (initial_value) { + mf.setVal(*initial_value); + } + + return &mf; + } + + amrex::MultiFab* + MultiFabRegister::alloc_init ( + std::string name, + Direction dir, int level, + amrex::BoxArray const & ba, + amrex::DistributionMapping const & dm, + int ncomp, + amrex::IntVect const & ngrow, std::optional initial_value, - bool redistribute + bool redistribute, + bool redistribute_on_remake ) { name = mf_name(name, level); @@ -34,7 +77,7 @@ namespace ablastr::fields auto [it, success] = m_mf_register.emplace( std::make_pair( name, - MultiFabOwner{{ba, dm, ncomp, ngrow, tag}, level, redistribute} + MultiFabOwner{{ba, dm, ncomp, ngrow, tag}, dir, level, redistribute, redistribute_on_remake} ) ); if (!success) { @@ -77,6 +120,18 @@ namespace ablastr::fields return m_mf_register.count(name) > 0; } + bool + MultiFabRegister::has ( + std::string name, + Direction dir, + int level + ) + { + name = mf_name(name, dir, level); + + return m_mf_register.count(name) > 0; + } + amrex::MultiFab* MultiFabRegister::get ( std::string name, @@ -118,25 +173,48 @@ namespace ablastr::fields { std::vector field_on_level; field_on_level.reserve(finest_level+1); - for (int lvl = 0; lvl<= finest_level; lvl++) + for (int lvl = 0; lvl <= finest_level; lvl++) { field_on_level.push_back(get(name, lvl)); } return field_on_level; } - std::vector - MultiFabRegister::get_mr_levels ( + std::vector< + std::map< + Direction, + amrex::MultiFab* + > + > + MultiFabRegister::get_mr_levels_alldirs ( std::string name, - Direction dir, int finest_level ) { - std::vector field_on_level; + std::vector< + std::map< + Direction, + amrex::MultiFab* + > + > field_on_level; field_on_level.reserve(finest_level+1); - for (int lvl = 0; lvl<= finest_level; lvl++) + + // TODO: Technically, we should search field_on_level via std::unique_copy + std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; + + for (int lvl = 0; lvl <= finest_level; lvl++) { - field_on_level.push_back(get(name, dir, lvl)); + // insert a new level + field_on_level.push_back(std::map< + Direction, + amrex::MultiFab* + >{}); + + // insert components + for (Direction dir : all_dirs) + { + field_on_level[lvl][dir] = get(name, dir, lvl); + } } return field_on_level; } From 7bf5e7d77f15abbd0c2d43005fb2f1f5ef38f8b4 Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Tue, 10 Sep 2024 09:33:29 -0700 Subject: [PATCH 029/314] initial refactor of phi_fp to use m_fields. --- .../FlushFormats/FlushFormatPlotfile.cpp | 4 +- Source/Diagnostics/FullDiagnostics.cpp | 4 +- Source/Diagnostics/ParticleIO.cpp | 2 +- Source/FieldSolver/ElectrostaticSolver.cpp | 34 ++++++++---- Source/Parallelization/WarpXRegrid.cpp | 52 +++++++++---------- Source/WarpX.H | 11 ++-- Source/WarpX.cpp | 14 +++-- 7 files changed, 70 insertions(+), 51 deletions(-) diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index f6c73d9fa7e..3aca35dd66a 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -593,8 +593,8 @@ FlushFormatPlotfile::WriteAllRawFields( const MultiFab rho_new(warpx.getField(FieldType::rho_fp, lev), amrex::make_alias, nstart, WarpX::ncomps); WriteRawMF(rho_new, dm, raw_pltname, default_level_prefix, "rho_fp", lev, plot_raw_fields_guards); } - if (warpx.isFieldInitialized(FieldType::phi_fp, lev)) { - WriteRawMF(warpx.getField(FieldType::phi_fp, lev), dm, raw_pltname, default_level_prefix, "phi_fp", lev, plot_raw_fields_guards); + if (warpx.m_fields.has("phi_fp", lev)) { + WriteRawMF(warpx.m_fields.get("phi_fp", lev), dm, raw_pltname, default_level_prefix, "phi_fp", lev, plot_raw_fields_guards); } // Averaged fields on fine patch diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index 1ee725e7312..a043ab1ce7b 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -326,7 +326,7 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) AddRZModesToOutputNames(std::string("G"), ncomp); } } else if ( m_varnames_fields[comp] == "phi" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::phi_fp, lev), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("phi_fp", lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("phi"), ncomp); @@ -688,7 +688,7 @@ FullDiagnostics::InitializeFieldFunctors (int lev) } else if ( m_varnames[comp] == "G" ){ m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("G_fp", lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "phi" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::phi_fp, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("phi_fp", lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "part_per_cell" ){ m_all_field_functors[lev][comp] = std::make_unique(nullptr, lev, m_crse_ratio); } else if ( m_varnames[comp] == "part_per_grid" ){ diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index bfb1867e741..b83fda0863c 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -268,7 +268,7 @@ storePhiOnParticles ( PinnedMemoryParticleContainer& tmp, const amrex::Geometry& geom = warpx.Geom(lev); auto plo = geom.ProbLoArray(); auto dxi = geom.InvCellSizeArray(); - amrex::MultiFab const& phi = warpx.getField( FieldType::phi_fp, lev, 0 ); + anrex::MultiFab const& phi = *m_fields.get("phi_fp", lev); for (PinnedParIter pti(tmp, lev); pti.isValid(); ++pti) { diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index 80110f5eb18..505870f0a3a 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -114,7 +114,7 @@ WarpX::AddBoundaryField () // Allocate fields for charge and potential const int num_levels = max_level + 1; Vector > rho(num_levels); - Vector > phi(num_levels); + std::vector phi(num_levels); // Use number of guard cells used for local deposition of rho const amrex::IntVect ng = guard_cells.ng_depos_rho; for (int lev = 0; lev <= max_level; lev++) { @@ -122,7 +122,7 @@ WarpX::AddBoundaryField () nba.surroundingNodes(); rho[lev] = std::make_unique(nba, DistributionMap(lev), 1, ng); rho[lev]->setVal(0.); - phi[lev] = std::make_unique(nba, DistributionMap(lev), 1, 1); + phi[lev] = warpx.m_fields.alloc_init("phi_temp", nba, DistributionMap(lev), 1, 1, lev); phi[lev]->setVal(0.); } @@ -140,6 +140,12 @@ WarpX::AddBoundaryField () // Compute the corresponding electric and magnetic field, from the potential phi. computeE( Efield_fp, phi, beta ); computeB( Bfield_fp, phi, beta ); + + // de-allocate temporary + for (int lev = 0; lev <= max_level; lev++) { + warpx.m_fields.erase("phi_temp",lev); + } + } void @@ -166,7 +172,7 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) const int num_levels = max_level + 1; Vector > rho(num_levels); Vector > rho_coarse(num_levels); // Used in order to interpolate between levels - Vector > phi(num_levels); + std::vector phi(num_levels); // Use number of guard cells used for local deposition of rho const amrex::IntVect ng = guard_cells.ng_depos_rho; for (int lev = 0; lev <= max_level; lev++) { @@ -174,7 +180,7 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) nba.surroundingNodes(); rho[lev] = std::make_unique(nba, DistributionMap(lev), 1, ng); rho[lev]->setVal(0.); - phi[lev] = std::make_unique(nba, DistributionMap(lev), 1, 1); + phi[lev] = warpx.m_fields.alloc_init("phi_temp", nba, DistributionMap(lev), 1, 1, lev); phi[lev]->setVal(0.); if (lev > 0) { // For MR levels: allocated the coarsened version of rho @@ -220,6 +226,11 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) // Compute the corresponding electric and magnetic field, from the potential phi computeE( Efield_fp, phi, beta ); computeB( Bfield_fp, phi, beta ); + + // de-allocate temporary + for (int lev = 0; lev <= max_level; lev++) { + warpx.m_fields.erase("phi_temp",lev); + } } @@ -265,6 +276,7 @@ WarpX::AddSpaceChargeFieldLabFrame () const std::array beta = {0._rt}; // set the boundary potentials appropriately + auto phi_fp = warpx.m_fields.get_mr_levels("phi_fp",max_level); setPhiBC(phi_fp); // Compute the potential phi, by solving the Poisson equation @@ -317,7 +329,7 @@ WarpX::AddSpaceChargeFieldLabFrame () */ void WarpX::computePhi (const amrex::Vector >& rho, - amrex::Vector >& phi, + std::vector& phi, std::array const beta, Real const required_precision, Real absolute_tolerance, @@ -325,10 +337,10 @@ WarpX::computePhi (const amrex::Vector >& rho, int const verbosity) const { // create a vector to our fields, sorted by level amrex::Vector sorted_rho; - amrex::Vector sorted_phi; + std::vector sorted_phi; for (int lev = 0; lev <= finest_level; ++lev) { sorted_rho.emplace_back(rho[lev].get()); - sorted_phi.emplace_back(phi[lev].get()); + sorted_phi.emplace_back(phi[lev]); } std::optional post_phi_calculation; @@ -421,7 +433,7 @@ WarpX::computePhi (const amrex::Vector >& rho, \param[in] idim The dimension for which the Dirichlet boundary condition is set */ void -WarpX::setPhiBC ( amrex::Vector>& phi ) const +WarpX::setPhiBC ( std::vector& phi ) const { // check if any dimension has non-periodic boundary conditions if (!m_poisson_boundary_handler.has_non_periodic) { return; } @@ -503,7 +515,7 @@ WarpX::setPhiBC ( amrex::Vector>& phi ) const */ void WarpX::computeE (amrex::Vector, 3> >& E, - const amrex::Vector >& phi, + const std::vector& phi, std::array const beta ) const { for (int lev = 0; lev <= max_level; lev++) { @@ -680,7 +692,7 @@ WarpX::computeE (amrex::Vector, 3> > */ void WarpX::computeB (amrex::Vector, 3> >& B, - const amrex::Vector >& phi, + const std::vector& phi, std::array const beta ) const { // return early if beta is 0 since there will be no B-field @@ -849,7 +861,7 @@ WarpX::computeB (amrex::Vector, 3> > */ void WarpX::computePhiTriDiagonal (const amrex::Vector >& rho, - amrex::Vector >& phi) const + std::vector& phi) const { WARPX_ALWAYS_ASSERT_WITH_MESSAGE(max_level == 0, diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index 112db68f488..a5bc585d6f0 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -168,7 +168,7 @@ void WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const DistributionMapping& dm) { - const auto RemakeMultiFab = [&](auto& mf, const bool redistribute){ + const auto RemakeMultiFab = [&](MultiFab* mf, const bool redistribute){ if (mf == nullptr) { return; } const IntVect& ng = mf->nGrowVect(); auto pmf = std::remove_reference_t{}; @@ -184,58 +184,58 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi // Fine patch for (int idim=0; idim < 3; ++idim) { - RemakeMultiFab(Bfield_fp[lev][idim], true); - RemakeMultiFab(Efield_fp[lev][idim], true); + RemakeMultiFab(Bfield_fp[lev][idim].get(), true); + RemakeMultiFab(Efield_fp[lev][idim].get(), true); if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { - RemakeMultiFab(Bfield_fp_external[lev][idim], true); + RemakeMultiFab(Bfield_fp_external[lev][idim].get(), true); } if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { - RemakeMultiFab(Efield_fp_external[lev][idim], true); + RemakeMultiFab(Efield_fp_external[lev][idim].get(), true); } if (mypc->m_B_ext_particle_s == "read_from_file") { - RemakeMultiFab(B_external_particle_field[lev][idim], true); + RemakeMultiFab(B_external_particle_field[lev][idim].get(), true); } if (mypc->m_E_ext_particle_s == "read_from_file") { - RemakeMultiFab(E_external_particle_field[lev][idim], true); + RemakeMultiFab(E_external_particle_field[lev][idim].get(), true); } - RemakeMultiFab(current_fp[lev][idim], false); - RemakeMultiFab(current_store[lev][idim], false); + RemakeMultiFab(current_fp[lev][idim].get(), false); + RemakeMultiFab(current_store[lev][idim].get(), false); if (current_deposition_algo == CurrentDepositionAlgo::Vay) { - RemakeMultiFab(current_fp_vay[lev][idim], false); + RemakeMultiFab(current_fp_vay[lev][idim].get(), false); } if (do_current_centering) { - RemakeMultiFab(current_fp_nodal[lev][idim], false); + RemakeMultiFab(current_fp_nodal[lev][idim].get(), false); } if (fft_do_time_averaging) { - RemakeMultiFab(Efield_avg_fp[lev][idim], true); - RemakeMultiFab(Bfield_avg_fp[lev][idim], true); + RemakeMultiFab(Efield_avg_fp[lev][idim].get(), true); + RemakeMultiFab(Bfield_avg_fp[lev][idim].get(), true); } if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { - RemakeMultiFab(m_hybrid_pic_model->current_fp_temp[lev][idim], true); - RemakeMultiFab(m_hybrid_pic_model->current_fp_ampere[lev][idim], false); - RemakeMultiFab(m_hybrid_pic_model->current_fp_external[lev][idim],true); + RemakeMultiFab(m_hybrid_pic_model->current_fp_temp[lev][idim].get(), true); + RemakeMultiFab(m_hybrid_pic_model->current_fp_ampere[lev][idim].get(), false); + RemakeMultiFab(m_hybrid_pic_model->current_fp_external[lev][idim].get(),true); } if (m_eb_enabled) { if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { - RemakeMultiFab(m_edge_lengths[lev][idim], false); - RemakeMultiFab(m_face_areas[lev][idim], false); + RemakeMultiFab(m_edge_lengths[lev][idim].get(), false); + RemakeMultiFab(m_face_areas[lev][idim].get(), false); if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { - RemakeMultiFab(Venl[lev][idim], false); - RemakeMultiFab(m_flag_info_face[lev][idim], false); - RemakeMultiFab(m_flag_ext_face[lev][idim], false); - RemakeMultiFab(m_area_mod[lev][idim], false); - RemakeMultiFab(ECTRhofield[lev][idim], false); + RemakeMultiFab(Venl[lev][idim].get(), false); + RemakeMultiFab(m_flag_info_face[lev][idim].get(), false); + RemakeMultiFab(m_flag_ext_face[lev][idim].get(), false); + RemakeMultiFab(m_area_mod[lev][idim].get(), false); + RemakeMultiFab(ECTRhofield[lev][idim].get(), false); m_borrowing[lev][idim] = std::make_unique>(amrex::convert(ba, Bfield_fp[lev][idim]->ixType().toIntVect()), dm); } } } } - RemakeMultiFab(F_fp[lev], true); - RemakeMultiFab(rho_fp[lev], false); + RemakeMultiFab(F_fp[lev].get(), true); + RemakeMultiFab(rho_fp[lev].get(), false); // phi_fp should be redistributed since we use the solution from // the last step as the initial guess for the next solve - RemakeMultiFab(phi_fp[lev], true); + RemakeMultiFab(warpx.m_fields.get("phi_fp",lev), true); if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { RemakeMultiFab(m_hybrid_pic_model->rho_fp_temp[lev], true); diff --git a/Source/WarpX.H b/Source/WarpX.H index 21109a2d10c..b1661d87315 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1007,23 +1007,23 @@ public: void AddSpaceChargeField (WarpXParticleContainer& pc); void AddSpaceChargeFieldLabFrame (); void computePhi (const amrex::Vector >& rho, - amrex::Vector >& phi, + std::vector& phi, std::array beta = {{0,0,0}}, amrex::Real required_precision=amrex::Real(1.e-11), amrex::Real absolute_tolerance=amrex::Real(0.0), int max_iters=200, int verbosity=2) const; - void setPhiBC (amrex::Vector >& phi ) const; + void setPhiBC (std::vector& phi ) const; void computeE (amrex::Vector, 3> >& E, - const amrex::Vector >& phi, + const std::vector& phi, std::array beta = {{0,0,0}} ) const; void computeB (amrex::Vector, 3> >& B, - const amrex::Vector >& phi, + const std::vector& phi, std::array beta = {{0,0,0}} ) const; void computePhiTriDiagonal (const amrex::Vector >& rho, - amrex::Vector >& phi) const; + std::vector& phi) const; // Magnetostatic Solver Interface MagnetostaticSolver::VectorPoissonBoundaryHandler m_vector_poisson_boundary_handler; @@ -1523,7 +1523,6 @@ private: // Fine patch amrex::Vector< std::unique_ptr > F_fp; amrex::Vector< std::unique_ptr > rho_fp; - amrex::Vector< std::unique_ptr > phi_fp; amrex::Vector, 3 > > current_fp; amrex::Vector, 3 > > current_fp_vay; amrex::Vector, 3 > > Efield_fp; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 147480ebfd4..ec147391285 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -322,7 +322,6 @@ WarpX::WarpX () F_fp.resize(nlevs_max); rho_fp.resize(nlevs_max); - phi_fp.resize(nlevs_max); current_fp.resize(nlevs_max); Efield_fp.resize(nlevs_max); Bfield_fp.resize(nlevs_max); @@ -2103,6 +2102,14 @@ WarpX::ClearLevel (int lev) m_hybrid_pic_model->ClearLevel(lev); } + for (int i = 0; i < 3; ++i) { + Efield_dotMask [lev][i].reset(); + Bfield_dotMask [lev][i].reset(); + Afield_dotMask [lev][i].reset(); + } + + phi_dotMask[lev].reset(); + #ifdef WARPX_USE_FFT if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { spectral_solver_fp[lev].reset(); @@ -2483,7 +2490,8 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) { const IntVect ngPhi = IntVect( AMREX_D_DECL(1,1,1) ); - AllocInitMultiFab(phi_fp[lev], amrex::convert(ba, phi_nodal_flag), dm, ncomps, ngPhi, lev, "phi_fp", 0.0_rt); + m_multifab_map.alloc_init( "phi_fp", amrex::convert(ba, phi_nodal_flag), dm, + ncomps, ngPhi, lev, 0.0_rt ); } if (do_subcycling && lev == 0) @@ -3493,7 +3501,7 @@ WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, cons field_pointer = F_fp[lev].get(); break; case FieldType::phi_fp : - field_pointer = phi_fp[lev].get(); + field_pointer = m_fields.get("phi_fp",lev); break; case FieldType::vector_potential_fp : field_pointer = vector_potential_fp_nodal[lev][direction].get(); From 03c02539a5cd19307abb9873b4fc45e02f2fa75f Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Tue, 10 Sep 2024 11:09:58 -0700 Subject: [PATCH 030/314] compiles. WIP. working on RemakeMultiFab refactor --- .../FlushFormats/FlushFormatPlotfile.cpp | 2 +- Source/Diagnostics/ParticleIO.cpp | 2 +- Source/FieldSolver/ElectrostaticSolver.cpp | 12 +++-- Source/Parallelization/WarpXRegrid.cpp | 52 +++++++++---------- Source/WarpX.cpp | 12 ++--- Source/ablastr/fields/PoissonSolver.H | 2 +- 6 files changed, 42 insertions(+), 40 deletions(-) diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index 3aca35dd66a..bbe2da8c7ab 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -594,7 +594,7 @@ FlushFormatPlotfile::WriteAllRawFields( WriteRawMF(rho_new, dm, raw_pltname, default_level_prefix, "rho_fp", lev, plot_raw_fields_guards); } if (warpx.m_fields.has("phi_fp", lev)) { - WriteRawMF(warpx.m_fields.get("phi_fp", lev), dm, raw_pltname, default_level_prefix, "phi_fp", lev, plot_raw_fields_guards); + WriteRawMF(*warpx.m_fields.get("phi_fp", lev), dm, raw_pltname, default_level_prefix, "phi_fp", lev, plot_raw_fields_guards); } // Averaged fields on fine patch diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index b83fda0863c..a06b5c2232f 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -268,7 +268,7 @@ storePhiOnParticles ( PinnedMemoryParticleContainer& tmp, const amrex::Geometry& geom = warpx.Geom(lev); auto plo = geom.ProbLoArray(); auto dxi = geom.InvCellSizeArray(); - anrex::MultiFab const& phi = *m_fields.get("phi_fp", lev); + amrex::MultiFab const& phi = *warpx.m_fields.get("phi_fp", lev); for (PinnedParIter pti(tmp, lev); pti.isValid(); ++pti) { diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index 505870f0a3a..a114895e93c 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -122,7 +122,8 @@ WarpX::AddBoundaryField () nba.surroundingNodes(); rho[lev] = std::make_unique(nba, DistributionMap(lev), 1, ng); rho[lev]->setVal(0.); - phi[lev] = warpx.m_fields.alloc_init("phi_temp", nba, DistributionMap(lev), 1, 1, lev); + phi[lev] = m_fields.alloc_init( "phi_temp", nba, DistributionMap(lev), 1, + IntVect::TheUnitVector(), lev); phi[lev]->setVal(0.); } @@ -143,7 +144,7 @@ WarpX::AddBoundaryField () // de-allocate temporary for (int lev = 0; lev <= max_level; lev++) { - warpx.m_fields.erase("phi_temp",lev); + m_fields.erase("phi_temp",lev); } } @@ -180,7 +181,8 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) nba.surroundingNodes(); rho[lev] = std::make_unique(nba, DistributionMap(lev), 1, ng); rho[lev]->setVal(0.); - phi[lev] = warpx.m_fields.alloc_init("phi_temp", nba, DistributionMap(lev), 1, 1, lev); + phi[lev] = m_fields.alloc_init( "phi_temp", nba, DistributionMap(lev), 1, + IntVect::TheUnitVector(), lev); phi[lev]->setVal(0.); if (lev > 0) { // For MR levels: allocated the coarsened version of rho @@ -229,7 +231,7 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) // de-allocate temporary for (int lev = 0; lev <= max_level; lev++) { - warpx.m_fields.erase("phi_temp",lev); + m_fields.erase("phi_temp",lev); } } @@ -276,7 +278,7 @@ WarpX::AddSpaceChargeFieldLabFrame () const std::array beta = {0._rt}; // set the boundary potentials appropriately - auto phi_fp = warpx.m_fields.get_mr_levels("phi_fp",max_level); + auto phi_fp = m_fields.get_mr_levels("phi_fp",max_level); setPhiBC(phi_fp); // Compute the potential phi, by solving the Poisson equation diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index a5bc585d6f0..b2cda1dd96d 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -168,7 +168,7 @@ void WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const DistributionMapping& dm) { - const auto RemakeMultiFab = [&](MultiFab* mf, const bool redistribute){ + const auto RemakeMultiFab = [&](auto& mf, const bool redistribute){ if (mf == nullptr) { return; } const IntVect& ng = mf->nGrowVect(); auto pmf = std::remove_reference_t{}; @@ -184,58 +184,58 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi // Fine patch for (int idim=0; idim < 3; ++idim) { - RemakeMultiFab(Bfield_fp[lev][idim].get(), true); - RemakeMultiFab(Efield_fp[lev][idim].get(), true); + RemakeMultiFab(Bfield_fp[lev][idim], true); + RemakeMultiFab(Efield_fp[lev][idim], true); if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { - RemakeMultiFab(Bfield_fp_external[lev][idim].get(), true); + RemakeMultiFab(Bfield_fp_external[lev][idim], true); } if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { - RemakeMultiFab(Efield_fp_external[lev][idim].get(), true); + RemakeMultiFab(Efield_fp_external[lev][idim], true); } if (mypc->m_B_ext_particle_s == "read_from_file") { - RemakeMultiFab(B_external_particle_field[lev][idim].get(), true); + RemakeMultiFab(B_external_particle_field[lev][idim], true); } if (mypc->m_E_ext_particle_s == "read_from_file") { - RemakeMultiFab(E_external_particle_field[lev][idim].get(), true); + RemakeMultiFab(E_external_particle_field[lev][idim], true); } - RemakeMultiFab(current_fp[lev][idim].get(), false); - RemakeMultiFab(current_store[lev][idim].get(), false); + RemakeMultiFab(current_fp[lev][idim], false); + RemakeMultiFab(current_store[lev][idim], false); if (current_deposition_algo == CurrentDepositionAlgo::Vay) { - RemakeMultiFab(current_fp_vay[lev][idim].get(), false); + RemakeMultiFab(current_fp_vay[lev][idim], false); } if (do_current_centering) { - RemakeMultiFab(current_fp_nodal[lev][idim].get(), false); + RemakeMultiFab(current_fp_nodal[lev][idim], false); } if (fft_do_time_averaging) { - RemakeMultiFab(Efield_avg_fp[lev][idim].get(), true); - RemakeMultiFab(Bfield_avg_fp[lev][idim].get(), true); + RemakeMultiFab(Efield_avg_fp[lev][idim], true); + RemakeMultiFab(Bfield_avg_fp[lev][idim], true); } if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { - RemakeMultiFab(m_hybrid_pic_model->current_fp_temp[lev][idim].get(), true); - RemakeMultiFab(m_hybrid_pic_model->current_fp_ampere[lev][idim].get(), false); - RemakeMultiFab(m_hybrid_pic_model->current_fp_external[lev][idim].get(),true); + RemakeMultiFab(m_hybrid_pic_model->current_fp_temp[lev][idim], true); + RemakeMultiFab(m_hybrid_pic_model->current_fp_ampere[lev][idim], false); + RemakeMultiFab(m_hybrid_pic_model->current_fp_external[lev][idim],true); } if (m_eb_enabled) { if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { - RemakeMultiFab(m_edge_lengths[lev][idim].get(), false); - RemakeMultiFab(m_face_areas[lev][idim].get(), false); + RemakeMultiFab(m_edge_lengths[lev][idim], false); + RemakeMultiFab(m_face_areas[lev][idim], false); if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { - RemakeMultiFab(Venl[lev][idim].get(), false); - RemakeMultiFab(m_flag_info_face[lev][idim].get(), false); - RemakeMultiFab(m_flag_ext_face[lev][idim].get(), false); - RemakeMultiFab(m_area_mod[lev][idim].get(), false); - RemakeMultiFab(ECTRhofield[lev][idim].get(), false); + RemakeMultiFab(Venl[lev][idim], false); + RemakeMultiFab(m_flag_info_face[lev][idim], false); + RemakeMultiFab(m_flag_ext_face[lev][idim], false); + RemakeMultiFab(m_area_mod[lev][idim], false); + RemakeMultiFab(ECTRhofield[lev][idim], false); m_borrowing[lev][idim] = std::make_unique>(amrex::convert(ba, Bfield_fp[lev][idim]->ixType().toIntVect()), dm); } } } } - RemakeMultiFab(F_fp[lev].get(), true); - RemakeMultiFab(rho_fp[lev].get(), false); + RemakeMultiFab(F_fp[lev], true); + RemakeMultiFab(rho_fp[lev], false); // phi_fp should be redistributed since we use the solution from // the last step as the initial guess for the next solve - RemakeMultiFab(warpx.m_fields.get("phi_fp",lev), true); + //RemakeMultiFab(phi_fp[lev], true); // JRA if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { RemakeMultiFab(m_hybrid_pic_model->rho_fp_temp[lev], true); diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index ec147391285..f4b3167f22c 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2490,8 +2490,8 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) { const IntVect ngPhi = IntVect( AMREX_D_DECL(1,1,1) ); - m_multifab_map.alloc_init( "phi_fp", amrex::convert(ba, phi_nodal_flag), dm, - ncomps, ngPhi, lev, 0.0_rt ); + m_fields.alloc_init( "phi_fp", amrex::convert(ba, phi_nodal_flag), dm, + ncomps, ngPhi, lev, 0.0_rt ); } if (do_subcycling && lev == 0) @@ -3500,9 +3500,9 @@ WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, cons case FieldType::F_fp : field_pointer = F_fp[lev].get(); break; - case FieldType::phi_fp : - field_pointer = m_fields.get("phi_fp",lev); - break; + //case FieldType::phi_fp : // JRA + // field_pointer = m_fields.get("phi_fp",lev); + // break; case FieldType::vector_potential_fp : field_pointer = vector_potential_fp_nodal[lev][direction].get(); break; @@ -3659,7 +3659,7 @@ WarpX::getFieldDotMaskPointer ( FieldType field_type, int lev, int dir ) const case FieldType::vector_potential_fp : SetDotMask( Afield_dotMask[lev][dir], field_type, lev, dir ); return Afield_dotMask[lev][dir].get(); - case FieldType::phi_fp : + case FieldType::phi_fp : // JRA SetDotMask( phi_dotMask[lev], field_type, lev, 0 ); return phi_dotMask[lev].get(); default: diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index fbd8e5f14be..b53698ac66c 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -97,7 +97,7 @@ template< > void computePhi (amrex::Vector const & rho, - amrex::Vector & phi, + std::vector & phi, std::array const beta, amrex::Real relative_tolerance, amrex::Real absolute_tolerance, From 18fda6342e87249d5656e50493705d2f3b28ca86 Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Tue, 10 Sep 2024 11:56:10 -0700 Subject: [PATCH 031/314] cleaning. --- Source/FieldSolver/ElectrostaticSolver.cpp | 8 +- Source/Parallelization/WarpXRegrid.cpp | 86 +--------------------- Source/WarpX.cpp | 9 +-- 3 files changed, 9 insertions(+), 94 deletions(-) diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index a114895e93c..d04042463f6 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -122,8 +122,8 @@ WarpX::AddBoundaryField () nba.surroundingNodes(); rho[lev] = std::make_unique(nba, DistributionMap(lev), 1, ng); rho[lev]->setVal(0.); - phi[lev] = m_fields.alloc_init( "phi_temp", nba, DistributionMap(lev), 1, - IntVect::TheUnitVector(), lev); + phi[lev] = m_fields.alloc_init( "phi_temp", lev, nba, DistributionMap(lev), 1, + IntVect::TheUnitVector()); phi[lev]->setVal(0.); } @@ -181,8 +181,8 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) nba.surroundingNodes(); rho[lev] = std::make_unique(nba, DistributionMap(lev), 1, ng); rho[lev]->setVal(0.); - phi[lev] = m_fields.alloc_init( "phi_temp", nba, DistributionMap(lev), 1, - IntVect::TheUnitVector(), lev); + phi[lev] = m_fields.alloc_init( "phi_temp", lev, nba, DistributionMap(lev), 1, + IntVect::TheUnitVector()); phi[lev]->setVal(0.); if (lev > 0) { // For MR levels: allocated the coarsened version of rho diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index b2cda1dd96d..717c80dfaf6 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -168,15 +168,6 @@ void WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const DistributionMapping& dm) { - const auto RemakeMultiFab = [&](auto& mf, const bool redistribute){ - if (mf == nullptr) { return; } - const IntVect& ng = mf->nGrowVect(); - auto pmf = std::remove_reference_t{}; - AllocInitMultiFab(pmf, mf->boxArray(), dm, mf->nComp(), ng, lev, mf->tags()[0]); - if (redistribute) { pmf->Redistribute(*mf, 0, 0, mf->nComp(), ng); } - mf = std::move(pmf); - }; - if (ba == boxArray(lev)) { if (ParallelDescriptor::NProcs() == 1) { return; } @@ -184,66 +175,16 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi // Fine patch for (int idim=0; idim < 3; ++idim) { - RemakeMultiFab(Bfield_fp[lev][idim], true); - RemakeMultiFab(Efield_fp[lev][idim], true); - if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { - RemakeMultiFab(Bfield_fp_external[lev][idim], true); - } - if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { - RemakeMultiFab(Efield_fp_external[lev][idim], true); - } - if (mypc->m_B_ext_particle_s == "read_from_file") { - RemakeMultiFab(B_external_particle_field[lev][idim], true); - } - if (mypc->m_E_ext_particle_s == "read_from_file") { - RemakeMultiFab(E_external_particle_field[lev][idim], true); - } - RemakeMultiFab(current_fp[lev][idim], false); - RemakeMultiFab(current_store[lev][idim], false); - if (current_deposition_algo == CurrentDepositionAlgo::Vay) { - RemakeMultiFab(current_fp_vay[lev][idim], false); - } - if (do_current_centering) { - RemakeMultiFab(current_fp_nodal[lev][idim], false); - } - if (fft_do_time_averaging) { - RemakeMultiFab(Efield_avg_fp[lev][idim], true); - RemakeMultiFab(Bfield_avg_fp[lev][idim], true); - } - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { - RemakeMultiFab(m_hybrid_pic_model->current_fp_temp[lev][idim], true); - RemakeMultiFab(m_hybrid_pic_model->current_fp_ampere[lev][idim], false); - RemakeMultiFab(m_hybrid_pic_model->current_fp_external[lev][idim],true); - } if (m_eb_enabled) { if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { - RemakeMultiFab(m_edge_lengths[lev][idim], false); - RemakeMultiFab(m_face_areas[lev][idim], false); if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { - RemakeMultiFab(Venl[lev][idim], false); - RemakeMultiFab(m_flag_info_face[lev][idim], false); - RemakeMultiFab(m_flag_ext_face[lev][idim], false); - RemakeMultiFab(m_area_mod[lev][idim], false); - RemakeMultiFab(ECTRhofield[lev][idim], false); m_borrowing[lev][idim] = std::make_unique>(amrex::convert(ba, Bfield_fp[lev][idim]->ixType().toIntVect()), dm); } } } } - RemakeMultiFab(F_fp[lev], true); - RemakeMultiFab(rho_fp[lev], false); - // phi_fp should be redistributed since we use the solution from - // the last step as the initial guess for the next solve - //RemakeMultiFab(phi_fp[lev], true); // JRA - - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { - RemakeMultiFab(m_hybrid_pic_model->rho_fp_temp[lev], true); - RemakeMultiFab(m_hybrid_pic_model->electron_pressure_fp[lev], false); - } - if (m_eb_enabled) { - RemakeMultiFab(m_distance_to_eb[lev], false); #ifdef AMREX_USE_EB int const max_guard = guard_cells.ng_FieldSolver.max(); @@ -300,25 +241,13 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi } else { for (int idim=0; idim < 3; ++idim) { - RemakeMultiFab(Bfield_aux[lev][idim], false); - RemakeMultiFab(Efield_aux[lev][idim], false); + //RemakeMultiFab(Bfield_aux[lev][idim], false); + //RemakeMultiFab(Efield_aux[lev][idim], false); } } // Coarse patch if (lev > 0) { - for (int idim=0; idim < 3; ++idim) - { - RemakeMultiFab(Bfield_cp[lev][idim], true); - RemakeMultiFab(Efield_cp[lev][idim], true); - RemakeMultiFab(current_cp[lev][idim], false); - if (fft_do_time_averaging) { - RemakeMultiFab(Efield_avg_cp[lev][idim], true); - RemakeMultiFab(Bfield_avg_cp[lev][idim], true); - } - } - RemakeMultiFab(F_cp[lev], true); - RemakeMultiFab(rho_cp[lev], false); #ifdef WARPX_USE_FFT if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { @@ -356,17 +285,6 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi } if (lev > 0 && (n_field_gather_buffer > 0 || n_current_deposition_buffer > 0)) { - for (int idim=0; idim < 3; ++idim) - { - RemakeMultiFab(Bfield_cax[lev][idim], false); - RemakeMultiFab(Efield_cax[lev][idim], false); - RemakeMultiFab(current_buf[lev][idim], false); - } - RemakeMultiFab(charge_buf[lev], false); - // we can avoid redistributing these since we immediately re-build the values via BuildBufferMasks() - RemakeMultiFab(current_buffer_masks[lev], false); - RemakeMultiFab(gather_buffer_masks[lev], false); - if (current_buffer_masks[lev] || gather_buffer_masks[lev]) { BuildBufferMasks(); } diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index f4b3167f22c..52e54c7306b 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2490,8 +2490,8 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) { const IntVect ngPhi = IntVect( AMREX_D_DECL(1,1,1) ); - m_fields.alloc_init( "phi_fp", amrex::convert(ba, phi_nodal_flag), dm, - ncomps, ngPhi, lev, 0.0_rt ); + m_fields.alloc_init( "phi_fp", lev, amrex::convert(ba, phi_nodal_flag), dm, + ncomps, ngPhi, 0.0_rt ); } if (do_subcycling && lev == 0) @@ -3500,9 +3500,6 @@ WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, cons case FieldType::F_fp : field_pointer = F_fp[lev].get(); break; - //case FieldType::phi_fp : // JRA - // field_pointer = m_fields.get("phi_fp",lev); - // break; case FieldType::vector_potential_fp : field_pointer = vector_potential_fp_nodal[lev][direction].get(); break; @@ -3659,7 +3656,7 @@ WarpX::getFieldDotMaskPointer ( FieldType field_type, int lev, int dir ) const case FieldType::vector_potential_fp : SetDotMask( Afield_dotMask[lev][dir], field_type, lev, dir ); return Afield_dotMask[lev][dir].get(); - case FieldType::phi_fp : // JRA + case FieldType::phi_fp : SetDotMask( phi_dotMask[lev], field_type, lev, 0 ); return phi_dotMask[lev].get(); default: From 1d652060474af75cf0468fcc5406a43a567ef78d Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 10 Sep 2024 13:21:05 -0700 Subject: [PATCH 032/314] `MultiFabRegister::remake_level` --- Source/Parallelization/WarpXRegrid.cpp | 5 +++- Source/ablastr/fields/MultiFabRegister.H | 13 +++++++++++ Source/ablastr/fields/MultiFabRegister.cpp | 27 ++++++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index 717c80dfaf6..10c1c7c6c90 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -20,6 +20,8 @@ #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXProfilerWrapper.H" +#include + #include #include #include @@ -167,11 +169,12 @@ WarpX::LoadBalance () void WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const DistributionMapping& dm) { - if (ba == boxArray(lev)) { if (ParallelDescriptor::NProcs() == 1) { return; } + m_fields.remake_level(lev, dm); + // Fine patch for (int idim=0; idim < 3; ++idim) { diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index f303c8ab5d2..4634354f28c 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -269,6 +269,19 @@ namespace ablastr::fields int level ); + /** Remake all (i)MultiFab with a new distribution mapping. + * + * If redistribute is true, we also copy from the old data into the new. + * + * @param level the MR level to erase all MultiFabs from + * @param new_dm new distribution mapping + */ + void + remake_level ( + int other_level, + amrex::DistributionMapping const & new_dm + ); + /** title * * body body diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index da7c73522b4..9da2d88ada5 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -109,6 +109,33 @@ namespace ablastr::fields // TODO: does the other_name already exist? error } + void + MultiFabRegister::remake_level ( + int level, + amrex::DistributionMapping const & new_dm + ) + { + for (auto & element : m_mf_register ) + { + MultiFabOwner & mf_owner = element.second; + if (mf_owner.level == level) { + amrex::MultiFab & mf = mf_owner.m_mf; + amrex::IntVect const & ng = mf.nGrowVect(); + const auto tag = amrex::MFInfo().SetTag(mf.tags()[0]); + amrex::MultiFab new_mf(mf.boxArray(), new_dm, mf.nComp(), ng, tag); + + // copy data to new MultiFab: Only done for persistent data like E and B field, not for + // temporary things like currents, etc. + if (mf_owner.redistribute_on_remake) { + new_mf.Redistribute(mf, 0, 0, mf.nComp(), ng); + } + + // replace old MultiFab with new one, deallocate old one + mf_owner.m_mf = std::move(new_mf); + } + } + } + bool MultiFabRegister::has ( std::string name, From 171e97d85216f34a78e747721f4b7519392a09e1 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 10 Sep 2024 13:25:52 -0700 Subject: [PATCH 033/314] Use new get function --- Source/BoundaryConditions/PML.cpp | 14 ++++++++------ Source/BoundaryConditions/WarpXEvolvePML.cpp | 14 ++++++++------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index e479e43b638..ea228792a35 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -570,6 +570,8 @@ PML::PML (const int lev, const BoxArray& grid_ba, WARPX_ALWAYS_ASSERT_WITH_MESSAGE(!eb_enabled, "PML: eb_enabled is true but was not compiled in."); #endif + using ablastr::fields::Direction; + // When `do_pml_in_domain` is true, the PML overlap with the last `ncell` of the physical domain or fine patch(es) // (instead of extending `ncell` outside of the physical domain or fine patch(es)) // In order to implement this, we define a new reduced Box Array ensuring that it does not @@ -711,9 +713,9 @@ PML::PML (const int lev, const BoxArray& grid_ba, WarpX::AllocInitMultiFab(pml_B_fp[1], ba_By, dm, ncompb, ngb, lev, "pml_B_fp[y]", 0.0_rt); WarpX::AllocInitMultiFab(pml_B_fp[2], ba_Bz, dm, ncompb, ngb, lev, "pml_B_fp[z]", 0.0_rt); - const amrex::BoxArray ba_jx = amrex::convert(ba, WarpX::GetInstance().m_fields.get("current_fp[x]", 0)->ixType().toIntVect()); - const amrex::BoxArray ba_jy = amrex::convert(ba, WarpX::GetInstance().m_fields.get("current_fp[y]", 0)->ixType().toIntVect()); - const amrex::BoxArray ba_jz = amrex::convert(ba, WarpX::GetInstance().m_fields.get("current_fp[z]", 0)->ixType().toIntVect()); + const amrex::BoxArray ba_jx = amrex::convert(ba, WarpX::GetInstance().m_fields.get("current_fp", Direction{0}, 0)->ixType().toIntVect()); + const amrex::BoxArray ba_jy = amrex::convert(ba, WarpX::GetInstance().m_fields.get("current_fp", Direction{1}, 0)->ixType().toIntVect()); + const amrex::BoxArray ba_jz = amrex::convert(ba, WarpX::GetInstance().m_fields.get("current_fp", Direction{2}, 0)->ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_j_fp[0], ba_jx, dm, 1, ngb, lev, "pml_j_fp[x]", 0.0_rt); WarpX::AllocInitMultiFab(pml_j_fp[1], ba_jy, dm, 1, ngb, lev, "pml_j_fp[y]", 0.0_rt); WarpX::AllocInitMultiFab(pml_j_fp[2], ba_jz, dm, 1, ngb, lev, "pml_j_fp[z]", 0.0_rt); @@ -864,9 +866,9 @@ PML::PML (const int lev, const BoxArray& grid_ba, WarpX::AllocInitMultiFab( pml_G_cp, cba_G_nodal, cdm, 3, ngf, lev, "pml_G_cp", 0.0_rt); } - const amrex::BoxArray cba_jx = amrex::convert(cba, WarpX::GetInstance().m_fields.get("current_cp[x]", 0)->ixType().toIntVect()); - const amrex::BoxArray cba_jy = amrex::convert(cba, WarpX::GetInstance().m_fields.get("current_cp[y]", 0)->ixType().toIntVect()); - const amrex::BoxArray cba_jz = amrex::convert(cba, WarpX::GetInstance().m_fields.get("current_cp[z]", 0)->ixType().toIntVect()); + const amrex::BoxArray cba_jx = amrex::convert(cba, WarpX::GetInstance().m_fields.get("current_cp", Direction{0}, 0)->ixType().toIntVect()); + const amrex::BoxArray cba_jy = amrex::convert(cba, WarpX::GetInstance().m_fields.get("current_cp", Direction{1}, 0)->ixType().toIntVect()); + const amrex::BoxArray cba_jz = amrex::convert(cba, WarpX::GetInstance().m_fields.get("current_cp", Direction{2}, 0)->ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_j_cp[0], cba_jx, cdm, 1, ngb, lev, "pml_j_cp[x]", 0.0_rt); WarpX::AllocInitMultiFab(pml_j_cp[1], cba_jy, cdm, 1, ngb, lev, "pml_j_cp[y]", 0.0_rt); WarpX::AllocInitMultiFab(pml_j_cp[2], cba_jz, cdm, 1, ngb, lev, "pml_j_cp[z]", 0.0_rt); diff --git a/Source/BoundaryConditions/WarpXEvolvePML.cpp b/Source/BoundaryConditions/WarpXEvolvePML.cpp index f3a3f88ba45..f469f194282 100644 --- a/Source/BoundaryConditions/WarpXEvolvePML.cpp +++ b/Source/BoundaryConditions/WarpXEvolvePML.cpp @@ -336,17 +336,19 @@ WarpX::DampJPML (int lev, PatchType patch_type) void WarpX::CopyJPML () { + using ablastr::fields::Direction; + for (int lev = 0; lev <= finest_level; ++lev) { if (pml[lev] && pml[lev]->ok()){ pml[lev]->CopyJtoPMLs({ - m_fields.get("current_fp[x]", lev), - m_fields.get("current_fp[y]", lev), - m_fields.get("current_fp[z]", lev) + m_fields.get("current_fp", Direction{0}, lev), + m_fields.get("current_fp", Direction{1}, lev), + m_fields.get("current_fp", Direction{2}, lev) }, { - m_fields.get("current_cp[x]", lev), - m_fields.get("current_cp[y]", lev), - m_fields.get("current_cp[z]", lev) + m_fields.get("current_cp", Direction{0}, lev), + m_fields.get("current_cp", Direction{1}, lev), + m_fields.get("current_cp", Direction{2}, lev) } ); } From c31a05b641700a923405f4da6c045f8d75f7f381 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 10 Sep 2024 13:38:50 -0700 Subject: [PATCH 034/314] Allocate current_buf --- Source/WarpX.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index eff83eaaa7f..a5b6544c1db 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -395,7 +395,6 @@ WarpX::WarpX () Bfield_cax.resize(nlevs_max); current_buffer_masks.resize(nlevs_max); gather_buffer_masks.resize(nlevs_max); - current_buf.resize(nlevs_max); charge_buf.resize(nlevs_max); pml.resize(nlevs_max); @@ -2804,9 +2803,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm } if (n_current_deposition_buffer > 0) { - AllocInitMultiFab(current_buf[lev][0], amrex::convert(cba,jx_nodal_flag),dm,ncomps,ngJ,lev, "current_buf[x]"); - AllocInitMultiFab(current_buf[lev][1], amrex::convert(cba,jy_nodal_flag),dm,ncomps,ngJ,lev, "current_buf[y]"); - AllocInitMultiFab(current_buf[lev][2], amrex::convert(cba,jz_nodal_flag),dm,ncomps,ngJ,lev, "current_buf[z]"); + m_fields.alloc_init("current_buf", Direction{0}, lev, amrex::convert(cba,jx_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init("current_buf", Direction{1}, lev, amrex::convert(cba,jy_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init("current_buf", Direction{2}, lev, amrex::convert(cba,jz_nodal_flag), dm, ncomps, ngJ, 0.0_rt); if (rho_cp[lev]) { AllocInitMultiFab(charge_buf[lev], amrex::convert(cba,rho_nodal_flag),dm,2*ncomps,ngRho,lev, "charge_buf"); } From 38c61fe87e2ec2468429cbd0e41f80e80b760555 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Tue, 10 Sep 2024 13:41:38 -0700 Subject: [PATCH 035/314] Move F fields to MultiFab map --- .../FlushFormats/FlushFormatPlotfile.cpp | 8 +++--- Source/Diagnostics/FullDiagnostics.cpp | 4 +-- Source/FieldSolver/Fields.H | 2 -- .../FiniteDifferenceSolver/EvolveE.cpp | 10 +++---- .../FiniteDifferenceSolver/EvolveF.cpp | 6 ++--- .../FiniteDifferenceSolver.H | 12 ++++----- .../ImplicitSolvers/WarpXImplicitOps.cpp | 4 +-- Source/FieldSolver/WarpXPushFieldsEM.cpp | 26 +++++++++++-------- Source/Parallelization/WarpXComm.cpp | 16 ++++++------ Source/Utils/WarpXMovingWindow.cpp | 6 ++--- Source/WarpX.H | 2 -- Source/WarpX.cpp | 16 +++++------- 12 files changed, 54 insertions(+), 58 deletions(-) diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index b861b994d75..bbc8baabd71 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -583,9 +583,9 @@ FlushFormatPlotfile::WriteAllRawFields( WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 0), dm, raw_pltname, default_level_prefix, "Bx_fp", lev, plot_raw_fields_guards); WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 1), dm, raw_pltname, default_level_prefix, "By_fp", lev, plot_raw_fields_guards); WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 2), dm, raw_pltname, default_level_prefix, "Bz_fp", lev, plot_raw_fields_guards); - if (warpx.isFieldInitialized(FieldType::F_fp, lev)) + if (warpx.m_fields.has("F_fp", lev)) { - WriteRawMF(warpx.getField(FieldType::F_fp, lev), dm, raw_pltname, default_level_prefix, "F_fp", lev, plot_raw_fields_guards); + WriteRawMF(*warpx.m_fields.get("F_fp", lev), dm, raw_pltname, default_level_prefix, "F_fp", lev, plot_raw_fields_guards); } if (warpx.isFieldInitialized(FieldType::rho_fp, lev)) { @@ -635,9 +635,9 @@ FlushFormatPlotfile::WriteAllRawFields( warpx.m_fields.get("current_cp", Direction{0}, lev), warpx.m_fields.get("current_cp", Direction{1}, lev), warpx.m_fields.get("current_cp", Direction{2}, lev), warpx.m_fields.get("current_fp", Direction{0}, lev), warpx.m_fields.get("current_fp", Direction{1}, lev), warpx.m_fields.get("current_fp", Direction{2}, lev), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards); - if (warpx.isFieldInitialized(FieldType::F_fp, lev) && warpx.isFieldInitialized(FieldType::F_cp, lev)) + if (warpx.m_fields.has("F_fp", lev) && warpx.m_fields.has("F_cp", lev)) { - WriteCoarseScalar("F", warpx.getFieldPointer(FieldType::F_cp, lev), warpx.getFieldPointer(FieldType::F_fp, lev), + WriteCoarseScalar("F", warpx.m_fields.get("F_cp", lev), warpx.m_fields.get("F_fp", lev), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards, 0); } if (warpx.isFieldInitialized(FieldType::rho_fp, lev) && warpx.isFieldInitialized(FieldType::rho_cp, lev)) diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index fa90bb86e41..cc955c4e2e8 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -315,7 +315,7 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) } i_T_species++; } else if ( m_varnames_fields[comp] == "F" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::F_fp, lev), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("F_fp", lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("F"), ncomp); @@ -686,7 +686,7 @@ FullDiagnostics::InitializeFieldFunctors (int lev) m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, m_T_per_species_index[i_T_species]); i_T_species++; } else if ( m_varnames[comp] == "F" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::F_fp, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("F_fp", lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "G" ){ m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("G_fp", lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "phi" ){ diff --git a/Source/FieldSolver/Fields.H b/Source/FieldSolver/Fields.H index 55b37a2182c..56597d5030d 100644 --- a/Source/FieldSolver/Fields.H +++ b/Source/FieldSolver/Fields.H @@ -24,14 +24,12 @@ namespace warpx::fields current_fp, current_fp_nodal, rho_fp, - F_fp, phi_fp, vector_potential_fp, Efield_cp, Bfield_cp, current_cp, rho_cp, - F_cp, edge_lengths, face_areas, Efield_avg_fp, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp index 566a81da021..8a7b952a9c9 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp @@ -53,7 +53,7 @@ void FiniteDifferenceSolver::EvolveE ( std::array< std::unique_ptr, 3 > const& edge_lengths, std::array< std::unique_ptr, 3 > const& face_areas, std::array< std::unique_ptr, 3 >& ECTRhofield, - std::unique_ptr const& Ffield, + amrex::MultiFab const* Ffield, int lev, amrex::Real const dt ) { if (m_fdtd_algo != ElectromagneticSolverAlgo::ECT) { @@ -94,7 +94,7 @@ void FiniteDifferenceSolver::EvolveECartesian ( std::array< std::unique_ptr, 3 > const& Bfield, std::array< std::unique_ptr, 3 > const& Jfield, std::array< std::unique_ptr, 3 > const& edge_lengths, - std::unique_ptr const& Ffield, + amrex::MultiFab const* Ffield, int lev, amrex::Real const dt ) { #ifndef AMREX_USE_EB @@ -191,7 +191,7 @@ void FiniteDifferenceSolver::EvolveECartesian ( if (Ffield) { // Extract field data for this grid/tile - const Array4 F = Ffield->array(mfi); + Array4 F = Ffield->array(mfi); // Loop over the cells and update the fields amrex::ParallelFor(tex, tey, tez, @@ -228,7 +228,7 @@ void FiniteDifferenceSolver::EvolveECylindrical ( std::array< std::unique_ptr, 3 > const& Bfield, std::array< std::unique_ptr, 3 > const& Jfield, std::array< std::unique_ptr, 3 > const& edge_lengths, - std::unique_ptr const& Ffield, + amrex::MultiFab const* Ffield, int lev, amrex::Real const dt ) { #ifndef AMREX_USE_EB @@ -391,7 +391,7 @@ void FiniteDifferenceSolver::EvolveECylindrical ( if (Ffield) { // Extract field data for this grid/tile - const Array4 F = Ffield->array(mfi); + Array4 F = Ffield->array(mfi); // Loop over the cells and update the fields amrex::ParallelFor(ter, tet, tez, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp index 8ce578bb52a..7a29a1db732 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp @@ -44,7 +44,7 @@ using namespace amrex; * \brief Update the F field, over one timestep */ void FiniteDifferenceSolver::EvolveF ( - std::unique_ptr& Ffield, + amrex::MultiFab* Ffield, std::array< std::unique_ptr, 3 > const& Efield, std::unique_ptr const& rhofield, int const rhocomp, @@ -82,7 +82,7 @@ void FiniteDifferenceSolver::EvolveF ( template void FiniteDifferenceSolver::EvolveFCartesian ( - std::unique_ptr& Ffield, + amrex::MultiFab* Ffield, std::array< std::unique_ptr, 3 > const& Efield, std::unique_ptr const& rhofield, int const rhocomp, @@ -135,7 +135,7 @@ void FiniteDifferenceSolver::EvolveFCartesian ( template void FiniteDifferenceSolver::EvolveFCylindrical ( - std::unique_ptr& Ffield, + amrex::MultiFab* Ffield, std::array< std::unique_ptr, 3 > const& Efield, std::unique_ptr const& rhofield, int const rhocomp, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index b9b8104ecf2..ea5cb8c66da 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -68,10 +68,10 @@ class FiniteDifferenceSolver std::array< std::unique_ptr, 3 > const& edge_lengths, std::array< std::unique_ptr, 3 > const& face_areas, std::array< std::unique_ptr, 3 >& ECTRhofield, - std::unique_ptr const& Ffield, + amrex::MultiFab const* Ffield, int lev, amrex::Real dt ); - void EvolveF ( std::unique_ptr& Ffield, + void EvolveF ( amrex::MultiFab* Ffield, std::array< std::unique_ptr, 3 > const& Efield, std::unique_ptr const& rhofield, int rhocomp, @@ -217,13 +217,13 @@ class FiniteDifferenceSolver std::array< std::unique_ptr, 3 > const& Bfield, std::array< std::unique_ptr, 3 > const& Jfield, std::array< std::unique_ptr, 3 > const& edge_lengths, - std::unique_ptr const& Ffield, + amrex::MultiFab const* Ffield, int lev, amrex::Real dt ); template< typename T_Algo > void EvolveFCylindrical ( - std::unique_ptr& Ffield, + amrex::MultiFab* Ffield, std::array< std::unique_ptr, 3 > const& Efield, std::unique_ptr const& rhofield, int rhocomp, @@ -269,12 +269,12 @@ class FiniteDifferenceSolver std::array< std::unique_ptr, 3 > const& Bfield, std::array< std::unique_ptr, 3 > const& Jfield, std::array< std::unique_ptr, 3 > const& edge_lengths, - std::unique_ptr const& Ffield, + amrex::MultiFab const* Ffield, int lev, amrex::Real dt ); template< typename T_Algo > void EvolveFCartesian ( - std::unique_ptr& Ffield, + amrex::MultiFab* Ffield, std::array< std::unique_ptr, 3 > const& Efield, std::unique_ptr const& rhofield, int rhocomp, diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp index 8dd97ed5525..9aedfb3d97f 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp @@ -338,12 +338,12 @@ WarpX::ImplicitComputeRHSE (int lev, PatchType patch_type, amrex::Real a_dt, War m_fdtd_solver_fp[lev]->EvolveE( a_Erhs_vec.getArrayVec()[lev], Bfield_fp[lev], current_fp[lev], m_edge_lengths[lev], m_face_areas[lev], ECTRhofield[lev], - F_fp[lev], lev, a_dt ); + m_fields.get("F_fp", lev), lev, a_dt ); } else { m_fdtd_solver_cp[lev]->EvolveE( a_Erhs_vec.getArrayVec()[lev], Bfield_cp[lev], current_cp[lev], m_edge_lengths[lev], m_face_areas[lev], ECTRhofield[lev], - F_cp[lev], lev, a_dt ); + m_fields.get("F_cp", lev), lev, a_dt ); } // Compute Efield_rhs in PML cells by calling EvolveEPML diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 33948bb17d3..34950a14072 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -184,11 +184,15 @@ WarpX::PSATDForwardTransformF () for (int lev = 0; lev <= finest_level; ++lev) { - if (F_fp[lev]) { spectral_solver_fp[lev]->ForwardTransform(lev, *F_fp[lev], Idx.F); } + if (m_fields.has("F_fp", lev)) { + spectral_solver_fp[lev]->ForwardTransform(lev, *m_fields.get("F_fp", lev), Idx.F); + } if (spectral_solver_cp[lev]) { - if (F_cp[lev]) { spectral_solver_cp[lev]->ForwardTransform(lev, *F_cp[lev], Idx.F); } + if (m_fields.has("F_cp", lev)) { + spectral_solver_cp[lev]->ForwardTransform(lev, *m_fields.get("F_cp", lev), Idx.F); + } } } } @@ -201,17 +205,17 @@ WarpX::PSATDBackwardTransformF () for (int lev = 0; lev <= finest_level; ++lev) { #ifdef WARPX_DIM_RZ - if (F_fp[lev]) { spectral_solver_fp[lev]->BackwardTransform(lev, *F_fp[lev], Idx.F); } + if (m_fields.has("F_fp", lev)) { spectral_solver_fp[lev]->BackwardTransform(lev, *m_fields.get("F_fp", lev), Idx.F); } #else - if (F_fp[lev]) { spectral_solver_fp[lev]->BackwardTransform(lev, *F_fp[lev], Idx.F, m_fill_guards_fields); } + if (m_fields.has("F_fp", lev)) { spectral_solver_fp[lev]->BackwardTransform(lev, *m_fields.get("F_fp", lev), Idx.F, m_fill_guards_fields); } #endif if (spectral_solver_cp[lev]) { #ifdef WARPX_DIM_RZ - if (F_cp[lev]) { spectral_solver_cp[lev]->BackwardTransform(lev, *F_cp[lev], Idx.F); } + if (m_fields.has("F_cp", lev)) { spectral_solver_cp[lev]->BackwardTransform(lev, *m_fields.get("F_cp", lev), Idx.F); } #else - if (F_cp[lev]) { spectral_solver_cp[lev]->BackwardTransform(lev, *F_cp[lev], Idx.F, m_fill_guards_fields); } + if (m_fields.has("F_cp", lev)) { spectral_solver_cp[lev]->BackwardTransform(lev, *m_fields.get("F_cp", lev), Idx.F, m_fill_guards_fields); } #endif } } @@ -219,7 +223,7 @@ WarpX::PSATDBackwardTransformF () // Damp the field in the guard cells for (int lev = 0; lev <= finest_level; ++lev) { - DampFieldsInGuards(lev, F_fp[lev].get()); + DampFieldsInGuards(lev, m_fields.get("F_fp", lev)); } } @@ -884,12 +888,12 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) m_fdtd_solver_fp[lev]->EvolveE(Efield_fp[lev], Bfield_fp[lev], current_fp[lev], m_edge_lengths[lev], m_face_areas[lev], ECTRhofield[lev], - F_fp[lev], lev, a_dt ); + m_fields.get("F_fp", lev), lev, a_dt ); } else { m_fdtd_solver_cp[lev]->EvolveE(Efield_cp[lev], Bfield_cp[lev], current_cp[lev], m_edge_lengths[lev], m_face_areas[lev], ECTRhofield[lev], - F_cp[lev], lev, a_dt ); + m_fields.get("F_cp", lev), lev, a_dt ); } // Evolve E field in PML cells @@ -960,10 +964,10 @@ WarpX::EvolveF (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_typ // Evolve F field in regular cells if (patch_type == PatchType::fine) { - m_fdtd_solver_fp[lev]->EvolveF( F_fp[lev], Efield_fp[lev], + m_fdtd_solver_fp[lev]->EvolveF( m_fields.get("F_fp", lev), Efield_fp[lev], rho_fp[lev], rhocomp, a_dt ); } else { - m_fdtd_solver_cp[lev]->EvolveF( F_cp[lev], Efield_cp[lev], + m_fdtd_solver_cp[lev]->EvolveF( m_fields.get("F_cp", lev), Efield_cp[lev], rho_cp[lev], rhocomp, a_dt ); } diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index d967ca2a40d..4d62b95060c 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -888,30 +888,30 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng, std::optionalok()) { - if (F_fp[lev]) { pml[lev]->ExchangeF(patch_type, F_fp[lev].get(), do_pml_in_domain); } + if (m_fields.has("F_fp", lev)) { pml[lev]->ExchangeF(patch_type, m_fields.get("F_fp", lev), do_pml_in_domain); } pml[lev]->FillBoundaryF(patch_type, nodal_sync); } - if (F_fp[lev]) + if (m_fields.has("F_fp", lev)) { const amrex::Periodicity& period = Geom(lev).periodicity(); - const amrex::IntVect& nghost = (safe_guard_cells) ? F_fp[lev]->nGrowVect() : ng; - ablastr::utils::communication::FillBoundary(*F_fp[lev], nghost, WarpX::do_single_precision_comms, period, nodal_sync); + const amrex::IntVect& nghost = (safe_guard_cells) ? m_fields.get("F_fp", lev)->nGrowVect() : ng; + ablastr::utils::communication::FillBoundary(*m_fields.get("F_fp", lev), nghost, WarpX::do_single_precision_comms, period, nodal_sync); } } else if (patch_type == PatchType::coarse) { if (do_pml && pml[lev] && pml[lev]->ok()) { - if (F_cp[lev]) { pml[lev]->ExchangeF(patch_type, F_cp[lev].get(), do_pml_in_domain); } + if (m_fields.has("F_cp", lev)) { pml[lev]->ExchangeF(patch_type, m_fields.get("F_cp", lev), do_pml_in_domain); } pml[lev]->FillBoundaryF(patch_type, nodal_sync); } - if (F_cp[lev]) + if (m_fields.has("F_cp", lev)) { const amrex::Periodicity& period = Geom(lev-1).periodicity(); - const amrex::IntVect& nghost = (safe_guard_cells) ? F_cp[lev]->nGrowVect() : ng; - ablastr::utils::communication::FillBoundary(*F_cp[lev], nghost, WarpX::do_single_precision_comms, period, nodal_sync); + const amrex::IntVect& nghost = (safe_guard_cells) ? m_fields.get("F_cp", lev)->nGrowVect() : ng; + ablastr::utils::communication::FillBoundary(*m_fields.get("F_cp", lev), nghost, WarpX::do_single_precision_comms, period, nodal_sync); } } } diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 3747268f381..3667e01d751 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -289,14 +289,14 @@ WarpX::MoveWindow (const int step, bool move_j) // Shift scalar field F with div(E) cleaning in valid domain // TODO: shift F from pml_rz for RZ geometry with PSATD, once implemented - if (F_fp[lev]) + if (m_fields.has("F_fp", lev)) { // Fine grid - shiftMF(*F_fp[lev], geom[lev], num_shift, dir, lev, do_update_cost); + shiftMF(*m_fields.get("F_fp", lev), geom[lev], num_shift, dir, lev, do_update_cost); if (lev > 0) { // Coarse grid - shiftMF(*F_cp[lev], geom[lev-1], num_shift_crse, dir, lev, do_update_cost); + shiftMF(*m_fields.get("F_cp", lev), geom[lev-1], num_shift_crse, dir, lev, do_update_cost); } } diff --git a/Source/WarpX.H b/Source/WarpX.H index b1661d87315..2dfffbf2ad1 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1521,7 +1521,6 @@ private: amrex::Vector, 3 > > Bfield_aux; // Fine patch - amrex::Vector< std::unique_ptr > F_fp; amrex::Vector< std::unique_ptr > rho_fp; amrex::Vector, 3 > > current_fp; amrex::Vector, 3 > > current_fp_vay; @@ -1600,7 +1599,6 @@ private: amrex::Vector,3>> current_fp_nodal; // Coarse patch - amrex::Vector< std::unique_ptr > F_cp; amrex::Vector< std::unique_ptr > rho_cp ; amrex::Vector, 3 > > current_cp; amrex::Vector, 3 > > Efield_cp; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index eff83eaaa7f..5a8cd79e883 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -320,7 +320,6 @@ WarpX::WarpX () Efield_aux.resize(nlevs_max); Bfield_aux.resize(nlevs_max); - F_fp.resize(nlevs_max); rho_fp.resize(nlevs_max); Efield_fp.resize(nlevs_max); Bfield_fp.resize(nlevs_max); @@ -379,7 +378,6 @@ WarpX::WarpX () m_hybrid_pic_model = std::make_unique(nlevs_max); } - F_cp.resize(nlevs_max); rho_cp.resize(nlevs_max); current_cp.resize(nlevs_max); Efield_cp.resize(nlevs_max); @@ -2504,7 +2502,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (do_dive_cleaning) { - AllocInitMultiFab(F_fp[lev], amrex::convert(ba, F_nodal_flag), dm, ncomps, ngF, lev, "F_fp", 0.0_rt); + m_fields.alloc_init( + "F_fp", lev, amrex::convert(ba, F_nodal_flag), dm, + ncomps, ngF, 0.0_rt); } if (do_divb_cleaning) @@ -2707,7 +2707,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (do_dive_cleaning) { - AllocInitMultiFab(F_cp[lev], amrex::convert(cba, IntVect::TheUnitVector()), dm, ncomps, ngF, lev, "F_cp", 0.0_rt); + m_fields.alloc_init( + "F_cp", lev, amrex::convert(cba, IntVect::TheUnitVector()), dm, + ncomps, ngF, 0.0_rt); } if (do_divb_cleaning) @@ -3498,9 +3500,6 @@ WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, cons case FieldType::rho_fp : field_pointer = rho_fp[lev].get(); break; - case FieldType::F_fp : - field_pointer = F_fp[lev].get(); - break; case FieldType::vector_potential_fp : field_pointer = vector_potential_fp_nodal[lev][direction].get(); break; @@ -3516,9 +3515,6 @@ WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, cons case FieldType::rho_cp : field_pointer = rho_cp[lev].get(); break; - case FieldType::F_cp : - field_pointer = F_cp[lev].get(); - break; case FieldType::edge_lengths : field_pointer = m_edge_lengths[lev][direction].get(); break; From 4e4f0ee811016df6992be4af301f8dddd0874ed7 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 10 Sep 2024 13:50:25 -0700 Subject: [PATCH 036/314] Add: `amrex::fields::va2vm` (temporary) --- Source/ablastr/fields/MultiFabRegister.H | 16 +++++++++ Source/ablastr/fields/MultiFabRegister.cpp | 38 ++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 4634354f28c..11fab77ef37 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -322,6 +322,22 @@ namespace ablastr::fields > m_mf_register; }; + /** TODO: temporary, remove me + * + * Convert get_mr_levels_alldirs type to legacy amrex::Vector, 3 > >. + * + * @return + */ + std::vector< + std::map< + Direction, + amrex::MultiFab* + > + > + va2vm ( + amrex::Vector, 3 > > const & old_vector_on_levels + ); + } // namespace ablastr::fields #endif // ABLASTR_FIELDS_MF_REGISTER_H diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 9da2d88ada5..440917ec15f 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -314,4 +314,42 @@ namespace ablastr::fields ); } + std::vector< + std::map< + Direction, + amrex::MultiFab* + > + > + va2vm ( + amrex::Vector, 3 > > const & old_vector_on_levels + ) + { + int const finest_level = old_vector_on_levels.size() - 1u; + + std::vector< + std::map< + Direction, + amrex::MultiFab* + > + > field_on_level; + field_on_level.reserve(finest_level+1); + + std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; + + for (int lvl = 0; lvl <= finest_level; lvl++) + { + // insert a new level + field_on_level.push_back(std::map< + Direction, + amrex::MultiFab* + >{}); + + // insert components + for (auto dir : {0, 1, 2}) + { + field_on_level[lvl][Direction{dir}] = old_vector_on_levels[lvl][dir].get(); + } + } + return field_on_level; + } } // namespace ablastr::fields From 066097701672f4ebd43da2ad4c7afdbf03de6ba4 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 10 Sep 2024 14:34:54 -0700 Subject: [PATCH 037/314] Aliases: `VectorField`, `MultiLevelScalar/VectorField` --- Source/ablastr/fields/MultiFabRegister.H | 32 ++++++++++++---------- Source/ablastr/fields/MultiFabRegister.cpp | 25 ++++------------- 2 files changed, 23 insertions(+), 34 deletions(-) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 11fab77ef37..8654470a151 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -23,7 +23,6 @@ namespace ablastr::fields { - /** Components (base vector directions) of vector/tensor fields. * * Because of different staggering, the components of vector/tensor fields are stored @@ -39,6 +38,21 @@ namespace ablastr::fields } }; + /** A vector field of three MultiFab */ + using VectorField = std::map; + + /** A multi-level scalar field + * + * TODO: use amrex::Vector again for bound checks in debug mode + */ + using MultiLevelScalarField = std::vector; + + /** A multi-level vector field + * + * TODO: use amrex::Vector again for bound checks in debug mode + */ + using MultiLevelVectorField = std::vector>; + /** title * * body body @@ -208,7 +222,7 @@ namespace ablastr::fields * @param finest_level ... * @return ... */ - std::vector + MultiLevelScalarField get_mr_levels ( std::string name, int finest_level @@ -223,12 +237,7 @@ namespace ablastr::fields * @param finest_level ... * @return ... */ - std::vector< - std::map< - Direction, - amrex::MultiFab* - > - > + MultiLevelVectorField get_mr_levels_alldirs ( std::string name, int finest_level @@ -328,12 +337,7 @@ namespace ablastr::fields * * @return */ - std::vector< - std::map< - Direction, - amrex::MultiFab* - > - > + MultiLevelVectorField va2vm ( amrex::Vector, 3 > > const & old_vector_on_levels ); diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 440917ec15f..4cf9ca139cb 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -192,13 +192,13 @@ namespace ablastr::fields return &mf; } - std::vector + MultiLevelScalarField MultiFabRegister::get_mr_levels ( std::string name, int finest_level ) { - std::vector field_on_level; + MultiLevelScalarField field_on_level; field_on_level.reserve(finest_level+1); for (int lvl = 0; lvl <= finest_level; lvl++) { @@ -207,23 +207,13 @@ namespace ablastr::fields return field_on_level; } - std::vector< - std::map< - Direction, - amrex::MultiFab* - > - > + MultiLevelVectorField MultiFabRegister::get_mr_levels_alldirs ( std::string name, int finest_level ) { - std::vector< - std::map< - Direction, - amrex::MultiFab* - > - > field_on_level; + MultiLevelVectorField field_on_level; field_on_level.reserve(finest_level+1); // TODO: Technically, we should search field_on_level via std::unique_copy @@ -314,12 +304,7 @@ namespace ablastr::fields ); } - std::vector< - std::map< - Direction, - amrex::MultiFab* - > - > + MultiLevelVectorField va2vm ( amrex::Vector, 3 > > const & old_vector_on_levels ) From 0c7d2e30aec9fe46ca422a249fbe5e4b73d129a9 Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Tue, 10 Sep 2024 23:31:42 +0200 Subject: [PATCH 038/314] fix merge conflict --- .../FlushFormats/FlushFormatPlotfile.cpp | 10 +++---- Source/Evolve/WarpXEvolve.cpp | 30 +++++++++---------- Source/FieldSolver/ElectrostaticSolver.cpp | 6 ++-- Source/FieldSolver/Fields.H | 2 -- .../HybridPICModel/HybridPICModel.cpp | 2 +- Source/FieldSolver/WarpXPushFieldsEM.cpp | 4 +-- Source/Parallelization/WarpXComm.cpp | 11 +++---- Source/Utils/WarpXMovingWindow.cpp | 2 +- Source/WarpX.H | 7 +---- Source/WarpX.cpp | 18 +++++------ 10 files changed, 39 insertions(+), 53 deletions(-) diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index bbc8baabd71..15d9f27241e 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -587,12 +587,12 @@ FlushFormatPlotfile::WriteAllRawFields( { WriteRawMF(*warpx.m_fields.get("F_fp", lev), dm, raw_pltname, default_level_prefix, "F_fp", lev, plot_raw_fields_guards); } - if (warpx.isFieldInitialized(FieldType::rho_fp, lev)) + if (warpx.m_fields.has("rho_fp", lev)) { // rho_fp will have either ncomps or 2*ncomps (2 being the old and new). When 2, return the new so // there is time synchronization. - const int nstart = warpx.getField(FieldType::rho_fp, lev).nComp() - WarpX::ncomps; - const MultiFab rho_new(warpx.getField(FieldType::rho_fp, lev), amrex::make_alias, nstart, WarpX::ncomps); + const int nstart = warpx.m_fields.get("rho_fp", lev)->nComp() - WarpX::ncomps; + const MultiFab rho_new(*warpx.m_fields.get("rho_fp", lev), amrex::make_alias, nstart, WarpX::ncomps); WriteRawMF(rho_new, dm, raw_pltname, default_level_prefix, "rho_fp", lev, plot_raw_fields_guards); } if (warpx.m_fields.has("phi_fp", lev)) { @@ -640,10 +640,10 @@ FlushFormatPlotfile::WriteAllRawFields( WriteCoarseScalar("F", warpx.m_fields.get("F_cp", lev), warpx.m_fields.get("F_fp", lev), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards, 0); } - if (warpx.isFieldInitialized(FieldType::rho_fp, lev) && warpx.isFieldInitialized(FieldType::rho_cp, lev)) + if (warpx.m_fields.has("rho_fp", lev) && warpx.m_fields.has("rho_cp", lev)) { // Use the component 1 of `rho_cp`, i.e. rho_new for time synchronization - WriteCoarseScalar("rho", warpx.getFieldPointer(FieldType::rho_cp, lev), warpx.getFieldPointer(FieldType::rho_fp, lev), + WriteCoarseScalar("rho", warpx.m_fields.get("rho_cp", lev), warpx.m_fields.get("rho_fp", lev), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards, 1); } } diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index e09d9f2253d..1b56e502b74 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -553,12 +553,12 @@ void WarpX::SyncCurrentAndRho () { // TODO Replace current_cp with current_cp_vay once Vay deposition is implemented with MR SyncCurrent(current_fp_vay, current_cp, current_buf); - SyncRho(rho_fp, rho_cp, charge_buf); + SyncRho(); } else { SyncCurrent(current_fp, current_cp, current_buf); - SyncRho(rho_fp, rho_cp, charge_buf); + SyncRho(); } } else // no periodic single box @@ -570,7 +570,7 @@ void WarpX::SyncCurrentAndRho () current_deposition_algo != CurrentDepositionAlgo::Vay) { SyncCurrent(current_fp, current_cp, current_buf); - SyncRho(rho_fp, rho_cp, charge_buf); + SyncRho(); } if (current_deposition_algo == CurrentDepositionAlgo::Vay) @@ -584,22 +584,22 @@ void WarpX::SyncCurrentAndRho () else // FDTD { SyncCurrent(current_fp, current_cp, current_buf); - SyncRho(rho_fp, rho_cp, charge_buf); + SyncRho(); } // Reflect charge and current density over PEC boundaries, if needed. for (int lev = 0; lev <= finest_level; ++lev) { - if (rho_fp[lev]) { - ApplyRhofieldBoundary(lev, rho_fp[lev].get(), PatchType::fine); + if (m_fields.has("rho_fp", lev)) { + ApplyRhofieldBoundary(lev, m_fields.get("rho_fp",lev), PatchType::fine); } ApplyJfieldBoundary( lev, current_fp[lev][0].get(), current_fp[lev][1].get(), current_fp[lev][2].get(), PatchType::fine ); if (lev > 0) { - if (rho_cp[lev]) { - ApplyRhofieldBoundary(lev, rho_cp[lev].get(), PatchType::coarse); + if (m_fields.has("rho_cp", lev)) { + ApplyRhofieldBoundary(lev, m_fields.get("rho_cp",lev), PatchType::coarse); } ApplyJfieldBoundary( lev, current_cp[lev][0].get(), current_cp[lev][1].get(), @@ -645,7 +645,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // (dt[0] denotes the time step on mesh refinement level 0) mypc->DepositCharge(rho_fp, -dt[0]); // Filter, exchange boundary, and interpolate across levels - SyncRho(rho_fp, rho_cp, charge_buf); + SyncRho(); // Forward FFT of rho PSATDForwardTransformRho(rho_fp, rho_cp, 0, rho_new); } @@ -709,7 +709,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Deposit rho at relative time t_deposit_charge mypc->DepositCharge(rho_fp, t_deposit_charge); // Filter, exchange boundary, and interpolate across levels - SyncRho(rho_fp, rho_cp, charge_buf); + SyncRho(; // Forward FFT of rho const int rho_idx = (rho_in_time == RhoInTime::Linear) ? rho_new : rho_mid; PSATDForwardTransformRho(rho_fp, rho_cp, 0, rho_idx); @@ -810,7 +810,7 @@ WarpX::OneStep_sub1 (Real cur_time) // i) Push particles and fields on the fine patch (first fine step) PushParticlesandDeposit(fine_lev, cur_time, DtType::FirstHalf); RestrictCurrentFromFineToCoarsePatch(current_fp, current_cp, fine_lev); - RestrictRhoFromFineToCoarsePatch(rho_fp, rho_cp, fine_lev); + RestrictRhoFromFineToCoarsePatch(fine_lev); if (use_filter) { ApplyFilterJ(current_fp, fine_lev); } SumBoundaryJ(current_fp, fine_lev, Geom(fine_lev).periodicity()); ApplyFilterandSumBoundaryRho(rho_fp, rho_cp, fine_lev, PatchType::fine, 0, 2*ncomps); @@ -1029,7 +1029,7 @@ WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, *Bfield_aux[lev][0], *Bfield_aux[lev][1], *Bfield_aux[lev][2], *current_x, *current_y, *current_z, current_buf[lev][0].get(), current_buf[lev][1].get(), current_buf[lev][2].get(), - rho_fp[lev].get(), charge_buf[lev].get(), + m_fields.get("rho_fp",lev), charge_buf[lev].get(), Efield_cax[lev][0].get(), Efield_cax[lev][1].get(), Efield_cax[lev][2].get(), Bfield_cax[lev][0].get(), Bfield_cax[lev][1].get(), Bfield_cax[lev][2].get(), cur_time, dt[lev], a_dt_type, skip_current, push_type); @@ -1040,8 +1040,8 @@ WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, if (current_buf[lev][0].get()) { ApplyInverseVolumeScalingToCurrentDensity(current_buf[lev][0].get(), current_buf[lev][1].get(), current_buf[lev][2].get(), lev-1); } - if (rho_fp[lev]) { - ApplyInverseVolumeScalingToChargeDensity(rho_fp[lev].get(), lev); + if (m_fields.has("rho_fp", lev)) { + ApplyInverseVolumeScalingToChargeDensity(m_fields.get("rho_fp", lev).get(), lev); if (charge_buf[lev]) { ApplyInverseVolumeScalingToChargeDensity(charge_buf[lev].get(), lev-1); } @@ -1058,7 +1058,7 @@ WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, myfl->Evolve(lev, *Efield_aux[lev][0], *Efield_aux[lev][1], *Efield_aux[lev][2], *Bfield_aux[lev][0], *Bfield_aux[lev][1], *Bfield_aux[lev][2], - rho_fp[lev].get(), *current_x, *current_y, *current_z, cur_time, skip_current); + m_fields.get("rho_fp", lev), *current_x, *current_y, *current_z, cur_time, skip_current); } } } diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index 12127098abe..77989f60c39 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -254,7 +254,7 @@ WarpX::AddSpaceChargeFieldLabFrame () #endif // Deposit particle charge density (source of Poisson solver) - mypc->DepositCharge(rho_fp, 0.0_rt); + mypc->DepositCharge(m_fields.get("rho_fp", lev), 0.0_rt); if (do_fluid_species) { int const lev = 0; myfl->DepositCharge( lev, *rho_fp[lev] ); @@ -266,11 +266,11 @@ WarpX::AddSpaceChargeFieldLabFrame () } } } - SyncRho(rho_fp, rho_cp, charge_buf); // Apply filter, perform MPI exchange, interpolate across levels + SyncRho(); // Apply filter, perform MPI exchange, interpolate across levels #ifndef WARPX_DIM_RZ for (int lev = 0; lev <= finestLevel(); lev++) { // Reflect density over PEC boundaries, if needed. - ApplyRhofieldBoundary(lev, rho_fp[lev].get(), PatchType::fine); + ApplyRhofieldBoundary(lev, m_fields.get("rho_fp", lev).get(), PatchType::fine); } #endif diff --git a/Source/FieldSolver/Fields.H b/Source/FieldSolver/Fields.H index 56597d5030d..964398cecbc 100644 --- a/Source/FieldSolver/Fields.H +++ b/Source/FieldSolver/Fields.H @@ -23,13 +23,11 @@ namespace warpx::fields Bfield_fp_external, current_fp, current_fp_nodal, - rho_fp, phi_fp, vector_potential_fp, Efield_cp, Bfield_cp, current_cp, - rho_cp, edge_lengths, face_areas, Efield_avg_fp, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 70efc04e259..379f4d28fa5 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -496,7 +496,7 @@ void HybridPICModel::CalculateElectronPressure(const int lev) auto& warpx = WarpX::GetInstance(); // Calculate the electron pressure using rho^{n+1}. FillElectronPressureMF( - electron_pressure_fp[lev], warpx.getFieldPointer(FieldType::rho_fp, lev) + electron_pressure_fp[lev], warpx.m_fields.get("rho_fp", lev) ); warpx.ApplyElectronPressureBoundary(lev, PatchType::fine); electron_pressure_fp[lev]->FillBoundary(warpx.Geom(lev).periodicity()); diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 34950a14072..7d639154bb0 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -736,7 +736,7 @@ WarpX::PushPSATD () // Synchronize J and rho SyncCurrent(current_fp, current_cp, current_buf); - SyncRho(rho_fp, rho_cp, charge_buf); + SyncRho(); } else if (current_deposition_algo == CurrentDepositionAlgo::Vay) { @@ -757,7 +757,7 @@ WarpX::PushPSATD () // TODO This works only without mesh refinement const int lev = 0; SumBoundaryJ(current_fp, lev, Geom(lev).periodicity()); - SyncRho(rho_fp, rho_cp, charge_buf); + SyncRho(); } // FFT of J and rho (if used) diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 4d62b95060c..68a0bbb4c67 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -1421,15 +1421,12 @@ void WarpX::AddCurrentFromFineLevelandSumBoundary ( } } -void WarpX::RestrictRhoFromFineToCoarsePatch ( - const amrex::Vector>& charge_fp, - const amrex::Vector>& charge_cp, - const int lev) +void WarpX::RestrictRhoFromFineToCoarsePatch ( const int lev ) { - if (charge_fp[lev]) { - charge_cp[lev]->setVal(0.0); + if (m_fields.has("rho_fp", lev)) { + m_fields.get("rho_cp", lev)->setVal(0.0); const IntVect& refinement_ratio = refRatio(lev-1); - ablastr::coarsen::average::Coarsen(*charge_cp[lev], *charge_fp[lev], refinement_ratio ); + ablastr::coarsen::average::Coarsen(*m_fields.get("rho_cp", lev), *m_fields.get("rho_fp", lev), refinement_ratio ); } } diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 3667e01d751..524ecc24763 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -355,7 +355,7 @@ WarpX::MoveWindow (const int step, bool move_j) // Shift scalar component rho if (move_j) { - if (rho_fp[lev]){ + if (m_fields.has("rho_fp", lev)) { // Fine grid shiftMF(*rho_fp[lev], geom[lev], num_shift, dir, lev, do_update_cost); if (lev > 0){ diff --git a/Source/WarpX.H b/Source/WarpX.H index 3ff6f6cc366..4f7a7e86794 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1365,10 +1365,7 @@ private: int lev, PatchType patch_type); - void RestrictRhoFromFineToCoarsePatch ( - const amrex::Vector>& charge_fp, - const amrex::Vector>& charge_cp, - int lev); + void RestrictRhoFromFineToCoarsePatch (int lev ); void ApplyFilterandSumBoundaryRho ( const amrex::Vector>& charge_fp, const amrex::Vector>& charge_cp, @@ -1520,7 +1517,6 @@ private: amrex::Vector, 3 > > Bfield_aux; // Fine patch - amrex::Vector< std::unique_ptr > rho_fp; amrex::Vector, 3 > > current_fp; amrex::Vector, 3 > > current_fp_vay; amrex::Vector, 3 > > Efield_fp; @@ -1598,7 +1594,6 @@ private: amrex::Vector,3>> current_fp_nodal; // Coarse patch - amrex::Vector< std::unique_ptr > rho_cp ; amrex::Vector, 3 > > current_cp; amrex::Vector, 3 > > Efield_cp; amrex::Vector, 3 > > Bfield_cp; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 82a40adf18f..6fc67fb1123 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -320,7 +320,6 @@ WarpX::WarpX () Efield_aux.resize(nlevs_max); Bfield_aux.resize(nlevs_max); - rho_fp.resize(nlevs_max); Efield_fp.resize(nlevs_max); Bfield_fp.resize(nlevs_max); @@ -378,7 +377,6 @@ WarpX::WarpX () m_hybrid_pic_model = std::make_unique(nlevs_max); } - rho_cp.resize(nlevs_max); current_cp.resize(nlevs_max); Efield_cp.resize(nlevs_max); Bfield_cp.resize(nlevs_max); @@ -2481,7 +2479,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm } if (rho_ncomps > 0) { - AllocInitMultiFab(rho_fp[lev], amrex::convert(ba, rho_nodal_flag), dm, rho_ncomps, ngRho, lev, "rho_fp", 0.0_rt); + m_fields.alloc_init( + "rho_fp", amrex::convert(ba, rho_nodal_flag), dm, + rho_ncomps, ngRho, lev, 0.0_rt); } if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrame || @@ -2701,7 +2701,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm m_fields.alloc_init( "current_cp", Direction{2}, lev, amrex::convert(cba, jz_nodal_flag), dm, ncomps, ngJ, 0.0_rt); if (rho_ncomps > 0) { - AllocInitMultiFab(rho_cp[lev], amrex::convert(cba, rho_nodal_flag), dm, rho_ncomps, ngRho, lev, "rho_cp", 0.0_rt); + m_fields.alloc_init( + "rho_cp", amrex::convert(cba, rho_nodal_flag), dm, + rho_ncomps, ngRho, lev, 0.0_rt); } if (do_dive_cleaning) @@ -2808,7 +2810,7 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm m_fields.alloc_init("current_buf", Direction{0}, lev, amrex::convert(cba,jx_nodal_flag), dm, ncomps, ngJ, 0.0_rt); m_fields.alloc_init("current_buf", Direction{1}, lev, amrex::convert(cba,jy_nodal_flag), dm, ncomps, ngJ, 0.0_rt); m_fields.alloc_init("current_buf", Direction{2}, lev, amrex::convert(cba,jz_nodal_flag), dm, ncomps, ngJ, 0.0_rt); - if (rho_cp[lev]) { + if (m_fields.has("rho_cp", lev)) { AllocInitMultiFab(charge_buf[lev], amrex::convert(cba,rho_nodal_flag),dm,2*ncomps,ngRho,lev, "charge_buf"); } AllocInitMultiFab(current_buffer_masks[lev], ba, dm, ncomps, amrex::IntVect(1), lev, "current_buffer_masks"); @@ -3496,9 +3498,6 @@ WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, cons case FieldType::current_fp_nodal : field_pointer = current_fp_nodal[lev][direction].get(); break; - case FieldType::rho_fp : - field_pointer = rho_fp[lev].get(); - break; case FieldType::vector_potential_fp : field_pointer = vector_potential_fp_nodal[lev][direction].get(); break; @@ -3511,9 +3510,6 @@ WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, cons case FieldType::current_cp : field_pointer = current_cp[lev][direction].get(); break; - case FieldType::rho_cp : - field_pointer = rho_cp[lev].get(); - break; case FieldType::edge_lengths : field_pointer = m_edge_lengths[lev][direction].get(); break; From b7c15d0eae25fefbedb3996015c60230224961e4 Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Tue, 10 Sep 2024 23:32:32 +0200 Subject: [PATCH 039/314] fix merge conflict --- Source/Evolve/WarpXEvolve.cpp | 21 ++++++++++++---- Source/FieldSolver/ElectrostaticSolver.cpp | 25 +++++++++++-------- .../FiniteDifferenceSolver/EvolveF.cpp | 4 +-- .../FiniteDifferenceSolver.H | 10 ++++---- .../HybridPICModel/HybridPICModel.H | 12 ++++----- .../HybridPICModel/HybridPICModel.cpp | 12 ++++----- .../HybridPICSolveE.cpp | 6 ++--- Source/FieldSolver/WarpXPushFieldsEM.cpp | 6 ++--- .../FieldSolver/WarpXPushFieldsHybridPIC.cpp | 18 ++++++------- Source/Parallelization/WarpXComm.cpp | 19 ++++++++------ Source/Particles/MultiParticleContainer.H | 2 +- Source/Particles/MultiParticleContainer.cpp | 2 +- Source/Particles/WarpXParticleContainer.H | 4 +-- Source/Particles/WarpXParticleContainer.cpp | 2 +- Source/WarpX.H | 14 +++++------ 15 files changed, 88 insertions(+), 69 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 1b56e502b74..90f2e0a5da0 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -813,7 +813,11 @@ WarpX::OneStep_sub1 (Real cur_time) RestrictRhoFromFineToCoarsePatch(fine_lev); if (use_filter) { ApplyFilterJ(current_fp, fine_lev); } SumBoundaryJ(current_fp, fine_lev, Geom(fine_lev).periodicity()); - ApplyFilterandSumBoundaryRho(rho_fp, rho_cp, fine_lev, PatchType::fine, 0, 2*ncomps); + + ApplyFilterandSumBoundaryRho( + m_fields.get_mr_levels("rho_fp", finest_level), + m_fields.get_mr_levels("rho_cp", finest_level), + fine_lev, PatchType::fine, 0, 2*ncomps); EvolveB(fine_lev, PatchType::fine, 0.5_rt*dt[fine_lev], DtType::FirstHalf); EvolveF(fine_lev, PatchType::fine, 0.5_rt*dt[fine_lev], DtType::FirstHalf); @@ -842,7 +846,9 @@ WarpX::OneStep_sub1 (Real cur_time) PushParticlesandDeposit(coarse_lev, cur_time, DtType::Full); StoreCurrent(coarse_lev); AddCurrentFromFineLevelandSumBoundary(current_fp, current_cp, current_buf, coarse_lev); - AddRhoFromFineLevelandSumBoundary(rho_fp, rho_cp, charge_buf, coarse_lev, 0, ncomps); + AddRhoFromFineLevelandSumBoundary( + m_fields.get_mr_levels("rho_fp", finest_level), + m_fields.get_mr_levels("rho_cp", finest_level), charge_buf, coarse_lev, 0, ncomps); EvolveB(fine_lev, PatchType::coarse, dt[fine_lev], DtType::FirstHalf); EvolveF(fine_lev, PatchType::coarse, dt[fine_lev], DtType::FirstHalf); @@ -871,10 +877,13 @@ WarpX::OneStep_sub1 (Real cur_time) // iv) Push particles and fields on the fine patch (second fine step) PushParticlesandDeposit(fine_lev, cur_time + dt[fine_lev], DtType::SecondHalf); RestrictCurrentFromFineToCoarsePatch(current_fp, current_cp, fine_lev); - RestrictRhoFromFineToCoarsePatch(rho_fp, rho_cp, fine_lev); + RestrictRhoFromFineToCoarsePatch(fine_lev); if (use_filter) { ApplyFilterJ(current_fp, fine_lev); } SumBoundaryJ(current_fp, fine_lev, Geom(fine_lev).periodicity()); - ApplyFilterandSumBoundaryRho(rho_fp, rho_cp, fine_lev, PatchType::fine, 0, ncomps); + ApplyFilterandSumBoundaryRho( + m_fields.get_mr_levels("rho_fp", finest_level), + m_fields.get_mr_levels("rho_cp", finest_level), + fine_lev, PatchType::fine, 0, ncomps); EvolveB(fine_lev, PatchType::fine, 0.5_rt*dt[fine_lev], DtType::FirstHalf); EvolveF(fine_lev, PatchType::fine, 0.5_rt*dt[fine_lev], DtType::FirstHalf); @@ -902,7 +911,9 @@ WarpX::OneStep_sub1 (Real cur_time) // by only half a coarse step (second half) RestoreCurrent(coarse_lev); AddCurrentFromFineLevelandSumBoundary(current_fp, current_cp, current_buf, coarse_lev); - AddRhoFromFineLevelandSumBoundary(rho_fp, rho_cp, charge_buf, coarse_lev, ncomps, ncomps); + AddRhoFromFineLevelandSumBoundary( + m_fields.get_mr_levels("rho_fp", finest_level), + m_fields.get_mr_levels("rho_cp", finest_level), charge_buf, coarse_lev, ncomps, ncomps); EvolveE(fine_lev, PatchType::coarse, dt[fine_lev]); FillBoundaryE(fine_lev, PatchType::coarse, guard_cells.ng_FieldSolver, diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index 77989f60c39..0dd12e0c271 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -114,8 +114,13 @@ WarpX::AddBoundaryField () // Allocate fields for charge and potential const int num_levels = max_level + 1; +<<<<<<< HEAD Vector > rho(num_levels); std::vector phi(num_levels); +======= + Vector> rho(num_levels); + Vector > phi(num_levels); +>>>>>>> f719adaa9 (continue implementing new strategy to get rho) // Use number of guard cells used for local deposition of rho const amrex::IntVect ng = guard_cells.ng_depos_rho; for (int lev = 0; lev <= max_level; lev++) { @@ -135,7 +140,7 @@ WarpX::AddBoundaryField () const std::array beta = {0._rt}; // Compute the potential phi, by solving the Poisson equation - computePhi( rho, phi, beta, self_fields_required_precision, + computePhi( amrex::GetVecOfPtrs(rho), phi, beta, self_fields_required_precision, self_fields_absolute_tolerance, self_fields_max_iters, self_fields_verbosity ); @@ -201,7 +206,7 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) bool const apply_boundary_and_scale_volume = true; bool const interpolate_across_levels = false; if ( !pc.do_not_deposit) { - pc.DepositCharge(rho, local, reset, apply_boundary_and_scale_volume, + pc.DepositCharge(amrex::GetVecOfPtrs(rho), local, reset, apply_boundary_and_scale_volume, interpolate_across_levels); } for (int lev = 0; lev <= max_level; lev++) { @@ -211,7 +216,7 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) } } } - SyncRho(rho, rho_coarse, charge_buf); // Apply filter, perform MPI exchange, interpolate across levels + SyncRho(amrex::GetVecOfPtrs(rho), amrex::GetVecOfPtrs(rho_coarse), charge_buf); // Apply filter, perform MPI exchange, interpolate across levels // Get the particle beta vector bool const local_average = false; // Average across all MPI ranks @@ -222,7 +227,7 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) } // Compute the potential phi, by solving the Poisson equation - computePhi( rho, phi, beta, pc.self_fields_required_precision, + computePhi( amrex::GetVecOfPtrs(rho), phi, beta, pc.self_fields_required_precision, pc.self_fields_absolute_tolerance, pc.self_fields_max_iters, pc.self_fields_verbosity ); @@ -254,10 +259,10 @@ WarpX::AddSpaceChargeFieldLabFrame () #endif // Deposit particle charge density (source of Poisson solver) - mypc->DepositCharge(m_fields.get("rho_fp", lev), 0.0_rt); + mypc->DepositCharge(m_fields.get_mr_levels("rho_fp", finest_level), 0.0_rt); if (do_fluid_species) { int const lev = 0; - myfl->DepositCharge( lev, *rho_fp[lev] ); + myfl->DepositCharge( lev, *m_fields.get("rho_fp", lev)); } for (int lev = 0; lev <= max_level; lev++) { if (lev > 0) { @@ -270,7 +275,7 @@ WarpX::AddSpaceChargeFieldLabFrame () #ifndef WARPX_DIM_RZ for (int lev = 0; lev <= finestLevel(); lev++) { // Reflect density over PEC boundaries, if needed. - ApplyRhofieldBoundary(lev, m_fields.get("rho_fp", lev).get(), PatchType::fine); + ApplyRhofieldBoundary(lev, m_fields.get("rho_fp", lev), PatchType::fine); } #endif @@ -295,7 +300,7 @@ WarpX::AddSpaceChargeFieldLabFrame () computePhiTriDiagonal(rho_fp, phi_fp); #else // Use the AMREX MLMG or the FFT (IGF) solver otherwise - computePhi(rho_fp, phi_fp, beta, self_fields_required_precision, + computePhi(m_fields.get_mr_levels("rho_fp", finest_level), phi_fp, beta, self_fields_required_precision, self_fields_absolute_tolerance, self_fields_max_iters, self_fields_verbosity); #endif @@ -331,7 +336,7 @@ WarpX::AddSpaceChargeFieldLabFrame () \param[in] verbosity The verbosity setting for the MLMG solver */ void -WarpX::computePhi (const amrex::Vector >& rho, +WarpX::computePhi (const std::vector& rho, std::vector& phi, std::array const beta, Real const required_precision, @@ -343,7 +348,7 @@ WarpX::computePhi (const amrex::Vector >& rho, std::vector sorted_phi; for (int lev = 0; lev <= finest_level; ++lev) { sorted_rho.emplace_back(rho[lev].get()); - sorted_phi.emplace_back(phi[lev]); + sorted_phi.emplace_back(amrex::GetVecOfPtrs(phi)[lev]); } std::optional post_phi_calculation; diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp index 7a29a1db732..381b014b342 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp @@ -46,7 +46,7 @@ using namespace amrex; void FiniteDifferenceSolver::EvolveF ( amrex::MultiFab* Ffield, std::array< std::unique_ptr, 3 > const& Efield, - std::unique_ptr const& rhofield, + amrex::MultiFab* const rhofield, int const rhocomp, amrex::Real const dt ) { @@ -84,7 +84,7 @@ template void FiniteDifferenceSolver::EvolveFCartesian ( amrex::MultiFab* Ffield, std::array< std::unique_ptr, 3 > const& Efield, - std::unique_ptr const& rhofield, + amrex::MultiFab* const rhofield, int const rhocomp, amrex::Real const dt ) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index ea5cb8c66da..9573b1a2d05 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -73,7 +73,7 @@ class FiniteDifferenceSolver void EvolveF ( amrex::MultiFab* Ffield, std::array< std::unique_ptr, 3 > const& Efield, - std::unique_ptr const& rhofield, + amrex::MultiFab* const rhofield, int rhocomp, amrex::Real dt ); @@ -156,7 +156,7 @@ class FiniteDifferenceSolver std::array< std::unique_ptr, 3 > const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3> const& Bfield, - std::unique_ptr const& rhofield, + amrex::MultiFab* const rhofield, std::unique_ptr const& Pefield, std::array< std::unique_ptr, 3 > const& edge_lengths, int lev, HybridPICModel const* hybrid_model, @@ -241,7 +241,7 @@ class FiniteDifferenceSolver std::array< std::unique_ptr, 3> const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3> const& Bfield, - std::unique_ptr const& rhofield, + amrex::MultiFab* const rhofield, std::unique_ptr const& Pefield, std::array< std::unique_ptr, 3 > const& edge_lengths, int lev, HybridPICModel const* hybrid_model, @@ -276,7 +276,7 @@ class FiniteDifferenceSolver void EvolveFCartesian ( amrex::MultiFab* Ffield, std::array< std::unique_ptr, 3 > const& Efield, - std::unique_ptr const& rhofield, + amrex::MultiFab* const rhofield, int rhocomp, amrex::Real dt ); @@ -346,7 +346,7 @@ class FiniteDifferenceSolver std::array< std::unique_ptr, 3> const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3> const& Bfield, - std::unique_ptr const& rhofield, + amrex::MultiFab* const rhofield, std::unique_ptr const& Pefield, std::array< std::unique_ptr, 3 > const& edge_lengths, int lev, HybridPICModel const* hybrid_model, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index 3a49d5fad4b..890342a5fab 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -89,7 +89,7 @@ public: amrex::Vector, 3>>& Efield, amrex::Vector, 3>> const& Jfield, amrex::Vector, 3>> const& Bfield, - amrex::Vector> const& rhofield, + std::vector const& rhofield, amrex::Vector, 3>> const& edge_lengths, bool solve_for_Faraday); @@ -97,7 +97,7 @@ public: std::array< std::unique_ptr, 3>& Efield, std::array< std::unique_ptr, 3> const& Jfield, std::array< std::unique_ptr, 3> const& Bfield, - std::unique_ptr const& rhofield, + amrex::MultiFab* const rhofield, std::array< std::unique_ptr, 3> const& edge_lengths, int lev, bool solve_for_Faraday); @@ -105,7 +105,7 @@ public: std::array< std::unique_ptr, 3>& Efield, std::array< std::unique_ptr, 3> const& Jfield, std::array< std::unique_ptr, 3> const& Bfield, - std::unique_ptr const& rhofield, + amrex::MultiFab* const rhofield, std::array< std::unique_ptr, 3> const& edge_lengths, int lev, PatchType patch_type, bool solve_for_Faraday); @@ -113,7 +113,7 @@ public: amrex::Vector, 3>>& Bfield, amrex::Vector, 3>>& Efield, amrex::Vector, 3>> const& Jfield, - amrex::Vector> const& rhofield, + std::vector const& rhofield, amrex::Vector, 3>> const& edge_lengths, amrex::Real dt, DtType a_dt_type, amrex::IntVect ng, std::optional nodal_sync); @@ -122,7 +122,7 @@ public: amrex::Vector, 3>>& Bfield, amrex::Vector, 3>>& Efield, amrex::Vector, 3>> const& Jfield, - amrex::Vector> const& rhofield, + std::vector const& rhofield, amrex::Vector, 3>> const& edge_lengths, amrex::Real dt, int lev, DtType dt_type, amrex::IntVect ng, std::optional nodal_sync); @@ -131,7 +131,7 @@ public: amrex::Vector, 3>>& Bfield, amrex::Vector, 3>>& Efield, amrex::Vector, 3>> const& Jfield, - amrex::Vector> const& rhofield, + std::vector const& rhofield, amrex::Vector, 3>> const& edge_lengths, amrex::Real dt, DtType dt_type, amrex::IntVect ng, std::optional nodal_sync); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 379f4d28fa5..5609d331775 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -424,7 +424,7 @@ void HybridPICModel::HybridPICSolveE ( amrex::Vector, 3>> & Efield, amrex::Vector, 3>> const& Jfield, amrex::Vector, 3>> const& Bfield, - amrex::Vector> const& rhofield, + std::vector const& rhofield, amrex::Vector, 3>> const& edge_lengths, const bool solve_for_Faraday) { @@ -442,7 +442,7 @@ void HybridPICModel::HybridPICSolveE ( std::array< std::unique_ptr, 3> & Efield, std::array< std::unique_ptr, 3> const& Jfield, std::array< std::unique_ptr, 3> const& Bfield, - std::unique_ptr const& rhofield, + amrex::MultiFab* const rhofield, std::array< std::unique_ptr, 3> const& edge_lengths, const int lev, const bool solve_for_Faraday) { @@ -463,7 +463,7 @@ void HybridPICModel::HybridPICSolveE ( std::array< std::unique_ptr, 3> & Efield, std::array< std::unique_ptr, 3> const& Jfield, std::array< std::unique_ptr, 3> const& Bfield, - std::unique_ptr const& rhofield, + amrex::MultiFab* const rhofield, std::array< std::unique_ptr, 3> const& edge_lengths, const int lev, PatchType patch_type, const bool solve_for_Faraday) @@ -535,7 +535,7 @@ void HybridPICModel::BfieldEvolveRK ( amrex::Vector, 3>>& Bfield, amrex::Vector, 3>>& Efield, amrex::Vector, 3>> const& Jfield, - amrex::Vector> const& rhofield, + std::vector const& rhofield, amrex::Vector, 3>> const& edge_lengths, amrex::Real dt, DtType dt_type, IntVect ng, std::optional nodal_sync ) @@ -554,7 +554,7 @@ void HybridPICModel::BfieldEvolveRK ( amrex::Vector, 3>>& Bfield, amrex::Vector, 3>>& Efield, amrex::Vector, 3>> const& Jfield, - amrex::Vector> const& rhofield, + std::vector const& rhofield, amrex::Vector, 3>> const& edge_lengths, amrex::Real dt, int lev, DtType dt_type, IntVect ng, std::optional nodal_sync ) @@ -667,7 +667,7 @@ void HybridPICModel::FieldPush ( amrex::Vector, 3>>& Bfield, amrex::Vector, 3>>& Efield, amrex::Vector, 3>> const& Jfield, - amrex::Vector> const& rhofield, + std::vector const& rhofield, amrex::Vector, 3>> const& edge_lengths, amrex::Real dt, DtType dt_type, IntVect ng, std::optional nodal_sync ) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp index baeaf7a6c18..a229163a09e 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp @@ -356,7 +356,7 @@ void FiniteDifferenceSolver::HybridPICSolveE ( std::array< std::unique_ptr, 3 > const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3 > const& Bfield, - std::unique_ptr const& rhofield, + amrex::MultiFab* const rhofield, std::unique_ptr const& Pefield, std::array< std::unique_ptr, 3 > const& edge_lengths, int lev, HybridPICModel const* hybrid_model, @@ -394,7 +394,7 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( std::array< std::unique_ptr, 3 > const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3 > const& Bfield, - std::unique_ptr const& rhofield, + amrex::MultiFab* const rhofield, std::unique_ptr const& Pefield, std::array< std::unique_ptr, 3 > const& edge_lengths, int lev, HybridPICModel const* hybrid_model, @@ -709,7 +709,7 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( std::array< std::unique_ptr, 3 > const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3 > const& Bfield, - std::unique_ptr const& rhofield, + amrex::MultiFab* const rhofield, std::unique_ptr const& Pefield, std::array< std::unique_ptr, 3 > const& edge_lengths, int lev, HybridPICModel const* hybrid_model, diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 7d639154bb0..76ef48e9bda 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -964,11 +964,11 @@ WarpX::EvolveF (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_typ // Evolve F field in regular cells if (patch_type == PatchType::fine) { - m_fdtd_solver_fp[lev]->EvolveF( m_fields.get("F_fp", lev), Efield_fp[lev], - rho_fp[lev], rhocomp, a_dt ); + m_fdtd_solver_fp[lev]->EvolveF( m_fields.get("F_fp", lev),, Efield_fp[lev], + m_fields.get("rho_fp",lev), rhocomp, a_dt ); } else { m_fdtd_solver_cp[lev]->EvolveF( m_fields.get("F_cp", lev), Efield_cp[lev], - rho_cp[lev], rhocomp, a_dt ); + m_fields.get("rho_cp",lev), rhocomp, a_dt ); } // Evolve F field in PML cells diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index c16f0193b8d..2e8f02ed198 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -28,14 +28,14 @@ void WarpX::HybridPICEvolveFields () // The particles have now been pushed to their t_{n+1} positions. // Perform charge deposition in component 0 of rho_fp at t_{n+1}. - mypc->DepositCharge(rho_fp, 0._rt); + mypc->DepositCharge(m_fields.get_mr_levels("rho_fp", finest_level), 0._rt); // Perform current deposition at t_{n+1/2}. mypc->DepositCurrent(current_fp, dt[0], -0.5_rt * dt[0]); // Deposit cold-relativistic fluid charge and current if (do_fluid_species) { int const lev = 0; - myfl->DepositCharge(lev, *rho_fp[lev]); + myfl->DepositCharge(lev, *m_fields.get("rho_fp", lev)); myfl->DepositCurrent(lev, *current_fp[lev][0], *current_fp[lev][1], *current_fp[lev][2]); } @@ -94,7 +94,7 @@ void WarpX::HybridPICEvolveFields () for (int sub_step = 0; sub_step < sub_steps; sub_step++) { m_hybrid_pic_model->BfieldEvolveRK( - Bfield_fp, Efield_fp, current_fp_temp, rho_fp_temp, + Bfield_fp, Efield_fp, current_fp_temp, amrex::GetVecOfPtrs(rho_fp_temp), m_edge_lengths, 0.5_rt/sub_steps*dt[0], DtType::FirstHalf, guard_cells.ng_FieldSolver, WarpX::sync_nodal_points @@ -109,7 +109,7 @@ void WarpX::HybridPICEvolveFields () // the result into the 0'th index of `rho_fp_temp[lev]` MultiFab::LinComb( *rho_fp_temp[lev], 0.5_rt, *rho_fp_temp[lev], 0, - 0.5_rt, *rho_fp[lev], 0, 0, 1, rho_fp_temp[lev]->nGrowVect() + 0.5_rt, *m_fields.get("rho_fp", lev), 0, 0, 1, rho_fp_temp[lev]->nGrowVect() ); } @@ -117,7 +117,7 @@ void WarpX::HybridPICEvolveFields () for (int sub_step = 0; sub_step < sub_steps; sub_step++) { m_hybrid_pic_model->BfieldEvolveRK( - Bfield_fp, Efield_fp, current_fp, rho_fp_temp, + Bfield_fp, Efield_fp, current_fp, amrex::GetVecOfPtrs(rho_fp_temp), m_edge_lengths, 0.5_rt/sub_steps*dt[0], DtType::SecondHalf, guard_cells.ng_FieldSolver, WarpX::sync_nodal_points @@ -148,7 +148,7 @@ void WarpX::HybridPICEvolveFields () // Update the E field to t=n+1 using the extrapolated J_i^n+1 value m_hybrid_pic_model->CalculateCurrentAmpere(Bfield_fp, m_edge_lengths); m_hybrid_pic_model->HybridPICSolveE( - Efield_fp, current_fp_temp, Bfield_fp, rho_fp, m_edge_lengths, false + Efield_fp, current_fp_temp, Bfield_fp, m_fields.get_mr_levels("rho_fp", finest_level), m_edge_lengths, false ); FillBoundaryE(guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); @@ -158,7 +158,7 @@ void WarpX::HybridPICEvolveFields () for (int lev = 0; lev <= finest_level; ++lev) { // copy 1 component value starting at index 0 to index 0 - MultiFab::Copy(*rho_fp_temp[lev], *rho_fp[lev], + MultiFab::Copy(*rho_fp_temp[lev], *m_fields.get("rho_fp", lev), 0, 0, 1, rho_fp_temp[lev]->nGrowVect()); for (int idim = 0; idim < 3; ++idim) { MultiFab::Copy(*current_fp_temp[lev][idim], *current_fp[lev][idim], @@ -171,9 +171,9 @@ void WarpX::HybridPICDepositInitialRhoAndJ () { auto& rho_fp_temp = m_hybrid_pic_model->rho_fp_temp; auto& current_fp_temp = m_hybrid_pic_model->current_fp_temp; - mypc->DepositCharge(rho_fp_temp, 0._rt); + mypc->DepositCharge(amrex::GetVecOfPtrs(rho_fp_temp), 0._rt); mypc->DepositCurrent(current_fp_temp, dt[0], 0._rt); - SyncRho(rho_fp_temp, rho_cp, charge_buf); + SyncRho(amrex::GetVecOfPtrs(rho_fp_temp), m_fields.get_mr_levels("rho_cp", finest_level), charge_buf); SyncCurrent(current_fp_temp, current_cp, current_buf); for (int lev=0; lev <= finest_level; ++lev) { // SyncCurrent does not include a call to FillBoundary, but it is needed diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 68a0bbb4c67..813b2370eb5 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -1151,13 +1151,16 @@ WarpX::SyncCurrent ( void WarpX::SyncRho () { - SyncRho(rho_fp, rho_cp, charge_buf); + SyncRho( + m_fields.get_mr_levels("rho_fp", finest_level), + m_fields.get_mr_levels("rho_cp", finest_level), + charge_buf); } void WarpX::SyncRho ( - const amrex::Vector>& charge_fp, - const amrex::Vector>& charge_cp, + const std::vector& charge_fp, + const std::vector& charge_cp, const amrex::Vector>& charge_buffer) { WARPX_PROFILE("WarpX::SyncRho()"); @@ -1431,15 +1434,15 @@ void WarpX::RestrictRhoFromFineToCoarsePatch ( const int lev ) } void WarpX::ApplyFilterandSumBoundaryRho ( - const amrex::Vector>& charge_fp, - const amrex::Vector>& charge_cp, + const std::vector& charge_fp, + const std::vector& charge_cp, const int lev, PatchType patch_type, const int icomp, const int ncomp) { const int glev = (patch_type == PatchType::fine) ? lev : lev-1; - const std::unique_ptr& rho = (patch_type == PatchType::fine) ? + amrex::MultiFab* rho = (patch_type == PatchType::fine) ? charge_fp[lev] : charge_cp[lev]; if (rho == nullptr) { return; } ApplyFilterandSumBoundaryRho(lev, glev, *rho, icomp, ncomp); @@ -1477,8 +1480,8 @@ void WarpX::ApplyFilterandSumBoundaryRho (int /*lev*/, int glev, amrex::MultiFab * patch (and buffer region) of `lev+1` */ void WarpX::AddRhoFromFineLevelandSumBoundary ( - const amrex::Vector>& charge_fp, - const amrex::Vector>& charge_cp, + const std::vector& charge_fp, + const std::vector& charge_cp, const amrex::Vector>& charge_buffer, const int lev, const int icomp, diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index 97e4e1bc4da..4a34f2ea04f 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -147,7 +147,7 @@ public: * the time of the deposition. */ void - DepositCharge (amrex::Vector >& rho, + DepositCharge (const std::vector& rho, amrex::Real relative_time); /** diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 23af4177228..8fd3211f6e6 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -557,7 +557,7 @@ MultiParticleContainer::DepositCurrent ( void MultiParticleContainer::DepositCharge ( - amrex::Vector >& rho, + std::vector& rho, const amrex::Real relative_time) { // Reset the rho array diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 7e882c151e8..f396d01e717 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -212,12 +212,12 @@ public: * \param[in] interpolate_across_levels whether to average down from the fine patch to the coarse patch * \param[in] icomp component of the MultiFab where rho is deposited (old, new) */ - void DepositCharge (amrex::Vector >& rho, + void DepositCharge (const std::vector& rho, bool local = false, bool reset = false, bool apply_boundary_and_scale_volume = false, bool interpolate_across_levels = true, int icomp = 0); - void DepositCharge (std::unique_ptr& rho, int lev, + void DepositCharge (const std::vector& rho, int lev, bool local = false, bool reset = false, bool apply_boundary_and_scale_volume = false, int icomp = 0); diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 591190a7a19..8709ff3fe80 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -1170,7 +1170,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, } void -WarpXParticleContainer::DepositCharge (amrex::Vector >& rho, +WarpXParticleContainer::DepositCharge (std::vector& rho, const bool local, const bool reset, const bool apply_boundary_and_scale_volume, const bool interpolate_across_levels, diff --git a/Source/WarpX.H b/Source/WarpX.H index 4f7a7e86794..c5a9342797a 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -882,8 +882,8 @@ public: void SyncRho (); void SyncRho ( - const amrex::Vector>& charge_fp, - const amrex::Vector>& charge_cp, + const std::vector& charge_fp, + const std::vector& charge_cp, const amrex::Vector>& charge_buffer); [[nodiscard]] amrex::Vector getnsubsteps () const {return nsubsteps;} @@ -1005,7 +1005,7 @@ public: void AddBoundaryField (); void AddSpaceChargeField (WarpXParticleContainer& pc); void AddSpaceChargeFieldLabFrame (); - void computePhi (const amrex::Vector >& rho, + void computePhi (const std::vector& rho, std::vector& phi, std::array beta = {{0,0,0}}, amrex::Real required_precision=amrex::Real(1.e-11), @@ -1367,15 +1367,15 @@ private: void RestrictRhoFromFineToCoarsePatch (int lev ); void ApplyFilterandSumBoundaryRho ( - const amrex::Vector>& charge_fp, - const amrex::Vector>& charge_cp, + const std::vector& charge_fp, + const std::vector& charge_cp, int lev, PatchType patch_type, int icomp, int ncomp); void AddRhoFromFineLevelandSumBoundary ( - const amrex::Vector>& charge_fp, - const amrex::Vector>& charge_cp, + const std::vector& charge_fp, + const std::vector& charge_cp, const amrex::Vector>& charge_buffer, int lev, int icomp, From f6264fe8e1c5b2ee6fe6b76b0fd38ac95d371e90 Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Tue, 10 Sep 2024 23:33:45 +0200 Subject: [PATCH 040/314] fix merge conflict --- Source/FieldSolver/ElectrostaticSolver.cpp | 20 ++++---------------- Source/WarpX.cpp | 8 ++++---- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index 0dd12e0c271..b2ea3700c90 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -114,13 +114,8 @@ WarpX::AddBoundaryField () // Allocate fields for charge and potential const int num_levels = max_level + 1; -<<<<<<< HEAD - Vector > rho(num_levels); - std::vector phi(num_levels); -======= - Vector> rho(num_levels); - Vector > phi(num_levels); ->>>>>>> f719adaa9 (continue implementing new strategy to get rho) + amrex::Vector> rho(num_levels); + amrex::Vector> phi(num_levels); // Use number of guard cells used for local deposition of rho const amrex::IntVect ng = guard_cells.ng_depos_rho; for (int lev = 0; lev <= max_level; lev++) { @@ -128,8 +123,7 @@ WarpX::AddBoundaryField () nba.surroundingNodes(); rho[lev] = std::make_unique(nba, DistributionMap(lev), 1, ng); rho[lev]->setVal(0.); - phi[lev] = m_fields.alloc_init( "phi_temp", lev, nba, DistributionMap(lev), 1, - IntVect::TheUnitVector()); + phi[lev] = std::make_unique(nba, DistributionMap(lev), 1, 1); phi[lev]->setVal(0.); } @@ -140,19 +134,13 @@ WarpX::AddBoundaryField () const std::array beta = {0._rt}; // Compute the potential phi, by solving the Poisson equation - computePhi( amrex::GetVecOfPtrs(rho), phi, beta, self_fields_required_precision, + computePhi( amrex::GetVecOfPtrs(rho), amrex::GetVecOfPtrs(phi), beta, self_fields_required_precision, self_fields_absolute_tolerance, self_fields_max_iters, self_fields_verbosity ); // Compute the corresponding electric and magnetic field, from the potential phi. computeE( Efield_fp, phi, beta ); computeB( Bfield_fp, phi, beta ); - - // de-allocate temporary - for (int lev = 0; lev <= max_level; lev++) { - m_fields.erase("phi_temp",lev); - } - } void diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 6fc67fb1123..c3f1e0067b2 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2480,8 +2480,8 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (rho_ncomps > 0) { m_fields.alloc_init( - "rho_fp", amrex::convert(ba, rho_nodal_flag), dm, - rho_ncomps, ngRho, lev, 0.0_rt); + "rho_fp", lev, amrex::convert(ba, rho_nodal_flag), dm, + rho_ncomps, ngRho, 0.0_rt); } if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrame || @@ -2702,8 +2702,8 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (rho_ncomps > 0) { m_fields.alloc_init( - "rho_cp", amrex::convert(cba, rho_nodal_flag), dm, - rho_ncomps, ngRho, lev, 0.0_rt); + "rho_cp", lev, amrex::convert(cba, rho_nodal_flag), dm, + rho_ncomps, ngRho, 0.0_rt); } if (do_dive_cleaning) From 3fc848ec88f610a6fdb063e19e7eb71dcf7b2a5f Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Tue, 10 Sep 2024 23:05:46 +0200 Subject: [PATCH 041/314] fix merge conflict --- Source/FieldSolver/ElectrostaticSolver.cpp | 2 +- Source/FieldSolver/WarpXPushFieldsEM.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index b2ea3700c90..286688a109a 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -128,7 +128,7 @@ WarpX::AddBoundaryField () } // Set the boundary potentials appropriately - setPhiBC(phi); + setPhiBC(amrex::GetVecOfPtrs(phi)); // beta is zero for boundaries const std::array beta = {0._rt}; diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 76ef48e9bda..0b096418063 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -964,7 +964,7 @@ WarpX::EvolveF (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_typ // Evolve F field in regular cells if (patch_type == PatchType::fine) { - m_fdtd_solver_fp[lev]->EvolveF( m_fields.get("F_fp", lev),, Efield_fp[lev], + m_fdtd_solver_fp[lev]->EvolveF( m_fields.get("F_fp", lev), Efield_fp[lev], m_fields.get("rho_fp",lev), rhocomp, a_dt ); } else { m_fdtd_solver_cp[lev]->EvolveF( m_fields.get("F_cp", lev), Efield_cp[lev], From d53450494ffbee7884c471537082da7ef6279e56 Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Tue, 10 Sep 2024 23:30:04 +0200 Subject: [PATCH 042/314] continue implementing new strategy to manage rho field --- Source/Evolve/WarpXEvolve.cpp | 2 +- Source/FieldSolver/ElectrostaticSolver.cpp | 25 ++++++++++++------- .../FiniteDifferenceSolver/EvolveF.cpp | 2 +- .../FiniteDifferenceSolver.H | 2 +- Source/Particles/MultiParticleContainer.cpp | 2 +- Source/Particles/WarpXParticleContainer.cpp | 4 +-- Source/WarpX.H | 2 +- 7 files changed, 23 insertions(+), 16 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 90f2e0a5da0..54df99f32c6 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -1052,7 +1052,7 @@ WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, ApplyInverseVolumeScalingToCurrentDensity(current_buf[lev][0].get(), current_buf[lev][1].get(), current_buf[lev][2].get(), lev-1); } if (m_fields.has("rho_fp", lev)) { - ApplyInverseVolumeScalingToChargeDensity(m_fields.get("rho_fp", lev).get(), lev); + ApplyInverseVolumeScalingToChargeDensity(m_fields.get("rho_fp", lev), lev); if (charge_buf[lev]) { ApplyInverseVolumeScalingToChargeDensity(charge_buf[lev].get(), lev-1); } diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index 286688a109a..935884f3956 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -114,8 +114,8 @@ WarpX::AddBoundaryField () // Allocate fields for charge and potential const int num_levels = max_level + 1; - amrex::Vector> rho(num_levels); - amrex::Vector> phi(num_levels); + Vector > rho(num_levels); + std::vector phi(num_levels); // Use number of guard cells used for local deposition of rho const amrex::IntVect ng = guard_cells.ng_depos_rho; for (int lev = 0; lev <= max_level; lev++) { @@ -123,24 +123,31 @@ WarpX::AddBoundaryField () nba.surroundingNodes(); rho[lev] = std::make_unique(nba, DistributionMap(lev), 1, ng); rho[lev]->setVal(0.); - phi[lev] = std::make_unique(nba, DistributionMap(lev), 1, 1); + phi[lev] = m_fields.alloc_init( "phi_temp", lev, nba, DistributionMap(lev), 1, + IntVect::TheUnitVector()); phi[lev]->setVal(0.); } // Set the boundary potentials appropriately - setPhiBC(amrex::GetVecOfPtrs(phi)); + setPhiBC(phi); // beta is zero for boundaries const std::array beta = {0._rt}; // Compute the potential phi, by solving the Poisson equation - computePhi( amrex::GetVecOfPtrs(rho), amrex::GetVecOfPtrs(phi), beta, self_fields_required_precision, + computePhi( amrex::GetVecOfPtrs(rho), phi, beta, self_fields_required_precision, self_fields_absolute_tolerance, self_fields_max_iters, self_fields_verbosity ); // Compute the corresponding electric and magnetic field, from the potential phi. computeE( Efield_fp, phi, beta ); computeB( Bfield_fp, phi, beta ); + + // de-allocate temporary + for (int lev = 0; lev <= max_level; lev++) { + m_fields.erase("phi_temp",lev); + } + } void @@ -285,7 +292,7 @@ WarpX::AddSpaceChargeFieldLabFrame () #if defined(WARPX_DIM_1D_Z) // Use the tridiag solver with 1D - computePhiTriDiagonal(rho_fp, phi_fp); + computePhiTriDiagonal(m_fields.get_mr_levels("rho_fp", finest_level), phi_fp); #else // Use the AMREX MLMG or the FFT (IGF) solver otherwise computePhi(m_fields.get_mr_levels("rho_fp", finest_level), phi_fp, beta, self_fields_required_precision, @@ -335,8 +342,8 @@ WarpX::computePhi (const std::vector& rho, amrex::Vector sorted_rho; std::vector sorted_phi; for (int lev = 0; lev <= finest_level; ++lev) { - sorted_rho.emplace_back(rho[lev].get()); - sorted_phi.emplace_back(amrex::GetVecOfPtrs(phi)[lev]); + sorted_rho.emplace_back(rho[lev]); + sorted_phi.emplace_back(phi[lev]); } std::optional post_phi_calculation; @@ -855,7 +862,7 @@ WarpX::computeB (amrex::Vector, 3> > \param[out] phi The potential to be computed by this function */ void -WarpX::computePhiTriDiagonal (const amrex::Vector >& rho, +WarpX::computePhiTriDiagonal (const std::vector& rho, std::vector& phi) const { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp index 381b014b342..71676aecf29 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp @@ -137,7 +137,7 @@ template void FiniteDifferenceSolver::EvolveFCylindrical ( amrex::MultiFab* Ffield, std::array< std::unique_ptr, 3 > const& Efield, - std::unique_ptr const& rhofield, + amrex::MultiFab* const rhofield, int const rhocomp, amrex::Real const dt ) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 9573b1a2d05..e12d134a7cf 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -225,7 +225,7 @@ class FiniteDifferenceSolver void EvolveFCylindrical ( amrex::MultiFab* Ffield, std::array< std::unique_ptr, 3 > const& Efield, - std::unique_ptr const& rhofield, + amrex::MultiFab* const rhofield, int rhocomp, amrex::Real dt ); diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 8fd3211f6e6..c092e271702 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -557,7 +557,7 @@ MultiParticleContainer::DepositCurrent ( void MultiParticleContainer::DepositCharge ( - std::vector& rho, + const std::vector& rho, const amrex::Real relative_time) { // Reset the rho array diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 8709ff3fe80..7777c53dd00 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -1211,7 +1211,7 @@ WarpXParticleContainer::DepositCharge (std::vector& rho, } void -WarpXParticleContainer::DepositCharge (std::unique_ptr& rho, +WarpXParticleContainer::DepositCharge (amrex::MultiFab* rho, const int lev, const bool local, const bool reset, const bool apply_boundary_and_scale_volume, const int icomp) @@ -1245,7 +1245,7 @@ WarpXParticleContainer::DepositCharge (std::unique_ptr& rho, ion_lev = pti.GetiAttribs(particle_icomps["ionizationLevel"]).dataPtr(); } - DepositCharge(pti, wp, ion_lev, rho.get(), icomp, 0, np, thread_num, lev, lev); + DepositCharge(pti, wp, ion_lev, rho, icomp, 0, np, thread_num, lev, lev); } #ifdef AMREX_USE_OMP } diff --git a/Source/WarpX.H b/Source/WarpX.H index c5a9342797a..e59b08928d2 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1021,7 +1021,7 @@ public: void computeB (amrex::Vector, 3> >& B, const std::vector& phi, std::array beta = {{0,0,0}} ) const; - void computePhiTriDiagonal (const amrex::Vector >& rho, + void computePhiTriDiagonal (const std::vector& rho, std::vector& phi) const; // Magnetostatic Solver Interface From a523fca19103657e95bdbadbbd62feef1e631714 Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Wed, 11 Sep 2024 00:07:28 +0200 Subject: [PATCH 043/314] finish adding rho to MultiFabRegister --- Source/Particles/MultiParticleContainer.cpp | 2 +- Source/Particles/WarpXParticleContainer.H | 2 +- Source/Particles/WarpXParticleContainer.cpp | 8 ++++---- Source/Utils/WarpXMovingWindow.cpp | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index c092e271702..cb323fa7933 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -587,7 +587,7 @@ MultiParticleContainer::DepositCharge ( #ifdef WARPX_DIM_RZ for (int lev = 0; lev < rho.size(); ++lev) { - WarpX::GetInstance().ApplyInverseVolumeScalingToChargeDensity(rho[lev].get(), lev); + WarpX::GetInstance().ApplyInverseVolumeScalingToChargeDensity(rho[lev], lev); } #endif } diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index f396d01e717..58c01b2c9ff 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -217,7 +217,7 @@ public: bool apply_boundary_and_scale_volume = false, bool interpolate_across_levels = true, int icomp = 0); - void DepositCharge (const std::vector& rho, int lev, + void DepositCharge (amrex::MultiFab* rho, int lev, bool local = false, bool reset = false, bool apply_boundary_and_scale_volume = false, int icomp = 0); diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 7777c53dd00..057e32f2cd3 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -1170,7 +1170,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, } void -WarpXParticleContainer::DepositCharge (std::vector& rho, +WarpXParticleContainer::DepositCharge (const std::vector& rho, const bool local, const bool reset, const bool apply_boundary_and_scale_volume, const bool interpolate_across_levels, @@ -1254,7 +1254,7 @@ WarpXParticleContainer::DepositCharge (amrex::MultiFab* rho, #ifdef WARPX_DIM_RZ if (apply_boundary_and_scale_volume) { - WarpX::GetInstance().ApplyInverseVolumeScalingToChargeDensity(rho.get(), lev); + WarpX::GetInstance().ApplyInverseVolumeScalingToChargeDensity(rho, lev); } #endif @@ -1273,7 +1273,7 @@ WarpXParticleContainer::DepositCharge (amrex::MultiFab* rho, if (apply_boundary_and_scale_volume) { // Reflect density over PEC boundaries, if needed. - WarpX::GetInstance().ApplyRhofieldBoundary(lev, rho.get(), PatchType::fine); + WarpX::GetInstance().ApplyRhofieldBoundary(lev, rho, PatchType::fine); } #endif } @@ -1300,7 +1300,7 @@ WarpXParticleContainer::GetChargeDensity (int lev, bool local) const int ng_rho = warpx.get_ng_depos_rho().max(); auto rho = std::make_unique(nba, dm, WarpX::ncomps,ng_rho); - DepositCharge(rho, lev, local, true, true, 0); + DepositCharge(rho.get(), lev, local, true, true, 0); return rho; } diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 524ecc24763..5298405ccaf 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -357,10 +357,10 @@ WarpX::MoveWindow (const int step, bool move_j) if (move_j) { if (m_fields.has("rho_fp", lev)) { // Fine grid - shiftMF(*rho_fp[lev], geom[lev], num_shift, dir, lev, do_update_cost); + shiftMF(*m_fields.get("rho_fp",lev), geom[lev], num_shift, dir, lev, do_update_cost); if (lev > 0){ // Coarse grid - shiftMF(*rho_cp[lev], geom[lev-1], num_shift_crse, dir, lev, do_update_cost); + shiftMF(*m_fields.get("rho_cp",lev), geom[lev-1], num_shift_crse, dir, lev, do_update_cost); } } } From ea8952e02ccab7f7ffb19937bb27014976ab707e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 10 Sep 2024 22:10:07 +0000 Subject: [PATCH 044/314] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- Source/FieldSolver/ElectrostaticSolver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index 935884f3956..10fbe7dd1f1 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -142,7 +142,7 @@ WarpX::AddBoundaryField () // Compute the corresponding electric and magnetic field, from the potential phi. computeE( Efield_fp, phi, beta ); computeB( Bfield_fp, phi, beta ); - + // de-allocate temporary for (int lev = 0; lev <= max_level; lev++) { m_fields.erase("phi_temp",lev); From 36ebbed07fddc8c72fb0c6984e2a87937ed69636 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 10 Sep 2024 15:25:47 -0700 Subject: [PATCH 045/314] Fix PSATD Compile --- Source/Evolve/WarpXEvolve.cpp | 12 +++++++++--- Source/FieldSolver/WarpXPushFieldsEM.cpp | 6 ++++-- Source/WarpX.H | 4 ++-- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 54df99f32c6..cca7bc9ce77 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -639,8 +639,11 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // 3) Deposit rho (in rho_new, since it will be moved during the loop) // (after checking that pointer to rho_fp on MR level 0 is not null) - if (rho_fp[0] && rho_in_time == RhoInTime::Linear) + if (m_fields.has("rho_fp", 0) && rho_in_time == RhoInTime::Linear) { + ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); + ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_fp", finest_level); + // Deposit rho at relative time -dt // (dt[0] denotes the time step on mesh refinement level 0) mypc->DepositCharge(rho_fp, -dt[0]); @@ -701,15 +704,18 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Deposit new rho // (after checking that pointer to rho_fp on MR level 0 is not null) - if (rho_fp[0]) + if (m_fields.has("rho_fp", 0)) { + ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); + ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_cp", finest_level); + // Move rho from new to old if rho is linear in time if (rho_in_time == RhoInTime::Linear) { PSATDMoveRhoNewToRhoOld(); } // Deposit rho at relative time t_deposit_charge mypc->DepositCharge(rho_fp, t_deposit_charge); // Filter, exchange boundary, and interpolate across levels - SyncRho(; + SyncRho(); // Forward FFT of rho const int rho_idx = (rho_in_time == RhoInTime::Linear) ? rho_new : rho_mid; PSATDForwardTransformRho(rho_fp, rho_cp, 0, rho_idx); diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 0b096418063..03dddf10979 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -377,8 +377,8 @@ void WarpX::PSATDBackwardTransformJ ( } void WarpX::PSATDForwardTransformRho ( - const amrex::Vector>& charge_fp, - const amrex::Vector>& charge_cp, + std::vector const & charge_fp, + std::vector const & charge_cp, const int icomp, const int dcomp, const bool apply_kspace_filter) { if (charge_fp[0] == nullptr) { return; } @@ -667,6 +667,8 @@ WarpX::PushPSATD () const int rho_old = spectral_solver_fp[0]->m_spectral_index.rho_old; const int rho_new = spectral_solver_fp[0]->m_spectral_index.rho_new; + ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); + ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_fp", finest_level); if (fft_periodic_single_box) { diff --git a/Source/WarpX.H b/Source/WarpX.H index e59b08928d2..b84627045e3 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1910,8 +1910,8 @@ private: * (only used in RZ geometry to avoid double filtering) */ void PSATDForwardTransformRho ( - const amrex::Vector>& charge_fp, - const amrex::Vector>& charge_cp, + std::vector const & charge_fp, + std::vector const & charge_cp, int icomp, int dcomp, bool apply_kspace_filter=true); /** From a5449a1f27e8fbf287ebd5572443fb55c3c5d661 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 10 Sep 2024 15:30:15 -0700 Subject: [PATCH 046/314] Add TODOs on `std::map` --- Source/ablastr/fields/MultiFabRegister.H | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 8654470a151..da7b6581690 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -38,7 +38,11 @@ namespace ablastr::fields } }; - /** A vector field of three MultiFab */ + /** A vector field of three MultiFab + * + * TODO: std::map has two issues: operator[] has no const overload and access with missing keys inserts new + * values. We want to use another type. + */ using VectorField = std::map; /** A multi-level scalar field @@ -50,6 +54,8 @@ namespace ablastr::fields /** A multi-level vector field * * TODO: use amrex::Vector again for bound checks in debug mode + * TODO: std::map has two issues: operator[] has no const overload and access with missing keys inserts new + * values. We want to use another type. */ using MultiLevelVectorField = std::vector>; From ec3d00cd6a9c742d7c38f8eb07468cf2b6f5cde1 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 10 Sep 2024 15:50:33 -0700 Subject: [PATCH 047/314] Introduce `ablastr::utils::ConstMap` Key-checked `operator[]` with `const` support. --- Source/ablastr/fields/MultiFabRegister.H | 11 ++--- Source/ablastr/fields/MultiFabRegister.cpp | 17 ++------ Source/ablastr/utils/ConstMap.H | 48 ++++++++++++++++++++++ 3 files changed, 55 insertions(+), 21 deletions(-) create mode 100644 Source/ablastr/utils/ConstMap.H diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index da7b6581690..f618e75e643 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -8,6 +8,8 @@ #ifndef ABLASTR_FIELDS_MF_REGISTER_H #define ABLASTR_FIELDS_MF_REGISTER_H +#include "ablastr/utils/ConstMap.H" + #include #include #include @@ -39,11 +41,8 @@ namespace ablastr::fields }; /** A vector field of three MultiFab - * - * TODO: std::map has two issues: operator[] has no const overload and access with missing keys inserts new - * values. We want to use another type. */ - using VectorField = std::map; + using VectorField = ablastr::utils::ConstMap; /** A multi-level scalar field * @@ -54,10 +53,8 @@ namespace ablastr::fields /** A multi-level vector field * * TODO: use amrex::Vector again for bound checks in debug mode - * TODO: std::map has two issues: operator[] has no const overload and access with missing keys inserts new - * values. We want to use another type. */ - using MultiLevelVectorField = std::vector>; + using MultiLevelVectorField = std::vector>; /** title * diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 4cf9ca139cb..e67cebb8df3 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -222,10 +222,7 @@ namespace ablastr::fields for (int lvl = 0; lvl <= finest_level; lvl++) { // insert a new level - field_on_level.push_back(std::map< - Direction, - amrex::MultiFab* - >{}); + field_on_level.push_back(VectorField{}); // insert components for (Direction dir : all_dirs) @@ -311,12 +308,7 @@ namespace ablastr::fields { int const finest_level = old_vector_on_levels.size() - 1u; - std::vector< - std::map< - Direction, - amrex::MultiFab* - > - > field_on_level; + MultiLevelVectorField field_on_level; field_on_level.reserve(finest_level+1); std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; @@ -324,10 +316,7 @@ namespace ablastr::fields for (int lvl = 0; lvl <= finest_level; lvl++) { // insert a new level - field_on_level.push_back(std::map< - Direction, - amrex::MultiFab* - >{}); + field_on_level.push_back(VectorField{}); // insert components for (auto dir : {0, 1, 2}) diff --git a/Source/ablastr/utils/ConstMap.H b/Source/ablastr/utils/ConstMap.H new file mode 100644 index 00000000000..e11f806b011 --- /dev/null +++ b/Source/ablastr/utils/ConstMap.H @@ -0,0 +1,48 @@ +/* Copyright 2024 The ABLASTR Community + * + * This file is part of ABLASTR. + * + * License: BSD-3-Clause-LBNL + * Authors: Axel Huebl + */ + +#ifndef ABLASTR_CONST_MAP_H_ +#define ABLASTR_CONST_MAP_H_ + +#include "TextMsg.H" + +#include +#include + + +namespace ablastr::utils +{ + /** This is a std::map where operator[] throws if the key is missing + */ + template< typename Key, typename T > + struct ConstMap : public std::map< Key, T > + { + T& operator[]( const Key& key ) + { + return this->at(key); + } + + T& operator[]( Key&& key ) + { + return this->at(key); + } + + T& operator[]( const Key& key ) const + { + return this->at(key); + } + + T& operator[]( Key&& key ) const + { + return this->at(key); + } + }; + +} // ablastr::utils + +#endif // ABLASTR_CONST_MAP_H_ \ No newline at end of file From 8b3a5bd5e3300793ca131efc12f570c871a56945 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Tue, 10 Sep 2024 16:13:34 -0700 Subject: [PATCH 048/314] Move vector potential to MultiFab map --- .../MagnetostaticSolver.cpp | 40 +++++++++++-------- Source/WarpX.H | 5 +-- Source/WarpX.cpp | 13 ++---- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp index 305ccf02eb3..dd17463b1cf 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp @@ -19,6 +19,7 @@ #include "Utils/WarpXProfilerWrapper.H" #include "Parallelization/WarpXComm_K.H" +#include #include #include #include @@ -56,6 +57,8 @@ using namespace amrex; +//using ablastr::fields::Direction; // FIXME + void WarpX::ComputeMagnetostaticField() { @@ -110,7 +113,7 @@ WarpX::AddMagnetostaticFieldLabFrame() SyncCurrent(current_fp, current_cp, current_buf); // Apply filter, perform MPI exchange, interpolate across levels // set the boundary and current density potentials - setVectorPotentialBC(vector_potential_fp_nodal); + setVectorPotentialBC(m_fields.get_mr_levels_alldirs("vector_potential_fp_nodal", finest_level)); // Compute the vector potential A, by solving the Poisson equation WARPX_ALWAYS_ASSERT_WITH_MESSAGE( !IsPythonCallbackInstalled("poissonsolver"), @@ -118,9 +121,10 @@ WarpX::AddMagnetostaticFieldLabFrame() const amrex::Real magnetostatic_absolute_tolerance = self_fields_absolute_tolerance*PhysConst::c; - computeVectorPotential( current_fp, vector_potential_fp_nodal, self_fields_required_precision, - magnetostatic_absolute_tolerance, self_fields_max_iters, - self_fields_verbosity ); + computeVectorPotential( + current_fp, m_fields.get_mr_levels_alldirs("vector_potential_fp_nodal", finest_level), + self_fields_required_precision, magnetostatic_absolute_tolerance, self_fields_max_iters, + self_fields_verbosity); } /* Compute the vector potential `A` by solving the Poisson equation with `J` as @@ -141,12 +145,14 @@ WarpX::AddMagnetostaticFieldLabFrame() */ void WarpX::computeVectorPotential (const amrex::Vector,3> >& curr, - amrex::Vector,3> >& A, - Real const required_precision, - Real absolute_tolerance, - int const max_iters, - int const verbosity) const + std::vector> A, + Real const required_precision, + Real absolute_tolerance, + int const max_iters, + int const verbosity) const { + using ablastr::fields::Direction; + // create a vector to our fields, sorted by level amrex::Vector> sorted_curr; amrex::Vector> sorted_A; @@ -154,9 +160,9 @@ WarpX::computeVectorPotential (const amrex::Vector ({curr[lev][0].get(), curr[lev][1].get(), curr[lev][2].get()})); - sorted_A.emplace_back(amrex::Array ({A[lev][0].get(), - A[lev][1].get(), - A[lev][2].get()})); + sorted_A.emplace_back(amrex::Array ({A[lev][Direction{0}], + A[lev][Direction{1}], + A[lev][Direction{2}]})); } #if defined(AMREX_USE_EB) @@ -205,8 +211,10 @@ WarpX::computeVectorPotential (const amrex::Vector,3>>& A ) const +WarpX::setVectorPotentialBC (std::vector> A) const { + using ablastr::fields::Direction; + // check if any dimension has non-periodic boundary conditions if (!m_vector_poisson_boundary_handler.has_non_periodic) { return; } @@ -221,11 +229,11 @@ WarpX::setVectorPotentialBC ( amrex::Vectorarray(mfi); + auto A_arr = A[lev][Direction{adim}]->array(mfi); // Extract tileboxes for which to loop - const Box& tb = mfi.tilebox( A[lev][adim]->ixType().toIntVect()); + const Box& tb = mfi.tilebox( A[lev][Direction{adim}]->ixType().toIntVect()); // loop over dimensions for (int idim=0; idim, 3> >& curr, - amrex::Vector, 3> >& A, + std::vector> A, amrex::Real required_precision=amrex::Real(1.e-11), amrex::Real absolute_tolerance=amrex::Real(0.0), int max_iters=200, int verbosity=2) const; - void setVectorPotentialBC (amrex::Vector, 3> >& A) const; + void setVectorPotentialBC (std::vector> A) const; /** * \brief @@ -1533,7 +1533,6 @@ private: // Memory buffers for computing magnetostatic fields // Vector Potential A and previous step. Time buffer needed for computing dA/dt to first order - amrex::Vector, 3 > > vector_potential_fp_nodal; amrex::Vector, 3 > > vector_potential_grad_buf_e_stag; amrex::Vector, 3 > > vector_potential_grad_buf_b_stag; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index c3f1e0067b2..c24e603d01b 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -331,7 +331,6 @@ WarpX::WarpX () // Only allocate vector potential arrays when using the Magnetostatic Solver if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) { - vector_potential_fp_nodal.resize(nlevs_max); vector_potential_grad_buf_e_stag.resize(nlevs_max); vector_potential_grad_buf_b_stag.resize(nlevs_max); } @@ -2329,12 +2328,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) { - AllocInitMultiFab(vector_potential_fp_nodal[lev][0], amrex::convert(ba, rho_nodal_flag), - dm, ncomps, ngRho, lev, "vector_potential_fp_nodal[x]", 0.0_rt); - AllocInitMultiFab(vector_potential_fp_nodal[lev][1], amrex::convert(ba, rho_nodal_flag), - dm, ncomps, ngRho, lev, "vector_potential_fp_nodal[y]", 0.0_rt); - AllocInitMultiFab(vector_potential_fp_nodal[lev][2], amrex::convert(ba, rho_nodal_flag), - dm, ncomps, ngRho, lev, "vector_potential_fp_nodal[z]", 0.0_rt); + m_fields.alloc_init( "vector_potential_fp_nodal", Direction{0}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngRho, 0.0_rt); + m_fields.alloc_init( "vector_potential_fp_nodal", Direction{1}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngRho, 0.0_rt); + m_fields.alloc_init( "vector_potential_fp_nodal", Direction{2}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngRho, 0.0_rt); AllocInitMultiFab(vector_potential_grad_buf_e_stag[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "vector_potential_grad_buf_e_stag[x]", 0.0_rt); @@ -3498,9 +3494,6 @@ WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, cons case FieldType::current_fp_nodal : field_pointer = current_fp_nodal[lev][direction].get(); break; - case FieldType::vector_potential_fp : - field_pointer = vector_potential_fp_nodal[lev][direction].get(); - break; case FieldType::Efield_cp : field_pointer = Efield_cp[lev][direction].get(); break; From d0835c2b8cf8c860fb230a62682b2009be3d8a07 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 10 Sep 2024 16:03:17 -0700 Subject: [PATCH 049/314] `MultiFabRegister::alias_init` --- Source/ablastr/fields/MultiFabRegister.H | 40 ++++++++-- Source/ablastr/fields/MultiFabRegister.cpp | 88 ++++++++++++++++++++-- 2 files changed, 118 insertions(+), 10 deletions(-) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index f618e75e643..c477e9f1a35 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -66,20 +66,34 @@ namespace ablastr::fields { // TODO: also add iMultiFab via std::variant - /** owned (i)MultiFab data */ + /** owned (i)MultiFab */ amrex::MultiFab m_mf; /** Components (base vector directions) of this (i)MultiFab */ - std::optional dir = std::nullopt; + std::optional m_dir = std::nullopt; /** the MR level of this (i)MultiFab */ - int level = 0; + int m_level = 0; /** redistribute */ - bool redistribute = true; + bool m_redistribute = true; /** redistribute on @see amrex::AmrCore::RemakeLevel */ - bool redistribute_on_remake = true; + bool m_redistribute_on_remake = true; + + /** if m_mf is a non-owning alias, this string tracks the name of the owner */ + std::string m_owner; + + /** Is this part of a vector/tensor? */ + bool + is_vector () { return m_dir.has_value(); } + + /** Is this an alias MultiFab? + * + * If yes, that means we do not own the memory. + */ + bool + is_alias () { return !m_owner.empty(); } }; /** This is a register of fields aka amrex::(i)MultiFabs. @@ -139,6 +153,22 @@ namespace ablastr::fields bool redistribute_on_remake = true ); + /** Create an alias + * + * @param new_name new name + * @param alias_name owner to alias to + * @param level ... + * @param initial_value ... + * @return + */ + amrex::MultiFab* + alias_init ( + std::string new_name, + std::string alias_name, + int level, + std::optional initial_value = std::nullopt + ); + /** title * * body body diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index e67cebb8df3..3b610105ec4 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -7,6 +7,8 @@ */ #include "MultiFabRegister.H" +#include "ablastr/utils/TextMsg.H" + #include @@ -35,7 +37,14 @@ namespace ablastr::fields auto [it, success] = m_mf_register.emplace( std::make_pair( name, - MultiFabOwner{{ba, dm, ncomp, ngrow, tag}, std::nullopt, level, redistribute, redistribute_on_remake} + MultiFabOwner{ + {ba, dm, ncomp, ngrow, tag}, + std::nullopt, // scalar: no direction + level, + redistribute, + redistribute_on_remake, + "" // we own the memory + } ) ); if (!success) { @@ -77,7 +86,14 @@ namespace ablastr::fields auto [it, success] = m_mf_register.emplace( std::make_pair( name, - MultiFabOwner{{ba, dm, ncomp, ngrow, tag}, dir, level, redistribute, redistribute_on_remake} + MultiFabOwner{ + {ba, dm, ncomp, ngrow, tag}, + dir, + level, + redistribute, + redistribute_on_remake, + "" // we own the memory + } ) ); if (!success) { @@ -109,16 +125,63 @@ namespace ablastr::fields // TODO: does the other_name already exist? error } + amrex::MultiFab* + MultiFabRegister::alias_init ( + std::string new_name, + std::string alias_name, + int level, + std::optional initial_value + ) + { + new_name = mf_name(new_name, level); + alias_name = mf_name(alias_name, level); + + // Checks + // TODO: does the key already exist? error + + MultiFabOwner & alias = m_mf_register[alias_name]; + amrex::MultiFab & mf_alias = alias.m_mf; + + // allocate + auto [it, success] = m_mf_register.emplace( + std::make_pair( + new_name, + MultiFabOwner{ + {mf_alias, amrex::make_alias, 0, mf_alias.nComp()}, + std::nullopt, // scalar: no direction + level, + alias.m_redistribute, + alias.m_redistribute_on_remake, + alias_name + } + ) + ); + if (!success) { + throw std::runtime_error("MultiFabRegister::alias_init failed for " + new_name); + } + + // a short-hand alias for the code below + amrex::MultiFab & mf = it->second.m_mf; + + // initialize with value + if (initial_value) { + mf.setVal(*initial_value); + } + + return &mf; + } + void MultiFabRegister::remake_level ( int level, amrex::DistributionMapping const & new_dm ) { + // Owning MultiFabs for (auto & element : m_mf_register ) { MultiFabOwner & mf_owner = element.second; - if (mf_owner.level == level) { + if (mf_owner.m_level == level && !mf_owner.is_alias()) { amrex::MultiFab & mf = mf_owner.m_mf; amrex::IntVect const & ng = mf.nGrowVect(); const auto tag = amrex::MFInfo().SetTag(mf.tags()[0]); @@ -126,7 +189,7 @@ namespace ablastr::fields // copy data to new MultiFab: Only done for persistent data like E and B field, not for // temporary things like currents, etc. - if (mf_owner.redistribute_on_remake) { + if (mf_owner.m_redistribute_on_remake) { new_mf.Redistribute(mf, 0, 0, mf.nComp(), ng); } @@ -134,6 +197,21 @@ namespace ablastr::fields mf_owner.m_mf = std::move(new_mf); } } + + // Aliases + for (auto & element : m_mf_register ) + { + MultiFabOwner & mf_owner = element.second; + if (mf_owner.m_level == level && mf_owner.is_alias()) { + amrex::MultiFab & mf = mf_owner.m_mf; + amrex::MultiFab new_mf(mf, amrex::make_alias, 0, mf.nComp()); + + // no copy via Redistribute: the owner was already redistributed + + // replace old MultiFab with new one, deallocate old one + mf_owner.m_mf = std::move(new_mf); + } + } } bool @@ -265,7 +343,7 @@ namespace ablastr::fields // C++20: Replace with std::erase_if for (auto first = m_mf_register.begin(), last = m_mf_register.end(); first != last;) { - if (first->second.level == level) + if (first->second.m_level == level) first = m_mf_register.erase(first); else ++first; From c30b6f9b89519c55be7701f8a884c8ed2569c847 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 10 Sep 2024 23:14:06 +0000 Subject: [PATCH 050/314] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- Source/ablastr/utils/ConstMap.H | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/ablastr/utils/ConstMap.H b/Source/ablastr/utils/ConstMap.H index e11f806b011..58d0519fc6d 100644 --- a/Source/ablastr/utils/ConstMap.H +++ b/Source/ablastr/utils/ConstMap.H @@ -45,4 +45,4 @@ namespace ablastr::utils } // ablastr::utils -#endif // ABLASTR_CONST_MAP_H_ \ No newline at end of file +#endif // ABLASTR_CONST_MAP_H_ From c8bf7e0c6533f8e02581ba76c77d0d9ee2d71945 Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Tue, 10 Sep 2024 13:43:49 -0700 Subject: [PATCH 051/314] initial pass with Efield_fp. Compiles. --- Source/BoundaryConditions/PML.cpp | 15 +++++++++------ Source/WarpX.cpp | 18 +++++++++--------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index ea228792a35..f154e4323e6 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -699,12 +699,15 @@ PML::PML (const int lev, const BoxArray& grid_ba, const int ncompe = (m_dive_cleaning) ? 3 : 2; const int ncompb = (m_divb_cleaning) ? 3 : 2; - const amrex::BoxArray ba_Ex = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Efield_fp, 0,0).ixType().toIntVect()); - const amrex::BoxArray ba_Ey = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Efield_fp, 0,1).ixType().toIntVect()); - const amrex::BoxArray ba_Ez = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Efield_fp, 0,2).ixType().toIntVect()); - WarpX::AllocInitMultiFab(pml_E_fp[0], ba_Ex, dm, ncompe, nge, lev, "pml_E_fp[x]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_E_fp[1], ba_Ey, dm, ncompe, nge, lev, "pml_E_fp[y]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_E_fp[2], ba_Ez, dm, ncompe, nge, lev, "pml_E_fp[z]", 0.0_rt); + auto& warpx = WarpX::GetInstance(); + using ablastr::fields::Direction; + + const amrex::BoxArray ba_Ex = amrex::convert(ba, warpx.m_fields.get("Efield_fp",Direction{0},0)->ixType().toIntVect()); + const amrex::BoxArray ba_Ey = amrex::convert(ba, warpx.m_fields.get("Efield_fp",Direction{1},0)->ixType().toIntVect()); + const amrex::BoxArray ba_Ez = amrex::convert(ba, warpx.m_fields.get("Efield_fp",Direction{2},0)->ixType().toIntVect()); + warpx.m_fields.alloc_init("pml_E_fp", Direction{0}, lev, ba_Ex, dm, ncompe, nge, 0.0_rt); + warpx.m_fields.alloc_init("pml_E_fp", Direction{1}, lev, ba_Ey, dm, ncompe, nge, 0.0_rt); + warpx.m_fields.alloc_init("pml_E_fp", Direction{2}, lev, ba_Ez, dm, ncompe, nge, 0.0_rt); const amrex::BoxArray ba_Bx = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Bfield_fp, 0,0).ixType().toIntVect()); const amrex::BoxArray ba_By = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Bfield_fp, 0,1).ixType().toIntVect()); diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index c3f1e0067b2..928fa0c8d54 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2304,9 +2304,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm AllocInitMultiFab(Bfield_fp[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp[y]", 0.0_rt); AllocInitMultiFab(Bfield_fp[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp[z]", 0.0_rt); - AllocInitMultiFab(Efield_fp[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp[x]", 0.0_rt); - AllocInitMultiFab(Efield_fp[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp[y]", 0.0_rt); - AllocInitMultiFab(Efield_fp[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp[z]", 0.0_rt); + m_fields.alloc_init( "Efield_fp", Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Efield_fp", Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Efield_fp", Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); m_fields.alloc_init( "current_fp", Direction{0}, lev, amrex::convert(ba, jx_nodal_flag), dm, ncomps, ngJ, 0.0_rt); m_fields.alloc_init( "current_fp", Direction{1}, lev, amrex::convert(ba, jy_nodal_flag), dm, ncomps, ngJ, 0.0_rt); @@ -2612,9 +2612,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm AllocInitMultiFab(Efield_aux[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[z]"); } else { // In this case, the aux grid is simply an alias of the fp grid (most common case in WarpX) - AliasInitMultiFab(Efield_aux[lev][0], *Efield_fp[lev][0], 0, ncomps, lev, "Efield_aux[x]", 0.0_rt); - AliasInitMultiFab(Efield_aux[lev][1], *Efield_fp[lev][1], 0, ncomps, lev, "Efield_aux[y]", 0.0_rt); - AliasInitMultiFab(Efield_aux[lev][2], *Efield_fp[lev][2], 0, ncomps, lev, "Efield_aux[z]", 0.0_rt); + AliasInitMultiFab(Efield_aux[lev][0], *m_fields.get("Efield_fp",Direction{0},lev), 0, ncomps, lev, "Efield_aux[x]", 0.0_rt); + AliasInitMultiFab(Efield_aux[lev][1], *m_fields.get("Efield_fp",Direction{1},lev), 0, ncomps, lev, "Efield_aux[y]", 0.0_rt); + AliasInitMultiFab(Efield_aux[lev][2], *m_fields.get("Efield_fp",Direction{2},lev), 0, ncomps, lev, "Efield_aux[z]", 0.0_rt); } } } else { @@ -2648,11 +2648,11 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm } if (m_p_ext_field_params->E_ext_grid_type != ExternalFieldType::default_zero && m_p_ext_field_params->E_ext_grid_type != ExternalFieldType::constant) { // These fields will be added directly to the grid, i.e. to fp, and need to match the index type - AllocInitMultiFab(Efield_fp_external[lev][0], amrex::convert(ba, Efield_fp[lev][0]->ixType()), + AllocInitMultiFab(Efield_fp_external[lev][0], amrex::convert(ba, m_fields.get("Efield_fp",Direction{0},lev)->ixType()), dm, ncomps, ngEB, lev, "Efield_fp_external[x]", 0.0_rt); - AllocInitMultiFab(Efield_fp_external[lev][1], amrex::convert(ba, Efield_fp[lev][1]->ixType()), + AllocInitMultiFab(Efield_fp_external[lev][1], amrex::convert(ba, m_fields.get("Efield_fp",Direction{1},lev)->ixType()), dm, ncomps, ngEB, lev, "Efield_fp_external[y]", 0.0_rt); - AllocInitMultiFab(Efield_fp_external[lev][2], amrex::convert(ba, Efield_fp[lev][2]->ixType()), + AllocInitMultiFab(Efield_fp_external[lev][2], amrex::convert(ba, m_fields.get("Efield_fp",Direction{2},lev)->ixType()), dm, ncomps, ngEB, lev, "Efield_fp_external[z]", 0.0_rt); } if (mypc->m_E_ext_particle_s == "read_from_file") { From 4b9d10beed930c5a9e246f10bfd8de1ffe93fa05 Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Tue, 10 Sep 2024 13:54:13 -0700 Subject: [PATCH 052/314] Efield_fp changes to PML. Compiles. --- Source/BoundaryConditions/PML_RZ.cpp | 10 ++++++---- Source/BoundaryConditions/WarpXEvolvePML.cpp | 7 ++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Source/BoundaryConditions/PML_RZ.cpp b/Source/BoundaryConditions/PML_RZ.cpp index 78f3cf24987..0033359fc43 100644 --- a/Source/BoundaryConditions/PML_RZ.cpp +++ b/Source/BoundaryConditions/PML_RZ.cpp @@ -42,13 +42,15 @@ PML_RZ::PML_RZ (const int lev, const amrex::BoxArray& grid_ba, const amrex::Dist m_do_pml_in_domain(do_pml_in_domain), m_geom(geom) { + auto& warpx = WarpX::GetInstance(); + using ablastr::fields::Direction; - const amrex::MultiFab & Er_fp = WarpX::GetInstance().getField(FieldType::Efield_fp, lev,0); - const amrex::MultiFab & Et_fp = WarpX::GetInstance().getField(FieldType::Efield_fp, lev,1); + const amrex::MultiFab & Er_fp = *warpx.m_fields.get("Efield_fp",Direction{0},lev); + const amrex::MultiFab & Et_fp = *warpx.m_fields.get("Efield_fp",Direction{1},lev); const amrex::BoxArray ba_Er = amrex::convert(grid_ba, Er_fp.ixType().toIntVect()); const amrex::BoxArray ba_Et = amrex::convert(grid_ba, Et_fp.ixType().toIntVect()); - WarpX::AllocInitMultiFab(pml_E_fp[0], ba_Er, grid_dm, Er_fp.nComp(), Er_fp.nGrowVect(), lev, "pml_E_fp[0]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_E_fp[1], ba_Et, grid_dm, Et_fp.nComp(), Et_fp.nGrowVect(), lev, "pml_E_fp[1]", 0.0_rt); + warpx.m_fields.alloc_init("pml_E_fp", Direction{0}, lev, ba_Er, dm, Er_fp.nComp(), Er_fp.nGrowVect(), 0.0_rt); + warpx.m_fields.alloc_init("pml_E_fp", Direction{1}, lev, ba_Et, dm, Et_fp.nComp(), Et_fp.nGrowVect(), 0.0_rt); const amrex::MultiFab & Br_fp = WarpX::GetInstance().getField(FieldType::Bfield_fp, lev,0); const amrex::MultiFab & Bt_fp = WarpX::GetInstance().getField(FieldType::Bfield_fp, lev,1); diff --git a/Source/BoundaryConditions/WarpXEvolvePML.cpp b/Source/BoundaryConditions/WarpXEvolvePML.cpp index 8d481af69d6..5dc8bc2be17 100644 --- a/Source/BoundaryConditions/WarpXEvolvePML.cpp +++ b/Source/BoundaryConditions/WarpXEvolvePML.cpp @@ -63,9 +63,10 @@ WarpX::DampPML (const int lev, PatchType patch_type) WARPX_PROFILE("WarpX::DampPML()"); #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev]) { - pml_rz[lev]->ApplyDamping(Efield_fp[lev][1].get(), Efield_fp[lev][2].get(), - Bfield_fp[lev][1].get(), Bfield_fp[lev][2].get(), - dt[lev]); + pml_rz[lev]->ApplyDamping( m_fields.get("Efield_fp",Direction{1},lev), + m_fields.get("Efield_fp",Direction{2},lev), + Bfield_fp[lev][1].get(), Bfield_fp[lev][2].get(), + dt[lev]); } #endif if (pml[lev]) { From 674117e498d2c6672db8e9dabd1723eefa886f29 Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Tue, 10 Sep 2024 14:21:07 -0700 Subject: [PATCH 053/314] Efield_fp refactored for WarpXFieldBoundaries --- .../WarpXFieldBoundaries.cpp | 17 +++++++++------- .../ApplySilverMuellerBoundary.cpp | 20 ++++++++++--------- .../FiniteDifferenceSolver.H | 3 ++- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index 2b063b99a15..26638778cdf 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -50,12 +50,14 @@ namespace void WarpX::ApplyEfieldBoundary(const int lev, PatchType patch_type) { + using ablastr::fields::Direction; + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { if (patch_type == PatchType::fine) { PEC::ApplyPECtoEfield( - {getFieldPointer(FieldType::Efield_fp, lev, 0), - getFieldPointer(FieldType::Efield_fp, lev, 1), - getFieldPointer(FieldType::Efield_fp, lev, 2)}, + {m_fields.get("Efield_fp",Direction{0},lev), + m_fields.get("Efield_fp",Direction{1},lev), + m_fields.get("Efield_fp",Direction{2},lev)}, field_boundary_lo, field_boundary_hi, get_ng_fieldgather(), Geom(lev), lev, patch_type, ref_ratio); @@ -92,9 +94,9 @@ void WarpX::ApplyEfieldBoundary(const int lev, PatchType patch_type) #ifdef WARPX_DIM_RZ if (patch_type == PatchType::fine) { - ApplyFieldBoundaryOnAxis(getFieldPointer(FieldType::Efield_fp, lev, 0), - getFieldPointer(FieldType::Efield_fp, lev, 1), - getFieldPointer(FieldType::Efield_fp, lev, 2), lev); + ApplyFieldBoundaryOnAxis(m_fields.get("Efield_fp",Direction{0},lev), + m_fields.get("Efield_fp",Direction{1},lev), + m_fields.get("Efield_fp",Direction{2},lev), lev); } else { ApplyFieldBoundaryOnAxis(getFieldPointer(FieldType::Efield_cp, lev, 0), getFieldPointer(FieldType::Efield_cp, lev, 1), @@ -131,8 +133,9 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d if (lev == 0) { if (a_dt_type == DtType::FirstHalf) { if(::isAnyBoundary(field_boundary_lo, field_boundary_hi)){ + auto Efield_fp_new = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); m_fdtd_solver_fp[0]->ApplySilverMuellerBoundary( - Efield_fp[lev], Bfield_fp[lev], + Efield_fp_new[lev], Bfield_fp[lev], Geom(lev).Domain(), dt[lev], field_boundary_lo, field_boundary_hi); } diff --git a/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp b/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp index 5e75698903e..5a9e694a010 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp @@ -35,13 +35,15 @@ using namespace amrex; * \brief Update the B field at the boundary, using the Silver-Mueller condition */ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( - std::array< std::unique_ptr, 3 >& Efield, + std::map< ablastr::fields::Direction, amrex::MultiFab* >& Efield, std::array< std::unique_ptr, 3 >& Bfield, amrex::Box domain_box, amrex::Real const dt, amrex::Vector field_boundary_lo, amrex::Vector field_boundary_hi) { + using ablastr::fields::Direction; + // Ensure that we are using the Yee solver WARPX_ALWAYS_ASSERT_WITH_MESSAGE( m_fdtd_algo == ElectromagneticSolverAlgo::Yee, @@ -79,11 +81,11 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( // tiling is usually set by TilingIfNotGPU() // but here, we set it to false because of potential race condition, // since we grow the tiles by one guard cell after creating them. - for ( MFIter mfi(*Efield[0], false); mfi.isValid(); ++mfi ) { + for ( MFIter mfi(*Efield[Direction{0}], false); mfi.isValid(); ++mfi ) { // Extract field data for this grid/tile - Array4 const& Er = Efield[0]->array(mfi); - Array4 const& Et = Efield[1]->array(mfi); - Array4 const& Ez = Efield[2]->array(mfi); + Array4 const& Er = Efield[Direction{0}]->array(mfi); + Array4 const& Et = Efield[Direction{1}]->array(mfi); + Array4 const& Ez = Efield[Direction{2}]->array(mfi); Array4 const& Br = Bfield[0]->array(mfi); Array4 const& Bt = Bfield[1]->array(mfi); Array4 const& Bz = Bfield[2]->array(mfi); @@ -203,13 +205,13 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( // tiling is usually set by TilingIfNotGPU() // but here, we set it to false because of potential race condition, // since we grow the tiles by one guard cell after creating them. - for ( MFIter mfi(*Efield[0], false); mfi.isValid(); ++mfi ) { + for ( MFIter mfi(*Efield[Direction{0}], false); mfi.isValid(); ++mfi ) { // Extract field data for this grid/tile - Array4 const& Ex = Efield[0]->array(mfi); - Array4 const& Ey = Efield[1]->array(mfi); + Array4 const& Ex = Efield[Direction{0}]->array(mfi); + Array4 const& Ey = Efield[Direction{1}]->array(mfi); #ifndef WARPX_DIM_1D_Z - Array4 const& Ez = Efield[2]->array(mfi); + Array4 const& Ez = Efield[Direction{2}]->array(mfi); #endif Array4 const& Bx = Bfield[0]->array(mfi); Array4 const& By = Bfield[1]->array(mfi); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index e12d134a7cf..e5ed7fce39b 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -18,6 +18,7 @@ #include "MacroscopicProperties/MacroscopicProperties_fwd.H" #include +#include #include #include @@ -88,7 +89,7 @@ class FiniteDifferenceSolver int lev ); void ApplySilverMuellerBoundary( - std::array< std::unique_ptr, 3 >& Efield, + std::map< ablastr::fields::Direction, amrex::MultiFab* >& Efield, std::array< std::unique_ptr, 3 >& Bfield, amrex::Box domain_box, amrex::Real dt, From 0cc41b74002278c2555564590a5c0f1e62cf1fd5 Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Tue, 10 Sep 2024 14:39:07 -0700 Subject: [PATCH 054/314] Efield_fp refactor for FlushFormats --- .../FlushFormats/FlushFormatCheckpoint.cpp | 8 +++-- .../FlushFormats/FlushFormatPlotfile.cpp | 35 +++++++++++++------ 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index ac367121970..0d3a6cad839 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -11,6 +11,8 @@ #include "Utils/WarpXProfilerWrapper.H" #include "WarpX.H" +#include + #include #include #include @@ -66,11 +68,11 @@ FlushFormatCheckpoint::WriteToFile ( for (int lev = 0; lev < nlev; ++lev) { - VisMF::Write(warpx.getField(FieldType::Efield_fp, lev, 0), + VisMF::Write(*warpx.m_fields.get("Efield_fp", Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ex_fp")); - VisMF::Write(warpx.getField(FieldType::Efield_fp, lev, 1), + VisMF::Write(*warpx.m_fields.get("Efield_fp", Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ey_fp")); - VisMF::Write(warpx.getField(FieldType::Efield_fp, lev, 2), + VisMF::Write(*warpx.m_fields.get("Efield_fp", Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ez_fp")); VisMF::Write(warpx.getField(FieldType::Bfield_fp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bx_fp")); diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index 15d9f27241e..fd836295e3f 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -13,6 +13,8 @@ #include "Utils/WarpXProfilerWrapper.H" #include "WarpX.H" +#include + #include #include #include @@ -574,18 +576,28 @@ FlushFormatPlotfile::WriteAllRawFields( WriteRawMF( warpx.getField(FieldType::Bfield_aux, lev, 2), dm, raw_pltname, default_level_prefix, "Bz_aux", lev, plot_raw_fields_guards); // fine patch - WriteRawMF( warpx.getField(FieldType::Efield_fp, lev, 0), dm, raw_pltname, default_level_prefix, "Ex_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getField(FieldType::Efield_fp, lev, 1), dm, raw_pltname, default_level_prefix, "Ey_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getField(FieldType::Efield_fp, lev, 2), dm, raw_pltname, default_level_prefix, "Ez_fp", lev, plot_raw_fields_guards); - WriteRawMF( *warpx.m_fields.get("current_fp",Direction{0}, lev), dm, raw_pltname, default_level_prefix, "jx_fp", lev,plot_raw_fields_guards); - WriteRawMF( *warpx.m_fields.get("current_fp",Direction{1}, lev), dm, raw_pltname, default_level_prefix, "jy_fp", lev,plot_raw_fields_guards); - WriteRawMF( *warpx.m_fields.get("current_fp",Direction{2}, lev), dm, raw_pltname, default_level_prefix, "jz_fp", lev,plot_raw_fields_guards); - WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 0), dm, raw_pltname, default_level_prefix, "Bx_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 1), dm, raw_pltname, default_level_prefix, "By_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 2), dm, raw_pltname, default_level_prefix, "Bz_fp", lev, plot_raw_fields_guards); + WriteRawMF( *warpx.m_fields.get("Efield_fp", Direction{0}, lev), dm, raw_pltname, + default_level_prefix, "Ex_fp", lev, plot_raw_fields_guards ); + WriteRawMF( *warpx.m_fields.get("Efield_fp", Direction{1}, lev), dm, raw_pltname, + default_level_prefix, "Ey_fp", lev, plot_raw_fields_guards ); + WriteRawMF( *warpx.m_fields.get("Efield_fp", Direction{2}, lev), dm, raw_pltname, + default_level_prefix, "Ez_fp", lev, plot_raw_fields_guards ); + WriteRawMF( *warpx.m_fields.get("current_fp",Direction{0}, lev), dm, raw_pltname, + default_level_prefix, "jx_fp", lev,plot_raw_fields_guards ); + WriteRawMF( *warpx.m_fields.get("current_fp",Direction{1}, lev), dm, raw_pltname, + default_level_prefix, "jy_fp", lev,plot_raw_fields_guards ); + WriteRawMF( *warpx.m_fields.get("current_fp",Direction{2}, lev), dm, raw_pltname, + default_level_prefix, "jz_fp", lev,plot_raw_fields_guards ); + WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 0), dm, raw_pltname, + default_level_prefix, "Bx_fp", lev, plot_raw_fields_guards ); + WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 1), dm, raw_pltname, + default_level_prefix, "By_fp", lev, plot_raw_fields_guards ); + WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 2), dm, raw_pltname, + default_level_prefix, "Bz_fp", lev, plot_raw_fields_guards ); if (warpx.m_fields.has("F_fp", lev)) { - WriteRawMF(*warpx.m_fields.get("F_fp", lev), dm, raw_pltname, default_level_prefix, "F_fp", lev, plot_raw_fields_guards); + WriteRawMF( *warpx.m_fields.get("F_fp", lev), dm, raw_pltname, + default_level_prefix, "F_fp", lev, plot_raw_fields_guards ); } if (warpx.m_fields.has("rho_fp", lev)) { @@ -596,7 +608,8 @@ FlushFormatPlotfile::WriteAllRawFields( WriteRawMF(rho_new, dm, raw_pltname, default_level_prefix, "rho_fp", lev, plot_raw_fields_guards); } if (warpx.m_fields.has("phi_fp", lev)) { - WriteRawMF(*warpx.m_fields.get("phi_fp", lev), dm, raw_pltname, default_level_prefix, "phi_fp", lev, plot_raw_fields_guards); + WriteRawMF( *warpx.m_fields.get("phi_fp", lev), dm, raw_pltname, + default_level_prefix, "phi_fp", lev, plot_raw_fields_guards ); } // Averaged fields on fine patch From bdbe74024284d0656680052b1426dbfac1f3089c Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Tue, 10 Sep 2024 14:45:09 -0700 Subject: [PATCH 055/314] Efield_fp refactored for WarpXIO and ChargeOnEB --- Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp | 6 +++--- Source/Diagnostics/WarpXIO.cpp | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp b/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp index 01a2b5d8b23..3376accceef 100644 --- a/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp +++ b/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp @@ -105,9 +105,9 @@ void ChargeOnEB::ComputeDiags (const int step) int const lev = 0; // get MultiFab data at lev - const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_fp, lev,0); - const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_fp, lev,1); - const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_fp, lev,2); + const amrex::MultiFab & Ex = *warpx.m_fields.get("Efield_fp",Direction{0},lev); + const amrex::MultiFab & Ey = *warpx.m_fields.get("Efield_fp",Direction{1},lev); + const amrex::MultiFab & Ez = *warpx.m_fields.get("Efield_fp",Direction{2},lev); // get EB structures amrex::EBFArrayBoxFactory const& eb_box_factory = warpx.fieldEBFactory(lev); diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp index 7da32a85666..9f84933a188 100644 --- a/Source/Diagnostics/WarpXIO.cpp +++ b/Source/Diagnostics/WarpXIO.cpp @@ -282,7 +282,7 @@ WarpX::InitFromCheckpoint () { for (int i = 0; i < 3; ++i) { m_fields.get("current_fp",Direction{i},lev)->setVal(0.0); - Efield_fp[lev][i]->setVal(0.0); + m_fields.get("Efield_fp",Direction{i},lev)->setVal(0.0); Bfield_fp[lev][i]->setVal(0.0); } @@ -297,11 +297,11 @@ WarpX::InitFromCheckpoint () } } - VisMF::Read(*Efield_fp[lev][0], + VisMF::Read(*m_fields.get("Efield_fp", Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ex_fp")); - VisMF::Read(*Efield_fp[lev][1], + VisMF::Read(*m_fields.get("Efield_fp", Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ey_fp")); - VisMF::Read(*Efield_fp[lev][2], + VisMF::Read(*m_fields.get("Efield_fp", Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ez_fp")); VisMF::Read(*Bfield_fp[lev][0], From ab0defa70c04124d9d9b8bfaa0fa521e140f4759 Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Tue, 10 Sep 2024 14:58:35 -0700 Subject: [PATCH 056/314] Efield_fp refactor WarpXFieldBoundaries WarpXEvolve --- Source/BoundaryConditions/WarpXFieldBoundaries.cpp | 2 +- Source/Evolve/WarpXEvolve.cpp | 5 +++-- Source/WarpX.H | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index 26638778cdf..8caa0f0091a 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -133,7 +133,7 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d if (lev == 0) { if (a_dt_type == DtType::FirstHalf) { if(::isAnyBoundary(field_boundary_lo, field_boundary_hi)){ - auto Efield_fp_new = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); + auto Efield_fp_new = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); // JRA, new to prevent shadow m_fdtd_solver_fp[0]->ApplySilverMuellerBoundary( Efield_fp_new[lev], Bfield_fp[lev], Geom(lev).Domain(), dt[lev], diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index cca7bc9ce77..5b2da6b4202 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -630,7 +630,8 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Initialize multi-J loop: // 1) Prepare E,B,F,G fields in spectral space - PSATDForwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); + auto const Efield_fp_new = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); // JRA, new to prevent shadow + PSATDForwardTransformEB(Efield_fp_new, Bfield_fp, Efield_cp, Bfield_cp); if (WarpX::do_dive_cleaning) { PSATDForwardTransformF(); } if (WarpX::do_divb_cleaning) { PSATDForwardTransformG(); } @@ -734,7 +735,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // (the relative time reached here coincides with an integer full time step) if (i_deposit == n_deposit-1) { - PSATDBackwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); + PSATDBackwardTransformEB(Efield_fp_new, Bfield_fp, Efield_cp, Bfield_cp); if (WarpX::do_dive_cleaning) { PSATDBackwardTransformF(); } if (WarpX::do_divb_cleaning) { PSATDBackwardTransformG(); } } diff --git a/Source/WarpX.H b/Source/WarpX.H index b84627045e3..8c669510548 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1828,7 +1828,7 @@ private: * storing the coarse patch magnetic field to be transformed */ void PSATDForwardTransformEB ( - const amrex::Vector,3>>& E_fp, + const std::vector>& E_fp, const amrex::Vector,3>>& B_fp, const amrex::Vector,3>>& E_cp, const amrex::Vector,3>>& B_cp); From 6421739a5739841206e86bc42fe7b9552358e328 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 10 Sep 2024 16:46:02 -0700 Subject: [PATCH 057/314] Merge --- Source/Evolve/WarpXEvolve.cpp | 22 ++++---- .../MagnetostaticSolver.cpp | 4 +- Source/FieldSolver/WarpXPushFieldsEM.cpp | 2 +- .../FieldSolver/WarpXPushFieldsHybridPIC.cpp | 6 ++- Source/Parallelization/WarpXComm.cpp | 54 ++++++++++--------- Source/WarpX.H | 15 +++--- 6 files changed, 59 insertions(+), 44 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index cca7bc9ce77..220a36122b2 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -543,6 +543,8 @@ void WarpX::HandleParticlesAtBoundaries (int step, amrex::Real cur_time, int num void WarpX::SyncCurrentAndRho () { + using ablastr::fields::va2vm; + if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { if (fft_periodic_single_box) @@ -552,12 +554,12 @@ void WarpX::SyncCurrentAndRho () if (current_deposition_algo == CurrentDepositionAlgo::Vay) { // TODO Replace current_cp with current_cp_vay once Vay deposition is implemented with MR - SyncCurrent(current_fp_vay, current_cp, current_buf); + SyncCurrent(va2vm(current_fp_vay), va2vm(current_cp), va2vm(current_buf)); SyncRho(); } else { - SyncCurrent(current_fp, current_cp, current_buf); + SyncCurrent(va2vm(current_fp), va2vm(current_cp), va2vm(current_buf)); SyncRho(); } } @@ -569,7 +571,7 @@ void WarpX::SyncCurrentAndRho () if (!current_correction && current_deposition_algo != CurrentDepositionAlgo::Vay) { - SyncCurrent(current_fp, current_cp, current_buf); + SyncCurrent(va2vm(current_fp), va2vm(current_cp), va2vm(current_buf)); SyncRho(); } @@ -577,13 +579,13 @@ void WarpX::SyncCurrentAndRho () { // TODO This works only without mesh refinement const int lev = 0; - if (use_filter) { ApplyFilterJ(current_fp_vay, lev); } + if (use_filter) { ApplyFilterJ(va2vm(current_fp_vay), lev); } } } } else // FDTD { - SyncCurrent(current_fp, current_cp, current_buf); + SyncCurrent(va2vm(current_fp), va2vm(current_cp), va2vm(current_buf)); SyncRho(); } @@ -664,7 +666,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // namely 'current_fp_nodal': SyncCurrent stores the result of its centering // into 'current_fp' and then performs both filtering, if used, and exchange // of guard cells. - SyncCurrent(current_fp, current_cp, current_buf); + SyncCurrent(va2vm(current_fp), va2vm(current_cp), va2vm(current_buf)); // Forward FFT of J PSATDForwardTransformJ(current_fp, current_cp); } @@ -698,7 +700,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // namely 'current_fp_nodal': SyncCurrent stores the result of its centering // into 'current_fp' and then performs both filtering, if used, and exchange // of guard cells. - SyncCurrent(current_fp, current_cp, current_buf); + SyncCurrent(va2vm(current_fp), va2vm(current_cp), va2vm(current_buf)); // Forward FFT of J PSATDForwardTransformJ(current_fp, current_cp); @@ -802,6 +804,8 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) void WarpX::OneStep_sub1 (Real cur_time) { + using ablastr::fields::va2vm; + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( electrostatic_solver_id == ElectrostaticSolverAlgo::None, "Electrostatic solver cannot be used with sub-cycling." @@ -817,8 +821,8 @@ WarpX::OneStep_sub1 (Real cur_time) PushParticlesandDeposit(fine_lev, cur_time, DtType::FirstHalf); RestrictCurrentFromFineToCoarsePatch(current_fp, current_cp, fine_lev); RestrictRhoFromFineToCoarsePatch(fine_lev); - if (use_filter) { ApplyFilterJ(current_fp, fine_lev); } - SumBoundaryJ(current_fp, fine_lev, Geom(fine_lev).periodicity()); + if (use_filter) { ApplyFilterJ( va2vm(current_fp), fine_lev); } + SumBoundaryJ( va2vm(current_fp), fine_lev, Geom(fine_lev).periodicity()); ApplyFilterandSumBoundaryRho( m_fields.get_mr_levels("rho_fp", finest_level), diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp index 305ccf02eb3..46eae8b8643 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp @@ -71,6 +71,8 @@ WarpX::ComputeMagnetostaticField() void WarpX::AddMagnetostaticFieldLabFrame() { + using ablastr::fields::va2vm; + WARPX_PROFILE("WarpX::AddMagnetostaticFieldLabFrame"); // Store the boundary conditions for the field solver if they haven't been @@ -107,7 +109,7 @@ WarpX::AddMagnetostaticFieldLabFrame() } #endif - SyncCurrent(current_fp, current_cp, current_buf); // Apply filter, perform MPI exchange, interpolate across levels + SyncCurrent(va2vm(current_fp), va2vm(current_cp), va2vm(current_buf)); // Apply filter, perform MPI exchange, interpolate across levels // set the boundary and current density potentials setVectorPotentialBC(vector_potential_fp_nodal); diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 03dddf10979..503ffa50b72 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -737,7 +737,7 @@ WarpX::PushPSATD () PSATDBackwardTransformJ(current_fp, current_cp); // Synchronize J and rho - SyncCurrent(current_fp, current_cp, current_buf); + SyncCurrent(va2vm(current_fp), va2vm(current_cp), va2vm(current_buf)); SyncRho(); } else if (current_deposition_algo == CurrentDepositionAlgo::Vay) diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index 2e8f02ed198..12e898b79ef 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -15,6 +15,8 @@ #include "Utils/WarpXProfilerWrapper.H" #include "WarpX.H" +#include + using namespace amrex; void WarpX::HybridPICEvolveFields () @@ -169,12 +171,14 @@ void WarpX::HybridPICEvolveFields () void WarpX::HybridPICDepositInitialRhoAndJ () { + using ablastr::fields::va2vm; + auto& rho_fp_temp = m_hybrid_pic_model->rho_fp_temp; auto& current_fp_temp = m_hybrid_pic_model->current_fp_temp; mypc->DepositCharge(amrex::GetVecOfPtrs(rho_fp_temp), 0._rt); mypc->DepositCurrent(current_fp_temp, dt[0], 0._rt); SyncRho(amrex::GetVecOfPtrs(rho_fp_temp), m_fields.get_mr_levels("rho_cp", finest_level), charge_buf); - SyncCurrent(current_fp_temp, current_cp, current_buf); + SyncCurrent(va2vm(current_fp_temp), va2vm(current_cp), va2vm(current_buf)); for (int lev=0; lev <= finest_level; ++lev) { // SyncCurrent does not include a call to FillBoundary, but it is needed // for the hybrid-PIC solver since current values are interpolated to diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 813b2370eb5..4d7374f04e0 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -989,22 +989,26 @@ WarpX::FillBoundaryAux (int lev, IntVect ng) void WarpX::SyncCurrent ( - const amrex::Vector,3>>& J_fp, - const amrex::Vector,3>>& J_cp, - const amrex::Vector,3>>& J_buffer) + const ablastr::fields::MultiLevelVectorField& J_fp, + const ablastr::fields::MultiLevelVectorField& J_cp, + const ablastr::fields::MultiLevelVectorField& J_buffer) { + using ablastr::fields::Direction; + WARPX_PROFILE("WarpX::SyncCurrent()"); // If warpx.do_current_centering = 1, center currents from nodal grid to staggered grid if (do_current_centering) { + std::vector> J_fp_nodal = m_fields.get_mr_levels_alldirs("current_fp_nodal", finest_level+1); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(finest_level <= 1, "warpx.do_current_centering=1 not supported with more than one fine levels"); for (int lev = 0; lev <= finest_level; lev++) { - WarpX::UpdateCurrentNodalToStag(*J_fp[lev][0], *current_fp_nodal[lev][0]); - WarpX::UpdateCurrentNodalToStag(*J_fp[lev][1], *current_fp_nodal[lev][1]); - WarpX::UpdateCurrentNodalToStag(*J_fp[lev][2], *current_fp_nodal[lev][2]); + WarpX::UpdateCurrentNodalToStag(*J_fp[lev][Direction{0}], *J_fp_nodal[lev][Direction{0}]); + WarpX::UpdateCurrentNodalToStag(*J_fp[lev][Direction{1}], *J_fp_nodal[lev][Direction{1}]); + WarpX::UpdateCurrentNodalToStag(*J_fp[lev][Direction{2}], *J_fp_nodal[lev][Direction{2}]); } } @@ -1079,7 +1083,7 @@ WarpX::SyncCurrent ( { for (int lev = finest_level; lev >= 0; --lev) { - const int ncomp = J_fp[lev][idim]->nComp(); + const int ncomp = J_fp[lev][Direction{idim}]->nComp(); auto const& period = Geom(lev).periodicity(); if (lev < finest_level) @@ -1087,8 +1091,8 @@ WarpX::SyncCurrent ( // On a coarse level, the data in mf_comm comes from the // coarse patch of the fine level. They are unfiltered and uncommunicated. // We need to add it to the fine patch of the current level. - MultiFab fine_lev_cp(J_fp[lev][idim]->boxArray(), - J_fp[lev][idim]->DistributionMap(), + MultiFab fine_lev_cp(J_fp[lev][Direction{idim}]->boxArray(), + J_fp[lev][Direction{idim}]->DistributionMap(), ncomp, 0); fine_lev_cp.setVal(0.0); fine_lev_cp.ParallelAdd(*mf_comm, 0, 0, ncomp, mf_comm->nGrowVect(), @@ -1097,7 +1101,7 @@ WarpX::SyncCurrent ( auto owner_mask = amrex::OwnerMask(fine_lev_cp, period); auto const& mma = owner_mask->const_arrays(); auto const& sma = fine_lev_cp.const_arrays(); - auto const& dma = J_fp[lev][idim]->arrays(); + auto const& dma = J_fp[lev][Direction{idim}]->arrays(); amrex::ParallelFor(fine_lev_cp, IntVect(0), ncomp, [=] AMREX_GPU_DEVICE (int bno, int i, int j, int k, int n) { @@ -1120,23 +1124,23 @@ WarpX::SyncCurrent ( // filtering depends on the level. This is also done before any // same-level communication because it's easier this way to // avoid double counting. - J_cp[lev][idim]->setVal(0.0); - ablastr::coarsen::average::Coarsen(*J_cp[lev][idim], - *J_fp[lev][idim], + J_cp[lev][Direction{idim}]->setVal(0.0); + ablastr::coarsen::average::Coarsen(*J_cp[lev][Direction{idim}], + *J_fp[lev][Direction{idim}], refRatio(lev-1)); - if (J_buffer[lev][idim]) + if (J_buffer[lev][Direction{idim}]) { - IntVect const& ng = J_cp[lev][idim]->nGrowVect(); - AMREX_ASSERT(ng.allLE(J_buffer[lev][idim]->nGrowVect())); - MultiFab::Add(*J_buffer[lev][idim], *J_cp[lev][idim], + IntVect const& ng = J_cp[lev][Direction{idim}]->nGrowVect(); + AMREX_ASSERT(ng.allLE(J_buffer[lev][Direction{idim}]->nGrowVect())); + MultiFab::Add(*J_buffer[lev][Direction{idim}], *J_cp[lev][Direction{idim}], 0, 0, ncomp, ng); mf_comm = std::make_unique - (*J_buffer[lev][idim], amrex::make_alias, 0, ncomp); + (*J_buffer[lev][Direction{idim}], amrex::make_alias, 0, ncomp); } else { mf_comm = std::make_unique - (*J_cp[lev][idim], amrex::make_alias, 0, ncomp); + (*J_cp[lev][Direction{idim}], amrex::make_alias, 0, ncomp); } } @@ -1258,11 +1262,13 @@ void WarpX::RestrictCurrentFromFineToCoarsePatch ( } void WarpX::ApplyFilterJ ( - const amrex::Vector,3>>& current, + const std::vector>& current, const int lev, const int idim) { - amrex::MultiFab& J = *current[lev][idim]; + using ablastr::fields::Direction; + + amrex::MultiFab& J = *current[lev][Direction{idim}]; const int ncomp = J.nComp(); const amrex::IntVect ngrow = J.nGrowVect(); @@ -1275,7 +1281,7 @@ void WarpX::ApplyFilterJ ( } void WarpX::ApplyFilterJ ( - const amrex::Vector,3>>& current, + const std::vector>& current, const int lev) { for (int idim=0; idim<3; ++idim) @@ -1285,7 +1291,7 @@ void WarpX::ApplyFilterJ ( } void WarpX::SumBoundaryJ ( - const amrex::Vector,3>>& current, + const std::vector>& current, const int lev, const int idim, const amrex::Periodicity& period) @@ -1323,7 +1329,7 @@ void WarpX::SumBoundaryJ ( } void WarpX::SumBoundaryJ ( - const amrex::Vector,3>>& current, + const std::vector>& current, const int lev, const amrex::Periodicity& period) { diff --git a/Source/WarpX.H b/Source/WarpX.H index b84627045e3..8c349297334 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -85,7 +85,6 @@ class WARPX_EXPORT WarpX : public amrex::AmrCore { public: - static WarpX& GetInstance (); static void ResetInstance (); @@ -875,9 +874,9 @@ public: * \param[in,out] J_buffer reference to buffer current \c MultiFab (all MR levels) */ void SyncCurrent ( - const amrex::Vector,3>>& J_fp, - const amrex::Vector,3>>& J_cp, - const amrex::Vector,3>>& J_buffer); + const ablastr::fields::MultiLevelVectorField& J_fp, + const ablastr::fields::MultiLevelVectorField& J_cp, + const ablastr::fields::MultiLevelVectorField& J_buffer); void SyncRho (); @@ -1344,19 +1343,19 @@ private: void StoreCurrent (int lev); void RestoreCurrent (int lev); void ApplyFilterJ ( - const amrex::Vector,3>>& current, + const std::vector>& current, int lev, int idim); void ApplyFilterJ ( - const amrex::Vector,3>>& current, + const std::vector>& current, int lev); void SumBoundaryJ ( - const amrex::Vector,3>>& current, + const std::vector>& current, int lev, int idim, const amrex::Periodicity& period); void SumBoundaryJ ( - const amrex::Vector,3>>& current, + const std::vector>& current, int lev, const amrex::Periodicity& period); void NodalSyncJ ( From fc60a1f2e4b0dec2a9c38ec862b9a128cd0b567e Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Tue, 10 Sep 2024 16:50:03 -0700 Subject: [PATCH 058/314] Fix merge bugs --- .../FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp | 6 +++--- Source/WarpX.H | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp index dd17463b1cf..c6535987a8f 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp @@ -144,8 +144,8 @@ WarpX::AddMagnetostaticFieldLabFrame() \param[in] verbosity The verbosity setting for the MLMG solver */ void -WarpX::computeVectorPotential (const amrex::Vector,3> >& curr, - std::vector> A, +WarpX::computeVectorPotential (const amrex::Vector,3>>& curr, + std::vector> const& A, Real const required_precision, Real absolute_tolerance, int const max_iters, @@ -211,7 +211,7 @@ WarpX::computeVectorPotential (const amrex::Vector> A) const +WarpX::setVectorPotentialBC (std::vector> const& A) const { using ablastr::fields::Direction; diff --git a/Source/WarpX.H b/Source/WarpX.H index 43cec84a191..8cd451eeb0c 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1029,13 +1029,13 @@ public: void ComputeMagnetostaticField (); void AddMagnetostaticFieldLabFrame (); void computeVectorPotential (const amrex::Vector, 3> >& curr, - std::vector> A, + std::vector> const& A, amrex::Real required_precision=amrex::Real(1.e-11), amrex::Real absolute_tolerance=amrex::Real(0.0), int max_iters=200, int verbosity=2) const; - void setVectorPotentialBC (std::vector> A) const; + void setVectorPotentialBC (std::vector> const& A) const; /** * \brief From dab5c3f399b270599ee9c47f35ce59baf8144743 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 10 Sep 2024 17:02:04 -0700 Subject: [PATCH 059/314] Fix Compile --- Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp | 2 ++ Source/Evolve/WarpXEvolve.cpp | 5 ++--- Source/WarpX.H | 2 +- Source/ablastr/fields/MultiFabRegister.H | 2 +- Source/ablastr/fields/MultiFabRegister.cpp | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp b/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp index 3376accceef..a26a292c359 100644 --- a/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp +++ b/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp @@ -98,6 +98,8 @@ void ChargeOnEB::ComputeDiags (const int step) throw std::runtime_error("ChargeOnEB::ComputeDiags only works when EBs are enabled at runtime"); } #if ((defined WARPX_DIM_3D) && (defined AMREX_USE_EB)) + using ablastr::fields::Direction; + // get a reference to WarpX instance auto & warpx = WarpX::GetInstance(); diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 5b2da6b4202..cca7bc9ce77 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -630,8 +630,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Initialize multi-J loop: // 1) Prepare E,B,F,G fields in spectral space - auto const Efield_fp_new = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); // JRA, new to prevent shadow - PSATDForwardTransformEB(Efield_fp_new, Bfield_fp, Efield_cp, Bfield_cp); + PSATDForwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); if (WarpX::do_dive_cleaning) { PSATDForwardTransformF(); } if (WarpX::do_divb_cleaning) { PSATDForwardTransformG(); } @@ -735,7 +734,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // (the relative time reached here coincides with an integer full time step) if (i_deposit == n_deposit-1) { - PSATDBackwardTransformEB(Efield_fp_new, Bfield_fp, Efield_cp, Bfield_cp); + PSATDBackwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); if (WarpX::do_dive_cleaning) { PSATDBackwardTransformF(); } if (WarpX::do_divb_cleaning) { PSATDBackwardTransformG(); } } diff --git a/Source/WarpX.H b/Source/WarpX.H index 8c669510548..b84627045e3 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1828,7 +1828,7 @@ private: * storing the coarse patch magnetic field to be transformed */ void PSATDForwardTransformEB ( - const std::vector>& E_fp, + const amrex::Vector,3>>& E_fp, const amrex::Vector,3>>& B_fp, const amrex::Vector,3>>& E_cp, const amrex::Vector,3>>& B_cp); diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index c477e9f1a35..c51a54ce543 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -372,7 +372,7 @@ namespace ablastr::fields */ MultiLevelVectorField va2vm ( - amrex::Vector, 3 > > const & old_vector_on_levels + amrex::Vector, 3 > > old_vector_on_levels ); } // namespace ablastr::fields diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 3b610105ec4..e4e58e3f00f 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -381,7 +381,7 @@ namespace ablastr::fields MultiLevelVectorField va2vm ( - amrex::Vector, 3 > > const & old_vector_on_levels + amrex::Vector, 3 > > old_vector_on_levels ) { int const finest_level = old_vector_on_levels.size() - 1u; From c745aa46f89f96f29300e86d3bc21adf6503c25d Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 10 Sep 2024 17:03:01 -0700 Subject: [PATCH 060/314] Broke compilation --- Source/Parallelization/WarpXComm.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 4d7374f04e0..78a3e725939 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -1000,7 +1000,7 @@ WarpX::SyncCurrent ( // If warpx.do_current_centering = 1, center currents from nodal grid to staggered grid if (do_current_centering) { - std::vector> J_fp_nodal = m_fields.get_mr_levels_alldirs("current_fp_nodal", finest_level+1); + ablastr::fields::MultiLevelVectorField J_fp_nodal = m_fields.get_mr_levels_alldirs("current_fp_nodal", finest_level+1); AMREX_ALWAYS_ASSERT_WITH_MESSAGE(finest_level <= 1, "warpx.do_current_centering=1 not supported with more than one fine levels"); @@ -1262,7 +1262,7 @@ void WarpX::RestrictCurrentFromFineToCoarsePatch ( } void WarpX::ApplyFilterJ ( - const std::vector>& current, + const ablastr::fields::MultiLevelVectorField& current, const int lev, const int idim) { @@ -1281,7 +1281,7 @@ void WarpX::ApplyFilterJ ( } void WarpX::ApplyFilterJ ( - const std::vector>& current, + const ablastr::fields::MultiLevelVectorField& current, const int lev) { for (int idim=0; idim<3; ++idim) @@ -1291,7 +1291,7 @@ void WarpX::ApplyFilterJ ( } void WarpX::SumBoundaryJ ( - const std::vector>& current, + const ablastr::fields::MultiLevelVectorField& current, const int lev, const int idim, const amrex::Periodicity& period) @@ -1329,7 +1329,7 @@ void WarpX::SumBoundaryJ ( } void WarpX::SumBoundaryJ ( - const std::vector>& current, + const ablastr::fields::MultiLevelVectorField& current, const int lev, const amrex::Periodicity& period) { From 6ebefcd293bba483dc20b56b8cecfb54e8654335 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 10 Sep 2024 17:20:12 -0700 Subject: [PATCH 061/314] Scalars: Use Cool Aliases --- Source/FieldSolver/ElectrostaticSolver.cpp | 18 +++++------ .../HybridPICModel/HybridPICModel.H | 8 ++--- .../HybridPICModel/HybridPICModel.cpp | 8 ++--- Source/FieldSolver/WarpXPushFieldsEM.cpp | 4 +-- Source/Parallelization/WarpXComm.cpp | 12 ++++---- Source/Particles/MultiParticleContainer.H | 2 +- Source/Particles/MultiParticleContainer.cpp | 2 +- Source/Particles/WarpXParticleContainer.H | 4 ++- Source/Particles/WarpXParticleContainer.cpp | 2 +- Source/WarpX.H | 30 +++++++++---------- Source/ablastr/fields/MultiFabRegister.H | 2 +- Source/ablastr/fields/PoissonSolver.H | 2 +- 12 files changed, 48 insertions(+), 46 deletions(-) diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index 10fbe7dd1f1..a7df2bd1e2d 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -115,7 +115,7 @@ WarpX::AddBoundaryField () // Allocate fields for charge and potential const int num_levels = max_level + 1; Vector > rho(num_levels); - std::vector phi(num_levels); + ablastr::fields::MultiLevelScalarField phi(num_levels); // Use number of guard cells used for local deposition of rho const amrex::IntVect ng = guard_cells.ng_depos_rho; for (int lev = 0; lev <= max_level; lev++) { @@ -174,7 +174,7 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) const int num_levels = max_level + 1; Vector > rho(num_levels); Vector > rho_coarse(num_levels); // Used in order to interpolate between levels - std::vector phi(num_levels); + ablastr::fields::MultiLevelScalarField phi(num_levels); // Use number of guard cells used for local deposition of rho const amrex::IntVect ng = guard_cells.ng_depos_rho; for (int lev = 0; lev <= max_level; lev++) { @@ -331,8 +331,8 @@ WarpX::AddSpaceChargeFieldLabFrame () \param[in] verbosity The verbosity setting for the MLMG solver */ void -WarpX::computePhi (const std::vector& rho, - std::vector& phi, +WarpX::computePhi (const ablastr::fields::MultiLevelScalarField& rho, + ablastr::fields::MultiLevelScalarField& phi, std::array const beta, Real const required_precision, Real absolute_tolerance, @@ -435,7 +435,7 @@ WarpX::computePhi (const std::vector& rho, \param[in] idim The dimension for which the Dirichlet boundary condition is set */ void -WarpX::setPhiBC ( std::vector& phi ) const +WarpX::setPhiBC ( ablastr::fields::MultiLevelScalarField& phi ) const { // check if any dimension has non-periodic boundary conditions if (!m_poisson_boundary_handler.has_non_periodic) { return; } @@ -517,7 +517,7 @@ WarpX::setPhiBC ( std::vector& phi ) const */ void WarpX::computeE (amrex::Vector, 3> >& E, - const std::vector& phi, + const ablastr::fields::MultiLevelScalarField& phi, std::array const beta ) const { for (int lev = 0; lev <= max_level; lev++) { @@ -694,7 +694,7 @@ WarpX::computeE (amrex::Vector, 3> > */ void WarpX::computeB (amrex::Vector, 3> >& B, - const std::vector& phi, + const ablastr::fields::MultiLevelScalarField& phi, std::array const beta ) const { // return early if beta is 0 since there will be no B-field @@ -862,8 +862,8 @@ WarpX::computeB (amrex::Vector, 3> > \param[out] phi The potential to be computed by this function */ void -WarpX::computePhiTriDiagonal (const std::vector& rho, - std::vector& phi) const +WarpX::computePhiTriDiagonal (const ablastr::fields::MultiLevelScalarField& rho, + ablastr::fields::MultiLevelScalarField& phi) const { WARPX_ALWAYS_ASSERT_WITH_MESSAGE(max_level == 0, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index 890342a5fab..dfd2c3d9653 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -89,7 +89,7 @@ public: amrex::Vector, 3>>& Efield, amrex::Vector, 3>> const& Jfield, amrex::Vector, 3>> const& Bfield, - std::vector const& rhofield, + ablastr::fields::MultiLevelScalarField const& rhofield, amrex::Vector, 3>> const& edge_lengths, bool solve_for_Faraday); @@ -113,7 +113,7 @@ public: amrex::Vector, 3>>& Bfield, amrex::Vector, 3>>& Efield, amrex::Vector, 3>> const& Jfield, - std::vector const& rhofield, + ablastr::fields::MultiLevelScalarField const& rhofield, amrex::Vector, 3>> const& edge_lengths, amrex::Real dt, DtType a_dt_type, amrex::IntVect ng, std::optional nodal_sync); @@ -122,7 +122,7 @@ public: amrex::Vector, 3>>& Bfield, amrex::Vector, 3>>& Efield, amrex::Vector, 3>> const& Jfield, - std::vector const& rhofield, + ablastr::fields::MultiLevelScalarField const& rhofield, amrex::Vector, 3>> const& edge_lengths, amrex::Real dt, int lev, DtType dt_type, amrex::IntVect ng, std::optional nodal_sync); @@ -131,7 +131,7 @@ public: amrex::Vector, 3>>& Bfield, amrex::Vector, 3>>& Efield, amrex::Vector, 3>> const& Jfield, - std::vector const& rhofield, + ablastr::fields::MultiLevelScalarField const& rhofield, amrex::Vector, 3>> const& edge_lengths, amrex::Real dt, DtType dt_type, amrex::IntVect ng, std::optional nodal_sync); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 5609d331775..8005cb28240 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -424,7 +424,7 @@ void HybridPICModel::HybridPICSolveE ( amrex::Vector, 3>> & Efield, amrex::Vector, 3>> const& Jfield, amrex::Vector, 3>> const& Bfield, - std::vector const& rhofield, + ablastr::fields::MultiLevelScalarField const& rhofield, amrex::Vector, 3>> const& edge_lengths, const bool solve_for_Faraday) { @@ -535,7 +535,7 @@ void HybridPICModel::BfieldEvolveRK ( amrex::Vector, 3>>& Bfield, amrex::Vector, 3>>& Efield, amrex::Vector, 3>> const& Jfield, - std::vector const& rhofield, + ablastr::fields::MultiLevelScalarField const& rhofield, amrex::Vector, 3>> const& edge_lengths, amrex::Real dt, DtType dt_type, IntVect ng, std::optional nodal_sync ) @@ -554,7 +554,7 @@ void HybridPICModel::BfieldEvolveRK ( amrex::Vector, 3>>& Bfield, amrex::Vector, 3>>& Efield, amrex::Vector, 3>> const& Jfield, - std::vector const& rhofield, + ablastr::fields::MultiLevelScalarField const& rhofield, amrex::Vector, 3>> const& edge_lengths, amrex::Real dt, int lev, DtType dt_type, IntVect ng, std::optional nodal_sync ) @@ -667,7 +667,7 @@ void HybridPICModel::FieldPush ( amrex::Vector, 3>>& Bfield, amrex::Vector, 3>>& Efield, amrex::Vector, 3>> const& Jfield, - std::vector const& rhofield, + ablastr::fields::MultiLevelScalarField const& rhofield, amrex::Vector, 3>> const& edge_lengths, amrex::Real dt, DtType dt_type, IntVect ng, std::optional nodal_sync ) diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 03dddf10979..fe060285797 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -377,8 +377,8 @@ void WarpX::PSATDBackwardTransformJ ( } void WarpX::PSATDForwardTransformRho ( - std::vector const & charge_fp, - std::vector const & charge_cp, + ablastr::fields::MultiLevelScalarField const & charge_fp, + ablastr::fields::MultiLevelScalarField const & charge_cp, const int icomp, const int dcomp, const bool apply_kspace_filter) { if (charge_fp[0] == nullptr) { return; } diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 813b2370eb5..ee7bef8d640 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -1159,8 +1159,8 @@ WarpX::SyncRho () { void WarpX::SyncRho ( - const std::vector& charge_fp, - const std::vector& charge_cp, + const ablastr::fields::MultiLevelScalarField& charge_fp, + const ablastr::fields::MultiLevelScalarField& charge_cp, const amrex::Vector>& charge_buffer) { WARPX_PROFILE("WarpX::SyncRho()"); @@ -1434,8 +1434,8 @@ void WarpX::RestrictRhoFromFineToCoarsePatch ( const int lev ) } void WarpX::ApplyFilterandSumBoundaryRho ( - const std::vector& charge_fp, - const std::vector& charge_cp, + const ablastr::fields::MultiLevelScalarField& charge_fp, + const ablastr::fields::MultiLevelScalarField& charge_cp, const int lev, PatchType patch_type, const int icomp, @@ -1480,8 +1480,8 @@ void WarpX::ApplyFilterandSumBoundaryRho (int /*lev*/, int glev, amrex::MultiFab * patch (and buffer region) of `lev+1` */ void WarpX::AddRhoFromFineLevelandSumBoundary ( - const std::vector& charge_fp, - const std::vector& charge_cp, + const ablastr::fields::MultiLevelScalarField& charge_fp, + const ablastr::fields::MultiLevelScalarField& charge_cp, const amrex::Vector>& charge_buffer, const int lev, const int icomp, diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index 4a34f2ea04f..2dfe6a404fe 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -147,7 +147,7 @@ public: * the time of the deposition. */ void - DepositCharge (const std::vector& rho, + DepositCharge (const ablastr::fields::MultiLevelScalarField& rho, amrex::Real relative_time); /** diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index cb323fa7933..db3a05c8b10 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -557,7 +557,7 @@ MultiParticleContainer::DepositCurrent ( void MultiParticleContainer::DepositCharge ( - const std::vector& rho, + const ablastr::fields::MultiLevelScalarField& rho, const amrex::Real relative_time) { // Reset the rho array diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 58c01b2c9ff..3664e670983 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -25,6 +25,8 @@ #include "MultiParticleContainer_fwd.H" #include "NamedComponentParticleContainer.H" +#include + #include #include #include @@ -212,7 +214,7 @@ public: * \param[in] interpolate_across_levels whether to average down from the fine patch to the coarse patch * \param[in] icomp component of the MultiFab where rho is deposited (old, new) */ - void DepositCharge (const std::vector& rho, + void DepositCharge (const ablastr::fields::MultiLevelScalarField& rho, bool local = false, bool reset = false, bool apply_boundary_and_scale_volume = false, bool interpolate_across_levels = true, diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 057e32f2cd3..b30ee355f55 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -1170,7 +1170,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, } void -WarpXParticleContainer::DepositCharge (const std::vector& rho, +WarpXParticleContainer::DepositCharge (const ablastr::fields::MultiLevelScalarField& rho, const bool local, const bool reset, const bool apply_boundary_and_scale_volume, const bool interpolate_across_levels, diff --git a/Source/WarpX.H b/Source/WarpX.H index b84627045e3..6bc488ce561 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -882,8 +882,8 @@ public: void SyncRho (); void SyncRho ( - const std::vector& charge_fp, - const std::vector& charge_cp, + const ablastr::fields::MultiLevelScalarField& charge_fp, + const ablastr::fields::MultiLevelScalarField& charge_cp, const amrex::Vector>& charge_buffer); [[nodiscard]] amrex::Vector getnsubsteps () const {return nsubsteps;} @@ -1005,24 +1005,24 @@ public: void AddBoundaryField (); void AddSpaceChargeField (WarpXParticleContainer& pc); void AddSpaceChargeFieldLabFrame (); - void computePhi (const std::vector& rho, - std::vector& phi, + void computePhi (const ablastr::fields::MultiLevelScalarField& rho, + ablastr::fields::MultiLevelScalarField& phi, std::array beta = {{0,0,0}}, amrex::Real required_precision=amrex::Real(1.e-11), amrex::Real absolute_tolerance=amrex::Real(0.0), int max_iters=200, int verbosity=2) const; - void setPhiBC (std::vector& phi ) const; + void setPhiBC (ablastr::fields::MultiLevelScalarField& phi ) const; void computeE (amrex::Vector, 3> >& E, - const std::vector& phi, + const ablastr::fields::MultiLevelScalarField& phi, std::array beta = {{0,0,0}} ) const; void computeB (amrex::Vector, 3> >& B, - const std::vector& phi, + const ablastr::fields::MultiLevelScalarField& phi, std::array beta = {{0,0,0}} ) const; - void computePhiTriDiagonal (const std::vector& rho, - std::vector& phi) const; + void computePhiTriDiagonal (const ablastr::fields::MultiLevelScalarField& rho, + ablastr::fields::MultiLevelScalarField& phi) const; // Magnetostatic Solver Interface MagnetostaticSolver::VectorPoissonBoundaryHandler m_vector_poisson_boundary_handler; @@ -1367,15 +1367,15 @@ private: void RestrictRhoFromFineToCoarsePatch (int lev ); void ApplyFilterandSumBoundaryRho ( - const std::vector& charge_fp, - const std::vector& charge_cp, + const ablastr::fields::MultiLevelScalarField& charge_fp, + const ablastr::fields::MultiLevelScalarField& charge_cp, int lev, PatchType patch_type, int icomp, int ncomp); void AddRhoFromFineLevelandSumBoundary ( - const std::vector& charge_fp, - const std::vector& charge_cp, + const ablastr::fields::MultiLevelScalarField& charge_fp, + const ablastr::fields::MultiLevelScalarField& charge_cp, const amrex::Vector>& charge_buffer, int lev, int icomp, @@ -1910,8 +1910,8 @@ private: * (only used in RZ geometry to avoid double filtering) */ void PSATDForwardTransformRho ( - std::vector const & charge_fp, - std::vector const & charge_cp, + ablastr::fields::MultiLevelScalarField const & charge_fp, + ablastr::fields::MultiLevelScalarField const & charge_cp, int icomp, int dcomp, bool apply_kspace_filter=true); /** diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index c51a54ce543..ccb59cdd299 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -54,7 +54,7 @@ namespace ablastr::fields * * TODO: use amrex::Vector again for bound checks in debug mode */ - using MultiLevelVectorField = std::vector>; + using MultiLevelVectorField = std::vector; /** title * diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index b53698ac66c..cea1539843e 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -97,7 +97,7 @@ template< > void computePhi (amrex::Vector const & rho, - std::vector & phi, + ablastr::fields::MultiLevelScalarField & phi, std::array const beta, amrex::Real relative_tolerance, amrex::Real absolute_tolerance, From 0fa1e26ccb7ed46467079de6788b88706723a78c Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 10 Sep 2024 18:09:21 -0700 Subject: [PATCH 062/314] Fix compilation --- Source/BoundaryConditions/PML_RZ.cpp | 4 ++-- Source/Evolve/WarpXEvolve.cpp | 4 ++-- Source/WarpX.H | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Source/BoundaryConditions/PML_RZ.cpp b/Source/BoundaryConditions/PML_RZ.cpp index 0033359fc43..8c07a775e14 100644 --- a/Source/BoundaryConditions/PML_RZ.cpp +++ b/Source/BoundaryConditions/PML_RZ.cpp @@ -49,8 +49,8 @@ PML_RZ::PML_RZ (const int lev, const amrex::BoxArray& grid_ba, const amrex::Dist const amrex::MultiFab & Et_fp = *warpx.m_fields.get("Efield_fp",Direction{1},lev); const amrex::BoxArray ba_Er = amrex::convert(grid_ba, Er_fp.ixType().toIntVect()); const amrex::BoxArray ba_Et = amrex::convert(grid_ba, Et_fp.ixType().toIntVect()); - warpx.m_fields.alloc_init("pml_E_fp", Direction{0}, lev, ba_Er, dm, Er_fp.nComp(), Er_fp.nGrowVect(), 0.0_rt); - warpx.m_fields.alloc_init("pml_E_fp", Direction{1}, lev, ba_Et, dm, Et_fp.nComp(), Et_fp.nGrowVect(), 0.0_rt); + warpx.m_fields.alloc_init("pml_E_fp", Direction{0}, lev, ba_Er, grid_dm, Er_fp.nComp(), Er_fp.nGrowVect(), 0.0_rt); + warpx.m_fields.alloc_init("pml_E_fp", Direction{1}, lev, ba_Et, grid_dm, Et_fp.nComp(), Et_fp.nGrowVect(), 0.0_rt); const amrex::MultiFab & Br_fp = WarpX::GetInstance().getField(FieldType::Bfield_fp, lev,0); const amrex::MultiFab & Bt_fp = WarpX::GetInstance().getField(FieldType::Bfield_fp, lev,1); diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 220a36122b2..4f84c6b275a 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -888,8 +888,8 @@ WarpX::OneStep_sub1 (Real cur_time) PushParticlesandDeposit(fine_lev, cur_time + dt[fine_lev], DtType::SecondHalf); RestrictCurrentFromFineToCoarsePatch(current_fp, current_cp, fine_lev); RestrictRhoFromFineToCoarsePatch(fine_lev); - if (use_filter) { ApplyFilterJ(current_fp, fine_lev); } - SumBoundaryJ(current_fp, fine_lev, Geom(fine_lev).periodicity()); + if (use_filter) { ApplyFilterJ( va2vm(current_fp), fine_lev); } + SumBoundaryJ( va2vm(current_fp), fine_lev, Geom(fine_lev).periodicity()); ApplyFilterandSumBoundaryRho( m_fields.get_mr_levels("rho_fp", finest_level), m_fields.get_mr_levels("rho_cp", finest_level), diff --git a/Source/WarpX.H b/Source/WarpX.H index 34dc2732332..9f8744a69bf 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1343,19 +1343,19 @@ private: void StoreCurrent (int lev); void RestoreCurrent (int lev); void ApplyFilterJ ( - const std::vector>& current, + const ablastr::fields::MultiLevelVectorField& current, int lev, int idim); void ApplyFilterJ ( - const std::vector>& current, + const ablastr::fields::MultiLevelVectorField& current, int lev); void SumBoundaryJ ( - const std::vector>& current, + const ablastr::fields::MultiLevelVectorField& current, int lev, int idim, const amrex::Periodicity& period); void SumBoundaryJ ( - const std::vector>& current, + const ablastr::fields::MultiLevelVectorField& current, int lev, const amrex::Periodicity& period); void NodalSyncJ ( From eca176017cd36504c7ca53bb95920795c422286a Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 10 Sep 2024 18:44:59 -0700 Subject: [PATCH 063/314] Apply suggestions from code review --- Source/BoundaryConditions/PML_RZ.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/BoundaryConditions/PML_RZ.cpp b/Source/BoundaryConditions/PML_RZ.cpp index 0033359fc43..8c07a775e14 100644 --- a/Source/BoundaryConditions/PML_RZ.cpp +++ b/Source/BoundaryConditions/PML_RZ.cpp @@ -49,8 +49,8 @@ PML_RZ::PML_RZ (const int lev, const amrex::BoxArray& grid_ba, const amrex::Dist const amrex::MultiFab & Et_fp = *warpx.m_fields.get("Efield_fp",Direction{1},lev); const amrex::BoxArray ba_Er = amrex::convert(grid_ba, Er_fp.ixType().toIntVect()); const amrex::BoxArray ba_Et = amrex::convert(grid_ba, Et_fp.ixType().toIntVect()); - warpx.m_fields.alloc_init("pml_E_fp", Direction{0}, lev, ba_Er, dm, Er_fp.nComp(), Er_fp.nGrowVect(), 0.0_rt); - warpx.m_fields.alloc_init("pml_E_fp", Direction{1}, lev, ba_Et, dm, Et_fp.nComp(), Et_fp.nGrowVect(), 0.0_rt); + warpx.m_fields.alloc_init("pml_E_fp", Direction{0}, lev, ba_Er, grid_dm, Er_fp.nComp(), Er_fp.nGrowVect(), 0.0_rt); + warpx.m_fields.alloc_init("pml_E_fp", Direction{1}, lev, ba_Et, grid_dm, Et_fp.nComp(), Et_fp.nGrowVect(), 0.0_rt); const amrex::MultiFab & Br_fp = WarpX::GetInstance().getField(FieldType::Bfield_fp, lev,0); const amrex::MultiFab & Bt_fp = WarpX::GetInstance().getField(FieldType::Bfield_fp, lev,1); From 0e1a37aa8da292e1e91bdcb2c7ff274b659bca6d Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 10 Sep 2024 20:42:21 -0700 Subject: [PATCH 064/314] AppAvoid compilation error with deleted copy constructor --- Source/ablastr/fields/MultiFabRegister.H | 2 +- Source/ablastr/fields/MultiFabRegister.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index ccb59cdd299..924b5760cec 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -372,7 +372,7 @@ namespace ablastr::fields */ MultiLevelVectorField va2vm ( - amrex::Vector, 3 > > old_vector_on_levels + const amrex::Vector, 3 > >& old_vector_on_levels ); } // namespace ablastr::fields diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index e4e58e3f00f..87191218f51 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -381,7 +381,7 @@ namespace ablastr::fields MultiLevelVectorField va2vm ( - amrex::Vector, 3 > > old_vector_on_levels + const amrex::Vector, 3 > >& old_vector_on_levels ) { int const finest_level = old_vector_on_levels.size() - 1u; From 63475fee30ff2dad1b623bc2aacffb91cfb42a71 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 10 Sep 2024 20:54:04 -0700 Subject: [PATCH 065/314] More compilation fixes --- Source/Parallelization/WarpXComm.cpp | 18 +++++++++++------- Source/ablastr/fields/MultiFabRegister.H | 2 +- Source/ablastr/fields/MultiFabRegister.cpp | 2 +- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 0d9c0faf2a3..7e6c466ba6a 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -1296,7 +1296,9 @@ void WarpX::SumBoundaryJ ( const int idim, const amrex::Periodicity& period) { - amrex::MultiFab& J = *current[lev][idim]; + using ablastr::fields::Direction; + + amrex::MultiFab& J = *current[lev][Direction{idim}]; const amrex::IntVect ng = J.nGrowVect(); amrex::IntVect ng_depos_J = get_ng_depos_J(); @@ -1358,13 +1360,15 @@ void WarpX::AddCurrentFromFineLevelandSumBoundary ( const amrex::Vector,3>>& J_buffer, const int lev) { + using ablastr::fields::va2vm; + const amrex::Periodicity& period = Geom(lev).periodicity(); if (use_filter) { - ApplyFilterJ(J_fp, lev); + ApplyFilterJ(va2vm(J_fp), lev); } - SumBoundaryJ(J_fp, lev, period); + SumBoundaryJ(va2vm(J_fp), lev, period); if (lev < finest_level) { @@ -1381,8 +1385,8 @@ void WarpX::AddCurrentFromFineLevelandSumBoundary ( if (use_filter && J_buffer[lev+1][idim]) { - ApplyFilterJ(J_cp, lev+1, idim); - ApplyFilterJ(J_buffer, lev+1, idim); + ApplyFilterJ(va2vm(J_cp), lev+1, idim); + ApplyFilterJ(va2vm(J_buffer), lev+1, idim); MultiFab::Add( *J_buffer[lev+1][idim], *J_cp[lev+1][idim], @@ -1396,7 +1400,7 @@ void WarpX::AddCurrentFromFineLevelandSumBoundary ( } else if (use_filter) // but no buffer { - ApplyFilterJ(J_cp, lev+1, idim); + ApplyFilterJ(va2vm(J_cp), lev+1, idim); ablastr::utils::communication::ParallelAdd( mf, *J_cp[lev+1][idim], 0, 0, @@ -1424,7 +1428,7 @@ void WarpX::AddCurrentFromFineLevelandSumBoundary ( ng, amrex::IntVect(0), do_single_precision_comms, period); } - SumBoundaryJ(J_cp, lev+1, idim, period); + SumBoundaryJ(va2vm(J_cp), lev+1, idim, period); MultiFab::Add(*J_fp[lev][idim], mf, 0, 0, J_fp[lev+1][idim]->nComp(), 0); } } diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index ccb59cdd299..924b5760cec 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -372,7 +372,7 @@ namespace ablastr::fields */ MultiLevelVectorField va2vm ( - amrex::Vector, 3 > > old_vector_on_levels + const amrex::Vector, 3 > >& old_vector_on_levels ); } // namespace ablastr::fields diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index e4e58e3f00f..87191218f51 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -381,7 +381,7 @@ namespace ablastr::fields MultiLevelVectorField va2vm ( - amrex::Vector, 3 > > old_vector_on_levels + const amrex::Vector, 3 > >& old_vector_on_levels ) { int const finest_level = old_vector_on_levels.size() - 1u; From 3b298fcc3016b05e4caf1d657567f678ecddbaf0 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Wed, 11 Sep 2024 09:28:57 -0700 Subject: [PATCH 066/314] Fix merge bugs --- .../FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp | 4 ++-- Source/WarpX.H | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp index c6535987a8f..a8ec76b5590 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp @@ -145,7 +145,7 @@ WarpX::AddMagnetostaticFieldLabFrame() */ void WarpX::computeVectorPotential (const amrex::Vector,3>>& curr, - std::vector> const& A, + ablastr::fields::MultiLevelVectorField const& A, Real const required_precision, Real absolute_tolerance, int const max_iters, @@ -211,7 +211,7 @@ WarpX::computeVectorPotential (const amrex::Vector> const& A) const +WarpX::setVectorPotentialBC (ablastr::fields::MultiLevelVectorField const& A) const { using ablastr::fields::Direction; diff --git a/Source/WarpX.H b/Source/WarpX.H index 36d24b0b4da..cbc0334f2fa 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1029,13 +1029,13 @@ public: void ComputeMagnetostaticField (); void AddMagnetostaticFieldLabFrame (); void computeVectorPotential (const amrex::Vector, 3> >& curr, - std::vector> const& A, + ablastr::fields::MultiLevelVectorField const& A, amrex::Real required_precision=amrex::Real(1.e-11), amrex::Real absolute_tolerance=amrex::Real(0.0), int max_iters=200, int verbosity=2) const; - void setVectorPotentialBC (std::vector> const& A) const; + void setVectorPotentialBC (ablastr::fields::MultiLevelVectorField const& A) const; /** * \brief From dc841ba103736110cd671da0db41a925943e0c0b Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 11 Sep 2024 09:34:01 -0700 Subject: [PATCH 067/314] `MultiFabRegister`: Simpler Compatibility Types --- Source/FieldSolver/ElectrostaticSolver.cpp | 2 +- .../ApplySilverMuellerBoundary.cpp | 2 +- .../FiniteDifferenceSolver.H | 4 ++-- Source/ablastr/fields/MultiFabRegister.H | 16 +++++++++------- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index a7df2bd1e2d..e7dc026eb86 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -340,7 +340,7 @@ WarpX::computePhi (const ablastr::fields::MultiLevelScalarField& rho, int const verbosity) const { // create a vector to our fields, sorted by level amrex::Vector sorted_rho; - std::vector sorted_phi; + ablastr::fields::MultiLevelScalarField sorted_phi; for (int lev = 0; lev <= finest_level; ++lev) { sorted_rho.emplace_back(rho[lev]); sorted_phi.emplace_back(phi[lev]); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp b/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp index 5a9e694a010..5020a062bd7 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp @@ -35,7 +35,7 @@ using namespace amrex; * \brief Update the B field at the boundary, using the Silver-Mueller condition */ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( - std::map< ablastr::fields::Direction, amrex::MultiFab* >& Efield, + ablastr::fields::VectorField& Efield, std::array< std::unique_ptr, 3 >& Bfield, amrex::Box domain_box, amrex::Real const dt, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index e5ed7fce39b..69d7630dd9e 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -88,8 +88,8 @@ class FiniteDifferenceSolver std::array< std::unique_ptr, 3 >& ECTRhofield, int lev ); - void ApplySilverMuellerBoundary( - std::map< ablastr::fields::Direction, amrex::MultiFab* >& Efield, + void ApplySilverMuellerBoundary ( + ablastr::fields::VectorField & Efield, std::array< std::unique_ptr, 3 >& Bfield, amrex::Box domain_box, amrex::Real dt, diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 924b5760cec..5c240e96d2f 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -10,11 +10,13 @@ #include "ablastr/utils/ConstMap.H" +#include #include #include #include #include +#include #include #include #include @@ -38,23 +40,23 @@ namespace ablastr::fields { return other.dir < this->dir; } + + /* TODO: just temporary int compatibility */ + operator int() { return dir; } }; /** A vector field of three MultiFab */ - using VectorField = ablastr::utils::ConstMap; + //using VectorField = ablastr::utils::ConstMap; + using VectorField = std::array; /** A multi-level scalar field - * - * TODO: use amrex::Vector again for bound checks in debug mode */ - using MultiLevelScalarField = std::vector; + using MultiLevelScalarField = amrex::Vector; /** A multi-level vector field - * - * TODO: use amrex::Vector again for bound checks in debug mode */ - using MultiLevelVectorField = std::vector; + using MultiLevelVectorField = amrex::Vector; /** title * From 8d6d66e3e93d26ac69183faed7ffb5280e43e349 Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Tue, 10 Sep 2024 17:21:06 -0700 Subject: [PATCH 068/314] Efield_fp refactor ElectrotaticSolver --- Source/FieldSolver/ElectrostaticSolver.cpp | 47 +++++++++++++--------- Source/WarpX.H | 7 +++- 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index e7dc026eb86..e260380e15c 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -62,12 +62,13 @@ void WarpX::ComputeSpaceChargeField (bool const reset_fields) { WARPX_PROFILE("WarpX::ComputeSpaceChargeField"); + using ablastr::fields::Direction; if (reset_fields) { // Reset all E and B fields to 0, before calculating space-charge fields WARPX_PROFILE("WarpX::ComputeSpaceChargeField::reset_fields"); for (int lev = 0; lev <= max_level; lev++) { for (int comp=0; comp<3; comp++) { - Efield_fp[lev][comp]->setVal(0); + m_fields.get("Efield_fp",Direction{comp},lev)->setVal(0); Bfield_fp[lev][comp]->setVal(0); } } @@ -140,7 +141,8 @@ WarpX::AddBoundaryField () self_fields_verbosity ); // Compute the corresponding electric and magnetic field, from the potential phi. - computeE( Efield_fp, phi, beta ); + auto Efield_fp_new = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); // JRA, new to prevent shadow + computeE( Efield_fp_new, phi, beta ); computeB( Bfield_fp, phi, beta ); // de-allocate temporary @@ -227,7 +229,8 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) pc.self_fields_verbosity ); // Compute the corresponding electric and magnetic field, from the potential phi - computeE( Efield_fp, phi, beta ); + auto Efield_fp_new = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); // JRA, new to prevent shadow + computeE( Efield_fp_new, phi, beta ); computeB( Bfield_fp, phi, beta ); // de-allocate temporary @@ -304,9 +307,10 @@ WarpX::AddSpaceChargeFieldLabFrame () // Compute the electric field. Note that if an EB is used the electric // field will be calculated in the computePhi call. - if (!EB::enabled()) { computeE( Efield_fp, phi_fp, beta ); } + auto Efield_fp_new = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); // JRA, new to prevent shadow + if (!EB::enabled()) { computeE( Efield_fp_new, phi_fp, beta ); } else { - if (IsPythonCallbackInstalled("poissonsolver")) { computeE(Efield_fp, phi_fp, beta); } + if (IsPythonCallbackInstalled("poissonsolver")) { computeE(Efield_fp_new, phi_fp, beta); } } // Compute the magnetic field @@ -337,7 +341,7 @@ WarpX::computePhi (const ablastr::fields::MultiLevelScalarField& rho, Real const required_precision, Real absolute_tolerance, int const max_iters, - int const verbosity) const { + int const verbosity) { // create a vector to our fields, sorted by level amrex::Vector sorted_rho; ablastr::fields::MultiLevelScalarField sorted_phi; @@ -345,6 +349,8 @@ WarpX::computePhi (const ablastr::fields::MultiLevelScalarField& rho, sorted_rho.emplace_back(rho[lev]); sorted_phi.emplace_back(phi[lev]); } + + using ablastr::fields::Direction; std::optional post_phi_calculation; #ifdef AMREX_USE_EB @@ -367,18 +373,18 @@ WarpX::computePhi (const ablastr::fields::MultiLevelScalarField& rho, e_field.push_back( # if defined(WARPX_DIM_1D_Z) amrex::Array{ - getFieldPointer(FieldType::Efield_fp, lev, 2) + m_fields.get("Efield_fp",Direction{2},lev) } # elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) amrex::Array{ - getFieldPointer(FieldType::Efield_fp, lev, 0), - getFieldPointer(FieldType::Efield_fp, lev, 2) + m_fields.get("Efield_fp",Direction{0},lev), + m_fields.get("Efield_fp",Direction{2},lev) } # elif defined(WARPX_DIM_3D) amrex::Array{ - getFieldPointer(FieldType::Efield_fp, lev, 0), - getFieldPointer(FieldType::Efield_fp, lev, 1), - getFieldPointer(FieldType::Efield_fp, lev, 2) + m_fields.get("Efield_fp",Direction{0},lev), + m_fields.get("Efield_fp",Direction{1},lev), + m_fields.get("Efield_fp",Direction{2},lev) } # endif ); @@ -516,10 +522,13 @@ WarpX::setPhiBC ( ablastr::fields::MultiLevelScalarField& phi ) const \param[in] beta Represents the velocity of the source of `phi` */ void -WarpX::computeE (amrex::Vector, 3> >& E, +WarpX::computeE (ablastr::fields::MultiLevelVectorField& E, const ablastr::fields::MultiLevelScalarField& phi, + const std::vector& phi, std::array const beta ) const { + using ablastr::fields::Direction; + for (int lev = 0; lev <= max_level; lev++) { const Real* dx = Geom(lev).CellSize(); @@ -539,18 +548,18 @@ WarpX::computeE (amrex::Vector, 3> > #else const Real inv_dz = 1._rt/dx[0]; #endif - const amrex::IntVect ex_type = E[lev][0]->ixType().toIntVect(); - const amrex::IntVect ey_type = E[lev][1]->ixType().toIntVect(); - const amrex::IntVect ez_type = E[lev][2]->ixType().toIntVect(); + const amrex::IntVect ex_type = E[lev][Direction{0}]->ixType().toIntVect(); + const amrex::IntVect ey_type = E[lev][Direction{1}]->ixType().toIntVect(); + const amrex::IntVect ez_type = E[lev][Direction{2}]->ixType().toIntVect(); const amrex::Box& tbx = mfi.tilebox(ex_type); const amrex::Box& tby = mfi.tilebox(ey_type); const amrex::Box& tbz = mfi.tilebox(ez_type); const auto& phi_arr = phi[lev]->array(mfi); - const auto& Ex_arr = (*E[lev][0])[mfi].array(); - const auto& Ey_arr = (*E[lev][1])[mfi].array(); - const auto& Ez_arr = (*E[lev][2])[mfi].array(); + const auto& Ex_arr = (*E[lev][Direction{0}])[mfi].array(); + const auto& Ey_arr = (*E[lev][Direction{1}])[mfi].array(); + const auto& Ez_arr = (*E[lev][Direction{2}])[mfi].array(); const Real beta_x = beta[0]; const Real beta_y = beta[1]; diff --git a/Source/WarpX.H b/Source/WarpX.H index 6bc488ce561..9967ed87f47 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1011,12 +1011,17 @@ public: amrex::Real required_precision=amrex::Real(1.e-11), amrex::Real absolute_tolerance=amrex::Real(0.0), int max_iters=200, - int verbosity=2) const; + int verbosity=2); void setPhiBC (ablastr::fields::MultiLevelScalarField& phi ) const; +<<<<<<< HEAD void computeE (amrex::Vector, 3> >& E, const ablastr::fields::MultiLevelScalarField& phi, +======= + void computeE (ablastr::fields::MultiLevelVectorField& E, + const std::vector& phi, +>>>>>>> 52d85c86c (Efield_fp refactor ElectrotaticSolver) std::array beta = {{0,0,0}} ) const; void computeB (amrex::Vector, 3> >& B, const ablastr::fields::MultiLevelScalarField& phi, From 549ea650f78302f49d1e3315de2c27b85fad72d8 Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Tue, 10 Sep 2024 20:36:11 -0700 Subject: [PATCH 069/314] fixed merge issue. --- Source/FieldSolver/ElectrostaticSolver.cpp | 1 - Source/WarpX.H | 7 +------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index e260380e15c..b53e9cdd67f 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -524,7 +524,6 @@ WarpX::setPhiBC ( ablastr::fields::MultiLevelScalarField& phi ) const void WarpX::computeE (ablastr::fields::MultiLevelVectorField& E, const ablastr::fields::MultiLevelScalarField& phi, - const std::vector& phi, std::array const beta ) const { using ablastr::fields::Direction; diff --git a/Source/WarpX.H b/Source/WarpX.H index 9967ed87f47..b0f866d8943 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1015,13 +1015,8 @@ public: void setPhiBC (ablastr::fields::MultiLevelScalarField& phi ) const; -<<<<<<< HEAD - void computeE (amrex::Vector, 3> >& E, - const ablastr::fields::MultiLevelScalarField& phi, -======= void computeE (ablastr::fields::MultiLevelVectorField& E, - const std::vector& phi, ->>>>>>> 52d85c86c (Efield_fp refactor ElectrotaticSolver) + const ablastr::fields::MultiLevelScalarField& phi, std::array beta = {{0,0,0}} ) const; void computeB (amrex::Vector, 3> >& B, const ablastr::fields::MultiLevelScalarField& phi, From b83fb575983ed0804ce8b5a772a336f11e3afe9c Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Tue, 10 Sep 2024 20:54:21 -0700 Subject: [PATCH 070/314] Efield_fp refeactor HybridPICModel and WarpXInitData --- .../HybridPICModel/HybridPICModel.cpp | 7 ++++--- Source/Initialization/WarpXInitData.cpp | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 8005cb28240..dc536a05a86 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -153,6 +153,7 @@ void HybridPICModel::InitData () } auto & warpx = WarpX::GetInstance(); + using ablastr::fields::Direction; // Get the grid staggering of the fields involved in calculating E amrex::IntVect Jx_stag = warpx.getField(FieldType::current_fp, 0,0).ixType().toIntVect(); @@ -161,9 +162,9 @@ void HybridPICModel::InitData () amrex::IntVect Bx_stag = warpx.getField(FieldType::Bfield_fp, 0,0).ixType().toIntVect(); amrex::IntVect By_stag = warpx.getField(FieldType::Bfield_fp, 0,1).ixType().toIntVect(); amrex::IntVect Bz_stag = warpx.getField(FieldType::Bfield_fp, 0,2).ixType().toIntVect(); - amrex::IntVect Ex_stag = warpx.getField(FieldType::Efield_fp, 0,0).ixType().toIntVect(); - amrex::IntVect Ey_stag = warpx.getField(FieldType::Efield_fp, 0,1).ixType().toIntVect(); - amrex::IntVect Ez_stag = warpx.getField(FieldType::Efield_fp, 0,2).ixType().toIntVect(); + amrex::IntVect Ex_stag = warpx.m_fields.get("Efield_fp",Direction{0},0)->ixType().toIntVect(); + amrex::IntVect Ey_stag = warpx.m_fields.get("Efield_fp",Direction{1},0)->ixType().toIntVect(); + amrex::IntVect Ez_stag = warpx.m_fields.get("Efield_fp",Direction{2},0)->ixType().toIntVect(); // Check that the grid types are appropriate const bool appropriate_grids = ( diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 36dbda77c64..041a4c2f6fd 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -503,6 +503,7 @@ WarpX::InitData () { WARPX_PROFILE("WarpX::InitData()"); ablastr::parallelization::check_mpi_thread_level(); + using ablastr::fields::Direction; #ifdef WARPX_QED Print() << "PICSAR (" << WarpX::PicsarVersion() << ")\n"; @@ -553,9 +554,9 @@ WarpX::InitData () const int lev_zero = 0; m_macroscopic_properties->InitData( Geom(lev_zero), - getField(warpx::fields::FieldType::Efield_fp, lev_zero,0).ixType().toIntVect(), - getField(warpx::fields::FieldType::Efield_fp, lev_zero,1).ixType().toIntVect(), - getField(warpx::fields::FieldType::Efield_fp, lev_zero,2).ixType().toIntVect() + m_fields.get("Efield_fp",Direction{0},lev_zero)->ixType().toIntVect(), + m_fields.get("Efield_fp",Direction{1},lev_zero)->ixType().toIntVect(), + m_fields.get("Efield_fp",Direction{2},lev_zero)->ixType().toIntVect() ); } From abf97555ba82cbdf6fd13a3b1d79affc92de2415 Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Wed, 11 Sep 2024 09:40:10 -0700 Subject: [PATCH 071/314] Efield_fp refactor for WarpXComm.cpp --- Source/Parallelization/WarpXComm.cpp | 53 ++++++++++++++++------------ 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index ee7bef8d640..5bb6db61b8d 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -87,11 +87,12 @@ WarpX::UpdateAuxilaryDataStagToNodal () "WarpX build with spectral solver support."); } #endif + using ablastr::fields::Direction; amrex::Vector,3>> const & Bmf = WarpX::fft_do_time_averaging ? Bfield_avg_fp : Bfield_fp; amrex::Vector,3>> const & Emf = WarpX::fft_do_time_averaging ? - Efield_avg_fp : Efield_fp; + Efield_avg_fp : Efield_fp; // JRA, do this one when doing Efield_avg_fp refactor to new style const amrex::IntVect& Bx_stag = Bmf[0][0]->ixType().toIntVect(); const amrex::IntVect& By_stag = Bmf[0][1]->ixType().toIntVect(); @@ -295,9 +296,9 @@ WarpX::UpdateAuxilaryDataStagToNodal () const amrex::IntVect& refinement_ratio = refRatio(lev-1); - const amrex::IntVect& Ex_fp_stag = Efield_fp[lev][0]->ixType().toIntVect(); - const amrex::IntVect& Ey_fp_stag = Efield_fp[lev][1]->ixType().toIntVect(); - const amrex::IntVect& Ez_fp_stag = Efield_fp[lev][2]->ixType().toIntVect(); + const amrex::IntVect& Ex_fp_stag = m_fields.get("Efield_fp",Direction{0},lev)->ixType().toIntVect(); + const amrex::IntVect& Ey_fp_stag = m_fields.get("Efield_fp",Direction{1},lev)->ixType().toIntVect(); + const amrex::IntVect& Ez_fp_stag = m_fields.get("Efield_fp",Direction{2},lev)->ixType().toIntVect(); const amrex::IntVect& Ex_cp_stag = Efield_cp[lev][0]->ixType().toIntVect(); const amrex::IntVect& Ey_cp_stag = Efield_cp[lev][1]->ixType().toIntVect(); @@ -311,9 +312,9 @@ WarpX::UpdateAuxilaryDataStagToNodal () Array4 const& ex_aux = Efield_aux[lev][0]->array(mfi); Array4 const& ey_aux = Efield_aux[lev][1]->array(mfi); Array4 const& ez_aux = Efield_aux[lev][2]->array(mfi); - Array4 const& ex_fp = Efield_fp[lev][0]->const_array(mfi); - Array4 const& ey_fp = Efield_fp[lev][1]->const_array(mfi); - Array4 const& ez_fp = Efield_fp[lev][2]->const_array(mfi); + Array4 const& ex_fp = m_fields.get("Efield_fp",Direction{0},lev)->const_array(mfi); + Array4 const& ey_fp = m_fields.get("Efield_fp",Direction{1},lev)->const_array(mfi); + Array4 const& ez_fp = m_fields.get("Efield_fp",Direction{2},lev)->const_array(mfi); Array4 const& ex_cp = Efield_cp[lev][0]->const_array(mfi); Array4 const& ey_cp = Efield_cp[lev][1]->const_array(mfi); Array4 const& ez_cp = Efield_cp[lev][2]->const_array(mfi); @@ -332,9 +333,9 @@ WarpX::UpdateAuxilaryDataStagToNodal () } } else { // electrostatic - const amrex::IntVect& Ex_fp_stag = Efield_fp[lev][0]->ixType().toIntVect(); - const amrex::IntVect& Ey_fp_stag = Efield_fp[lev][1]->ixType().toIntVect(); - const amrex::IntVect& Ez_fp_stag = Efield_fp[lev][2]->ixType().toIntVect(); + const amrex::IntVect& Ex_fp_stag = m_fields.get("Efield_fp",Direction{0},lev)->ixType().toIntVect(); + const amrex::IntVect& Ey_fp_stag = m_fields.get("Efield_fp",Direction{1},lev)->ixType().toIntVect(); + const amrex::IntVect& Ez_fp_stag = m_fields.get("Efield_fp",Direction{2},lev)->ixType().toIntVect(); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif @@ -343,9 +344,9 @@ WarpX::UpdateAuxilaryDataStagToNodal () Array4 const& ex_aux = Efield_aux[lev][0]->array(mfi); Array4 const& ey_aux = Efield_aux[lev][1]->array(mfi); Array4 const& ez_aux = Efield_aux[lev][2]->array(mfi); - Array4 const& ex_fp = Efield_fp[lev][0]->const_array(mfi); - Array4 const& ey_fp = Efield_fp[lev][1]->const_array(mfi); - Array4 const& ez_fp = Efield_fp[lev][2]->const_array(mfi); + Array4 const& ex_fp = m_fields.get("Efield_fp",Direction{0},lev)->const_array(mfi); + Array4 const& ey_fp = m_fields.get("Efield_fp",Direction{1},lev)->const_array(mfi); + Array4 const& ez_fp = m_fields.get("Efield_fp",Direction{2},lev)->const_array(mfi); const Box& bx = mfi.growntilebox(); amrex::ParallelFor(bx, @@ -366,6 +367,8 @@ WarpX::UpdateAuxilaryDataSameType () { // Update aux field, including guard cells, up to ng_FieldGather const amrex::IntVect& ng_src = guard_cells.ng_FieldGather; + + using ablastr::fields::Direction; // Level 0: Copy from fine to aux // Note: in some configurations, Efield_aux/Bfield_aux and Efield_fp/Bfield_fp are simply aliases to the @@ -381,9 +384,9 @@ WarpX::UpdateAuxilaryDataSameType () } else { - MultiFab::Copy(*Efield_aux[0][0], *Efield_fp[0][0], 0, 0, Efield_aux[0][0]->nComp(), ng_src); - MultiFab::Copy(*Efield_aux[0][1], *Efield_fp[0][1], 0, 0, Efield_aux[0][1]->nComp(), ng_src); - MultiFab::Copy(*Efield_aux[0][2], *Efield_fp[0][2], 0, 0, Efield_aux[0][2]->nComp(), ng_src); + MultiFab::Copy(*Efield_aux[0][0], *m_fields.get("Efield_fp",Direction{0},0), 0, 0, Efield_aux[0][0]->nComp(), ng_src); + MultiFab::Copy(*Efield_aux[0][1], *m_fields.get("Efield_fp",Direction{1},0), 0, 0, Efield_aux[0][1]->nComp(), ng_src); + MultiFab::Copy(*Efield_aux[0][2], *m_fields.get("Efield_fp",Direction{2},0), 0, 0, Efield_aux[0][2]->nComp(), ng_src); MultiFab::Copy(*Bfield_aux[0][0], *Bfield_fp[0][0], 0, 0, Bfield_aux[0][0]->nComp(), ng_src); MultiFab::Copy(*Bfield_aux[0][1], *Bfield_fp[0][1], 0, 0, Bfield_aux[0][1]->nComp(), ng_src); MultiFab::Copy(*Bfield_aux[0][2], *Bfield_fp[0][2], 0, 0, Bfield_aux[0][2]->nComp(), ng_src); @@ -521,9 +524,9 @@ WarpX::UpdateAuxilaryDataSameType () Array4 const& ex_aux = Efield_aux[lev][0]->array(mfi); Array4 const& ey_aux = Efield_aux[lev][1]->array(mfi); Array4 const& ez_aux = Efield_aux[lev][2]->array(mfi); - Array4 const& ex_fp = Efield_fp[lev][0]->const_array(mfi); - Array4 const& ey_fp = Efield_fp[lev][1]->const_array(mfi); - Array4 const& ez_fp = Efield_fp[lev][2]->const_array(mfi); + Array4 const& ex_fp = m_fields.get("Efield_fp",Direction{0},lev)->const_array(mfi); + Array4 const& ey_fp = m_fields.get("Efield_fp",Direction{1},lev)->const_array(mfi); + Array4 const& ez_fp = m_fields.get("Efield_fp",Direction{2},lev)->const_array(mfi); Array4 const& ex_c = dEx.const_array(mfi); Array4 const& ey_c = dEy.const_array(mfi); Array4 const& ez_c = dEz.const_array(mfi); @@ -545,9 +548,9 @@ WarpX::UpdateAuxilaryDataSameType () } else // electrostatic { - MultiFab::Copy(*Efield_aux[lev][0], *Efield_fp[lev][0], 0, 0, Efield_aux[lev][0]->nComp(), Efield_aux[lev][0]->nGrowVect()); - MultiFab::Copy(*Efield_aux[lev][1], *Efield_fp[lev][1], 0, 0, Efield_aux[lev][1]->nComp(), Efield_aux[lev][1]->nGrowVect()); - MultiFab::Copy(*Efield_aux[lev][2], *Efield_fp[lev][2], 0, 0, Efield_aux[lev][2]->nComp(), Efield_aux[lev][2]->nGrowVect()); + MultiFab::Copy(*Efield_aux[lev][0], *m_fields.get("Efield_fp",Direction{0},lev), 0, 0, Efield_aux[lev][0]->nComp(), Efield_aux[lev][0]->nGrowVect()); + MultiFab::Copy(*Efield_aux[lev][1], *m_fields.get("Efield_fp",Direction{1},lev), 0, 0, Efield_aux[lev][1]->nComp(), Efield_aux[lev][1]->nGrowVect()); + MultiFab::Copy(*Efield_aux[lev][2], *m_fields.get("Efield_fp",Direction{2},lev), 0, 0, Efield_aux[lev][2]->nComp(), Efield_aux[lev][2]->nGrowVect()); } } } @@ -667,10 +670,14 @@ WarpX::FillBoundaryE (const int lev, const PatchType patch_type, const amrex::In { std::array mf; amrex::Periodicity period; + + using ablastr::fields::Direction; if (patch_type == PatchType::fine) { - mf = {Efield_fp[lev][0].get(), Efield_fp[lev][1].get(), Efield_fp[lev][2].get()}; + mf = {m_fields.get("Efield_fp",Direction{0},lev), + m_fields.get("Efield_fp",Direction{1},lev), + m_fields.get("Efield_fp",Direction{2},lev)}; period = Geom(lev).periodicity(); } else // coarse patch From b312e22c06d5f087a9712751c0a8ee256ee838e2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 11 Sep 2024 16:43:39 +0000 Subject: [PATCH 072/314] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- Source/FieldSolver/ElectrostaticSolver.cpp | 2 +- Source/Parallelization/WarpXComm.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index b53e9cdd67f..9aea1e29e03 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -349,7 +349,7 @@ WarpX::computePhi (const ablastr::fields::MultiLevelScalarField& rho, sorted_rho.emplace_back(rho[lev]); sorted_phi.emplace_back(phi[lev]); } - + using ablastr::fields::Direction; std::optional post_phi_calculation; diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 5bb6db61b8d..d3d0558ac55 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -367,7 +367,7 @@ WarpX::UpdateAuxilaryDataSameType () { // Update aux field, including guard cells, up to ng_FieldGather const amrex::IntVect& ng_src = guard_cells.ng_FieldGather; - + using ablastr::fields::Direction; // Level 0: Copy from fine to aux @@ -670,7 +670,7 @@ WarpX::FillBoundaryE (const int lev, const PatchType patch_type, const amrex::In { std::array mf; amrex::Periodicity period; - + using ablastr::fields::Direction; if (patch_type == PatchType::fine) From d87898d4c7b70a5249e191a9c88df046debc5069 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 10:03:30 -0700 Subject: [PATCH 073/314] Start replacing calls to SyncCurrent --- Source/Evolve/WarpXEvolve.cpp | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 4f84c6b275a..e4a1e2b3882 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -551,17 +551,15 @@ void WarpX::SyncCurrentAndRho () { // With periodic single box, synchronize J and rho here, // even with current correction or Vay deposition - if (current_deposition_algo == CurrentDepositionAlgo::Vay) - { - // TODO Replace current_cp with current_cp_vay once Vay deposition is implemented with MR - SyncCurrent(va2vm(current_fp_vay), va2vm(current_cp), va2vm(current_buf)); - SyncRho(); - } - else - { - SyncCurrent(va2vm(current_fp), va2vm(current_cp), va2vm(current_buf)); - SyncRho(); - } + std::string const current_fp_string = (current_deposition_algo == CurrentDepositionAlgo::Vay) + ? "current_fp_vay" : "current_fp"; + // TODO Replace current_cp with current_cp_vay once Vay deposition is implemented with MR + + SyncCurrent( m_fields.get_mr_levels_alldirs(current_fp_string, finest_level), + m_fields.get_mr_levels_alldirs("current_cp", finest_level), + m_fields.get_mr_levels_alldirs("current_buf", finest_level) ); + SyncRho(); + } else // no periodic single box { From bd64eea726a386a777a1b80828bffe62f99a26c9 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 11 Sep 2024 09:53:18 -0700 Subject: [PATCH 074/314] `MultiFabRegister::alias_init`: `dir` --- Source/ablastr/fields/MultiFabRegister.H | 18 +++++++++ Source/ablastr/fields/MultiFabRegister.cpp | 47 ++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 5c240e96d2f..e2b2f0e6024 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -171,6 +171,24 @@ namespace ablastr::fields std::optional initial_value = std::nullopt ); + /** Create an alias + * + * @param new_name new name + * @param alias_name owner to alias to + * @param dir ... + * @param level ... + * @param initial_value ... + * @return + */ + amrex::MultiFab* + alias_init ( + std::string new_name, + std::string alias_name, + Direction dir, + int level, + std::optional initial_value = std::nullopt + ); + /** title * * body body diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 87191218f51..01c90d29c30 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -171,6 +171,53 @@ namespace ablastr::fields return &mf; } + amrex::MultiFab* + MultiFabRegister::alias_init ( + std::string new_name, + std::string alias_name, + Direction dir, + int level, + std::optional initial_value + ) + { + new_name = mf_name(new_name, dir, level); + alias_name = mf_name(alias_name, dir, level); + + // Checks + // TODO: does the key already exist? error + + MultiFabOwner & alias = m_mf_register[alias_name]; + amrex::MultiFab & mf_alias = alias.m_mf; + + // allocate + auto [it, success] = m_mf_register.emplace( + std::make_pair( + new_name, + MultiFabOwner{ + {mf_alias, amrex::make_alias, 0, mf_alias.nComp()}, + dir, + level, + alias.m_redistribute, + alias.m_redistribute_on_remake, + alias_name + } + ) + ); + if (!success) { + throw std::runtime_error("MultiFabRegister::alias_init failed for " + new_name); + } + + // a short-hand alias for the code below + amrex::MultiFab & mf = it->second.m_mf; + + // initialize with value + if (initial_value) { + mf.setVal(*initial_value); + } + + return &mf; + } + void MultiFabRegister::remake_level ( int level, From 607ad9013b64b362671525c4f580f4111201c3e6 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 11 Sep 2024 10:14:53 -0700 Subject: [PATCH 075/314] `MultiFabRegister::get_alldirs(name, lvl) -> VectorField` --- Source/ablastr/fields/MultiFabRegister.H | 14 ++++++++++++++ Source/ablastr/fields/MultiFabRegister.cpp | 20 ++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index e2b2f0e6024..d313e92c56c 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -281,6 +281,20 @@ namespace ablastr::fields int finest_level ); + /** title + * + * Same as get above, but returns all levels for a name. + * + * @param name ... + * @param level ... + * @return ... + */ + VectorField + get_alldirs ( + std::string name, + int level + ); + /** Return a vector field on all MR levels. * * Out loop: MR levels. diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 01c90d29c30..96f737343ad 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -332,6 +332,26 @@ namespace ablastr::fields return field_on_level; } + VectorField + MultiFabRegister::get_alldirs ( + std::string name, + int level + ) + { + // TODO: Technically, we should search field_on_level via std::unique_copy + std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; + + // insert a new level + VectorField vectorField; + + // insert components + for (Direction dir : all_dirs) + { + vectorField[dir] = get(name, dir, level); + } + return vectorField; + } + MultiLevelVectorField MultiFabRegister::get_mr_levels_alldirs ( std::string name, From fa5480436eaf8195de36a19ade4415fd2e17d61d Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 10:30:04 -0700 Subject: [PATCH 076/314] Update SyncCurrent code --- Source/Evolve/WarpXEvolve.cpp | 33 ++++++++++++++---------- Source/FieldSolver/WarpXPushFieldsEM.cpp | 4 +-- Source/WarpX.H | 4 +-- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index e4a1e2b3882..7feacd6e510 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -544,6 +544,7 @@ void WarpX::HandleParticlesAtBoundaries (int step, amrex::Real cur_time, int num void WarpX::SyncCurrentAndRho () { using ablastr::fields::va2vm; + using ablastr::fields::Direction; if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { @@ -555,9 +556,9 @@ void WarpX::SyncCurrentAndRho () ? "current_fp_vay" : "current_fp"; // TODO Replace current_cp with current_cp_vay once Vay deposition is implemented with MR - SyncCurrent( m_fields.get_mr_levels_alldirs(current_fp_string, finest_level), - m_fields.get_mr_levels_alldirs("current_cp", finest_level), - m_fields.get_mr_levels_alldirs("current_buf", finest_level) ); + SyncCurrent(m_fields.get_mr_levels_alldirs(current_fp_string, finest_level), + m_fields.get_mr_levels_alldirs("current_cp", finest_level), + m_fields.get_mr_levels_alldirs("current_buf", finest_level) ); SyncRho(); } @@ -569,7 +570,9 @@ void WarpX::SyncCurrentAndRho () if (!current_correction && current_deposition_algo != CurrentDepositionAlgo::Vay) { - SyncCurrent(va2vm(current_fp), va2vm(current_cp), va2vm(current_buf)); + SyncCurrent(m_fields.get_mr_levels_alldirs("current_fp", finest_level), + m_fields.get_mr_levels_alldirs("current_cp", finest_level), + m_fields.get_mr_levels_alldirs("current_buf", finest_level) ); SyncRho(); } @@ -583,7 +586,9 @@ void WarpX::SyncCurrentAndRho () } else // FDTD { - SyncCurrent(va2vm(current_fp), va2vm(current_cp), va2vm(current_buf)); + SyncCurrent(m_fields.get_mr_levels_alldirs("current_fp", finest_level), + m_fields.get_mr_levels_alldirs("current_cp", finest_level), + m_fields.get_mr_levels_alldirs("current_buf", finest_level) ); SyncRho(); } @@ -593,18 +598,20 @@ void WarpX::SyncCurrentAndRho () if (m_fields.has("rho_fp", lev)) { ApplyRhofieldBoundary(lev, m_fields.get("rho_fp",lev), PatchType::fine); } - ApplyJfieldBoundary( - lev, current_fp[lev][0].get(), current_fp[lev][1].get(), - current_fp[lev][2].get(), PatchType::fine - ); + ApplyJfieldBoundary(lev, + m_fields.get("current_fp",Direction{0},lev), + m_fields.get("current_fp",Direction{1},lev), + m_fields.get("current_fp",Direction{2},lev), + PatchType::fine); if (lev > 0) { if (m_fields.has("rho_cp", lev)) { ApplyRhofieldBoundary(lev, m_fields.get("rho_cp",lev), PatchType::coarse); } - ApplyJfieldBoundary( - lev, current_cp[lev][0].get(), current_cp[lev][1].get(), - current_cp[lev][2].get(), PatchType::coarse - ); + ApplyJfieldBoundary(lev, + m_fields.get("current_cp",Direction{0},lev), + m_fields.get("current_cp",Direction{1},lev), + m_fields.get("current_cp",Direction{2},lev), + PatchType::coarse); } } } diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 6b40947070b..3c518fa584d 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -341,8 +341,8 @@ void WarpX::PSATDForwardTransformJ ( } void WarpX::PSATDBackwardTransformJ ( - const amrex::Vector,3>>& J_fp, - const amrex::Vector,3>>& J_cp) + ablastr::fields::MultiLevelVectorField const & J_fp, + ablastr::fields::MultiLevelVectorField const & J_cp) { SpectralFieldIndex Idx; int idx_jx, idx_jy, idx_jz; diff --git a/Source/WarpX.H b/Source/WarpX.H index 3fd0772f369..3c35d0237e2 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1893,8 +1893,8 @@ private: * storing the coarse patch current to be transformed */ void PSATDBackwardTransformJ ( - const amrex::Vector,3>>& J_fp, - const amrex::Vector,3>>& J_cp); + ablastr::fields::MultiLevelVectorField const & J_fp, + ablastr::fields::MultiLevelVectorField const & J_cp); /** * \brief Forward FFT of rho on all mesh refinement levels, From c778237ad9e6f8edc6baab831e41f05be5cded68 Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Wed, 11 Sep 2024 00:55:35 +0200 Subject: [PATCH 077/314] initial work to add face_area to the new field register --- .../EmbeddedBoundary/WarpXFaceExtensions.cpp | 2 +- Source/WarpX.H | 8 ++--- Source/WarpX.cpp | 33 ++++++++++--------- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp b/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp index 717aa26b021..2a3f48fcf2e 100644 --- a/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp +++ b/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp @@ -456,7 +456,7 @@ WarpX::ComputeOneWayExtensions () for (amrex::MFIter mfi(*Bfield_fp[maxLevel()][idim]); mfi.isValid(); ++mfi) { amrex::Box const &box = mfi.validbox(); - + m_fields.get("face_areas", maxLevel()); auto const &S = m_face_areas[maxLevel()][idim]->array(mfi); auto const &flag_ext_face = m_flag_ext_face[maxLevel()][idim]->array(mfi); auto const &flag_info_face = m_flag_info_face[maxLevel()][idim]->array(mfi); diff --git a/Source/WarpX.H b/Source/WarpX.H index 74f3d0a5ab6..5244125c4d9 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1064,7 +1064,7 @@ public: amrex::ParserExecutor<3> const& yfield_parser, amrex::ParserExecutor<3> const& zfield_parser, std::array< std::unique_ptr, 3 > const& edge_lengths, - std::array< std::unique_ptr, 3 > const& face_areas, + std::array const& face_areas, [[maybe_unused]] char field, int lev, PatchType patch_type); @@ -1150,7 +1150,7 @@ public: * \brief Compute the area of the mesh faces. Here the area is a value in [0, 1]. * An edge of area 0 is fully covered. */ - static void ComputeFaceAreas (std::array< std::unique_ptr, 3 >& face_areas, + static void ComputeFaceAreas (std::array& face_areas, const amrex::EBFArrayBoxFactory& eb_fact); /** @@ -1161,7 +1161,7 @@ public: /** * \brief Scale the edges areas by the mesh width to obtain the real areas. */ - static void ScaleAreas (std::array< std::unique_ptr, 3 >& face_areas, + static void ScaleAreas (std::array& face_areas, const std::array& cell_size); /** * \brief Initialize information for cell extensions. @@ -1544,8 +1544,6 @@ private: //! EB: Lengths of the mesh edges amrex::Vector, 3 > > m_edge_lengths; - //! EB: Areas of the mesh faces - amrex::Vector, 3 > > m_face_areas; /** EB: for every mesh face flag_info_face contains a: * * 0 if the face needs to be extended diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index e016bd13536..c27e5e8419c 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -348,7 +348,6 @@ WarpX::WarpX () E_external_particle_field.resize(1); m_edge_lengths.resize(nlevs_max); - m_face_areas.resize(nlevs_max); m_distance_to_eb.resize(nlevs_max); m_flag_info_face.resize(nlevs_max); m_flag_ext_face.resize(nlevs_max); @@ -2397,12 +2396,14 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm guard_cells.ng_FieldSolver, lev, "m_edge_lengths[y]"); AllocInitMultiFab(m_edge_lengths[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_edge_lengths[z]"); - AllocInitMultiFab(m_face_areas[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, - guard_cells.ng_FieldSolver, lev, "m_face_areas[x]"); - AllocInitMultiFab(m_face_areas[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, - guard_cells.ng_FieldSolver, lev, "m_face_areas[y]"); - AllocInitMultiFab(m_face_areas[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, - guard_cells.ng_FieldSolver, lev, "m_face_areas[z]"); + + //! EB: Areas of the mesh faces + m_fields.alloc_init( "face_areas", Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + m_fields.alloc_init( "face_areas", Direction{1}, lev, amrex::convert(ba, By_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + m_fields.alloc_init( "face_areas", Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); } if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { AllocInitMultiFab(m_edge_lengths[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, @@ -2411,12 +2412,15 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm guard_cells.ng_FieldSolver, lev, "m_edge_lengths[y]"); AllocInitMultiFab(m_edge_lengths[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_edge_lengths[z]"); - AllocInitMultiFab(m_face_areas[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, - guard_cells.ng_FieldSolver, lev, "m_face_areas[x]"); - AllocInitMultiFab(m_face_areas[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, - guard_cells.ng_FieldSolver, lev, "m_face_areas[y]"); - AllocInitMultiFab(m_face_areas[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, - guard_cells.ng_FieldSolver, lev, "m_face_areas[z]"); + + //! EB: Areas of the mesh faces + m_fields.alloc_init( "face_areas", Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + m_fields.alloc_init( "face_areas", Direction{1}, lev, amrex::convert(ba, By_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + m_fields.alloc_init( "face_areas", Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + AllocInitMultiFab(m_flag_info_face[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_flag_info_face[x]"); AllocInitMultiFab(m_flag_info_face[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, @@ -3506,9 +3510,6 @@ WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, cons case FieldType::edge_lengths : field_pointer = m_edge_lengths[lev][direction].get(); break; - case FieldType::face_areas : - field_pointer = m_face_areas[lev][direction].get(); - break; case FieldType::Efield_avg_fp : field_pointer = Efield_avg_fp[lev][direction].get(); break; From b4ef2fe41804ffd11174efba4e5438ed4f9a16a1 Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Wed, 11 Sep 2024 01:08:30 +0200 Subject: [PATCH 078/314] continue adding face_areas to field register --- .../inputs_test_3d_embedded_boundary_picmi.py | 2 +- Python/pywarpx/fields.py | 6 +++--- Source/EmbeddedBoundary/WarpXFaceExtensions.cpp | 13 ++++++------- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/Examples/Tests/embedded_boundary_python_api/inputs_test_3d_embedded_boundary_picmi.py b/Examples/Tests/embedded_boundary_python_api/inputs_test_3d_embedded_boundary_picmi.py index 80ce483f2c7..1a42c6d9622 100755 --- a/Examples/Tests/embedded_boundary_python_api/inputs_test_3d_embedded_boundary_picmi.py +++ b/Examples/Tests/embedded_boundary_python_api/inputs_test_3d_embedded_boundary_picmi.py @@ -159,7 +159,7 @@ print("Perimeter of the middle z-slice:", perimeter_slice_z) assert np.isclose(perimeter_slice_z, perimeter_slice_z_true, rtol=1e-05, atol=1e-08) -print("======== Testing the wrappers of m_face_areas =========") +print("======== Testing the wrappers of face_areas =========") Sx_slice = np.sum(face_areas_x[nx // 2, :, :]) dx = (xmax - xmin) / nx diff --git a/Python/pywarpx/fields.py b/Python/pywarpx/fields.py index cbdd8d4517a..88ae8149753 100644 --- a/Python/pywarpx/fields.py +++ b/Python/pywarpx/fields.py @@ -847,19 +847,19 @@ def EdgeLengthszWrapper(level=0, include_ghosts=False): def FaceAreasxWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="m_face_areas[x]", level=level, include_ghosts=include_ghosts + mf_name="face_areas[x]", level=level, include_ghosts=include_ghosts ) def FaceAreasyWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="m_face_areas[y]", level=level, include_ghosts=include_ghosts + mf_name="face_areas[y]", level=level, include_ghosts=include_ghosts ) def FaceAreaszWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="m_face_areas[z]", level=level, include_ghosts=include_ghosts + mf_name="face_areas[z]", level=level, include_ghosts=include_ghosts ) diff --git a/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp b/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp index 2a3f48fcf2e..ddc4021cb9d 100644 --- a/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp +++ b/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp @@ -456,8 +456,7 @@ WarpX::ComputeOneWayExtensions () for (amrex::MFIter mfi(*Bfield_fp[maxLevel()][idim]); mfi.isValid(); ++mfi) { amrex::Box const &box = mfi.validbox(); - m_fields.get("face_areas", maxLevel()); - auto const &S = m_face_areas[maxLevel()][idim]->array(mfi); + auto const &S = m_fields.get("face_areas", Direction{idim}, maxLevel())->array(mfi); auto const &flag_ext_face = m_flag_ext_face[maxLevel()][idim]->array(mfi); auto const &flag_info_face = m_flag_info_face[maxLevel()][idim]->array(mfi); auto &borrowing = (*m_borrowing[maxLevel()][idim])[mfi]; @@ -585,7 +584,7 @@ WarpX::ComputeEightWaysExtensions () amrex::Box const &box = mfi.validbox(); - auto const &S = m_face_areas[maxLevel()][idim]->array(mfi); + auto const &S = m_fields.get("face_areas", Direction{idim}, maxLevel())->array(mfi); auto const &flag_ext_face = m_flag_ext_face[maxLevel()][idim]->array(mfi); auto const &flag_info_face = m_flag_info_face[maxLevel()][idim]->array(mfi); auto &borrowing = (*m_borrowing[maxLevel()][idim])[mfi]; @@ -737,10 +736,10 @@ WarpX::ApplyBCKCorrection (const int idim) const amrex::Box &box = mfi.tilebox(); const amrex::Array4 &flag_ext_face = m_flag_ext_face[maxLevel()][idim]->array(mfi); const amrex::Array4 &flag_info_face = m_flag_info_face[maxLevel()][idim]->array(mfi); - const amrex::Array4 &S = m_face_areas[maxLevel()][idim]->array(mfi); - const amrex::Array4 &lx = m_face_areas[maxLevel()][0]->array(mfi); - const amrex::Array4 &ly = m_face_areas[maxLevel()][1]->array(mfi); - const amrex::Array4 &lz = m_face_areas[maxLevel()][2]->array(mfi); + const amrex::Array4 &S = m_fields.get("face_areas", Direction{idim}, maxLevel())->array(mfi); + const amrex::Array4 &lx = m_fields.get("face_areas", Direction{0}, maxLevel())->array(mfi);; + const amrex::Array4 &ly = m_fields.get("face_areas", Direction{1}, maxLevel())->array(mfi);; + const amrex::Array4 &lz = m_fields.get("face_areas", Direction{2}, maxLevel())->array(mfi);; amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k) { if (flag_ext_face(i, j, k)) { From 0fb5a13177044685b72557e47ea2abf5f8279789 Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Wed, 11 Sep 2024 01:54:44 +0200 Subject: [PATCH 079/314] continue adding face_areas to field register --- Source/EmbeddedBoundary/WarpXInitEB.cpp | 10 ++++++---- Source/FieldSolver/WarpXPushFieldsEM.cpp | 13 +++++++------ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index b3e6290ad6c..00cf16c5097 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -332,11 +332,13 @@ WarpX::MarkCells(){ "MarkCells: Only implemented in 2D3V and 3D3V"); #endif for (amrex::MFIter mfi(*Bfield_fp[maxLevel()][idim]); mfi.isValid(); ++mfi) { - //amrex::Box const &box = mfi.tilebox(m_face_areas[maxLevel()][idim]->ixType().toIntVect()); - const amrex::Box& box = mfi.tilebox(m_face_areas[maxLevel()][idim]->ixType().toIntVect(), - m_face_areas[maxLevel()][idim]->nGrowVect() ); + auto* face_areas_idim_max_lev = + m_fields.get("face_areas", Direction{idim}, maxLevel()); - auto const &S = m_face_areas[maxLevel()][idim]->array(mfi); + const amrex::Box& box = mfi.tilebox(face_areas_idim_max_lev->ixType().toIntVect(), + face_areas_idim_max_lev->nGrowVect() ); + + auto const &S = face_areas_idim_max_lev->array(mfi); auto const &flag_info_face = m_flag_info_face[maxLevel()][idim]->array(mfi); auto const &flag_ext_face = m_flag_ext_face[maxLevel()][idim]->array(mfi); const auto &lx = m_edge_lengths[maxLevel()][0]->array(mfi); diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index fe060285797..09d39129bde 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -830,18 +830,19 @@ WarpX::EvolveB (int lev, amrex::Real a_dt, DtType a_dt_type) void WarpX::EvolveB (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_type) { + auto face_areas_lev = m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev]; // Evolve B field in regular cells if (patch_type == PatchType::fine) { m_fdtd_solver_fp[lev]->EvolveB(Bfield_fp[lev], Efield_fp[lev], - m_fields.get("G_fp", lev), - m_face_areas[lev], m_area_mod[lev], ECTRhofield[lev], Venl[lev], - m_flag_info_face[lev], m_borrowing[lev], lev, a_dt); + m_fields.get("G_fp", lev), + face_areas_lev, m_area_mod[lev], ECTRhofield[lev], Venl[lev], + m_flag_info_face[lev], m_borrowing[lev], lev, a_dt); } else { m_fdtd_solver_cp[lev]->EvolveB(Bfield_cp[lev], Efield_cp[lev], - m_fields.get("G_fp", lev), - m_face_areas[lev], m_area_mod[lev], ECTRhofield[lev], Venl[lev], - m_flag_info_face[lev], m_borrowing[lev], lev, a_dt); + m_fields.get("G_fp", lev), + face_areas_lev, m_area_mod[lev], ECTRhofield[lev], Venl[lev], + m_flag_info_face[lev], m_borrowing[lev], lev, a_dt); } // Evolve B field in PML cells From 88ac301c545a23640579aa640873bea2ea04fe23 Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Wed, 11 Sep 2024 18:43:06 +0200 Subject: [PATCH 080/314] continue work to use new field register for face_areas --- Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp | 2 +- .../FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp index 84d647eb387..6e419637e77 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp @@ -51,7 +51,7 @@ void FiniteDifferenceSolver::EvolveB ( [[maybe_unused]] std::array< std::unique_ptr, 3 >& Bfield, [[maybe_unused]] std::array< std::unique_ptr, 3 > const& Efield, [[maybe_unused]] amrex::MultiFab const * Gfield, - [[maybe_unused]] std::array< std::unique_ptr, 3 > const& face_areas, + [[maybe_unused]] ablastr::fields::VectorField const& face_areas, [[maybe_unused]] std::array< std::unique_ptr, 3 > const& area_mod, [[maybe_unused]] std::array< std::unique_ptr, 3 >& ECTRhofield, [[maybe_unused]] std::array< std::unique_ptr, 3 >& Venl, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 69d7630dd9e..9e7df1db1b3 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -55,7 +55,7 @@ class FiniteDifferenceSolver void EvolveB ( std::array< std::unique_ptr, 3 >& Bfield, std::array< std::unique_ptr, 3 > const& Efield, amrex::MultiFab const * Gfield, - std::array< std::unique_ptr, 3 > const& face_areas, + ablastr::fields::VectorField const& face_areas, std::array< std::unique_ptr, 3 > const& area_mod, std::array< std::unique_ptr, 3 >& ECTRhofield, std::array< std::unique_ptr, 3 >& Venl, From cabdb2879fa4f118c100fdccbf19fb852e06c2d3 Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Wed, 11 Sep 2024 18:52:31 +0200 Subject: [PATCH 081/314] continue work to use new field register for face_areas --- .../FiniteDifferenceSolver/EvolveECTRho.cpp | 11 ++++++----- .../FiniteDifferenceSolver/FiniteDifferenceSolver.H | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp index 8abdab71300..18960dee9a1 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp @@ -42,6 +42,7 @@ #include using namespace amrex; +using namespace ablastr::fields; /** * \brief Update the B field, over one timestep @@ -49,7 +50,7 @@ using namespace amrex; void FiniteDifferenceSolver::EvolveECTRho ( std::array< std::unique_ptr, 3 > const& Efield, std::array< std::unique_ptr, 3 > const& edge_lengths, - std::array< std::unique_ptr, 3 > const& face_areas, + ablastr::fields::VectorField const& face_areas, std::array< std::unique_ptr, 3 >& ECTRhofield, const int lev) { @@ -69,7 +70,7 @@ void FiniteDifferenceSolver::EvolveECTRho ( void FiniteDifferenceSolver::EvolveRhoCartesianECT ( std::array< std::unique_ptr, 3 > const& Efield, std::array< std::unique_ptr, 3 > const& edge_lengths, - std::array< std::unique_ptr, 3 > const& face_areas, + ablastr::fields::VectorField const& face_areas, std::array< std::unique_ptr, 3 >& ECTRhofield, const int lev ) { #ifdef AMREX_USE_EB @@ -100,9 +101,9 @@ void FiniteDifferenceSolver::EvolveRhoCartesianECT ( amrex::Array4 const &lx = edge_lengths[0]->array(mfi); amrex::Array4 const &ly = edge_lengths[1]->array(mfi); amrex::Array4 const &lz = edge_lengths[2]->array(mfi); - amrex::Array4 const &Sx = face_areas[0]->array(mfi); - amrex::Array4 const &Sy = face_areas[1]->array(mfi); - amrex::Array4 const &Sz = face_areas[2]->array(mfi); + amrex::Array4 const &Sx = face_areas[Direction{0}]->array(mfi); + amrex::Array4 const &Sy = face_areas[Direction{1}]->array(mfi); + amrex::Array4 const &Sz = face_areas[Direction{2}]->array(mfi); // Extract tileboxes for which to loop amrex::Box const &trhox = mfi.tilebox(ECTRhofield[0]->ixType().toIntVect()); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 9e7df1db1b3..1167ca7b30c 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -290,12 +290,12 @@ class FiniteDifferenceSolver void EvolveRhoCartesianECT ( std::array< std::unique_ptr, 3 > const& Efield, std::array< std::unique_ptr, 3 > const& edge_lengths, - std::array< std::unique_ptr, 3 > const& face_areas, + ablastr::fields::VectorField const& face_areas, std::array< std::unique_ptr, 3 >& ECTRhofield, int lev); void EvolveBCartesianECT ( std::array< std::unique_ptr, 3 >& Bfield, - std::array< std::unique_ptr, 3 > const& face_areas, + ablastr::fields::VectorField const& face_areas, std::array< std::unique_ptr, 3 > const& area_mod, std::array< std::unique_ptr, 3 >& ECTRhofield, std::array< std::unique_ptr, 3 >& Venl, From 89839a7c5835664abf4fbac5876257b075922ca7 Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Wed, 11 Sep 2024 19:46:16 +0200 Subject: [PATCH 082/314] complete using new field register for face_areas --- .../EmbeddedBoundary/WarpXFaceExtensions.cpp | 3 ++ Source/EmbeddedBoundary/WarpXInitEB.cpp | 6 ++-- .../FiniteDifferenceSolver/EvolveB.cpp | 2 +- .../FiniteDifferenceSolver/EvolveE.cpp | 5 +++- .../FiniteDifferenceSolver/EvolveECTRho.cpp | 6 ++-- .../FiniteDifferenceSolver.H | 4 +-- .../ImplicitSolvers/WarpXImplicitOps.cpp | 5 ++-- Source/FieldSolver/WarpXPushFieldsEM.cpp | 9 +++--- Source/Initialization/WarpXInitData.cpp | 30 +++++++++++-------- Source/WarpX.H | 4 +-- 10 files changed, 44 insertions(+), 30 deletions(-) diff --git a/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp b/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp index ddc4021cb9d..87592589716 100644 --- a/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp +++ b/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp @@ -11,11 +11,14 @@ #include "WarpX.H" #include +#include #include #include #include +using namespace ablastr::fields; + /** * \brief Get the value of arr in the neighbor (i_n, j_n) on the plane with normal 'dim'. * diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index 00cf16c5097..c1f46a7d34b 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -41,6 +41,8 @@ # include # include +using namespace ablastr::fields; + #endif #ifdef AMREX_USE_EB @@ -183,7 +185,7 @@ WarpX::ComputeEdgeLengths (std::array< std::unique_ptr, 3 >& ed void -WarpX::ComputeFaceAreas (std::array< std::unique_ptr, 3 >& face_areas, +WarpX::ComputeFaceAreas (VectorField& face_areas, const amrex::EBFArrayBoxFactory& eb_fact) { BL_PROFILE("ComputeFaceAreas"); @@ -270,7 +272,7 @@ WarpX::ScaleEdges (std::array< std::unique_ptr, 3 >& edge_lengt } void -WarpX::ScaleAreas(std::array< std::unique_ptr, 3 >& face_areas, +WarpX::ScaleAreas(ablastr::fields::VectorField& face_areas, const std::array& cell_size) { BL_PROFILE("ScaleAreas"); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp index 6e419637e77..5c6cae5c30b 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp @@ -194,7 +194,7 @@ void FiniteDifferenceSolver::EvolveBCartesian ( void FiniteDifferenceSolver::EvolveBCartesianECT ( std::array< std::unique_ptr, 3 >& Bfield, - std::array< std::unique_ptr, 3 > const& face_areas, + ablastr::fields::VectorField const& face_areas, std::array< std::unique_ptr, 3 > const& area_mod, std::array< std::unique_ptr, 3 >& ECTRhofield, std::array< std::unique_ptr, 3 >& Venl, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp index 8a7b952a9c9..3e7b37557a6 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp @@ -19,6 +19,8 @@ #include "Utils/WarpXConst.H" #include "WarpX.H" +#include + #include #include #include @@ -42,6 +44,7 @@ #include using namespace amrex; +using namespace ablastr::fields; /** * \brief Update the E field, over one timestep @@ -51,7 +54,7 @@ void FiniteDifferenceSolver::EvolveE ( std::array< std::unique_ptr, 3 > const& Bfield, std::array< std::unique_ptr, 3 > const& Jfield, std::array< std::unique_ptr, 3 > const& edge_lengths, - std::array< std::unique_ptr, 3 > const& face_areas, + VectorField const& face_areas, std::array< std::unique_ptr, 3 >& ECTRhofield, amrex::MultiFab const* Ffield, int lev, amrex::Real const dt ) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp index 18960dee9a1..3d6b5ca992b 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp @@ -101,9 +101,9 @@ void FiniteDifferenceSolver::EvolveRhoCartesianECT ( amrex::Array4 const &lx = edge_lengths[0]->array(mfi); amrex::Array4 const &ly = edge_lengths[1]->array(mfi); amrex::Array4 const &lz = edge_lengths[2]->array(mfi); - amrex::Array4 const &Sx = face_areas[Direction{0}]->array(mfi); - amrex::Array4 const &Sy = face_areas[Direction{1}]->array(mfi); - amrex::Array4 const &Sz = face_areas[Direction{2}]->array(mfi); + amrex::Array4 const &Sx = face_areas[0]->array(mfi); + amrex::Array4 const &Sy = face_areas[1]->array(mfi); + amrex::Array4 const &Sz = face_areas[2]->array(mfi); // Extract tileboxes for which to loop amrex::Box const &trhox = mfi.tilebox(ECTRhofield[0]->ixType().toIntVect()); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 1167ca7b30c..464c8703146 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -67,7 +67,7 @@ class FiniteDifferenceSolver std::array< std::unique_ptr, 3 > const& Bfield, std::array< std::unique_ptr, 3 > const& Jfield, std::array< std::unique_ptr, 3 > const& edge_lengths, - std::array< std::unique_ptr, 3 > const& face_areas, + ablastr::fields::VectorField const& face_areas, std::array< std::unique_ptr, 3 >& ECTRhofield, amrex::MultiFab const* Ffield, int lev, amrex::Real dt ); @@ -84,7 +84,7 @@ class FiniteDifferenceSolver void EvolveECTRho ( std::array< std::unique_ptr, 3 > const& Efield, std::array< std::unique_ptr, 3 > const& edge_lengths, - std::array< std::unique_ptr, 3 > const& face_areas, + ablastr::fields::VectorField const& face_areas, std::array< std::unique_ptr, 3 >& ECTRhofield, int lev ); diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp index 9aedfb3d97f..2fa2c26e3cf 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp @@ -334,15 +334,16 @@ WarpX::ImplicitComputeRHSE (int lev, PatchType patch_type, amrex::Real a_dt, War // a_Erhs_vec is set to zero above, calling EvolveE below results in // a_Erhs_vec storing only the RHS of the update equation. I.e., // c^2*dt*(curl(B^{n+theta} - mu0*J^{n+1/2}) + const auto face_area_lev = m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev]; if (patch_type == PatchType::fine) { m_fdtd_solver_fp[lev]->EvolveE( a_Erhs_vec.getArrayVec()[lev], Bfield_fp[lev], current_fp[lev], m_edge_lengths[lev], - m_face_areas[lev], ECTRhofield[lev], + face_area_lev, ECTRhofield[lev], m_fields.get("F_fp", lev), lev, a_dt ); } else { m_fdtd_solver_cp[lev]->EvolveE( a_Erhs_vec.getArrayVec()[lev], Bfield_cp[lev], current_cp[lev], m_edge_lengths[lev], - m_face_areas[lev], ECTRhofield[lev], + face_area_lev, ECTRhofield[lev], m_fields.get("F_cp", lev), lev, a_dt ); } diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 09d39129bde..9f19c94a85b 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -887,15 +887,16 @@ void WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) { // Evolve E field in regular cells + auto face_areas_lev = m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev]; if (patch_type == PatchType::fine) { m_fdtd_solver_fp[lev]->EvolveE(Efield_fp[lev], Bfield_fp[lev], current_fp[lev], m_edge_lengths[lev], - m_face_areas[lev], ECTRhofield[lev], + face_areas_lev, ECTRhofield[lev], m_fields.get("F_fp", lev), lev, a_dt ); } else { m_fdtd_solver_cp[lev]->EvolveE(Efield_cp[lev], Bfield_cp[lev], current_cp[lev], m_edge_lengths[lev], - m_face_areas[lev], ECTRhofield[lev], + face_areas_lev, ECTRhofield[lev], m_fields.get("F_cp", lev), lev, a_dt ); } @@ -926,10 +927,10 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { if (patch_type == PatchType::fine) { m_fdtd_solver_fp[lev]->EvolveECTRho(Efield_fp[lev], m_edge_lengths[lev], - m_face_areas[lev], ECTRhofield[lev], lev); + face_areas_lev, ECTRhofield[lev], lev); } else { m_fdtd_solver_cp[lev]->EvolveECTRho(Efield_cp[lev], m_edge_lengths[lev], - m_face_areas[lev], ECTRhofield[lev], lev); + face_areas_lev, ECTRhofield[lev], lev); } } #endif diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 041a4c2f6fd..00ba8bdf6a8 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -967,7 +967,7 @@ WarpX::InitLevelData (int lev, Real /*time*/) m_p_ext_field_params->Byfield_parser->compile<3>(), m_p_ext_field_params->Bzfield_parser->compile<3>(), m_edge_lengths[lev], - m_face_areas[lev], + m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev], 'B', lev, PatchType::fine); @@ -979,7 +979,7 @@ WarpX::InitLevelData (int lev, Real /*time*/) m_p_ext_field_params->Byfield_parser->compile<3>(), m_p_ext_field_params->Bzfield_parser->compile<3>(), m_edge_lengths[lev], - m_face_areas[lev], + m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev], 'B', lev, PatchType::coarse); } @@ -998,7 +998,8 @@ WarpX::InitLevelData (int lev, Real /*time*/) if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { m_fdtd_solver_fp[lev]->EvolveECTRho( Efield_fp[lev], m_edge_lengths[lev], - m_face_areas[lev], ECTRhofield[lev], lev); + m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev], + ECTRhofield[lev], lev); } } #endif @@ -1012,7 +1013,7 @@ WarpX::InitLevelData (int lev, Real /*time*/) m_p_ext_field_params->Eyfield_parser->compile<3>(), m_p_ext_field_params->Ezfield_parser->compile<3>(), m_edge_lengths[lev], - m_face_areas[lev], + m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev], 'E', lev, PatchType::fine); @@ -1024,16 +1025,17 @@ WarpX::InitLevelData (int lev, Real /*time*/) m_p_ext_field_params->Eyfield_parser->compile<3>(), m_p_ext_field_params->Ezfield_parser->compile<3>(), m_edge_lengths[lev], - m_face_areas[lev], + m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev], 'E', lev, PatchType::coarse); #ifdef AMREX_USE_EB if (eb_enabled) { if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { // We initialize ECTRhofield consistently with the Efield - m_fdtd_solver_cp[lev]->EvolveECTRho(Efield_cp[lev], m_edge_lengths[lev], - m_face_areas[lev], ECTRhofield[lev], lev); - + m_fdtd_solver_cp[lev]->EvolveECTRho( + Efield_cp[lev], m_edge_lengths[lev], + m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev], + ECTRhofield[lev], lev); } } #endif @@ -1058,7 +1060,7 @@ WarpX::InitializeExternalFieldsOnGridUsingParser ( ParserExecutor<3> const& xfield_parser, ParserExecutor<3> const& yfield_parser, ParserExecutor<3> const& zfield_parser, std::array< std::unique_ptr, 3 > const& edge_lengths, - std::array< std::unique_ptr, 3 > const& face_areas, + ablastr::fields::VectorField const& face_areas, [[maybe_unused]] const char field, const int lev, PatchType patch_type) { @@ -1276,8 +1278,10 @@ void WarpX::InitializeEBGridData (int lev) ComputeEdgeLengths(m_edge_lengths[lev], eb_fact); ScaleEdges(m_edge_lengths[lev], CellSize(lev)); - ComputeFaceAreas(m_face_areas[lev], eb_fact); - ScaleAreas(m_face_areas[lev], CellSize(lev)); + + auto face_areas_lev = m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev]; + ComputeFaceAreas(face_areas_lev, eb_fact); + ScaleAreas(face_areas_lev, CellSize(lev)); if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { MarkCells(); @@ -1366,7 +1370,7 @@ WarpX::LoadExternalFields (int const lev) m_p_ext_field_params->Byfield_parser->compile<3>(), m_p_ext_field_params->Bzfield_parser->compile<3>(), m_edge_lengths[lev], - m_face_areas[lev], + m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev], 'B', lev, PatchType::fine); } @@ -1394,7 +1398,7 @@ WarpX::LoadExternalFields (int const lev) m_p_ext_field_params->Eyfield_parser->compile<3>(), m_p_ext_field_params->Ezfield_parser->compile<3>(), m_edge_lengths[lev], - m_face_areas[lev], + m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev], 'E', lev, PatchType::fine); } diff --git a/Source/WarpX.H b/Source/WarpX.H index 5244125c4d9..2eb83301e1d 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1150,7 +1150,7 @@ public: * \brief Compute the area of the mesh faces. Here the area is a value in [0, 1]. * An edge of area 0 is fully covered. */ - static void ComputeFaceAreas (std::array& face_areas, + static void ComputeFaceAreas (ablastr::fields::VectorField& face_areas, const amrex::EBFArrayBoxFactory& eb_fact); /** @@ -1161,7 +1161,7 @@ public: /** * \brief Scale the edges areas by the mesh width to obtain the real areas. */ - static void ScaleAreas (std::array& face_areas, + static void ScaleAreas (ablastr::fields::VectorField& face_areas, const std::array& cell_size); /** * \brief Initialize information for cell extensions. From d268e720757f7f52cd386bf9b58e05e85cca1925 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 11:17:51 -0700 Subject: [PATCH 083/314] Start working on PSATD --- Source/Evolve/WarpXEvolve.cpp | 32 +++++++++++++++------- Source/FieldSolver/WarpXPushFieldsEM.cpp | 34 +++++++++++++----------- Source/WarpX.H | 20 +++++++------- 3 files changed, 52 insertions(+), 34 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 7feacd6e510..fdef46c7748 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -637,7 +637,11 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Initialize multi-J loop: // 1) Prepare E,B,F,G fields in spectral space - PSATDForwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); + PSATDForwardTransformEB( + m_fields.get_mr_levels_alldirs("Efield_fp", finest_level), + m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level), + m_fields.get_mr_levels_alldirs("Efield_cp", finest_level), + m_fields.get_mr_levels_alldirs("Bfield_cp", finest_level) ); if (WarpX::do_dive_cleaning) { PSATDForwardTransformF(); } if (WarpX::do_divb_cleaning) { PSATDForwardTransformG(); } @@ -664,16 +668,21 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // (dt[0] denotes the time step on mesh refinement level 0) if (J_in_time == JInTime::Linear) { - auto& current = (do_current_centering) ? current_fp_nodal : current_fp; - mypc->DepositCurrent(current, dt[0], -dt[0]); + std::string const current_string = (do_current_centering) ? "current_fp_nodal" : "current_fp"; + //RL TODO mypc->DepositCurrent( m_fields.get_mr_levels(current_string, finest_level), dt[0], -dt[0]); // Synchronize J: filter, exchange boundary, and interpolate across levels. // With current centering, the nodal current is deposited in 'current', // namely 'current_fp_nodal': SyncCurrent stores the result of its centering // into 'current_fp' and then performs both filtering, if used, and exchange // of guard cells. - SyncCurrent(va2vm(current_fp), va2vm(current_cp), va2vm(current_buf)); + SyncCurrent( + m_fields.get_mr_levels_alldirs( "current_fp", finest_level), + m_fields.get_mr_levels_alldirs( "current_cp", finest_level), + m_fields.get_mr_levels_alldirs( "current_buf", finest_level) ); // Forward FFT of J - PSATDForwardTransformJ(current_fp, current_cp); + PSATDForwardTransformJ( + m_fields.get_mr_levels_alldirs( "current_fp", finest_level), + m_fields.get_mr_levels_alldirs( "current_cp", finest_level) ); } // Number of depositions for multi-J scheme @@ -698,16 +707,21 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Deposit new J at relative time t_deposit_current with time step dt // (dt[0] denotes the time step on mesh refinement level 0) - auto& current = (do_current_centering) ? current_fp_nodal : current_fp; - mypc->DepositCurrent(current, dt[0], t_deposit_current); + std::string const current_string = (do_current_centering) ? "current_fp_nodal" : "current_fp"; + //RL TODO mypc->DepositCurrent( m_fields.get_mr_levels(current_string, finest_level), dt[0], t_deposit_current); // Synchronize J: filter, exchange boundary, and interpolate across levels. // With current centering, the nodal current is deposited in 'current', // namely 'current_fp_nodal': SyncCurrent stores the result of its centering // into 'current_fp' and then performs both filtering, if used, and exchange // of guard cells. - SyncCurrent(va2vm(current_fp), va2vm(current_cp), va2vm(current_buf)); + SyncCurrent( + m_fields.get_mr_levels_alldirs( "current_fp", finest_level), + m_fields.get_mr_levels_alldirs( "current_cp", finest_level), + m_fields.get_mr_levels_alldirs( "current_buf", finest_level) ); // Forward FFT of J - PSATDForwardTransformJ(current_fp, current_cp); + PSATDForwardTransformJ( + m_fields.get_mr_levels_alldirs( "current_fp", finest_level), + m_fields.get_mr_levels_alldirs( "current_cp", finest_level) ); // Deposit new rho // (after checking that pointer to rho_fp on MR level 0 is not null) diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 3c518fa584d..b872b3d27b4 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -64,7 +64,7 @@ namespace { #else SpectralSolver& solver, #endif - const std::array,3>& vector_field, + const ablastr::fields::VectorField& vector_field, const int compx, const int compy, const int compz) { #ifdef WARPX_DIM_RZ @@ -84,7 +84,7 @@ namespace { #else SpectralSolver& solver, #endif - const std::array,3>& vector_field, + const ablastr::fields::VectorField& vector_field, const int compx, const int compy, const int compz, const amrex::IntVect& fill_guards) { @@ -101,10 +101,10 @@ namespace { } void WarpX::PSATDForwardTransformEB ( - const amrex::Vector,3>>& E_fp, - const amrex::Vector,3>>& B_fp, - const amrex::Vector,3>>& E_cp, - const amrex::Vector,3>>& B_cp) + const ablastr::fields::MultiLevelVectorField& E_fp, + const ablastr::fields::MultiLevelVectorField& B_fp, + const ablastr::fields::MultiLevelVectorField& E_cp, + const ablastr::fields::MultiLevelVectorField& B_cp) { const SpectralFieldIndex& Idx = spectral_solver_fp[0]->m_spectral_index; @@ -122,10 +122,10 @@ void WarpX::PSATDForwardTransformEB ( } void WarpX::PSATDBackwardTransformEB ( - const amrex::Vector,3>>& E_fp, - const amrex::Vector,3>>& B_fp, - const amrex::Vector,3>>& E_cp, - const amrex::Vector,3>>& B_cp) + const ablastr::fields::MultiLevelVectorField& E_fp, + const ablastr::fields::MultiLevelVectorField& B_fp, + const ablastr::fields::MultiLevelVectorField& E_cp, + const ablastr::fields::MultiLevelVectorField& B_cp) { const SpectralFieldIndex& Idx = spectral_solver_fp[0]->m_spectral_index; @@ -280,8 +280,8 @@ WarpX::PSATDBackwardTransformG () } void WarpX::PSATDForwardTransformJ ( - const amrex::Vector,3>>& J_fp, - const amrex::Vector,3>>& J_cp, + const ablastr::fields::MultiLevelVectorField& J_fp, + const ablastr::fields::MultiLevelVectorField& J_cp, const bool apply_kspace_filter) { SpectralFieldIndex Idx; @@ -769,7 +769,11 @@ WarpX::PushPSATD () } // FFT of E and B - PSATDForwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); + PSATDForwardTransformEB( + m_fields.get_mr_levels_alldirs("Efield_fp", finest_level), + m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level), + m_fields.get_mr_levels_alldirs("Efield_cp", finest_level), + m_fields.get_mr_levels_alldirs("Bfield_cp", finest_level) ); #ifdef WARPX_DIM_RZ if (pml_rz[0]) { pml_rz[0]->PushPSATD(0); } @@ -1090,8 +1094,8 @@ WarpX::MacroscopicEvolveE (int lev, PatchType patch_type, amrex::Real a_dt) { void WarpX::DampFieldsInGuards(const int lev, - const std::array,3>& Efield, - const std::array,3>& Bfield) { + const ablastr::fields::VectorField& Efield, + const ablastr::fields::VectorField& Bfield) { // Loop over dimensions for (int dampdir = 0 ; dampdir < AMREX_SPACEDIM ; dampdir++) diff --git a/Source/WarpX.H b/Source/WarpX.H index 3c35d0237e2..df1a7dbd45a 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -722,8 +722,8 @@ public: * when FieldBoundaryType is set to damped. Vector version. */ void DampFieldsInGuards (int lev, - const std::array,3>& Efield, - const std::array,3>& Bfield); + const ablastr::fields::VectorField& Efield, + const ablastr::fields::VectorField& Bfield); /** * \brief Private function for spectral solver @@ -1826,10 +1826,10 @@ private: * storing the coarse patch magnetic field to be transformed */ void PSATDForwardTransformEB ( - const amrex::Vector,3>>& E_fp, - const amrex::Vector,3>>& B_fp, - const amrex::Vector,3>>& E_cp, - const amrex::Vector,3>>& B_cp); + const ablastr::fields::MultiLevelVectorField& E_fp, + const ablastr::fields::MultiLevelVectorField& B_fp, + const ablastr::fields::MultiLevelVectorField& E_cp, + const ablastr::fields::MultiLevelVectorField& B_cp); /** * \brief Backward FFT of E,B on all mesh refinement levels, @@ -1845,10 +1845,10 @@ private: * storing the coarse patch magnetic field to be transformed */ void PSATDBackwardTransformEB ( - const amrex::Vector,3>>& E_fp, - const amrex::Vector,3>>& B_fp, - const amrex::Vector,3>>& E_cp, - const amrex::Vector,3>>& B_cp); + const ablastr::fields::MultiLevelVectorField& E_fp, + const ablastr::fields::MultiLevelVectorField& B_fp, + const ablastr::fields::MultiLevelVectorField& E_cp, + const ablastr::fields::MultiLevelVectorField& B_cp); /** * \brief Backward FFT of averaged E,B on all mesh refinement levels From ff96925ba2fb83460e7b06df325fb7b05bb017a5 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 11:27:32 -0700 Subject: [PATCH 084/314] Reactivate current deposition --- Source/Evolve/WarpXEvolve.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index fdef46c7748..c2a3b2d9d9f 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -669,7 +669,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) if (J_in_time == JInTime::Linear) { std::string const current_string = (do_current_centering) ? "current_fp_nodal" : "current_fp"; - //RL TODO mypc->DepositCurrent( m_fields.get_mr_levels(current_string, finest_level), dt[0], -dt[0]); + mypc->DepositCurrent( m_fields.get_mr_levels(current_string, finest_level), dt[0], -dt[0]); // Synchronize J: filter, exchange boundary, and interpolate across levels. // With current centering, the nodal current is deposited in 'current', // namely 'current_fp_nodal': SyncCurrent stores the result of its centering @@ -708,7 +708,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Deposit new J at relative time t_deposit_current with time step dt // (dt[0] denotes the time step on mesh refinement level 0) std::string const current_string = (do_current_centering) ? "current_fp_nodal" : "current_fp"; - //RL TODO mypc->DepositCurrent( m_fields.get_mr_levels(current_string, finest_level), dt[0], t_deposit_current); + mypc->DepositCurrent( m_fields.get_mr_levels(current_string, finest_level), dt[0], t_deposit_current); // Synchronize J: filter, exchange boundary, and interpolate across levels. // With current centering, the nodal current is deposited in 'current', // namely 'current_fp_nodal': SyncCurrent stores the result of its centering From ad28684fb4db25890fb5223868bbe87335b4e3f6 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 11 Sep 2024 11:41:09 -0700 Subject: [PATCH 085/314] `ScalarField` alias, TODO on AMReX Direction --- Source/ablastr/fields/MultiFabRegister.H | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index d313e92c56c..e407ce10420 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -30,7 +30,9 @@ namespace ablastr::fields /** Components (base vector directions) of vector/tensor fields. * * Because of different staggering, the components of vector/tensor fields are stored - * in separate (i)MultiFab. This + * in separate (i)MultiFab. + * + * @todo: synchronize with AMReX "enum class Direction" */ struct Direction { @@ -45,6 +47,12 @@ namespace ablastr::fields operator int() { return dir; } }; + /** A scalar field (a MultiFab) + * + * Note: might still have components, e.g., for copies at different times. + */ + using ScalarField = amrex::MultiFab*; + /** A vector field of three MultiFab */ //using VectorField = ablastr::utils::ConstMap; @@ -52,7 +60,7 @@ namespace ablastr::fields /** A multi-level scalar field */ - using MultiLevelScalarField = amrex::Vector; + using MultiLevelScalarField = amrex::Vector; /** A multi-level vector field */ From e336b2185ddefb923a89d1ce944809136f1394ae Mon Sep 17 00:00:00 2001 From: Marco Acciarri Date: Wed, 11 Sep 2024 11:48:50 -0700 Subject: [PATCH 086/314] Bfield_fp_external Efield_fp_external moved to register --- .../DivCleaner/ProjectionDivCleaner.H | 6 ++- .../DivCleaner/ProjectionDivCleaner.cpp | 33 ++++++++---- Source/Initialization/WarpXInitData.cpp | 52 ++++++++++--------- Source/WarpX.cpp | 27 +++++----- 4 files changed, 69 insertions(+), 49 deletions(-) diff --git a/Source/Initialization/DivCleaner/ProjectionDivCleaner.H b/Source/Initialization/DivCleaner/ProjectionDivCleaner.H index 7ee1fe57048..d44b3bfa73d 100644 --- a/Source/Initialization/DivCleaner/ProjectionDivCleaner.H +++ b/Source/Initialization/DivCleaner/ProjectionDivCleaner.H @@ -64,7 +64,8 @@ protected: amrex::Real m_rtol; amrex::Real m_atol; - warpx::fields::FieldType m_field_type; + //warpx::fields::FieldType m_field_type; + std::string m_field_name; public: amrex::Vector< std::unique_ptr > m_solution; @@ -83,7 +84,8 @@ public: amrex::Gpu::DeviceVector m_stencil_coefs_z; // Default Constructor - ProjectionDivCleaner (warpx::fields::FieldType a_field_type); + //ProjectionDivCleaner (warpx::fields::FieldType a_field_type); + ProjectionDivCleaner (std::string a_field_name); void ReadParameters (); diff --git a/Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp b/Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp index 40326aadc3c..fcbb8555e33 100644 --- a/Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp +++ b/Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp @@ -30,9 +30,11 @@ using namespace amrex; namespace warpx::initialization { -ProjectionDivCleaner::ProjectionDivCleaner(warpx::fields::FieldType a_field_type) : - m_field_type(a_field_type) +ProjectionDivCleaner::ProjectionDivCleaner(std::string a_field_name) : + //m_field_type(a_field_type) + m_field_name(a_field_name) { + using ablastr::fields::Direction; ReadParameters(); auto& warpx = WarpX::GetInstance(); @@ -48,7 +50,8 @@ ProjectionDivCleaner::ProjectionDivCleaner(warpx::fields::FieldType a_field_type m_source.resize(m_levels); const int ncomps = WarpX::ncomps; - auto const& ng = warpx.getFieldPointer(m_field_type, 0, 0)->nGrowVect(); + //auto const& ng = warpx.getFieldPointer(m_field_type, 0, 0)->nGrowVect(); + auto const& ng = warpx.m_fields.get(m_field_name,Direction{0},0)->nGrowVect(); for (int lev = 0; lev < m_levels; ++lev) { @@ -201,6 +204,8 @@ ProjectionDivCleaner::solve () void ProjectionDivCleaner::setSourceFromBfield () { + using ablastr::fields::Direction; + // Get WarpX object auto & warpx = WarpX::GetInstance(); const auto& geom = warpx.Geom(); @@ -211,7 +216,10 @@ ProjectionDivCleaner::setSourceFromBfield () WarpX::ComputeDivB( *m_source[ilev], 0, - warpx.getFieldPointerArray(m_field_type, ilev), + //warpx.getFieldPointerArray(m_field_type, ilev), + {warpx.m_fields.get(m_field_name,Direction{0},ilev), + warpx.m_fields.get(m_field_name,Direction{1},ilev), + warpx.m_fields.get(m_field_name,Direction{2},ilev)}, WarpX::CellSize(0) ); @@ -228,6 +236,8 @@ ProjectionDivCleaner::setSourceFromBfield () void ProjectionDivCleaner::correctBfield () { + using ablastr::fields::Direction; + // Get WarpX object auto & warpx = WarpX::GetInstance(); const auto& geom = warpx.Geom(); @@ -236,9 +246,12 @@ ProjectionDivCleaner::correctBfield () for (int ilev = 0; ilev < m_levels; ++ilev) { // Grab B-field multifabs at this level - amrex::MultiFab* Bx = warpx.getFieldPointer(m_field_type, ilev, 0); - amrex::MultiFab* By = warpx.getFieldPointer(m_field_type, ilev, 1); - amrex::MultiFab* Bz = warpx.getFieldPointer(m_field_type, ilev, 2); + //amrex::MultiFab* Bx = warpx.getFieldPointer(m_field_type, ilev, 0); + //amrex::MultiFab* By = warpx.getFieldPointer(m_field_type, ilev, 1); + //amrex::MultiFab* Bz = warpx.getFieldPointer(m_field_type, ilev, 2); + amrex::MultiFab* Bx = warpx.m_fields.get(m_field_name,Direction{0},ilev); + amrex::MultiFab* By = warpx.m_fields.get(m_field_name,Direction{1},ilev); + amrex::MultiFab* Bz = warpx.m_fields.get(m_field_name,Direction{2},ilev); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) @@ -337,8 +350,10 @@ WarpX::ProjectionCleanDivB() { ablastr::warn_manager::WarnPriority::low); } - warpx::initialization::ProjectionDivCleaner dc( - warpx::fields::FieldType::Bfield_fp_external); + //warpx::initialization::ProjectionDivCleaner dc( + // warpx::fields::FieldType::Bfield_fp_external); + warpx::initialization::ProjectionDivCleaner dc("Bfield_fp_external"); + dc.setSourceFromBfield(); dc.solve(); diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 36dbda77c64..8476eda0e95 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -626,6 +626,8 @@ WarpX::InitData () void WarpX::AddExternalFields (int const lev) { + using ablastr::fields::Direction; + // FIXME: RZ multimode has more than one component for all these if (m_p_ext_field_params->E_ext_grid_type != ExternalFieldType::default_zero) { if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::constant) { @@ -634,9 +636,9 @@ WarpX::AddExternalFields (int const lev) { Efield_fp[lev][2]->plus(m_p_ext_field_params->E_external_grid[2], guard_cells.ng_alloc_EB.min()); } else { - amrex::MultiFab::Add(*Efield_fp[lev][0], *Efield_fp_external[lev][0], 0, 0, 1, guard_cells.ng_alloc_EB); - amrex::MultiFab::Add(*Efield_fp[lev][1], *Efield_fp_external[lev][1], 0, 0, 1, guard_cells.ng_alloc_EB); - amrex::MultiFab::Add(*Efield_fp[lev][2], *Efield_fp_external[lev][2], 0, 0, 1, guard_cells.ng_alloc_EB); + amrex::MultiFab::Add(*Efield_fp[lev][0], *m_fields.get("Efield_fp_external",Direction{0},lev), 0, 0, 1, guard_cells.ng_alloc_EB); + amrex::MultiFab::Add(*Efield_fp[lev][1], *m_fields.get("Efield_fp_external",Direction{1},lev), 0, 0, 1, guard_cells.ng_alloc_EB); + amrex::MultiFab::Add(*Efield_fp[lev][2], *m_fields.get("Efield_fp_external",Direction{2},lev), 0, 0, 1, guard_cells.ng_alloc_EB); } } if (m_p_ext_field_params->B_ext_grid_type != ExternalFieldType::default_zero) { @@ -646,9 +648,9 @@ WarpX::AddExternalFields (int const lev) { Bfield_fp[lev][2]->plus(m_p_ext_field_params->B_external_grid[2], guard_cells.ng_alloc_EB.min()); } else { - amrex::MultiFab::Add(*Bfield_fp[lev][0], *Bfield_fp_external[lev][0], 0, 0, 1, guard_cells.ng_alloc_EB); - amrex::MultiFab::Add(*Bfield_fp[lev][1], *Bfield_fp_external[lev][1], 0, 0, 1, guard_cells.ng_alloc_EB); - amrex::MultiFab::Add(*Bfield_fp[lev][2], *Bfield_fp_external[lev][2], 0, 0, 1, guard_cells.ng_alloc_EB); + amrex::MultiFab::Add(*Bfield_fp[lev][0], *m_fields.get("Bfield_fp_external",Direction{0},lev), 0, 0, 1, guard_cells.ng_alloc_EB); + amrex::MultiFab::Add(*Bfield_fp[lev][1], *m_fields.get("Bfield_fp_external",Direction{1},lev), 0, 0, 1, guard_cells.ng_alloc_EB); + amrex::MultiFab::Add(*Bfield_fp[lev][2], *m_fields.get("Bfield_fp_external",Direction{2},lev), 0, 0, 1, guard_cells.ng_alloc_EB); } } } @@ -1341,6 +1343,8 @@ void WarpX::CheckKnownIssues() void WarpX::LoadExternalFields (int const lev) { + using ablastr::fields::Direction; + // External fields from file are currently not compatible with the moving window // In order to support the moving window, the MultiFab containing the external // fields should be updated every time the window moves. @@ -1358,9 +1362,9 @@ WarpX::LoadExternalFields (int const lev) if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::parse_ext_grid_function) { // Initialize Bfield_fp_external with external function InitializeExternalFieldsOnGridUsingParser( - Bfield_fp_external[lev][0].get(), - Bfield_fp_external[lev][1].get(), - Bfield_fp_external[lev][2].get(), + m_fields.get("Bfield_fp_external",Direction{0},lev), + m_fields.get("Bfield_fp_external",Direction{1},lev), + m_fields.get("Bfield_fp_external",Direction{2},lev), m_p_ext_field_params->Bxfield_parser->compile<3>(), m_p_ext_field_params->Byfield_parser->compile<3>(), m_p_ext_field_params->Bzfield_parser->compile<3>(), @@ -1373,22 +1377,22 @@ WarpX::LoadExternalFields (int const lev) #if defined(WARPX_DIM_RZ) WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, "External field reading is not implemented for more than one RZ mode (see #3829)"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][0].get(), "B", "r"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][1].get(), "B", "t"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][2].get(), "B", "z"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get("Bfield_fp_external",Direction{0},lev), "B", "r"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get("Bfield_fp_external",Direction{1},lev), "B", "t"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get("Bfield_fp_external",Direction{2},lev), "B", "z"); #else - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][0].get(), "B", "x"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][1].get(), "B", "y"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][2].get(), "B", "z"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get("Bfield_fp_external",Direction{0},lev), "B", "x"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get("Bfield_fp_external",Direction{1},lev), "B", "y"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get("Bfield_fp_external",Direction{2},lev), "B", "z"); #endif } if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::parse_ext_grid_function) { // Initialize Efield_fp_external with external function InitializeExternalFieldsOnGridUsingParser( - Efield_fp_external[lev][0].get(), - Efield_fp_external[lev][1].get(), - Efield_fp_external[lev][2].get(), + m_fields.get("Efield_fp_external",Direction{0},lev), + m_fields.get("Efield_fp_external",Direction{1},lev), + m_fields.get("Efield_fp_external",Direction{2},lev), m_p_ext_field_params->Exfield_parser->compile<3>(), m_p_ext_field_params->Eyfield_parser->compile<3>(), m_p_ext_field_params->Ezfield_parser->compile<3>(), @@ -1401,13 +1405,13 @@ WarpX::LoadExternalFields (int const lev) #if defined(WARPX_DIM_RZ) WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, "External field reading is not implemented for more than one RZ mode (see #3829)"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][0].get(), "E", "r"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][1].get(), "E", "t"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][2].get(), "E", "z"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get("Efield_fp_external",Direction{0},lev), "E", "r"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get("Efield_fp_external",Direction{1},lev), "E", "t"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get("Efield_fp_external",Direction{2},lev), "E", "z"); #else - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][0].get(), "E", "x"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][1].get(), "E", "y"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][2].get(), "E", "z"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get("Efield_fp_external",Direction{0},lev), "E", "x"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get("Efield_fp_external",Direction{1},lev), "E", "y"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get("Efield_fp_external",Direction{2},lev), "E", "z"); #endif } diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 928fa0c8d54..cb536c957b2 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -343,8 +343,6 @@ WarpX::WarpX () } // Same as Bfield_fp/Efield_fp for reading external field data - Bfield_fp_external.resize(nlevs_max); - Efield_fp_external.resize(nlevs_max); B_external_particle_field.resize(1); E_external_particle_field.resize(1); @@ -2630,12 +2628,13 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm // The external fields that are read from file if (m_p_ext_field_params->B_ext_grid_type != ExternalFieldType::default_zero && m_p_ext_field_params->B_ext_grid_type != ExternalFieldType::constant) { // These fields will be added directly to the grid, i.e. to fp, and need to match the index type - AllocInitMultiFab(Bfield_fp_external[lev][0], amrex::convert(ba, Bfield_fp[lev][0]->ixType()), - dm, ncomps, ngEB, lev, "Bfield_fp_external[x]", 0.0_rt); - AllocInitMultiFab(Bfield_fp_external[lev][1], amrex::convert(ba, Bfield_fp[lev][1]->ixType()), - dm, ncomps, ngEB, lev, "Bfield_fp_external[y]", 0.0_rt); - AllocInitMultiFab(Bfield_fp_external[lev][2], amrex::convert(ba, Bfield_fp[lev][2]->ixType()), - dm, ncomps, ngEB, lev, "Bfield_fp_external[z]", 0.0_rt); + m_fields.alloc_init( "Bfield_fp_external", Direction{0}, lev, amrex::convert(ba, Bfield_fp[lev][0]->ixType()), + dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Bfield_fp_external", Direction{1}, lev, amrex::convert(ba, Bfield_fp[lev][1]->ixType()), + dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Bfield_fp_external", Direction{2}, lev, amrex::convert(ba, Bfield_fp[lev][2]->ixType()), + dm, ncomps, ngEB, 0.0_rt); + } if (mypc->m_B_ext_particle_s == "read_from_file") { // These fields will be added to the fields that the particles see, and need to match the index type @@ -2648,12 +2647,12 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm } if (m_p_ext_field_params->E_ext_grid_type != ExternalFieldType::default_zero && m_p_ext_field_params->E_ext_grid_type != ExternalFieldType::constant) { // These fields will be added directly to the grid, i.e. to fp, and need to match the index type - AllocInitMultiFab(Efield_fp_external[lev][0], amrex::convert(ba, m_fields.get("Efield_fp",Direction{0},lev)->ixType()), - dm, ncomps, ngEB, lev, "Efield_fp_external[x]", 0.0_rt); - AllocInitMultiFab(Efield_fp_external[lev][1], amrex::convert(ba, m_fields.get("Efield_fp",Direction{1},lev)->ixType()), - dm, ncomps, ngEB, lev, "Efield_fp_external[y]", 0.0_rt); - AllocInitMultiFab(Efield_fp_external[lev][2], amrex::convert(ba, m_fields.get("Efield_fp",Direction{2},lev)->ixType()), - dm, ncomps, ngEB, lev, "Efield_fp_external[z]", 0.0_rt); + m_fields.alloc_init( "Efield_fp_external", Direction{0}, lev, amrex::convert(ba, m_fields.get("Efield_fp",Direction{0},lev)->ixType()), + dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Efield_fp_external", Direction{1}, lev, amrex::convert(ba, m_fields.get("Efield_fp",Direction{1},lev)->ixType()), + dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Efield_fp_external", Direction{2}, lev, amrex::convert(ba, m_fields.get("Efield_fp",Direction{2},lev)->ixType()), + dm, ncomps, ngEB, 0.0_rt); } if (mypc->m_E_ext_particle_s == "read_from_file") { // These fields will be added to the fields that the particles see, and need to match the index type From 94e0655b39751f97ed91355acc8deff18767627f Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 11:56:19 -0700 Subject: [PATCH 087/314] Current refactoring --- Source/Evolve/WarpXEvolve.cpp | 37 ++++++++++++++++++------- Source/Parallelization/WarpXComm.cpp | 40 +++++++++++++--------------- Source/WarpX.H | 26 +++++++++--------- Source/WarpX.cpp | 12 ++++----- 4 files changed, 66 insertions(+), 49 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index c2a3b2d9d9f..05eeb996b3f 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -580,7 +580,9 @@ void WarpX::SyncCurrentAndRho () { // TODO This works only without mesh refinement const int lev = 0; - if (use_filter) { ApplyFilterJ(va2vm(current_fp_vay), lev); } + if (use_filter) { + ApplyFilterJ(m_fields.get_mr_levels_alldirs("current_fp_vay", finest_level), lev); + } } } } @@ -838,10 +840,16 @@ WarpX::OneStep_sub1 (Real cur_time) // i) Push particles and fields on the fine patch (first fine step) PushParticlesandDeposit(fine_lev, cur_time, DtType::FirstHalf); - RestrictCurrentFromFineToCoarsePatch(current_fp, current_cp, fine_lev); + RestrictCurrentFromFineToCoarsePatch( + m_fields.get_mr_levels_alldirs("current_fp", finest_level), + m_fields.get_mr_levels_alldirs("current_cp", finest_level), fine_lev); RestrictRhoFromFineToCoarsePatch(fine_lev); - if (use_filter) { ApplyFilterJ( va2vm(current_fp), fine_lev); } - SumBoundaryJ( va2vm(current_fp), fine_lev, Geom(fine_lev).periodicity()); + if (use_filter) { + ApplyFilterJ( m_fields.get_mr_levels_alldirs("current_fp", finest_level), fine_lev); + } + SumBoundaryJ( + m_fields.get_mr_levels_alldirs("current_fp", finest_level), + fine_lev, Geom(fine_lev).periodicity()); ApplyFilterandSumBoundaryRho( m_fields.get_mr_levels("rho_fp", finest_level), @@ -874,7 +882,10 @@ WarpX::OneStep_sub1 (Real cur_time) // by only half a coarse step (first half) PushParticlesandDeposit(coarse_lev, cur_time, DtType::Full); StoreCurrent(coarse_lev); - AddCurrentFromFineLevelandSumBoundary(current_fp, current_cp, current_buf, coarse_lev); + AddCurrentFromFineLevelandSumBoundary( + m_fields.get_mr_levels_alldirs("current_fp", finest_level), + m_fields.get_mr_levels_alldirs("current_cp", finest_level), + m_fields.get_mr_levels_alldirs("current_buf", finest_level), coarse_lev); AddRhoFromFineLevelandSumBoundary( m_fields.get_mr_levels("rho_fp", finest_level), m_fields.get_mr_levels("rho_cp", finest_level), charge_buf, coarse_lev, 0, ncomps); @@ -905,10 +916,14 @@ WarpX::OneStep_sub1 (Real cur_time) // iv) Push particles and fields on the fine patch (second fine step) PushParticlesandDeposit(fine_lev, cur_time + dt[fine_lev], DtType::SecondHalf); - RestrictCurrentFromFineToCoarsePatch(current_fp, current_cp, fine_lev); + RestrictCurrentFromFineToCoarsePatch( + m_fields.get_mr_levels_alldirs("current_fp", finest_level), + m_fields.get_mr_levels_alldirs("current_cp", finest_level), fine_lev); RestrictRhoFromFineToCoarsePatch(fine_lev); - if (use_filter) { ApplyFilterJ( va2vm(current_fp), fine_lev); } - SumBoundaryJ( va2vm(current_fp), fine_lev, Geom(fine_lev).periodicity()); + if (use_filter) { + ApplyFilterJ( m_fields.get_mr_levels_alldirs("current_fp", finest_level), fine_lev); + } + SumBoundaryJ( m_fields.get_mr_levels_alldirs("current_fp", finest_level), fine_lev, Geom(fine_lev).periodicity()); ApplyFilterandSumBoundaryRho( m_fields.get_mr_levels("rho_fp", finest_level), m_fields.get_mr_levels("rho_cp", finest_level), @@ -939,7 +954,11 @@ WarpX::OneStep_sub1 (Real cur_time) // v) Push the fields on the coarse patch and mother grid // by only half a coarse step (second half) RestoreCurrent(coarse_lev); - AddCurrentFromFineLevelandSumBoundary(current_fp, current_cp, current_buf, coarse_lev); + AddCurrentFromFineLevelandSumBoundary( + m_fields.get_mr_levels_alldirs("current_fp", finest_level), + m_fields.get_mr_levels_alldirs("current_cp", finest_level), + m_fields.get_mr_levels_alldirs("current_buf", finest_level), + coarse_lev); AddRhoFromFineLevelandSumBoundary( m_fields.get_mr_levels("rho_fp", finest_level), m_fields.get_mr_levels("rho_cp", finest_level), charge_buf, coarse_lev, ncomps, ncomps); diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 71045d954bd..71abd8d44eb 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -1247,8 +1247,8 @@ WarpX::SyncRho ( * averaging the values of the current of the fine patch (on the same level). */ void WarpX::RestrictCurrentFromFineToCoarsePatch ( - const amrex::Vector,3>>& J_fp, - const amrex::Vector,3>>& J_cp, + const ablastr::fields::MultiLevelVectorField& J_fp, + const ablastr::fields::MultiLevelVectorField& J_cp, const int lev) { J_cp[lev][0]->setVal(0.0); @@ -1257,12 +1257,12 @@ void WarpX::RestrictCurrentFromFineToCoarsePatch ( const IntVect& refinement_ratio = refRatio(lev-1); - std::array fine { J_fp[lev][0].get(), - J_fp[lev][1].get(), - J_fp[lev][2].get() }; - std::array< MultiFab*,3> crse { J_cp[lev][0].get(), - J_cp[lev][1].get(), - J_cp[lev][2].get() }; + std::array fine { J_fp[lev][0], + J_fp[lev][1], + J_fp[lev][2] }; + std::array< MultiFab*,3> crse { J_cp[lev][0], + J_cp[lev][1], + J_cp[lev][2] }; ablastr::coarsen::average::Coarsen(*crse[0], *fine[0], refinement_ratio ); ablastr::coarsen::average::Coarsen(*crse[1], *fine[1], refinement_ratio ); ablastr::coarsen::average::Coarsen(*crse[2], *fine[2], refinement_ratio ); @@ -1362,20 +1362,18 @@ void WarpX::SumBoundaryJ ( * patch (and buffer region) of `lev+1` */ void WarpX::AddCurrentFromFineLevelandSumBoundary ( - const amrex::Vector,3>>& J_fp, - const amrex::Vector,3>>& J_cp, - const amrex::Vector,3>>& J_buffer, + const ablastr::fields::MultiLevelVectorField& J_fp, + const ablastr::fields::MultiLevelVectorField& J_cp, + const ablastr::fields::MultiLevelVectorField& J_buffer, const int lev) { - using ablastr::fields::va2vm; - const amrex::Periodicity& period = Geom(lev).periodicity(); if (use_filter) { - ApplyFilterJ(va2vm(J_fp), lev); + ApplyFilterJ(J_fp, lev); } - SumBoundaryJ(va2vm(J_fp), lev, period); + SumBoundaryJ(J_fp, lev, period); if (lev < finest_level) { @@ -1392,8 +1390,8 @@ void WarpX::AddCurrentFromFineLevelandSumBoundary ( if (use_filter && J_buffer[lev+1][idim]) { - ApplyFilterJ(va2vm(J_cp), lev+1, idim); - ApplyFilterJ(va2vm(J_buffer), lev+1, idim); + ApplyFilterJ(J_cp, lev+1, idim); + ApplyFilterJ(J_buffer, lev+1, idim); MultiFab::Add( *J_buffer[lev+1][idim], *J_cp[lev+1][idim], @@ -1407,7 +1405,7 @@ void WarpX::AddCurrentFromFineLevelandSumBoundary ( } else if (use_filter) // but no buffer { - ApplyFilterJ(va2vm(J_cp), lev+1, idim); + ApplyFilterJ(J_cp, lev+1, idim); ablastr::utils::communication::ParallelAdd( mf, *J_cp[lev+1][idim], 0, 0, @@ -1435,7 +1433,7 @@ void WarpX::AddCurrentFromFineLevelandSumBoundary ( ng, amrex::IntVect(0), do_single_precision_comms, period); } - SumBoundaryJ(va2vm(J_cp), lev+1, idim, period); + SumBoundaryJ(J_cp, lev+1, idim, period); MultiFab::Add(*J_fp[lev][idim], mf, 0, 0, J_fp[lev+1][idim]->nComp(), 0); } } @@ -1578,8 +1576,8 @@ void WarpX::AddRhoFromFineLevelandSumBoundary ( } void WarpX::NodalSyncJ ( - const amrex::Vector,3>>& J_fp, - const amrex::Vector,3>>& J_cp, + const ablastr::fields::MultiLevelVectorField& J_fp, + const ablastr::fields::MultiLevelVectorField& J_cp, const int lev, PatchType patch_type) { diff --git a/Source/WarpX.H b/Source/WarpX.H index 47da8ba0902..61d80499b2b 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1332,13 +1332,13 @@ private: void OneStep_multiJ (amrex::Real cur_time); void RestrictCurrentFromFineToCoarsePatch ( - const amrex::Vector,3>>& J_fp, - const amrex::Vector,3>>& J_cp, + const ablastr::fields::MultiLevelVectorField& J_fp, + const ablastr::fields::MultiLevelVectorField& J_cp, int lev); void AddCurrentFromFineLevelandSumBoundary ( - const amrex::Vector,3>>& J_fp, - const amrex::Vector,3>>& J_cp, - const amrex::Vector,3>>& J_buffer, + const ablastr::fields::MultiLevelVectorField& J_fp, + const ablastr::fields::MultiLevelVectorField& J_cp, + const ablastr::fields::MultiLevelVectorField& J_buffer, int lev); void StoreCurrent (int lev); void RestoreCurrent (int lev); @@ -1359,8 +1359,8 @@ private: int lev, const amrex::Periodicity& period); void NodalSyncJ ( - const amrex::Vector,3>>& J_fp, - const amrex::Vector,3>>& J_cp, + const ablastr::fields::MultiLevelVectorField& J_fp, + const ablastr::fields::MultiLevelVectorField& J_cp, int lev, PatchType patch_type); @@ -1861,10 +1861,10 @@ private: * storing the coarse patch averaged magnetic field to be transformed */ void PSATDBackwardTransformEBavg ( - const amrex::Vector,3>>& E_avg_fp, - const amrex::Vector,3>>& B_avg_fp, - const amrex::Vector,3>>& E_avg_cp, - const amrex::Vector,3>>& B_avg_cp); + const ablastr::fields::MultiLevelVectorField& E_avg_fp, + const ablastr::fields::MultiLevelVectorField& B_avg_fp, + const ablastr::fields::MultiLevelVectorField& E_avg_cp, + const ablastr::fields::MultiLevelVectorField& B_avg_cp); /** * \brief Forward FFT of J on all mesh refinement levels, @@ -1878,8 +1878,8 @@ private: * (only used in RZ geometry to avoid double filtering) */ void PSATDForwardTransformJ ( - const amrex::Vector,3>>& J_fp, - const amrex::Vector,3>>& J_cp, + ablastr::fields::MultiLevelVectorField const& J_fp, + ablastr::fields::MultiLevelVectorField const& J_cp, bool apply_kspace_filter=true); /** diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index c27e5e8419c..d3438d2d7b3 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2313,16 +2313,16 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (do_current_centering) { amrex::BoxArray const& nodal_ba = amrex::convert(ba, amrex::IntVect::TheNodeVector()); - AllocInitMultiFab(current_fp_nodal[lev][0], nodal_ba, dm, ncomps, ngJ, lev, "current_fp_nodal[x]", 0.0_rt); - AllocInitMultiFab(current_fp_nodal[lev][1], nodal_ba, dm, ncomps, ngJ, lev, "current_fp_nodal[y]", 0.0_rt); - AllocInitMultiFab(current_fp_nodal[lev][2], nodal_ba, dm, ncomps, ngJ, lev, "current_fp_nodal[z]", 0.0_rt); + m_fields.alloc_init( "current_fp_nodal", Direction{0}, lev, nodal_ba, dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init( "current_fp_nodal", Direction{1}, lev, nodal_ba, dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init( "current_fp_nodal", Direction{2}, lev, nodal_ba, dm, ncomps, ngJ, 0.0_rt); } if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Vay) { - AllocInitMultiFab(current_fp_vay[lev][0], amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngJ, lev, "current_fp_vay[x]", 0.0_rt); - AllocInitMultiFab(current_fp_vay[lev][1], amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngJ, lev, "current_fp_vay[y]", 0.0_rt); - AllocInitMultiFab(current_fp_vay[lev][2], amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngJ, lev, "current_fp_vay[z]", 0.0_rt); + m_fields.alloc_init( "current_fp_vay", Direction{0}, lev, amrex::convert(ba, jx_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init( "current_fp_vay", Direction{1}, lev, amrex::convert(ba, jy_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init( "current_fp_vay", Direction{2}, lev, amrex::convert(ba, jz_nodal_flag), dm, ncomps, ngJ, 0.0_rt); } if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) From 9bb2b05cdd48409b801e2d7259e12de4d709e8d4 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 12:04:03 -0700 Subject: [PATCH 088/314] Continue refactoring currents --- Source/Evolve/WarpXEvolve.cpp | 3 --- .../MagnetostaticSolver/MagnetostaticSolver.cpp | 7 ++++--- Source/FieldSolver/WarpXPushFieldsEM.cpp | 4 +++- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 05eeb996b3f..9b9f67fc683 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -543,7 +543,6 @@ void WarpX::HandleParticlesAtBoundaries (int step, amrex::Real cur_time, int num void WarpX::SyncCurrentAndRho () { - using ablastr::fields::va2vm; using ablastr::fields::Direction; if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) @@ -825,8 +824,6 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) void WarpX::OneStep_sub1 (Real cur_time) { - using ablastr::fields::va2vm; - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( electrostatic_solver_id == ElectrostaticSolverAlgo::None, "Electrostatic solver cannot be used with sub-cycling." diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp index a8f5aab9fdc..26ddbc476e4 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp @@ -74,8 +74,6 @@ WarpX::ComputeMagnetostaticField() void WarpX::AddMagnetostaticFieldLabFrame() { - using ablastr::fields::va2vm; - WARPX_PROFILE("WarpX::AddMagnetostaticFieldLabFrame"); // Store the boundary conditions for the field solver if they haven't been @@ -112,7 +110,10 @@ WarpX::AddMagnetostaticFieldLabFrame() } #endif - SyncCurrent(va2vm(current_fp), va2vm(current_cp), va2vm(current_buf)); // Apply filter, perform MPI exchange, interpolate across levels + SyncCurrent( + m_fields.get_mr_levels_alldirs("current_fp", finest_level), + m_fields.get_mr_levels_alldirs("current_cp", finest_level), + m_fields.get_mr_levels_alldirs("current_buf", finest_level) ); // set the boundary and current density potentials setVectorPotentialBC(m_fields.get_mr_levels_alldirs("vector_potential_fp_nodal", finest_level)); diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 647989b69d1..e8444efb453 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -669,6 +669,8 @@ WarpX::PushPSATD () const int rho_new = spectral_solver_fp[0]->m_spectral_index.rho_new; ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_fp", finest_level); + ablastr::fields::MultiLevelVectorField current_fp = m_fields.get_mr_levels_alldirs("current_fp", finest_level); + ablastr::fields::MultiLevelVectorField current_cp = m_fields.get_mr_levels_alldirs("current_cp", finest_level); if (fft_periodic_single_box) { @@ -737,7 +739,7 @@ WarpX::PushPSATD () PSATDBackwardTransformJ(current_fp, current_cp); // Synchronize J and rho - SyncCurrent(va2vm(current_fp), va2vm(current_cp), va2vm(current_buf)); + SyncCurrent(current_fp, current_cp, current_buf); SyncRho(); } else if (current_deposition_algo == CurrentDepositionAlgo::Vay) From 66c5d36a1860c4fef68c70fc0ef1ed67828d3eb9 Mon Sep 17 00:00:00 2001 From: David Grote Date: Wed, 11 Sep 2024 12:03:43 -0700 Subject: [PATCH 089/314] Update for PSATD solver --- Source/Evolve/WarpXEvolve.cpp | 5 ++++- Source/FieldSolver/WarpXPushFieldsEM.cpp | 8 ++++---- Source/WarpX.H | 12 ++++++------ 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index fdef46c7748..44c3b5648fb 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -755,7 +755,10 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // (the relative time reached here coincides with an integer full time step) if (i_deposit == n_deposit-1) { - PSATDBackwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); + PSATDBackwardTransformEB(m_fields.get_mr_levels_alldirs("Efield_fp", max_level), + m_fields.get_mr_levels_alldirs("Bfield_fp", max_level), + m_fields.get_mr_levels_alldirs("Efield_cp", max_level), + m_fields.get_mr_levels_alldirs("Bfield_cp", max_level)); if (WarpX::do_dive_cleaning) { PSATDBackwardTransformF(); } if (WarpX::do_divb_cleaning) { PSATDBackwardTransformG(); } } diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 647989b69d1..94902870905 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -153,10 +153,10 @@ void WarpX::PSATDBackwardTransformEB ( } void WarpX::PSATDBackwardTransformEBavg ( - const amrex::Vector,3>>& E_avg_fp, - const amrex::Vector,3>>& B_avg_fp, - const amrex::Vector,3>>& E_avg_cp, - const amrex::Vector,3>>& B_avg_cp) + const ablastr::fields::MultiLevelVectorField& E_avg_fp, + const ablastr::fields::MultiLevelVectorField& B_avg_fp, + const ablastr::fields::MultiLevelVectorField& E_avg_cp, + const ablastr::fields::MultiLevelVectorField& B_avg_cp) { const SpectralFieldIndex& Idx = spectral_solver_fp[0]->m_spectral_index; diff --git a/Source/WarpX.H b/Source/WarpX.H index 47da8ba0902..6908c67e92c 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1861,10 +1861,10 @@ private: * storing the coarse patch averaged magnetic field to be transformed */ void PSATDBackwardTransformEBavg ( - const amrex::Vector,3>>& E_avg_fp, - const amrex::Vector,3>>& B_avg_fp, - const amrex::Vector,3>>& E_avg_cp, - const amrex::Vector,3>>& B_avg_cp); + const ablastr::fields::MultiLevelVectorField& E_avg_fp, + const ablastr::fields::MultiLevelVectorField& B_avg_fp, + const ablastr::fields::MultiLevelVectorField& E_avg_cp, + const ablastr::fields::MultiLevelVectorField& B_avg_cp); /** * \brief Forward FFT of J on all mesh refinement levels, @@ -1878,8 +1878,8 @@ private: * (only used in RZ geometry to avoid double filtering) */ void PSATDForwardTransformJ ( - const amrex::Vector,3>>& J_fp, - const amrex::Vector,3>>& J_cp, + ablastr::fields::MultiLevelVectorField const& J_fp, + ablastr::fields::MultiLevelVectorField const& J_cp, bool apply_kspace_filter=true); /** From e9352d619ff0348c0fb5b784aa0592c4c31fe15c Mon Sep 17 00:00:00 2001 From: Marco Acciarri Date: Wed, 11 Sep 2024 12:14:09 -0700 Subject: [PATCH 090/314] added Efield_fp_external and Bfield_fp_external to m_fields --- Source/Initialization/DivCleaner/ProjectionDivCleaner.H | 2 -- Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp | 8 -------- 2 files changed, 10 deletions(-) diff --git a/Source/Initialization/DivCleaner/ProjectionDivCleaner.H b/Source/Initialization/DivCleaner/ProjectionDivCleaner.H index d44b3bfa73d..cde1c98bed2 100644 --- a/Source/Initialization/DivCleaner/ProjectionDivCleaner.H +++ b/Source/Initialization/DivCleaner/ProjectionDivCleaner.H @@ -64,7 +64,6 @@ protected: amrex::Real m_rtol; amrex::Real m_atol; - //warpx::fields::FieldType m_field_type; std::string m_field_name; public: @@ -84,7 +83,6 @@ public: amrex::Gpu::DeviceVector m_stencil_coefs_z; // Default Constructor - //ProjectionDivCleaner (warpx::fields::FieldType a_field_type); ProjectionDivCleaner (std::string a_field_name); void ReadParameters (); diff --git a/Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp b/Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp index fcbb8555e33..d153aed3db3 100644 --- a/Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp +++ b/Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp @@ -31,7 +31,6 @@ using namespace amrex; namespace warpx::initialization { ProjectionDivCleaner::ProjectionDivCleaner(std::string a_field_name) : - //m_field_type(a_field_type) m_field_name(a_field_name) { using ablastr::fields::Direction; @@ -50,7 +49,6 @@ ProjectionDivCleaner::ProjectionDivCleaner(std::string a_field_name) : m_source.resize(m_levels); const int ncomps = WarpX::ncomps; - //auto const& ng = warpx.getFieldPointer(m_field_type, 0, 0)->nGrowVect(); auto const& ng = warpx.m_fields.get(m_field_name,Direction{0},0)->nGrowVect(); for (int lev = 0; lev < m_levels; ++lev) @@ -216,7 +214,6 @@ ProjectionDivCleaner::setSourceFromBfield () WarpX::ComputeDivB( *m_source[ilev], 0, - //warpx.getFieldPointerArray(m_field_type, ilev), {warpx.m_fields.get(m_field_name,Direction{0},ilev), warpx.m_fields.get(m_field_name,Direction{1},ilev), warpx.m_fields.get(m_field_name,Direction{2},ilev)}, @@ -246,9 +243,6 @@ ProjectionDivCleaner::correctBfield () for (int ilev = 0; ilev < m_levels; ++ilev) { // Grab B-field multifabs at this level - //amrex::MultiFab* Bx = warpx.getFieldPointer(m_field_type, ilev, 0); - //amrex::MultiFab* By = warpx.getFieldPointer(m_field_type, ilev, 1); - //amrex::MultiFab* Bz = warpx.getFieldPointer(m_field_type, ilev, 2); amrex::MultiFab* Bx = warpx.m_fields.get(m_field_name,Direction{0},ilev); amrex::MultiFab* By = warpx.m_fields.get(m_field_name,Direction{1},ilev); amrex::MultiFab* Bz = warpx.m_fields.get(m_field_name,Direction{2},ilev); @@ -350,8 +344,6 @@ WarpX::ProjectionCleanDivB() { ablastr::warn_manager::WarnPriority::low); } - //warpx::initialization::ProjectionDivCleaner dc( - // warpx::fields::FieldType::Bfield_fp_external); warpx::initialization::ProjectionDivCleaner dc("Bfield_fp_external"); From 5f83941743397fa7354a279fee57ccb2a2387e8a Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 12:14:40 -0700 Subject: [PATCH 091/314] Update current for PSATD --- Source/FieldSolver/WarpXPushFieldsEM.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 516d7a65812..8321afdf997 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -671,6 +671,7 @@ WarpX::PushPSATD () ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_fp", finest_level); ablastr::fields::MultiLevelVectorField current_fp = m_fields.get_mr_levels_alldirs("current_fp", finest_level); ablastr::fields::MultiLevelVectorField current_cp = m_fields.get_mr_levels_alldirs("current_cp", finest_level); + ablastr::fields::MultiLevelVectorField current_buf = m_fields.get_mr_levels_alldirs("current_buf", finest_level); if (fft_periodic_single_box) { @@ -691,7 +692,8 @@ WarpX::PushPSATD () { // FFT of D and rho (if used) // TODO Replace current_cp with current_cp_vay once Vay deposition is implemented with MR - PSATDForwardTransformJ(current_fp_vay, current_cp); + PSATDForwardTransformJ( + m_fields.get_mr_levels_alldirs("current_fp_vay", finest_level), current_cp); PSATDForwardTransformRho(rho_fp, rho_cp, 0, rho_old); PSATDForwardTransformRho(rho_fp, rho_cp, 1, rho_new); @@ -745,7 +747,8 @@ WarpX::PushPSATD () else if (current_deposition_algo == CurrentDepositionAlgo::Vay) { // FFT of D - PSATDForwardTransformJ(current_fp_vay, current_cp); + PSATDForwardTransformJ( + m_fields.get_mr_levels_alldirs("current_fp_vay", finest_level), current_cp); // Compute J from D in k-space PSATDVayDeposition(); From ac505ebdf3eb7408038bd85c9f3fb671f5cd9cc2 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 11 Sep 2024 12:17:23 -0700 Subject: [PATCH 092/314] `Efield_aux` & `Bfield_aux` --- Source/Diagnostics/BTDiagnostics.cpp | 13 +-- .../ComputeDiagFunctors/DivBFunctor.H | 13 ++- .../ComputeDiagFunctors/DivBFunctor.cpp | 9 +- .../ComputeDiagFunctors/DivEFunctor.H | 13 ++- .../ComputeDiagFunctors/DivEFunctor.cpp | 10 +- .../FlushFormats/FlushFormatPlotfile.cpp | 12 +-- Source/Diagnostics/FullDiagnostics.cpp | 20 ++-- .../ReducedDiags/ColliderRelevant.cpp | 14 +-- .../Diagnostics/ReducedDiags/FieldEnergy.cpp | 16 +-- .../Diagnostics/ReducedDiags/FieldMaximum.cpp | 15 +-- .../ReducedDiags/FieldMomentum.cpp | 15 +-- .../Diagnostics/ReducedDiags/FieldProbe.cpp | 15 +-- .../Diagnostics/ReducedDiags/FieldReduction.H | 12 +-- .../ReducedDiags/LoadBalanceCosts.cpp | 6 +- .../ReducedDiags/ParticleExtrema.cpp | 16 +-- Source/Diagnostics/SliceDiagnostic.cpp | 17 +-- Source/Diagnostics/WarpXIO.cpp | 4 +- Source/Evolve/WarpXEvolve.cpp | 86 +++++++++++---- Source/FieldSolver/Fields.H | 4 +- .../FiniteDifferenceSolver/ComputeDivE.cpp | 11 +- .../FiniteDifferenceSolver.H | 8 +- .../SpectralBaseAlgorithm.H | 3 +- .../SpectralBaseAlgorithm.cpp | 7 +- .../SpectralSolver/SpectralSolver.H | 10 +- Source/Initialization/WarpXInitData.cpp | 15 +-- Source/Parallelization/WarpXComm.cpp | 30 ++++-- Source/Parallelization/WarpXRegrid.cpp | 16 --- Source/Particles/MultiParticleContainer.cpp | 14 +-- Source/Utils/WarpXMovingWindow.cpp | 3 + Source/WarpX.H | 8 +- Source/WarpX.cpp | 102 +++++++++--------- 31 files changed, 325 insertions(+), 212 deletions(-) diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 5d74bf1e718..a958fc018f5 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -21,6 +21,7 @@ #include "Utils/WarpXConst.H" #include "WarpX.H" +#include #include #include #include @@ -569,17 +570,17 @@ BTDiagnostics::InitializeFieldFunctors (int lev) m_cell_center_functors.at(lev).size()); for (int comp=0; comp(warpx.getFieldPointer(FieldType::Efield_aux, lev, 0), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Efield_aux", Direction{0}, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "Ey" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 1), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Efield_aux", Direction{1}, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "Ez" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 2), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Efield_aux", Direction{2}, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "Bx" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 0), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{0}, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "By" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 1), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{1}, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "Bz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 2), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{2}, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "jx" ){ m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp",Direction{0}, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "jy" ){ diff --git a/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.H index 3d04a56742b..1d36b434ae2 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.H @@ -3,6 +3,8 @@ #include "ComputeDiagFunctor.H" +#include + #include #include @@ -22,8 +24,13 @@ public: * (summing over modes) * \param[in] ncomp Number of component of mf_src to cell-center in dst multifab. */ - DivBFunctor(std::array arr_mf_src, int lev, amrex::IntVect crse_ratio, - bool convertRZmodes2cartesian=true, int ncomp=1); + DivBFunctor ( + ablastr::fields::VectorField const & arr_mf_src, + int lev, + amrex::IntVect crse_ratio, + bool convertRZmodes2cartesian=true, + int ncomp=1 + ); /** \brief Compute DivB directly into mf_dst. * @@ -34,7 +41,7 @@ public: void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer*/) const override; private: /** Vector of pointer to source multifab Bx, By, Bz */ - std::array m_arr_mf_src; + ablastr::fields::VectorField m_arr_mf_src; int const m_lev; /**< level on which mf_src is defined (used in cylindrical) */ /**< (for cylindrical) whether to average all modes into 1 comp */ bool m_convertRZmodes2cartesian; diff --git a/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.cpp index b5782e76ae6..224b74ba372 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.cpp @@ -7,8 +7,13 @@ #include #include -DivBFunctor::DivBFunctor(const std::array arr_mf_src, const int lev, const amrex::IntVect crse_ratio, - bool convertRZmodes2cartesian, const int ncomp) +DivBFunctor::DivBFunctor ( + ablastr::fields::VectorField const & arr_mf_src, + const int lev, + const amrex::IntVect crse_ratio, + bool convertRZmodes2cartesian, + const int ncomp +) : ComputeDiagFunctor(ncomp, crse_ratio), m_arr_mf_src(arr_mf_src), m_lev(lev), m_convertRZmodes2cartesian(convertRZmodes2cartesian) {} diff --git a/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.H index 312ccaa5cd6..e7691187f3a 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.H @@ -3,6 +3,8 @@ #include "ComputeDiagFunctor.H" +#include + #include #include @@ -21,8 +23,13 @@ public: * \param[in] convertRZmodes2cartesian if true, all RZ modes are averaged into one component * \param[in] ncomp Number of component of mf_src to cell-center in dst multifab. */ - DivEFunctor(std::array arr_mf_src, int lev, amrex::IntVect crse_ratio, - bool convertRZmodes2cartesian=true, int ncomp=1); + DivEFunctor ( + ablastr::fields::VectorField const & arr_mf_src, + int lev, + amrex::IntVect crse_ratio, + bool convertRZmodes2cartesian=true, + int ncomp=1 + ); /** \brief Compute DivE directly into mf_dst. * @@ -33,7 +40,7 @@ public: void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer=0*/) const override; private: /** Vector of pointer to source multifab Bx, By, Bz */ - std::array m_arr_mf_src; + ablastr::fields::VectorField m_arr_mf_src; int const m_lev; /**< level on which mf_src is defined (used in cylindrical) */ /**< (for cylindrical) whether to average all modes into 1 comp */ bool m_convertRZmodes2cartesian; diff --git a/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.cpp index 62801cd431a..e2c4d98c708 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.cpp @@ -13,9 +13,13 @@ #include #include -DivEFunctor::DivEFunctor(const std::array arr_mf_src, const int lev, - const amrex::IntVect crse_ratio, - bool convertRZmodes2cartesian, const int ncomp) +DivEFunctor::DivEFunctor ( + ablastr::fields::VectorField const & arr_mf_src, + const int lev, + const amrex::IntVect crse_ratio, + bool convertRZmodes2cartesian, + const int ncomp +) : ComputeDiagFunctor(ncomp, crse_ratio), m_arr_mf_src(arr_mf_src), m_lev(lev), m_convertRZmodes2cartesian(convertRZmodes2cartesian) { diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index fd836295e3f..3af9b6c91ec 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -568,12 +568,12 @@ FlushFormatPlotfile::WriteAllRawFields( // Auxiliary patch - WriteRawMF( warpx.getField(FieldType::Efield_aux, lev, 0), dm, raw_pltname, default_level_prefix, "Ex_aux", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getField(FieldType::Efield_aux, lev, 1), dm, raw_pltname, default_level_prefix, "Ey_aux", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getField(FieldType::Efield_aux, lev, 2), dm, raw_pltname, default_level_prefix, "Ez_aux", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getField(FieldType::Bfield_aux, lev, 0), dm, raw_pltname, default_level_prefix, "Bx_aux", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getField(FieldType::Bfield_aux, lev, 1), dm, raw_pltname, default_level_prefix, "By_aux", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getField(FieldType::Bfield_aux, lev, 2), dm, raw_pltname, default_level_prefix, "Bz_aux", lev, plot_raw_fields_guards); + WriteRawMF( *warpx.m_fields.get("Efield_aux", Direction{0}, lev), dm, raw_pltname, default_level_prefix, "Ex_aux", lev, plot_raw_fields_guards); + WriteRawMF( *warpx.m_fields.get("Efield_aux", Direction{1}, lev), dm, raw_pltname, default_level_prefix, "Ey_aux", lev, plot_raw_fields_guards); + WriteRawMF( *warpx.m_fields.get("Efield_aux", Direction{2}, lev), dm, raw_pltname, default_level_prefix, "Ez_aux", lev, plot_raw_fields_guards); + WriteRawMF( *warpx.m_fields.get("Bfield_aux", Direction{0}, lev), dm, raw_pltname, default_level_prefix, "Bx_aux", lev, plot_raw_fields_guards); + WriteRawMF( *warpx.m_fields.get("Bfield_aux", Direction{1}, lev), dm, raw_pltname, default_level_prefix, "By_aux", lev, plot_raw_fields_guards); + WriteRawMF( *warpx.m_fields.get("Bfield_aux", Direction{2}, lev), dm, raw_pltname, default_level_prefix, "Bz_aux", lev, plot_raw_fields_guards); // fine patch WriteRawMF( *warpx.m_fields.get("Efield_fp", Direction{0}, lev), dm, raw_pltname, diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index cc955c4e2e8..7c33365d61b 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -20,6 +20,8 @@ #include "Utils/WarpXAlgorithmSelection.H" #include "WarpX.H" +#include + #include #include #include @@ -660,13 +662,15 @@ FullDiagnostics::InitializeFieldFunctors (int lev) // diagnostic output bool deposit_current = !m_solver_deposits_current; + using ablastr::fields::Direction; + m_all_field_functors[lev].resize(ntot); // Fill vector of functors for all components except individual cylindrical modes. for (int comp=0; comp(warpx.getFieldPointer(FieldType::Efield_aux, lev, 2), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Efield_aux", Direction{2}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Bz" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 2), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{2}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "jz" ){ m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, true, deposit_current); deposit_current = false; @@ -696,9 +700,9 @@ FullDiagnostics::InitializeFieldFunctors (int lev) } else if ( m_varnames[comp] == "part_per_grid" ){ m_all_field_functors[lev][comp] = std::make_unique(nullptr, lev, m_crse_ratio); } else if ( m_varnames[comp] == "divB" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointerArray(FieldType::Bfield_aux, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get_alldirs("Bfield_aux", lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "divE" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointerArray(FieldType::Efield_aux, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get_alldirs("Efield_aux", lev), lev, m_crse_ratio); } else { @@ -731,13 +735,13 @@ FullDiagnostics::InitializeFieldFunctors (int lev) #else // Valid transverse fields in Cartesian coordinates if ( m_varnames[comp] == "Ex" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 0), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Efield_aux", Direction{0}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Ey" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 1), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Efield_aux", Direction{1}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Bx" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 0), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{0}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "By" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 1), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Efield_aux", Direction{1}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "jx" ){ m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, true, deposit_current); deposit_current = false; diff --git a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp index 1b0c74cf7fa..d1210c35e34 100644 --- a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp @@ -429,6 +429,8 @@ void ColliderRelevant::ComputeDiags (int step) amrex::Real chimax_f = 0.0_rt; amrex::Real chiave_f = 0.0_rt; + using ablastr::fields::Direction; + if (myspc.DoQED()) { // define variables in preparation for field gatheeduce_data.value()ring @@ -442,12 +444,12 @@ void ColliderRelevant::ComputeDiags (int step) // define variables in preparation for field gathering const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(lev, 0)); - const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); - const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); - const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); - const amrex::MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); - const amrex::MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); - const amrex::MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); + const amrex::MultiFab & Ex = *warpx.m_fields.get("Efield_aux", Direction{0}, lev); + const amrex::MultiFab & Ey = *warpx.m_fields.get("Efield_aux", Direction{1}, lev); + const amrex::MultiFab & Ez = *warpx.m_fields.get("Efield_aux", Direction{2}, lev); + const amrex::MultiFab & Bx = *warpx.m_fields.get("Bfield_aux", Direction{0}, lev); + const amrex::MultiFab & By = *warpx.m_fields.get("Bfield_aux", Direction{1}, lev); + const amrex::MultiFab & Bz = *warpx.m_fields.get("Bfield_aux", Direction{2}, lev); // declare reduce_op ReduceOps reduce_op; diff --git a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp index 9c0fac101a2..326bdd68c01 100644 --- a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp @@ -13,6 +13,8 @@ #include "Utils/WarpXConst.H" #include "WarpX.H" +#include + #include #include #include @@ -87,16 +89,18 @@ void FieldEnergy::ComputeDiags (int step) // get number of level const auto nLevel = warpx.finestLevel() + 1; + using ablastr::fields::Direction; + // loop over refinement levels for (int lev = 0; lev < nLevel; ++lev) { // get MultiFab data at lev - const MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); - const MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); - const MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); - const MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); - const MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); - const MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); + const MultiFab & Ex = *warpx.m_fields.get("Efield_aux", Direction{0}, lev); + const MultiFab & Ey = *warpx.m_fields.get("Efield_aux", Direction{1}, lev); + const MultiFab & Ez = *warpx.m_fields.get("Efield_aux", Direction{2}, lev); + const MultiFab & Bx = *warpx.m_fields.get("Bfield_aux", Direction{0}, lev); + const MultiFab & By = *warpx.m_fields.get("Bfield_aux", Direction{1}, lev); + const MultiFab & Bz = *warpx.m_fields.get("Bfield_aux", Direction{2}, lev); // get cell volume const std::array &dx = WarpX::CellSize(lev); diff --git a/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp b/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp index 24d9dda3ea6..21a8f252a31 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp @@ -11,6 +11,7 @@ #include "Utils/TextMsg.H" #include "WarpX.H" +#include #include #include @@ -112,16 +113,18 @@ void FieldMaximum::ComputeDiags (int step) // get number of level const auto nLevel = warpx.finestLevel() + 1; + using ablastr::fields::Direction; + // loop over refinement levels for (int lev = 0; lev < nLevel; ++lev) { // get MultiFab data at lev - const MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); - const MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); - const MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); - const MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); - const MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); - const MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); + const MultiFab & Ex = *warpx.m_fields.get("Efield_aux", Direction{0}, lev); + const MultiFab & Ey = *warpx.m_fields.get("Efield_aux", Direction{1}, lev); + const MultiFab & Ez = *warpx.m_fields.get("Efield_aux", Direction{2}, lev); + const MultiFab & Bx = *warpx.m_fields.get("Bfield_aux", Direction{0}, lev); + const MultiFab & By = *warpx.m_fields.get("Bfield_aux", Direction{1}, lev); + const MultiFab & Bz = *warpx.m_fields.get("Bfield_aux", Direction{2}, lev); constexpr int noutputs = 8; // max of Ex,Ey,Ez,|E|,Bx,By,Bz and |B| constexpr int index_Ex = 0; diff --git a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp index 7eb16efecff..80965edae11 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp @@ -12,6 +12,7 @@ #include "Utils/WarpXConst.H" #include "WarpX.H" +#include #include #include @@ -104,16 +105,18 @@ void FieldMomentum::ComputeDiags (int step) // Get number of refinement levels const auto nLevel = warpx.finestLevel() + 1; + using ablastr::fields::Direction; + // Loop over refinement levels for (int lev = 0; lev < nLevel; ++lev) { // Get MultiFab data at given refinement level - const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev, 0); - const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev, 1); - const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev, 2); - const amrex::MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev, 0); - const amrex::MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev, 1); - const amrex::MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev, 2); + const amrex::MultiFab & Ex = *warpx.m_fields.get("Efield_aux", Direction{0}, lev); + const amrex::MultiFab & Ey = *warpx.m_fields.get("Efield_aux", Direction{1}, lev); + const amrex::MultiFab & Ez = *warpx.m_fields.get("Efield_aux", Direction{2}, lev); + const amrex::MultiFab & Bx = *warpx.m_fields.get("Bfield_aux", Direction{0}, lev); + const amrex::MultiFab & By = *warpx.m_fields.get("Bfield_aux", Direction{1}, lev); + const amrex::MultiFab & Bz = *warpx.m_fields.get("Bfield_aux", Direction{2}, lev); // Cell-centered index type const amrex::GpuArray cc{0,0,0}; diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp index 1113075f4cd..e8d347289fd 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp @@ -17,6 +17,7 @@ #include "Utils/WarpXConst.H" #include "WarpX.H" +#include #include #include @@ -381,6 +382,8 @@ void FieldProbe::ComputeDiags (int step) // get number of mesh-refinement levels const auto nLevel = warpx.finestLevel() + 1; + using ablastr::fields::Direction; + // loop over refinement levels for (int lev = 0; lev < nLevel; ++lev) { @@ -398,12 +401,12 @@ void FieldProbe::ComputeDiags (int step) } // get MultiFab data at lev - const amrex::MultiFab &Ex = warpx.getField(FieldType::Efield_aux, lev, 0); - const amrex::MultiFab &Ey = warpx.getField(FieldType::Efield_aux, lev, 1); - const amrex::MultiFab &Ez = warpx.getField(FieldType::Efield_aux, lev, 2); - const amrex::MultiFab &Bx = warpx.getField(FieldType::Bfield_aux, lev, 0); - const amrex::MultiFab &By = warpx.getField(FieldType::Bfield_aux, lev, 1); - const amrex::MultiFab &Bz = warpx.getField(FieldType::Bfield_aux, lev, 2); + const amrex::MultiFab &Ex = *warpx.m_fields.get("Efield_aux", Direction{0}, lev); + const amrex::MultiFab &Ey = *warpx.m_fields.get("Efield_aux", Direction{1}, lev); + const amrex::MultiFab &Ez = *warpx.m_fields.get("Efield_aux", Direction{2}, lev); + const amrex::MultiFab &Bx = *warpx.m_fields.get("Bfield_aux", Direction{0}, lev); + const amrex::MultiFab &By = *warpx.m_fields.get("Bfield_aux", Direction{1}, lev); + const amrex::MultiFab &Bz = *warpx.m_fields.get("Bfield_aux", Direction{2}, lev); /* * Prepare interpolation of field components to probe_position diff --git a/Source/Diagnostics/ReducedDiags/FieldReduction.H b/Source/Diagnostics/ReducedDiags/FieldReduction.H index 111393441bc..b197ea6b767 100644 --- a/Source/Diagnostics/ReducedDiags/FieldReduction.H +++ b/Source/Diagnostics/ReducedDiags/FieldReduction.H @@ -100,12 +100,12 @@ public: const auto dx = geom.CellSizeArray(); // get MultiFab data - const amrex::MultiFab & Ex = warpx.getField(warpx::fields::FieldType::Efield_aux, lev,0); - const amrex::MultiFab & Ey = warpx.getField(warpx::fields::FieldType::Efield_aux, lev,1); - const amrex::MultiFab & Ez = warpx.getField(warpx::fields::FieldType::Efield_aux, lev,2); - const amrex::MultiFab & Bx = warpx.getField(warpx::fields::FieldType::Bfield_aux, lev,0); - const amrex::MultiFab & By = warpx.getField(warpx::fields::FieldType::Bfield_aux, lev,1); - const amrex::MultiFab & Bz = warpx.getField(warpx::fields::FieldType::Bfield_aux, lev,2); + const amrex::MultiFab & Ex = *warpx.m_fields.get("Efield_aux", Direction{0}, lev); + const amrex::MultiFab & Ey = *warpx.m_fields.get("Efield_aux", Direction{1}, lev); + const amrex::MultiFab & Ez = *warpx.m_fields.get("Efield_aux", Direction{2}, lev); + const amrex::MultiFab & Bx = *warpx.m_fields.get("Bfield_aux", Direction{0}, lev); + const amrex::MultiFab & By = *warpx.m_fields.get("Bfield_aux", Direction{1}, lev); + const amrex::MultiFab & Bz = *warpx.m_fields.get("Bfield_aux", Direction{2}, lev); const amrex::MultiFab & jx = *warpx.m_fields.get("current_fp",Direction{0},lev); const amrex::MultiFab & jy = *warpx.m_fields.get("current_fp",Direction{1},lev); const amrex::MultiFab & jz = *warpx.m_fields.get("current_fp",Direction{2},lev); diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp index 882c94b12eb..7231c2f3bdc 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp @@ -13,6 +13,8 @@ #include "Utils/WarpXAlgorithmSelection.H" #include "WarpX.H" +#include + #include #include #include @@ -123,11 +125,13 @@ void LoadBalanceCosts::ComputeDiags (int step) // shift index for m_data int shift_m_data = 0; + using ablastr::fields::Direction; + // save data for (int lev = 0; lev < nLevels; ++lev) { const amrex::DistributionMapping& dm = warpx.DistributionMap(lev); - const MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); + const MultiFab & Ex = *warpx.m_fields.get("Efield_aux", Direction{0}, lev); for (MFIter mfi(Ex, false); mfi.isValid(); ++mfi) { const Box& tbx = mfi.tilebox(); diff --git a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp index 811fee7a9de..c1c16c02e92 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp @@ -21,6 +21,8 @@ #include "Utils/WarpXConst.H" #include "WarpX.H" +#include + #include #include #include @@ -260,18 +262,20 @@ void ParticleExtrema::ComputeDiags (int step) const bool galerkin_interpolation = WarpX::galerkin_interpolation; const amrex::IntVect ngEB = warpx.getngEB(); + using ablastr::fields::Direction; + // loop over refinement levels for (int lev = 0; lev <= level_number; ++lev) { // define variables in preparation for field gathering const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(lev, 0)); - const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); - const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); - const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); - const amrex::MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); - const amrex::MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); - const amrex::MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); + const amrex::MultiFab & Ex = *warpx.m_fields.get("Efield_aux", Direction{0}, lev); + const amrex::MultiFab & Ey = *warpx.m_fields.get("Efield_aux", Direction{1}, lev); + const amrex::MultiFab & Ez = *warpx.m_fields.get("Efield_aux", Direction{2}, lev); + const amrex::MultiFab & Bx = *warpx.m_fields.get("Bfield_aux", Direction{0}, lev); + const amrex::MultiFab & By = *warpx.m_fields.get("Bfield_aux", Direction{1}, lev); + const amrex::MultiFab & Bz = *warpx.m_fields.get("Bfield_aux", Direction{2}, lev); // declare reduce_op amrex::ReduceOps reduce_op; diff --git a/Source/Diagnostics/SliceDiagnostic.cpp b/Source/Diagnostics/SliceDiagnostic.cpp index 97af967f2be..8e038e2bdf7 100644 --- a/Source/Diagnostics/SliceDiagnostic.cpp +++ b/Source/Diagnostics/SliceDiagnostic.cpp @@ -11,6 +11,7 @@ #include "Utils/TextMsg.H" #include "WarpX.H" +#include #include #include @@ -175,6 +176,10 @@ CreateSlice( const MultiFab& mf, const Vector &dom_geom, const MultiFab& mfSrc = *smf; MultiFab& mfDst = *cs_mf; + auto & warpx = WarpX::GetInstance(); + + using ablastr::fields::Direction; + MFIter mfi_dst(mfDst); for (MFIter mfi(mfSrc); mfi.isValid(); ++mfi) { @@ -196,27 +201,27 @@ CreateSlice( const MultiFab& mf, const Vector &dom_geom, amrex::amrex_avgdown_nodes(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio); } - if( SliceType == WarpX::GetInstance().getField(FieldType::Efield_aux, 0,0).ixType().toIntVect() ) { + if( SliceType == warpx.m_fields.get("Efield_aux", Direction{0}, 0)->ixType().toIntVect() ) { amrex::amrex_avgdown_edges(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 0); } - if( SliceType == WarpX::GetInstance().getField(FieldType::Efield_aux, 0,1).ixType().toIntVect() ) { + if( SliceType == warpx.m_fields.get("Efield_aux", Direction{1}, 0)->ixType().toIntVect() ) { amrex::amrex_avgdown_edges(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 1); } - if( SliceType == WarpX::GetInstance().getField(FieldType::Efield_aux, 0,2).ixType().toIntVect() ) { + if( SliceType == warpx.m_fields.get("Efield_aux", Direction{2}, 0)->ixType().toIntVect() ) { amrex::amrex_avgdown_edges(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 2); } - if( SliceType == WarpX::GetInstance().getField(FieldType::Bfield_aux, 0,0).ixType().toIntVect() ) { + if( SliceType == warpx.m_fields.get("Bfield_aux", Direction{0}, 0)->ixType().toIntVect() ) { amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 0); } - if( SliceType == WarpX::GetInstance().getField(FieldType::Bfield_aux, 0,1).ixType().toIntVect() ) { + if( SliceType == warpx.m_fields.get("Bfield_aux", Direction{1}, 0)->ixType().toIntVect() ) { amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 1); } - if( SliceType == WarpX::GetInstance().getField(FieldType::Bfield_aux, 0,2).ixType().toIntVect() ) { + if( SliceType == warpx.m_fields.get("Bfield_aux", Direction{2}, 0)->ixType().toIntVect() ) { amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 2); } diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp index 9f84933a188..d30a47f9aaf 100644 --- a/Source/Diagnostics/WarpXIO.cpp +++ b/Source/Diagnostics/WarpXIO.cpp @@ -288,8 +288,8 @@ WarpX::InitFromCheckpoint () if (lev > 0) { for (int i = 0; i < 3; ++i) { - Efield_aux[lev][i]->setVal(0.0); - Bfield_aux[lev][i]->setVal(0.0); + m_fields.get("Efield_aux", Direction{i}, lev)->setVal(0.0); + m_fields.get("Bfield_aux", Direction{i}, lev)->setVal(0.0); m_fields.get("current_cp",Direction{i},lev)->setVal(0.0); Efield_cp[lev][i]->setVal(0.0); diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index fa12e1eff0f..5301fe4268b 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -66,6 +66,8 @@ WarpX::Evolve (int numsteps) WARPX_PROFILE_REGION("WarpX::Evolve()"); WARPX_PROFILE("WarpX::Evolve()"); + using ablastr::fields::Direction; + Real cur_time = t_new[0]; // Note that the default argument is numsteps = -1 @@ -187,11 +189,16 @@ WarpX::Evolve (int numsteps) UpdateAuxilaryData(); FillBoundaryAux(guard_cells.ng_UpdateAux); for (int lev = 0; lev <= finest_level; ++lev) { - mypc->PushP(lev, 0.5_rt*dt[lev], - *Efield_aux[lev][0],*Efield_aux[lev][1], - *Efield_aux[lev][2], - *Bfield_aux[lev][0],*Bfield_aux[lev][1], - *Bfield_aux[lev][2]); + mypc->PushP( + lev, + 0.5_rt*dt[lev], + *m_fields.get("Efield_aux", Direction{0}, lev), + *m_fields.get("Efield_aux", Direction{1}, lev), + *m_fields.get("Efield_aux", Direction{2}, lev), + *m_fields.get("Bfield_aux", Direction{0}, lev), + *m_fields.get("Bfield_aux", Direction{1}, lev), + *m_fields.get("Bfield_aux", Direction{2}, lev) + ); } is_synchronized = true; } @@ -447,6 +454,8 @@ void WarpX::ExplicitFillBoundaryEBUpdateAux () WARPX_ALWAYS_ASSERT_WITH_MESSAGE(evolve_scheme == EvolveScheme::Explicit, "Cannot call WarpX::ExplicitFillBoundaryEBUpdateAux wihtout Explicit evolve scheme set!"); + using ablastr::fields::Direction; + // At the beginning, we have B^{n} and E^{n}. // Particles have p^{n} and x^{n}. // is_synchronized is true. @@ -461,9 +470,16 @@ void WarpX::ExplicitFillBoundaryEBUpdateAux () // on first step, push p by -0.5*dt for (int lev = 0; lev <= finest_level; ++lev) { - mypc->PushP(lev, -0.5_rt*dt[lev], - *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], - *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2]); + mypc->PushP( + lev, + -0.5_rt*dt[lev], + *m_fields.get("Efield_aux", Direction{0}, lev), + *m_fields.get("Efield_aux", Direction{1}, lev), + *m_fields.get("Efield_aux", Direction{2}, lev), + *m_fields.get("Bfield_aux", Direction{0}, lev), + *m_fields.get("Bfield_aux", Direction{1}, lev), + *m_fields.get("Bfield_aux", Direction{2}, lev) + ); } is_synchronized = false; @@ -1022,9 +1038,17 @@ WarpX::doFieldIonization () void WarpX::doFieldIonization (int lev) { - mypc->doFieldIonization(lev, - *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], - *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2]); + using ablastr::fields::Direction; + + mypc->doFieldIonization( + lev, + *m_fields.get("Efield_aux", Direction{0}, lev), + *m_fields.get("Efield_aux", Direction{1}, lev), + *m_fields.get("Efield_aux", Direction{2}, lev), + *m_fields.get("Bfield_aux", Direction{0}, lev), + *m_fields.get("Bfield_aux", Direction{1}, lev), + *m_fields.get("Bfield_aux", Direction{2}, lev) + ); } #ifdef WARPX_QED @@ -1039,9 +1063,17 @@ WarpX::doQEDEvents () void WarpX::doQEDEvents (int lev) { - mypc->doQedEvents(lev, - *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], - *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2]); + using ablastr::fields::Direction; + + mypc->doQedEvents( + lev, + *m_fields.get("Efield_aux", Direction{0}, lev), + *m_fields.get("Efield_aux", Direction{1}, lev), + *m_fields.get("Efield_aux", Direction{2}, lev), + *m_fields.get("Bfield_aux", Direction{0}, lev), + *m_fields.get("Bfield_aux", Direction{1}, lev), + *m_fields.get("Bfield_aux", Direction{2}, lev) + ); } #endif @@ -1059,6 +1091,8 @@ void WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, bool skip_current, PushType push_type) { + using ablastr::fields::Direction; + amrex::MultiFab* current_x = nullptr; amrex::MultiFab* current_y = nullptr; amrex::MultiFab* current_z = nullptr; @@ -1084,8 +1118,12 @@ WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, } mypc->Evolve(lev, - *Efield_aux[lev][0], *Efield_aux[lev][1], *Efield_aux[lev][2], - *Bfield_aux[lev][0], *Bfield_aux[lev][1], *Bfield_aux[lev][2], + *m_fields.get("Efield_aux", Direction{0}, lev), + *m_fields.get("Efield_aux", Direction{1}, lev), + *m_fields.get("Efield_aux", Direction{2}, lev), + *m_fields.get("Bfield_aux", Direction{0}, lev), + *m_fields.get("Bfield_aux", Direction{1}, lev), + *m_fields.get("Bfield_aux", Direction{2}, lev), *current_x, *current_y, *current_z, current_buf[lev][0].get(), current_buf[lev][1].get(), current_buf[lev][2].get(), m_fields.get("rho_fp",lev), charge_buf[lev].get(), @@ -1115,9 +1153,19 @@ WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, #endif if (do_fluid_species) { myfl->Evolve(lev, - *Efield_aux[lev][0], *Efield_aux[lev][1], *Efield_aux[lev][2], - *Bfield_aux[lev][0], *Bfield_aux[lev][1], *Bfield_aux[lev][2], - m_fields.get("rho_fp", lev), *current_x, *current_y, *current_z, cur_time, skip_current); + *m_fields.get("Efield_aux", Direction{0}, lev), + *m_fields.get("Efield_aux", Direction{1}, lev), + *m_fields.get("Efield_aux", Direction{2}, lev), + *m_fields.get("Bfield_aux", Direction{0}, lev), + *m_fields.get("Bfield_aux", Direction{1}, lev), + *m_fields.get("Bfield_aux", Direction{2}, lev), + m_fields.get("rho_fp", lev), + *current_x, + *current_y, + *current_z, + cur_time, + skip_current + ); } } } diff --git a/Source/FieldSolver/Fields.H b/Source/FieldSolver/Fields.H index 964398cecbc..e75657484fa 100644 --- a/Source/FieldSolver/Fields.H +++ b/Source/FieldSolver/Fields.H @@ -15,8 +15,6 @@ namespace warpx::fields enum struct FieldType : int { None, - Efield_aux, - Bfield_aux, Efield_fp, Bfield_fp, Efield_fp_external, @@ -37,7 +35,7 @@ namespace warpx::fields }; constexpr FieldType ArrayFieldTypes[] = { - FieldType::Efield_aux, FieldType::Bfield_aux, FieldType::Efield_fp, FieldType::Bfield_fp, + FieldType::Efield_fp, FieldType::Bfield_fp, FieldType::current_fp, FieldType::current_fp_nodal, FieldType::vector_potential_fp, FieldType::Efield_cp, FieldType::Bfield_cp, FieldType::current_cp, FieldType::Efield_avg_fp, FieldType::Bfield_avg_fp, FieldType::Efield_avg_cp, FieldType::Bfield_avg_cp}; diff --git a/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp index 0702b264874..a6a83d25417 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp @@ -16,6 +16,8 @@ # include "FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H" #endif +#include + #include #include #include @@ -40,9 +42,10 @@ using namespace amrex; * \brief Update the F field, over one timestep */ void FiniteDifferenceSolver::ComputeDivE ( - const std::array,3>& Efield, - amrex::MultiFab& divEfield ) { - + ablastr::fields::VectorField const & Efield, + amrex::MultiFab& divEfield +) +{ // Select algorithm (The choice of algorithm is a runtime option, // but we compile code for each algorithm, using templates) #ifdef WARPX_DIM_RZ @@ -77,7 +80,7 @@ void FiniteDifferenceSolver::ComputeDivE ( template void FiniteDifferenceSolver::ComputeDivECartesian ( - const std::array,3>& Efield, + ablastr::fields::VectorField const & Efield, amrex::MultiFab& divEfield ) { // Loop through the grids, and over the tiles within each grid diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 464c8703146..3b6e2aff197 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -96,8 +96,10 @@ class FiniteDifferenceSolver amrex::Vector field_boundary_lo, amrex::Vector field_boundary_hi); - void ComputeDivE ( const std::array,3>& Efield, - amrex::MultiFab& divE ); + void ComputeDivE ( + ablastr::fields::VectorField const & Efield, + amrex::MultiFab& divE + ); /** * \brief Macroscopic E-update for non-vacuum medium using the user-selected @@ -306,7 +308,7 @@ class FiniteDifferenceSolver template< typename T_Algo > void ComputeDivECartesian ( - const std::array,3>& Efield, + ablastr::fields::VectorField const & Efield, amrex::MultiFab& divE ); template< typename T_Algo, typename T_MacroAlgo > diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H index c72e7db250d..462bce23c23 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H @@ -13,6 +13,7 @@ #include "FieldSolver/SpectralSolver/SpectralFieldData_fwd.H" #include "FieldSolver/SpectralSolver/SpectralFieldData.H" +#include #include #include @@ -74,7 +75,7 @@ class SpectralBaseAlgorithm */ void ComputeSpectralDivE ( int lev, SpectralFieldData& field_data, - const std::array,3>& Efield, + ablastr::fields::VectorField const& Efield, amrex::MultiFab& divE ); protected: // Meant to be used in the subclasses diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.cpp index b3f18dd6912..069b724f96c 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.cpp @@ -9,6 +9,8 @@ #include "FieldSolver/SpectralSolver/SpectralFieldData.H" #include "Utils/WarpX_Complex.H" +#include + #include #include #include @@ -58,8 +60,9 @@ void SpectralBaseAlgorithm::ComputeSpectralDivE ( const int lev, SpectralFieldData& field_data, - const std::array,3>& Efield, - amrex::MultiFab& divE ) + ablastr::fields::VectorField const & Efield, + amrex::MultiFab& divE +) { const SpectralFieldIndex& Idx = m_spectral_index; diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolver.H b/Source/FieldSolver/SpectralSolver/SpectralSolver.H index 26581900c02..145ea317f7b 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralSolver.H +++ b/Source/FieldSolver/SpectralSolver/SpectralSolver.H @@ -12,6 +12,7 @@ #include "SpectralAlgorithms/SpectralBaseAlgorithm.H" #include "SpectralFieldData.H" +#include #include #include @@ -125,9 +126,12 @@ class SpectralSolver * \brief Public interface to call the member function ComputeSpectralDivE * of the base class SpectralBaseAlgorithm from objects of class SpectralSolver */ - void ComputeSpectralDivE ( int lev, - const std::array,3>& Efield, - amrex::MultiFab& divE ) { + void ComputeSpectralDivE ( + int lev, + ablastr::fields::VectorField const & Efield, + amrex::MultiFab& divE + ) + { algorithm->ComputeSpectralDivE( lev, field_data, Efield, divE ); } diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 76fb801fa5b..79174fafc37 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -900,6 +900,9 @@ WarpX::PostRestart () void WarpX::InitLevelData (int lev, Real /*time*/) { + ablastr::fields::MultiLevelVectorField Efield_aux = m_fields.get_mr_levels_alldirs("Efield_aux", finest_level); + ablastr::fields::MultiLevelVectorField Bfield_aux = m_fields.get_mr_levels_alldirs("Bfield_aux", finest_level); + // initialize the averaged fields only if the averaged algorithm // is activated ('psatd.do_time_averaging=1') const ParmParse pp_psatd("psatd"); @@ -962,9 +965,9 @@ WarpX::InitLevelData (int lev, Real /*time*/) && (lev > 0) && (lev <= maxlevel_extEMfield_init)) { InitializeExternalFieldsOnGridUsingParser( - Bfield_aux[lev][0].get(), - Bfield_aux[lev][1].get(), - Bfield_aux[lev][2].get(), + Bfield_aux[lev][0], + Bfield_aux[lev][1], + Bfield_aux[lev][2], m_p_ext_field_params->Bxfield_parser->compile<3>(), m_p_ext_field_params->Byfield_parser->compile<3>(), m_p_ext_field_params->Bzfield_parser->compile<3>(), @@ -1008,9 +1011,9 @@ WarpX::InitLevelData (int lev, Real /*time*/) if (lev > 0) { InitializeExternalFieldsOnGridUsingParser( - Efield_aux[lev][0].get(), - Efield_aux[lev][1].get(), - Efield_aux[lev][2].get(), + Efield_aux[lev][0], + Efield_aux[lev][1], + Efield_aux[lev][2], m_p_ext_field_params->Exfield_parser->compile<3>(), m_p_ext_field_params->Eyfield_parser->compile<3>(), m_p_ext_field_params->Ezfield_parser->compile<3>(), diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 71abd8d44eb..e08df14420d 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -55,7 +55,11 @@ WarpX::UpdateAuxilaryData () { WARPX_PROFILE("WarpX::UpdateAuxilaryData()"); - if (Bfield_aux[0][0]->ixType() == Bfield_fp[0][0]->ixType()) { + using ablastr::fields::Direction; + + auto Bfield_aux_lvl0_0 = m_fields.get("Bfield_aux", Direction{0}, 0); + + if (Bfield_aux_lvl0_0->ixType() == Bfield_fp[0][0]->ixType()) { UpdateAuxilaryDataSameType(); } else { UpdateAuxilaryDataStagToNodal(); @@ -63,15 +67,19 @@ WarpX::UpdateAuxilaryData () // When loading particle fields from file: add the external fields: for (int lev = 0; lev <= finest_level; ++lev) { + auto Bfield_aux_lvl_0 = m_fields.get("Bfield_aux", Direction{0}, lev); + auto Bfield_aux_lvl_1 = m_fields.get("Bfield_aux", Direction{1}, lev); + auto Bfield_aux_lvl_2 = m_fields.get("Bfield_aux", Direction{2}, lev); + if (mypc->m_E_ext_particle_s == "read_from_file") { - amrex::MultiFab::Add(*Efield_aux[lev][0], *E_external_particle_field[lev][0], 0, 0, E_external_particle_field[lev][0]->nComp(), guard_cells.ng_FieldGather); - amrex::MultiFab::Add(*Efield_aux[lev][1], *E_external_particle_field[lev][1], 0, 0, E_external_particle_field[lev][1]->nComp(), guard_cells.ng_FieldGather); - amrex::MultiFab::Add(*Efield_aux[lev][2], *E_external_particle_field[lev][2], 0, 0, E_external_particle_field[lev][2]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Bfield_aux_lvl_0, *E_external_particle_field[lev][0], 0, 0, E_external_particle_field[lev][0]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Bfield_aux_lvl_1, *E_external_particle_field[lev][1], 0, 0, E_external_particle_field[lev][1]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Bfield_aux_lvl_2, *E_external_particle_field[lev][2], 0, 0, E_external_particle_field[lev][2]->nComp(), guard_cells.ng_FieldGather); } if (mypc->m_B_ext_particle_s == "read_from_file") { - amrex::MultiFab::Add(*Bfield_aux[lev][0], *B_external_particle_field[lev][0], 0, 0, B_external_particle_field[lev][0]->nComp(), guard_cells.ng_FieldGather); - amrex::MultiFab::Add(*Bfield_aux[lev][1], *B_external_particle_field[lev][1], 0, 0, B_external_particle_field[lev][0]->nComp(), guard_cells.ng_FieldGather); - amrex::MultiFab::Add(*Bfield_aux[lev][2], *B_external_particle_field[lev][2], 0, 0, B_external_particle_field[lev][0]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Bfield_aux_lvl_0, *B_external_particle_field[lev][0], 0, 0, B_external_particle_field[lev][0]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Bfield_aux_lvl_1, *B_external_particle_field[lev][1], 0, 0, B_external_particle_field[lev][0]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Bfield_aux_lvl_2, *B_external_particle_field[lev][2], 0, 0, B_external_particle_field[lev][0]->nComp(), guard_cells.ng_FieldGather); } } @@ -89,6 +97,9 @@ WarpX::UpdateAuxilaryDataStagToNodal () #endif using ablastr::fields::Direction; + ablastr::fields::MultiLevelVectorField Efield_aux = m_fields.get_mr_levels_alldirs("Efield_aux", finest_level); + ablastr::fields::MultiLevelVectorField Bfield_aux = m_fields.get_mr_levels_alldirs("Bfield_aux", finest_level); + amrex::Vector,3>> const & Bmf = WarpX::fft_do_time_averaging ? Bfield_avg_fp : Bfield_fp; amrex::Vector,3>> const & Emf = WarpX::fft_do_time_averaging ? @@ -369,6 +380,8 @@ WarpX::UpdateAuxilaryDataSameType () const amrex::IntVect& ng_src = guard_cells.ng_FieldGather; using ablastr::fields::Direction; + ablastr::fields::MultiLevelVectorField Efield_aux = m_fields.get_mr_levels_alldirs("Efield_aux", finest_level); + ablastr::fields::MultiLevelVectorField Bfield_aux = m_fields.get_mr_levels_alldirs("Bfield_aux", finest_level); // Level 0: Copy from fine to aux // Note: in some configurations, Efield_aux/Bfield_aux and Efield_fp/Bfield_fp are simply aliases to the @@ -985,6 +998,9 @@ WarpX::FillBoundaryAux (IntVect ng) void WarpX::FillBoundaryAux (int lev, IntVect ng) { + ablastr::fields::MultiLevelVectorField Efield_aux = m_fields.get_mr_levels_alldirs("Efield_aux", finest_level); + ablastr::fields::MultiLevelVectorField Bfield_aux = m_fields.get_mr_levels_alldirs("Bfield_aux", finest_level); + const amrex::Periodicity& period = Geom(lev).periodicity(); ablastr::utils::communication::FillBoundary(*Efield_aux[lev][0], ng, WarpX::do_single_precision_comms, period); ablastr::utils::communication::FillBoundary(*Efield_aux[lev][1], ng, WarpX::do_single_precision_comms, period); diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index 1e3ed954fe2..4435ccbc10d 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -170,7 +170,6 @@ WarpX::LoadBalance () void WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const DistributionMapping& dm) { - bool const eb_enabled = EB::enabled(); if (ba == boxArray(lev)) { @@ -236,21 +235,6 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi } #endif - // Aux patch - if (lev == 0 && Bfield_aux[0][0]->ixType() == Bfield_fp[0][0]->ixType()) - { - for (int idim = 0; idim < 3; ++idim) { - Bfield_aux[lev][idim] = std::make_unique(*Bfield_fp[lev][idim], amrex::make_alias, 0, Bfield_aux[lev][idim]->nComp()); - Efield_aux[lev][idim] = std::make_unique(*Efield_fp[lev][idim], amrex::make_alias, 0, Efield_aux[lev][idim]->nComp()); - } - } else { - for (int idim=0; idim < 3; ++idim) - { - //RemakeMultiFab(Bfield_aux[lev][idim], false); - //RemakeMultiFab(Efield_aux[lev][idim], false); - } - } - // Coarse patch if (lev > 0) { diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index db3a05c8b10..56e59d66f86 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -44,6 +44,7 @@ #include "WarpX.H" +#include #include #include @@ -1358,12 +1359,13 @@ MultiParticleContainer::doQEDSchwinger () pc_product_ele->defineAllParticleTiles(); pc_product_pos->defineAllParticleTiles(); - const MultiFab & Ex = warpx.getField(FieldType::Efield_aux, level_0,0); - const MultiFab & Ey = warpx.getField(FieldType::Efield_aux, level_0,1); - const MultiFab & Ez = warpx.getField(FieldType::Efield_aux, level_0,2); - const MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, level_0,0); - const MultiFab & By = warpx.getField(FieldType::Bfield_aux, level_0,1); - const MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, level_0,2); + using ablastr::fields::Direction; + const MultiFab & Ex = *warpx.m_fields.get("Efield_aux", Direction{0}, level_0); + const MultiFab & Ey = *warpx.m_fields.get("Efield_aux", Direction{1}, level_0); + const MultiFab & Ez = *warpx.m_fields.get("Efield_aux", Direction{2}, level_0); + const MultiFab & Bx = *warpx.m_fields.get("Bfield_aux", Direction{0}, level_0); + const MultiFab & By = *warpx.m_fields.get("Bfield_aux", Direction{1}, level_0); + const MultiFab & Bz = *warpx.m_fields.get("Bfield_aux", Direction{2}, level_0); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 5298405ccaf..8e115887ba9 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -147,6 +147,9 @@ WarpX::MoveWindow (const int step, bool move_j) } if (!moving_window_active(step)) { return 0; } + ablastr::fields::MultiLevelVectorField Efield_aux = m_fields.get_mr_levels_alldirs("Efield_aux", finest_level); + ablastr::fields::MultiLevelVectorField Bfield_aux = m_fields.get_mr_levels_alldirs("Bfield_aux", finest_level); + // Update the continuous position of the moving window, // and of the plasma injection moving_window_x += (moving_window_v - WarpX::beta_boost * PhysConst::c)/(1 - moving_window_v * WarpX::beta_boost / PhysConst::c) * dt[0]; diff --git a/Source/WarpX.H b/Source/WarpX.H index 61d80499b2b..9b4c94ca086 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -970,11 +970,11 @@ public: // these should be private, but can't due to Cuda limitations static void ComputeDivB (amrex::MultiFab& divB, int dcomp, - const std::array& B, + ablastr::fields::VectorField const & B, const std::array& dx); static void ComputeDivB (amrex::MultiFab& divB, int dcomp, - const std::array& B, + ablastr::fields::VectorField const & B, const std::array& dx, amrex::IntVect ngrow); void ComputeDivE(amrex::MultiFab& divE, int lev); @@ -1511,10 +1511,6 @@ private: // Fields: First array for level, second for direction // - // Full solution - amrex::Vector, 3 > > Efield_aux; - amrex::Vector, 3 > > Bfield_aux; - // Fine patch amrex::Vector, 3 > > current_fp; amrex::Vector, 3 > > current_fp_vay; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 5b4e4b470eb..043bffc41b3 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -317,9 +317,6 @@ WarpX::WarpX () myfl = std::make_unique(nlevs_max); } - Efield_aux.resize(nlevs_max); - Bfield_aux.resize(nlevs_max); - Efield_fp.resize(nlevs_max); Bfield_fp.resize(nlevs_max); @@ -2577,52 +2574,52 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm // Create aux multifabs on Nodal Box Array BoxArray const nba = amrex::convert(ba,IntVect::TheNodeVector()); - AllocInitMultiFab(Bfield_aux[lev][0], nba, dm, ncomps, ngEB, lev, "Bfield_aux[x]", 0.0_rt); - AllocInitMultiFab(Bfield_aux[lev][1], nba, dm, ncomps, ngEB, lev, "Bfield_aux[y]", 0.0_rt); - AllocInitMultiFab(Bfield_aux[lev][2], nba, dm, ncomps, ngEB, lev, "Bfield_aux[z]", 0.0_rt); + m_fields.alloc_init("Bfield_aux", Direction{0}, lev, nba, dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Bfield_aux", Direction{1}, lev, nba, dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Bfield_aux", Direction{2}, lev, nba, dm, ncomps, ngEB, 0.0_rt); - AllocInitMultiFab(Efield_aux[lev][0], nba, dm, ncomps, ngEB, lev, "Efield_aux[x]", 0.0_rt); - AllocInitMultiFab(Efield_aux[lev][1], nba, dm, ncomps, ngEB, lev, "Efield_aux[y]", 0.0_rt); - AllocInitMultiFab(Efield_aux[lev][2], nba, dm, ncomps, ngEB, lev, "Efield_aux[z]", 0.0_rt); + m_fields.alloc_init("Efield_aux", Direction{0}, lev, nba, dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Efield_aux", Direction{1}, lev, nba, dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Efield_aux", Direction{2}, lev, nba, dm, ncomps, ngEB, 0.0_rt); } else if (lev == 0) { if (WarpX::fft_do_time_averaging) { - AliasInitMultiFab(Bfield_aux[lev][0], *Bfield_avg_fp[lev][0], 0, ncomps, lev, "Bfield_aux[x]", 0.0_rt); - AliasInitMultiFab(Bfield_aux[lev][1], *Bfield_avg_fp[lev][1], 0, ncomps, lev, "Bfield_aux[y]", 0.0_rt); - AliasInitMultiFab(Bfield_aux[lev][2], *Bfield_avg_fp[lev][2], 0, ncomps, lev, "Bfield_aux[z]", 0.0_rt); + m_fields.alias_init("Bfield_aux", "Bfield_avg_fp", Direction{0}, lev, 0.0_rt); + m_fields.alias_init("Bfield_aux", "Bfield_avg_fp", Direction{1}, lev, 0.0_rt); + m_fields.alias_init("Bfield_aux", "Bfield_avg_fp", Direction{2}, lev, 0.0_rt); - AliasInitMultiFab(Efield_aux[lev][0], *Efield_avg_fp[lev][0], 0, ncomps, lev, "Efield_aux[x]", 0.0_rt); - AliasInitMultiFab(Efield_aux[lev][1], *Efield_avg_fp[lev][1], 0, ncomps, lev, "Efield_aux[y]", 0.0_rt); - AliasInitMultiFab(Efield_aux[lev][2], *Efield_avg_fp[lev][2], 0, ncomps, lev, "Efield_aux[z]", 0.0_rt); + m_fields.alias_init("Efield_aux", "Efield_avg_fp", Direction{0}, lev, 0.0_rt); + m_fields.alias_init("Efield_aux", "Efield_avg_fp", Direction{1}, lev, 0.0_rt); + m_fields.alias_init("Efield_aux", "Efield_avg_fp", Direction{2}, lev, 0.0_rt); } else { if (mypc->m_B_ext_particle_s == "read_from_file") { - AllocInitMultiFab(Bfield_aux[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[x]"); - AllocInitMultiFab(Bfield_aux[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[y]"); - AllocInitMultiFab(Bfield_aux[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[z]"); + m_fields.alloc_init("Bfield_aux", Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Bfield_aux", Direction{1}, lev, amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Bfield_aux", Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); } else { // In this case, the aux grid is simply an alias of the fp grid (most common case in WarpX) - AliasInitMultiFab(Bfield_aux[lev][0], *Bfield_fp[lev][0], 0, ncomps, lev, "Bfield_aux[x]", 0.0_rt); - AliasInitMultiFab(Bfield_aux[lev][1], *Bfield_fp[lev][1], 0, ncomps, lev, "Bfield_aux[y]", 0.0_rt); - AliasInitMultiFab(Bfield_aux[lev][2], *Bfield_fp[lev][2], 0, ncomps, lev, "Bfield_aux[z]", 0.0_rt); + m_fields.alias_init("Bfield_aux", "Bfield_avg_fp", Direction{0}, lev, 0.0_rt); + m_fields.alias_init("Bfield_aux", "Bfield_avg_fp", Direction{1}, lev, 0.0_rt); + m_fields.alias_init("Bfield_aux", "Bfield_avg_fp", Direction{2}, lev, 0.0_rt); } if (mypc->m_E_ext_particle_s == "read_from_file") { - AllocInitMultiFab(Efield_aux[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[x]"); - AllocInitMultiFab(Efield_aux[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[y]"); - AllocInitMultiFab(Efield_aux[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[z]"); + m_fields.alloc_init("Efield_aux", Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Efield_aux", Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Efield_aux", Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); } else { // In this case, the aux grid is simply an alias of the fp grid (most common case in WarpX) - AliasInitMultiFab(Efield_aux[lev][0], *m_fields.get("Efield_fp",Direction{0},lev), 0, ncomps, lev, "Efield_aux[x]", 0.0_rt); - AliasInitMultiFab(Efield_aux[lev][1], *m_fields.get("Efield_fp",Direction{1},lev), 0, ncomps, lev, "Efield_aux[y]", 0.0_rt); - AliasInitMultiFab(Efield_aux[lev][2], *m_fields.get("Efield_fp",Direction{2},lev), 0, ncomps, lev, "Efield_aux[z]", 0.0_rt); + m_fields.alias_init("Efield_aux", "Efield_avg_fp", Direction{0}, lev, 0.0_rt); + m_fields.alias_init("Efield_aux", "Efield_avg_fp", Direction{1}, lev, 0.0_rt); + m_fields.alias_init("Efield_aux", "Efield_avg_fp", Direction{2}, lev, 0.0_rt); } } } else { - AllocInitMultiFab(Bfield_aux[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[x]", 0.0_rt); - AllocInitMultiFab(Bfield_aux[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[y]", 0.0_rt); - AllocInitMultiFab(Bfield_aux[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[z]", 0.0_rt); + m_fields.alloc_init("Bfield_aux", Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Bfield_aux", Direction{1}, lev, amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Bfield_aux", Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - AllocInitMultiFab(Efield_aux[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[x]", 0.0_rt); - AllocInitMultiFab(Efield_aux[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[y]", 0.0_rt); - AllocInitMultiFab(Efield_aux[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[z]", 0.0_rt); + m_fields.alloc_init("Efield_aux", Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Efield_aux", Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Efield_aux", Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); } // The external fields that are read from file @@ -2638,11 +2635,14 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm } if (mypc->m_B_ext_particle_s == "read_from_file") { // These fields will be added to the fields that the particles see, and need to match the index type - AllocInitMultiFab(B_external_particle_field[lev][0], amrex::convert(ba, Bfield_aux[lev][0]->ixType()), + auto Bfield_aux_levl_0 = m_fields.get("Bfield_aux", Direction{0}, lev); + auto Bfield_aux_levl_1 = m_fields.get("Bfield_aux", Direction{1}, lev); + auto Bfield_aux_levl_2 = m_fields.get("Bfield_aux", Direction{2}, lev); + AllocInitMultiFab(B_external_particle_field[lev][0], amrex::convert(ba, Bfield_aux_levl_0->ixType()), dm, ncomps, ngEB, lev, "B_external_particle_field[x]", 0.0_rt); - AllocInitMultiFab(B_external_particle_field[lev][1], amrex::convert(ba, Bfield_aux[lev][1]->ixType()), + AllocInitMultiFab(B_external_particle_field[lev][1], amrex::convert(ba, Bfield_aux_levl_1->ixType()), dm, ncomps, ngEB, lev, "B_external_particle_field[y]", 0.0_rt); - AllocInitMultiFab(B_external_particle_field[lev][2], amrex::convert(ba, Bfield_aux[lev][2]->ixType()), + AllocInitMultiFab(B_external_particle_field[lev][2], amrex::convert(ba, Bfield_aux_levl_2->ixType()), dm, ncomps, ngEB, lev, "B_external_particle_field[z]", 0.0_rt); } if (m_p_ext_field_params->E_ext_grid_type != ExternalFieldType::default_zero && m_p_ext_field_params->E_ext_grid_type != ExternalFieldType::constant) { @@ -2656,11 +2656,14 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm } if (mypc->m_E_ext_particle_s == "read_from_file") { // These fields will be added to the fields that the particles see, and need to match the index type - AllocInitMultiFab(E_external_particle_field[lev][0], amrex::convert(ba, Efield_aux[lev][0]->ixType()), + auto Efield_aux_levl_0 = m_fields.get("Efield_aux", Direction{0}, lev); + auto Efield_aux_levl_1 = m_fields.get("Efield_aux", Direction{1}, lev); + auto Efield_aux_levl_2 = m_fields.get("Efield_aux", Direction{2}, lev); + AllocInitMultiFab(E_external_particle_field[lev][0], amrex::convert(ba, Efield_aux_levl_0->ixType()), dm, ncomps, ngEB, lev, "E_external_particle_field[x]", 0.0_rt); - AllocInitMultiFab(E_external_particle_field[lev][1], amrex::convert(ba, Efield_aux[lev][1]->ixType()), + AllocInitMultiFab(E_external_particle_field[lev][1], amrex::convert(ba, Efield_aux_levl_1->ixType()), dm, ncomps, ngEB, lev, "E_external_particle_field[y]", 0.0_rt); - AllocInitMultiFab(E_external_particle_field[lev][2], amrex::convert(ba, Efield_aux[lev][2]->ixType()), + AllocInitMultiFab(E_external_particle_field[lev][2], amrex::convert(ba, Efield_aux_levl_2->ixType()), dm, ncomps, ngEB, lev, "E_external_particle_field[z]", 0.0_rt); } @@ -3013,7 +3016,7 @@ WarpX::RefRatio (int lev) void WarpX::ComputeDivB (amrex::MultiFab& divB, int const dcomp, - const std::array& B, + ablastr::fields::VectorField const& B, const std::array& dx) { ComputeDivB(divB, dcomp, B, dx, IntVect::TheZeroVector()); @@ -3021,7 +3024,7 @@ WarpX::ComputeDivB (amrex::MultiFab& divB, int const dcomp, void WarpX::ComputeDivB (amrex::MultiFab& divB, int const dcomp, - const std::array& B, + ablastr::fields::VectorField const& B, const std::array& dx, IntVect const ngrow) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE(grid_type != GridType::Collocated, @@ -3061,13 +3064,15 @@ WarpX::ComputeDivE(amrex::MultiFab& divE, const int lev) { if ( WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD ) { #ifdef WARPX_USE_FFT - spectral_solver_fp[lev]->ComputeSpectralDivE( lev, Efield_aux[lev], divE ); + ablastr::fields::VectorField Efield_aux_lev = m_fields.get_alldirs("Efield_aux", lev); + spectral_solver_fp[lev]->ComputeSpectralDivE(lev, Efield_aux_lev, divE); #else WARPX_ABORT_WITH_MESSAGE( "ComputeDivE: PSATD requested but not compiled"); #endif } else { - m_fdtd_solver_fp[lev]->ComputeDivE( Efield_aux[lev], divE ); + ablastr::fields::VectorField Efield_aux_lev = m_fields.get_alldirs("Efield_aux", lev); + m_fdtd_solver_fp[lev]->ComputeDivE(Efield_aux_lev, divE); } } @@ -3473,12 +3478,6 @@ WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, cons switch(field_type) { - case FieldType::Efield_aux : - field_pointer = Efield_aux[lev][direction].get(); - break; - case FieldType::Bfield_aux : - field_pointer = Bfield_aux[lev][direction].get(); - break; case FieldType::Efield_fp : field_pointer = Efield_fp[lev][direction].get(); break; @@ -3549,7 +3548,6 @@ std::array WarpX::getFieldPointerArray (const FieldType field_type, const int lev) const { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - (field_type == FieldType::Efield_aux) || (field_type == FieldType::Bfield_aux) || (field_type == FieldType::Efield_fp) || (field_type == FieldType::Bfield_fp) || (field_type == FieldType::Efield_fp_external) || (field_type == FieldType::Bfield_fp_external) || (field_type == FieldType::current_fp) || (field_type == FieldType::current_fp_nodal) || @@ -3599,10 +3597,6 @@ WarpX::getMultiLevelField(warpx::fields::FieldType field_type) const { switch(field_type) { - case FieldType::Efield_aux : - return Efield_aux; - case FieldType::Bfield_aux : - return Bfield_aux; case FieldType::Efield_fp : return Efield_fp; case FieldType::Efield_fp_external : From 6db6518aa734b3411effea8c056e22eedc59f62d Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 13:14:57 -0700 Subject: [PATCH 093/314] Update current --- Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index 12e898b79ef..2c7458a083e 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -171,14 +171,15 @@ void WarpX::HybridPICEvolveFields () void WarpX::HybridPICDepositInitialRhoAndJ () { - using ablastr::fields::va2vm; - auto& rho_fp_temp = m_hybrid_pic_model->rho_fp_temp; auto& current_fp_temp = m_hybrid_pic_model->current_fp_temp; mypc->DepositCharge(amrex::GetVecOfPtrs(rho_fp_temp), 0._rt); mypc->DepositCurrent(current_fp_temp, dt[0], 0._rt); SyncRho(amrex::GetVecOfPtrs(rho_fp_temp), m_fields.get_mr_levels("rho_cp", finest_level), charge_buf); - SyncCurrent(va2vm(current_fp_temp), va2vm(current_cp), va2vm(current_buf)); + SyncCurrent( + m_fields.get_mr_levels_alldirs("current_fp", finest_level), + m_fields.get_mr_levels_alldirs("current_cp", finest_level), + m_fields.get_mr_levels_alldirs("current_buf", finest_level) ); for (int lev=0; lev <= finest_level; ++lev) { // SyncCurrent does not include a call to FillBoundary, but it is needed // for the hybrid-PIC solver since current values are interpolated to From b8d27312ec3f91ea337988430ebe4cd35d3cdffb Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 13:24:17 -0700 Subject: [PATCH 094/314] Update Evolve --- Source/Evolve/WarpXEvolve.cpp | 51 ++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 5301fe4268b..747ac48833e 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -685,8 +685,8 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // (dt[0] denotes the time step on mesh refinement level 0) if (J_in_time == JInTime::Linear) { - std::string const current_string = (do_current_centering) ? "current_fp_nodal" : "current_fp"; - mypc->DepositCurrent( m_fields.get_mr_levels(current_string, finest_level), dt[0], -dt[0]); + std::string const current_fp_string = (do_current_centering) ? "current_fp_nodal" : "current_fp"; + mypc->DepositCurrent( m_fields.get_mr_levels(current_fp_string, finest_level), dt[0], -dt[0]); // Synchronize J: filter, exchange boundary, and interpolate across levels. // With current centering, the nodal current is deposited in 'current', // namely 'current_fp_nodal': SyncCurrent stores the result of its centering @@ -724,8 +724,8 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Deposit new J at relative time t_deposit_current with time step dt // (dt[0] denotes the time step on mesh refinement level 0) - std::string const current_string = (do_current_centering) ? "current_fp_nodal" : "current_fp"; - mypc->DepositCurrent( m_fields.get_mr_levels(current_string, finest_level), dt[0], t_deposit_current); + std::string const current_fp_string = (do_current_centering) ? "current_fp_nodal" : "current_fp"; + mypc->DepositCurrent( m_fields.get_mr_levels(current_fp_string, finest_level), dt[0], t_deposit_current); // Synchronize J: filter, exchange boundary, and interpolate across levels. // With current centering, the nodal current is deposited in 'current', // namely 'current_fp_nodal': SyncCurrent stores the result of its centering @@ -1093,28 +1093,19 @@ WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, { using ablastr::fields::Direction; - amrex::MultiFab* current_x = nullptr; - amrex::MultiFab* current_y = nullptr; - amrex::MultiFab* current_z = nullptr; + std::string current_fp_string; if (WarpX::do_current_centering) { - current_x = current_fp_nodal[lev][0].get(); - current_y = current_fp_nodal[lev][1].get(); - current_z = current_fp_nodal[lev][2].get(); + current_fp_string = "current_fp_nodal"; } else if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Vay) { - // Note that Vay deposition is supported only for PSATD and the code currently aborts otherwise - current_x = current_fp_vay[lev][0].get(); - current_y = current_fp_vay[lev][1].get(); - current_z = current_fp_vay[lev][2].get(); + current_fp_string = "current_fp_vay"; } else { - current_x = current_fp[lev][0].get(); - current_y = current_fp[lev][1].get(); - current_z = current_fp[lev][2].get(); + current_fp_string = "current_fp"; } mypc->Evolve(lev, @@ -1124,8 +1115,12 @@ WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, *m_fields.get("Bfield_aux", Direction{0}, lev), *m_fields.get("Bfield_aux", Direction{1}, lev), *m_fields.get("Bfield_aux", Direction{2}, lev), - *current_x, *current_y, *current_z, - current_buf[lev][0].get(), current_buf[lev][1].get(), current_buf[lev][2].get(), + *m_fields.get(current_fp_string, Direction{0}, lev), + *m_fields.get(current_fp_string, Direction{1}, lev), + *m_fields.get(current_fp_string, Direction{2}, lev), + m_fields.get("current_buf", Direction{0}, lev), + m_fields.get("current_buf", Direction{1}, lev), + m_fields.get("current_buf", Direction{2}, lev), m_fields.get("rho_fp",lev), charge_buf[lev].get(), Efield_cax[lev][0].get(), Efield_cax[lev][1].get(), Efield_cax[lev][2].get(), Bfield_cax[lev][0].get(), Bfield_cax[lev][1].get(), Bfield_cax[lev][2].get(), @@ -1133,9 +1128,17 @@ WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, if (! skip_current) { #ifdef WARPX_DIM_RZ // This is called after all particles have deposited their current and charge. - ApplyInverseVolumeScalingToCurrentDensity(current_fp[lev][0].get(), current_fp[lev][1].get(), current_fp[lev][2].get(), lev); + ApplyInverseVolumeScalingToCurrentDensity( + m_fields.get("current_fp", Direction{0}, lev), + m_fields.get("current_fp", Direction{1}, lev), + m_fields.get("current_fp", Direction{2}, lev), + lev); if (current_buf[lev][0].get()) { - ApplyInverseVolumeScalingToCurrentDensity(current_buf[lev][0].get(), current_buf[lev][1].get(), current_buf[lev][2].get(), lev-1); + ApplyInverseVolumeScalingToCurrentDensity( + m_fields.get("current_buf", Direction{0}, lev), + m_fields.get("current_buf", Direction{1}, lev), + m_fields.get("current_buf", Direction{2}, lev), + lev-1); } if (m_fields.has("rho_fp", lev)) { ApplyInverseVolumeScalingToChargeDensity(m_fields.get("rho_fp", lev), lev); @@ -1160,9 +1163,9 @@ WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, *m_fields.get("Bfield_aux", Direction{1}, lev), *m_fields.get("Bfield_aux", Direction{2}, lev), m_fields.get("rho_fp", lev), - *current_x, - *current_y, - *current_z, + *m_fields.get(current_fp_string, Direction{0}, lev), + *m_fields.get(current_fp_string, Direction{1}, lev), + *m_fields.get(current_fp_string, Direction{2}, lev), cur_time, skip_current ); From d026e31ca4e0da2e443b720499279b5bcf0ed39b Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 11 Sep 2024 13:27:55 -0700 Subject: [PATCH 095/314] Improve FFT Compile --- .../ComputeDiagFunctors/JFunctor.cpp | 27 +++++++++---------- Source/Evolve/WarpXEvolve.cpp | 4 +-- Source/FieldSolver/WarpXPushFieldsEM.cpp | 18 ++++++++++--- .../FieldSolver/WarpXPushFieldsHybridPIC.cpp | 2 +- Source/Particles/MultiParticleContainer.H | 4 ++- Source/ablastr/fields/MultiFabRegister.H | 16 +++++++++++ Source/ablastr/fields/MultiFabRegister.cpp | 15 +++++++++++ 7 files changed, 63 insertions(+), 23 deletions(-) diff --git a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp index 2cb1d8f710b..10d575a63af 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp @@ -10,6 +10,8 @@ #include "Particles/MultiParticleContainer.H" #include "WarpX.H" +#include + #include #include #include @@ -39,22 +41,12 @@ JFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/ if (m_deposit_current) { // allocate temporary multifab to deposit current density into - amrex::Vector, 3 > > current_fp_temp; - current_fp_temp.resize(1); - - const auto& current_fp_x = *warpx.m_fields.get("current_fp",Direction{0},m_lev); - current_fp_temp[0][0] = std::make_unique( - current_fp_x, amrex::make_alias, 0, current_fp_x.nComp() - ); + using ablastr::fields::Direction; + ablastr::fields::MultiLevelVectorField current_fp_temp; - const auto& current_fp_y = *warpx.m_fields.get("current_fp",Direction{1},m_lev); - current_fp_temp[0][1] = std::make_unique( - current_fp_y, amrex::make_alias, 0, current_fp_y.nComp() - ); - const auto& current_fp_z = *warpx.m_fields.get("current_fp",Direction{2},m_lev); - current_fp_temp[0][2] = std::make_unique( - current_fp_z, amrex::make_alias, 0, current_fp_z.nComp() - ); + warpx.m_fields.alias_init("current_fp_temp", "current_fp", Direction{0}, 0); + warpx.m_fields.alias_init("current_fp_temp", "current_fp", Direction{1}, 0); + warpx.m_fields.alias_init("current_fp_temp", "current_fp", Direction{2}, 0); auto& mypc = warpx.GetPartContainer(); mypc.DepositCurrent(current_fp_temp, warpx.getdt(m_lev), 0.0); @@ -64,6 +56,11 @@ JFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/ for (int idim = 0; idim < 3; ++idim) { current_fp_temp[0][idim]->FillBoundary(warpx.Geom(m_lev).periodicity()); } + + // remove aliases again + warpx.m_fields.erase("current_fp_temp", Direction{0}, 0); + warpx.m_fields.erase("current_fp_temp", Direction{1}, 0); + warpx.m_fields.erase("current_fp_temp", Direction{2}, 0); } WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_mf_src != nullptr, "m_mf_src can't be a nullptr."); diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 5301fe4268b..813f7420b65 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -686,7 +686,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) if (J_in_time == JInTime::Linear) { std::string const current_string = (do_current_centering) ? "current_fp_nodal" : "current_fp"; - mypc->DepositCurrent( m_fields.get_mr_levels(current_string, finest_level), dt[0], -dt[0]); + mypc->DepositCurrent( m_fields.get_mr_levels_alldirs(current_string, finest_level), dt[0], -dt[0]); // Synchronize J: filter, exchange boundary, and interpolate across levels. // With current centering, the nodal current is deposited in 'current', // namely 'current_fp_nodal': SyncCurrent stores the result of its centering @@ -725,7 +725,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Deposit new J at relative time t_deposit_current with time step dt // (dt[0] denotes the time step on mesh refinement level 0) std::string const current_string = (do_current_centering) ? "current_fp_nodal" : "current_fp"; - mypc->DepositCurrent( m_fields.get_mr_levels(current_string, finest_level), dt[0], t_deposit_current); + mypc->DepositCurrent( m_fields.get_mr_levels_alldirs(current_string, finest_level), dt[0], t_deposit_current); // Synchronize J: filter, exchange boundary, and interpolate across levels. // With current centering, the nodal current is deposited in 'current', // namely 'current_fp_nodal': SyncCurrent stores the result of its centering diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 8321afdf997..998a4d7a1dc 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -773,12 +773,18 @@ WarpX::PushPSATD () PSATDForwardTransformRho(rho_fp, rho_cp, 1, rho_new); } + auto Efield_fp = m_fields.get_mr_levels_alldirs("Efield_fp", finest_level); + auto Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); + auto Efield_cp = m_fields.get_mr_levels_alldirs("Efield_cp", finest_level); + auto Bfield_cp = m_fields.get_mr_levels_alldirs("Bfield_cp", finest_level); + // FFT of E and B PSATDForwardTransformEB( - m_fields.get_mr_levels_alldirs("Efield_fp", finest_level), - m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level), - m_fields.get_mr_levels_alldirs("Efield_cp", finest_level), - m_fields.get_mr_levels_alldirs("Bfield_cp", finest_level) ); + Efield_fp, + Bfield_fp, + Efield_cp, + Bfield_cp + ); #ifdef WARPX_DIM_RZ if (pml_rz[0]) { pml_rz[0]->PushPSATD(0); } @@ -794,6 +800,10 @@ WarpX::PushPSATD () // Inverse FFT of E, B, F, and G PSATDBackwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); if (WarpX::fft_do_time_averaging) { + auto Efield_avg_fp = m_fields.get_mr_levels_alldirs("Efield_avg_fp", finest_level); + auto Bfield_avg_fp = m_fields.get_mr_levels_alldirs("Bfield_avg_fp", finest_level); + auto Efield_avg_cp = m_fields.get_mr_levels_alldirs("Efield_avg_cp", finest_level); + auto Bfield_avg_cp = m_fields.get_mr_levels_alldirs("Bfield_avg_cp", finest_level); PSATDBackwardTransformEBavg(Efield_avg_fp, Bfield_avg_fp, Efield_avg_cp, Bfield_avg_cp); } if (WarpX::do_dive_cleaning) { PSATDBackwardTransformF(); } diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index 12e898b79ef..bee2e5d1be4 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -32,7 +32,7 @@ void WarpX::HybridPICEvolveFields () // Perform charge deposition in component 0 of rho_fp at t_{n+1}. mypc->DepositCharge(m_fields.get_mr_levels("rho_fp", finest_level), 0._rt); // Perform current deposition at t_{n+1/2}. - mypc->DepositCurrent(current_fp, dt[0], -0.5_rt * dt[0]); + mypc->DepositCurrent(m_fields.get_mr_levels_alldirs("current_fp", finest_level), dt[0], -0.5_rt * dt[0]); // Deposit cold-relativistic fluid charge and current if (do_fluid_species) { diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index 2dfe6a404fe..d8bd7931ad5 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -26,6 +26,8 @@ #include "WarpXParticleContainer.H" #include "ParticleBoundaries.H" +#include + #include #include #include @@ -162,7 +164,7 @@ public: * the time of the deposition. */ void - DepositCurrent (amrex::Vector, 3 > >& J, + DepositCurrent (ablastr::fields::MultiLevelVectorField J, amrex::Real dt, amrex::Real relative_time); /// diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index e407ce10420..a8647ec556f 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -342,6 +342,22 @@ namespace ablastr::fields int level ); + /** title + * + * body body + * body + * + * @param name ... + * @param dir ... + * @param level ... + */ + void + erase ( + std::string name, + Direction dir, + int level + ); + /** Erase all MultiFabs on a specific MR level. * * Calls @see erase for all MultiFabs on a specific level. diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 96f737343ad..1c2e5609dc9 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -402,6 +402,21 @@ namespace ablastr::fields m_mf_register.erase(name); } + void + MultiFabRegister::erase ( + std::string name, + Direction dir, + int level + ) + { + name = mf_name(name, dir, level); + + if (m_mf_register.count(name) != 1) { + throw std::runtime_error("MultiFabRegister::remove name does not exist in register: " + name); + } + m_mf_register.erase(name); + } + void MultiFabRegister::clear_level ( int level From 4927fd00f30882f75353db1359e18dc5fa7702e7 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 13:46:32 -0700 Subject: [PATCH 096/314] Fix compilation --- .../FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp | 4 +++- Source/Particles/WarpXParticleContainer.cpp | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp index 26ddbc476e4..4d576e5fa24 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp @@ -98,7 +98,9 @@ WarpX::AddMagnetostaticFieldLabFrame() for (int ispecies=0; ispeciesnSpecies(); ispecies++){ WarpXParticleContainer& species = mypc->GetParticleContainer(ispecies); if (!species.do_not_deposit) { - species.DepositCurrent(current_fp, dt[0], 0.); + species.DepositCurrent( + m_fields.get_mr_levels_alldirs("current_fp", finest_level), + dt[0], 0.); } } diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 490b376e482..42d58b8255a 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -853,7 +853,7 @@ WarpXParticleContainer::DepositCurrent ( } DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, - J[lev][0].get(), J[lev][1].get(), J[lev][2].get(), + J[lev][0], J[lev][1], J[lev][2], 0, np, thread_num, lev, lev, dt, relative_time, PushType::Explicit); } #ifdef AMREX_USE_OMP From e94d8bc7496be8c81e8bfa71f749f1fef290c902 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 13:54:50 -0700 Subject: [PATCH 097/314] Fix some of the RZ compilation --- Source/Diagnostics/BTDiagnostics.cpp | 2 +- Source/Diagnostics/FullDiagnostics.cpp | 50 +++++++++++++------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index a958fc018f5..84a79c7e451 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -601,7 +601,7 @@ BTDiagnostics::UpdateVarnamesForRZopenPMD () { #ifdef WARPX_DIM_RZ auto & warpx = WarpX::GetInstance(); - const int ncomp_multimodefab = warpx.getFieldPointer(FieldType::Efield_aux, 0,0)->nComp(); + const int ncomp_multimodefab = warpx.m_fields.get("Efield_aux", 0, 0)->nComp(); const int ncomp = ncomp_multimodefab; diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index 7c33365d61b..2e1dcccc62f 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -177,13 +177,13 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) using ablastr::fields::Direction; auto & warpx = WarpX::GetInstance(); - const int ncomp_multimodefab = warpx.getFieldPointer(FieldType::Efield_aux, 0, 0)->nComp(); + const int ncomp_multimodefab = warpx.m_fields.get("Efield_aux", Direction{0}, 0)->nComp(); // Make sure all multifabs have the same number of components for (int dim=0; dim<3; dim++){ AMREX_ALWAYS_ASSERT( - warpx.getFieldPointer(FieldType::Efield_aux, lev, dim)->nComp() == ncomp_multimodefab ); + warpx.m_fields.get("Efield_aux", Direction{lev}, dim)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( - warpx.getFieldPointer(FieldType::Bfield_aux, lev, dim)->nComp() == ncomp_multimodefab ); + warpx.m_fields.get("Bfield_aux", Direction{lev}, dim)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( warpx.m_fields.get("current_fp", Direction{dim}, lev)->nComp() == ncomp_multimodefab ); } @@ -220,37 +220,37 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) const auto m_varname_fields_size = static_cast(m_varnames_fields.size()); for (int comp=0; comp(warpx.getFieldPointer(FieldType::Efield_aux, lev, 0), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Efield_aux", Direction{0}, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Er"), ncomp); } } else if ( m_varnames_fields[comp] == "Et" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 1), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Efield_aux", Direction{1}, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Et"), ncomp); } } else if ( m_varnames_fields[comp] == "Ez" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 2), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Efield_aux", Direction{2}, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Ez"), ncomp); } } else if ( m_varnames_fields[comp] == "Br" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 0), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{0}, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Br"), ncomp); } } else if ( m_varnames_fields[comp] == "Bt" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 1), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{1}, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Bt"), ncomp); } } else if ( m_varnames_fields[comp] == "Bz" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 2), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{2}, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Bz"), ncomp); @@ -401,13 +401,13 @@ FullDiagnostics::AddRZModesToDiags (int lev) if (!m_dump_rz_modes) { return; } auto & warpx = WarpX::GetInstance(); - const int ncomp_multimodefab = warpx.getFieldPointer(FieldType::Efield_aux, 0, 0)->nComp(); + const int ncomp_multimodefab = warpx.m_fields.get("Efield_aux", Direction{0}, 0)->nComp(); // Make sure all multifabs have the same number of components for (int dim=0; dim<3; dim++){ AMREX_ALWAYS_ASSERT( - warpx.getFieldPointer(FieldType::Efield_aux, lev, dim)->nComp() == ncomp_multimodefab ); + warpx.m_fields.get("Efield_aux", Direction{dim}, lev)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( - warpx.getFieldPointer(FieldType::Bfield_aux, lev, dim)->nComp() == ncomp_multimodefab ); + warpx.m_fields.get("Bfield_aux", Direction{dim}, lev)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( warpx.m_fields.get("current_fp", Direction{dim}, lev)->nComp() == ncomp_multimodefab ); } @@ -444,19 +444,19 @@ FullDiagnostics::AddRZModesToDiags (int lev) for (int dim=0; dim<3; dim++){ // 3 components, r theta z m_all_field_functors[lev].push_back(std::make_unique( - warpx.getFieldPointer(FieldType::Efield_aux, lev, dim), lev, + warpx.m_fields.get("Efield_aux", Direction{dim}, lev), lev, m_crse_ratio, false, ncomp_multimodefab)); AddRZModesToOutputNames(std::string("E") + coord[dim], - warpx.getFieldPointer(FieldType::Efield_aux, 0, 0)->nComp()); + warpx.m_fields.get("Efield_aux", Direction{0}, 0)->nComp()); } // B for (int dim=0; dim<3; dim++){ // 3 components, r theta z m_all_field_functors[lev].push_back(std::make_unique( - warpx.getFieldPointer(FieldType::Bfield_aux, lev, dim), lev, + warpx.m_fields.get("Bfield_aux", Direction{dim}, lev), lev, m_crse_ratio, false, ncomp_multimodefab)); AddRZModesToOutputNames(std::string("B") + coord[dim], - warpx.getFieldPointer(FieldType::Bfield_aux, 0, 0)->nComp()); + warpx.m_fields.get("Bfield_aux", Direction{0}, 0)->nComp()); } // j for (int dim=0; dim<3; dim++){ @@ -677,7 +677,7 @@ FullDiagnostics::InitializeFieldFunctors (int lev) } else if ( m_varnames[comp] == "jz_displacement" ) { m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, true); } else if ( m_varnames[comp] == "Az" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::vector_potential_fp, lev, 2), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("vector_potential_fp", Direction{2}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "rho" ){ // Initialize rho functor to dump total rho m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true); @@ -708,13 +708,13 @@ FullDiagnostics::InitializeFieldFunctors (int lev) #ifdef WARPX_DIM_RZ if ( m_varnames[comp] == "Er" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 0), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Efield_aux", Direction{0}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Et" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 1), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Efield_aux", Direction{1}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Br" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 0), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{0}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Bt" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 1), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{1}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "jr" ){ m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, true, deposit_current); deposit_current = false; @@ -726,9 +726,9 @@ FullDiagnostics::InitializeFieldFunctors (int lev) } else if (m_varnames[comp] == "jt_displacement" ){ m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, true); } else if ( m_varnames[comp] == "Ar" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::vector_potential_fp, lev, 0), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("vector_potential_fp", Direction{0}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "At" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::vector_potential_fp, lev, 1), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("vector_potential_fp", Direction{1}, lev), lev, m_crse_ratio); } else { WARPX_ABORT_WITH_MESSAGE(m_varnames[comp] + " is not a known field output type for RZ geometry"); } @@ -753,9 +753,9 @@ FullDiagnostics::InitializeFieldFunctors (int lev) } else if ( m_varnames[comp] == "jy_displacement" ){ m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio); } else if ( m_varnames[comp] == "Ax" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::vector_potential_fp, lev, 0), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("vector_potential_fp", Direction{0}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Ay" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::vector_potential_fp, lev, 1), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("vector_potential_fp", Direction{1}, lev), lev, m_crse_ratio); } else { std::cout << "Error on component " << m_varnames[comp] << std::endl; WARPX_ABORT_WITH_MESSAGE(m_varnames[comp] + " is not a known field output type for this geometry"); From dc2f859c24564b41a3799bb03db0f5e6f6d97fdd Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 13:57:59 -0700 Subject: [PATCH 098/314] Remove current_fp_vay --- Source/FieldSolver/WarpXPushFieldsEM.cpp | 6 +++--- Source/WarpX.H | 1 - Source/WarpX.cpp | 5 ----- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 998a4d7a1dc..e61afce01b4 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -451,9 +451,9 @@ void WarpX::PSATDSubtractCurrentPartialSumsAvg () { const std::array& dx = WarpX::CellSize(lev); - amrex::MultiFab const& Dx = *current_fp_vay[lev][0]; - amrex::MultiFab const& Dy = *current_fp_vay[lev][1]; - amrex::MultiFab const& Dz = *current_fp_vay[lev][2]; + amrex::MultiFab const& Dx = *m_fields.get("current_fp_vay", Direction{0}, lev); + amrex::MultiFab const& Dy = *m_fields.get("current_fp_vay", Direction{1}, lev); + amrex::MultiFab const& Dz = *m_fields.get("current_fp_vay", Direction{2}, lev); #if defined (WARPX_DIM_XZ) amrex::ignore_unused(Dy); diff --git a/Source/WarpX.H b/Source/WarpX.H index 9b4c94ca086..1a39e9e9c9d 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1513,7 +1513,6 @@ private: // Fine patch amrex::Vector, 3 > > current_fp; - amrex::Vector, 3 > > current_fp_vay; amrex::Vector, 3 > > Efield_fp; amrex::Vector, 3 > > Bfield_fp; amrex::Vector, 3 > > Efield_avg_fp; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 043bffc41b3..86d1039ef89 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -359,11 +359,6 @@ WarpX::WarpX () current_fp_nodal.resize(nlevs_max); } - if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Vay) - { - current_fp_vay.resize(nlevs_max); - } - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { // Create hybrid-PIC model object if needed From 0de94257839101d0a06b7a2c64e034b4a899b76c Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 14:02:00 -0700 Subject: [PATCH 099/314] Fix some of the aux bugs --- Source/Diagnostics/FullDiagnostics.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index 2e1dcccc62f..6fd6e2fd463 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -346,14 +346,14 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) } } else if ( m_varnames_fields[comp] == "divB" ){ m_all_field_functors[lev][comp] = std::make_unique( - warpx.getFieldPointerArray(FieldType::Bfield_aux, lev), + warpx.m_fields.get_alldirs("Bfield_aux", lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("divB"), ncomp); } } else if ( m_varnames_fields[comp] == "divE" ){ m_all_field_functors[lev][comp] = std::make_unique( - warpx.getFieldPointerArray(FieldType::Efield_aux, lev), + warpx.m_fields.get_alldirs("Efield_aux", lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("divE"), ncomp); @@ -470,7 +470,7 @@ FullDiagnostics::AddRZModesToDiags (int lev) // divE if (divE_requested) { m_all_field_functors[lev].push_back(std::make_unique( - warpx.getFieldPointerArray(FieldType::Efield_aux, lev), + warpx.m_fields.get_alldirs("Efield_aux", lev), lev, m_crse_ratio, false, ncomp_multimodefab)); AddRZModesToOutputNames(std::string("divE"), ncomp_multimodefab); } From 4be9a60449b11bf5daccb03f90a479ef99c4a0da Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 14:08:04 -0700 Subject: [PATCH 100/314] Remove current_fp_nodal --- Source/WarpX.H | 3 --- Source/WarpX.cpp | 10 ---------- 2 files changed, 13 deletions(-) diff --git a/Source/WarpX.H b/Source/WarpX.H index 1a39e9e9c9d..413b4122ac6 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1581,9 +1581,6 @@ private: // store fine patch amrex::Vector, 3 > > current_store; - // Nodal MultiFab for nodal current deposition if warpx.do_current_centering = 1 - amrex::Vector,3>> current_fp_nodal; - // Coarse patch amrex::Vector, 3 > > current_cp; amrex::Vector, 3 > > Efield_cp; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 86d1039ef89..8773d79bb5d 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -354,11 +354,6 @@ WarpX::WarpX () current_store.resize(nlevs_max); - if (do_current_centering) - { - current_fp_nodal.resize(nlevs_max); - } - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { // Create hybrid-PIC model object if needed @@ -3488,9 +3483,6 @@ WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, cons case FieldType::current_fp : field_pointer = current_fp[lev][direction].get(); break; - case FieldType::current_fp_nodal : - field_pointer = current_fp_nodal[lev][direction].get(); - break; case FieldType::Efield_cp : field_pointer = Efield_cp[lev][direction].get(); break; @@ -3602,8 +3594,6 @@ WarpX::getMultiLevelField(warpx::fields::FieldType field_type) const return Bfield_fp_external; case FieldType::current_fp : return current_fp; - case FieldType::current_fp_nodal : - return current_fp_nodal; case FieldType::Efield_cp : return Efield_cp; case FieldType::Bfield_cp : From 84f58bada019ef8ba9b01873db577ce0a7a5413b Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 14:26:42 -0700 Subject: [PATCH 101/314] Remove current_cp --- Source/Diagnostics/WarpXIO.cpp | 6 +++--- .../FieldSolver/FiniteDifferenceSolver/EvolveE.cpp | 6 +++--- .../FiniteDifferenceSolver.H | 6 +++--- .../ImplicitSolvers/WarpXImplicitOps.cpp | 6 ++++-- Source/FieldSolver/WarpXPushFieldsEM.cpp | 6 ++++-- Source/FieldSolver/WarpX_QED_Field_Pushers.cpp | 14 ++++++++------ Source/Utils/WarpXMovingWindow.cpp | 6 ++++-- Source/WarpX.H | 1 - Source/WarpX.cpp | 6 ------ 9 files changed, 29 insertions(+), 28 deletions(-) diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp index d30a47f9aaf..9112714084d 100644 --- a/Source/Diagnostics/WarpXIO.cpp +++ b/Source/Diagnostics/WarpXIO.cpp @@ -371,11 +371,11 @@ WarpX::InitFromCheckpoint () } if (is_synchronized) { - VisMF::Read(*current_cp[lev][0], + VisMF::Read(*m_fields.get("current_cp", Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "jx_cp")); - VisMF::Read(*current_cp[lev][1], + VisMF::Read(*m_fields.get("current_cp", Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "jy_cp")); - VisMF::Read(*current_cp[lev][2], + VisMF::Read(*m_fields.get("current_cp", Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "jz_cp")); } } diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp index 3e7b37557a6..fa83b15672e 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp @@ -52,7 +52,7 @@ using namespace ablastr::fields; void FiniteDifferenceSolver::EvolveE ( std::array< std::unique_ptr, 3 >& Efield, std::array< std::unique_ptr, 3 > const& Bfield, - std::array< std::unique_ptr, 3 > const& Jfield, + ablastr::fields::VectorField const& Jfield, std::array< std::unique_ptr, 3 > const& edge_lengths, VectorField const& face_areas, std::array< std::unique_ptr, 3 >& ECTRhofield, @@ -95,7 +95,7 @@ template void FiniteDifferenceSolver::EvolveECartesian ( std::array< std::unique_ptr, 3 >& Efield, std::array< std::unique_ptr, 3 > const& Bfield, - std::array< std::unique_ptr, 3 > const& Jfield, + ablastr::fields::VectorField const& Jfield, std::array< std::unique_ptr, 3 > const& edge_lengths, amrex::MultiFab const* Ffield, int lev, amrex::Real const dt ) { @@ -229,7 +229,7 @@ template void FiniteDifferenceSolver::EvolveECylindrical ( std::array< std::unique_ptr, 3 >& Efield, std::array< std::unique_ptr, 3 > const& Bfield, - std::array< std::unique_ptr, 3 > const& Jfield, + ablastr::fields::VectorField const& Jfield, std::array< std::unique_ptr, 3 > const& edge_lengths, amrex::MultiFab const* Ffield, int lev, amrex::Real const dt ) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 3b6e2aff197..435f3b5ebe7 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -65,7 +65,7 @@ class FiniteDifferenceSolver void EvolveE ( std::array< std::unique_ptr, 3 >& Efield, std::array< std::unique_ptr, 3 > const& Bfield, - std::array< std::unique_ptr, 3 > const& Jfield, + ablastr::fields::VectorField const& Jfield, std::array< std::unique_ptr, 3 > const& edge_lengths, ablastr::fields::VectorField const& face_areas, std::array< std::unique_ptr, 3 >& ECTRhofield, @@ -218,7 +218,7 @@ class FiniteDifferenceSolver void EvolveECylindrical ( std::array< std::unique_ptr, 3 >& Efield, std::array< std::unique_ptr, 3 > const& Bfield, - std::array< std::unique_ptr, 3 > const& Jfield, + ablastr::fields::VectorField const& Jfield, std::array< std::unique_ptr, 3 > const& edge_lengths, amrex::MultiFab const* Ffield, int lev, @@ -270,7 +270,7 @@ class FiniteDifferenceSolver void EvolveECartesian ( std::array< std::unique_ptr, 3 >& Efield, std::array< std::unique_ptr, 3 > const& Bfield, - std::array< std::unique_ptr, 3 > const& Jfield, + ablastr::fields::VectorField const& Jfield, std::array< std::unique_ptr, 3 > const& edge_lengths, amrex::MultiFab const* Ffield, int lev, amrex::Real dt ); diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp index 2fa2c26e3cf..b3f12d4f8d3 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp @@ -337,12 +337,14 @@ WarpX::ImplicitComputeRHSE (int lev, PatchType patch_type, amrex::Real a_dt, War const auto face_area_lev = m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev]; if (patch_type == PatchType::fine) { m_fdtd_solver_fp[lev]->EvolveE( a_Erhs_vec.getArrayVec()[lev], Bfield_fp[lev], - current_fp[lev], m_edge_lengths[lev], + m_fields.get_alldirs("current_fp", lev), + m_edge_lengths[lev], face_area_lev, ECTRhofield[lev], m_fields.get("F_fp", lev), lev, a_dt ); } else { m_fdtd_solver_cp[lev]->EvolveE( a_Erhs_vec.getArrayVec()[lev], Bfield_cp[lev], - current_cp[lev], m_edge_lengths[lev], + m_fields.get_alldirs("current_cp", lev), + m_edge_lengths[lev], face_area_lev, ECTRhofield[lev], m_fields.get("F_cp", lev), lev, a_dt ); } diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index e61afce01b4..bc2f8cae3a2 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -909,12 +909,14 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) auto face_areas_lev = m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev]; if (patch_type == PatchType::fine) { m_fdtd_solver_fp[lev]->EvolveE(Efield_fp[lev], Bfield_fp[lev], - current_fp[lev], m_edge_lengths[lev], + m_fields.get_alldirs("current_fp", lev), + m_edge_lengths[lev], face_areas_lev, ECTRhofield[lev], m_fields.get("F_fp", lev), lev, a_dt ); } else { m_fdtd_solver_cp[lev]->EvolveE(Efield_cp[lev], Bfield_cp[lev], - current_cp[lev], m_edge_lengths[lev], + m_fields.get_alldirs("current_fp", lev), + m_edge_lengths[lev], face_areas_lev, ECTRhofield[lev], m_fields.get("F_cp", lev), lev, a_dt ); } diff --git a/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp b/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp index 9741d9b667b..4b8bcd62708 100644 --- a/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp +++ b/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp @@ -69,6 +69,8 @@ WarpX::Hybrid_QED_Push (int lev, amrex::Real a_dt) void WarpX::Hybrid_QED_Push (int lev, PatchType patch_type, amrex::Real a_dt) { + using ablastr::fields::Direction; + const int patch_level = (patch_type == PatchType::fine) ? lev : lev-1; const std::array& dx_vec= WarpX::CellSize(patch_level); const Real dx = dx_vec[0]; @@ -84,9 +86,9 @@ WarpX::Hybrid_QED_Push (int lev, PatchType patch_type, amrex::Real a_dt) Bx = Bfield_fp[lev][0].get(); By = Bfield_fp[lev][1].get(); Bz = Bfield_fp[lev][2].get(); - Jx = current_fp[lev][0].get(); - Jy = current_fp[lev][1].get(); - Jz = current_fp[lev][2].get(); + Jx = m_fields.get("current_fp", Direction{0}, lev); + Jy = m_fields.get("current_fp", Direction{1}, lev); + Jz = m_fields.get("current_fp", Direction{2}, lev); } else { @@ -96,9 +98,9 @@ WarpX::Hybrid_QED_Push (int lev, PatchType patch_type, amrex::Real a_dt) Bx = Bfield_cp[lev][0].get(); By = Bfield_cp[lev][1].get(); Bz = Bfield_cp[lev][2].get(); - Jx = current_cp[lev][0].get(); - Jy = current_cp[lev][1].get(); - Jz = current_cp[lev][2].get(); + Jx = m_fields.get("current_cp", Direction{0}, lev); + Jy = m_fields.get("current_cp", Direction{1}, lev); + Jz = m_fields.get("current_cp", Direction{2}, lev); } amrex::LayoutData* cost = WarpX::getCosts(lev); diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 8e115887ba9..26841d28587 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -137,6 +137,8 @@ WarpX::UpdateInjectionPosition (const amrex::Real a_dt) int WarpX::MoveWindow (const int step, bool move_j) { + using ablastr::fields::Direction; + WARPX_PROFILE("WarpX::MoveWindow"); if (step == start_moving_window_step) { @@ -248,7 +250,7 @@ WarpX::MoveWindow (const int step, bool move_j) m_p_ext_field_params-> E_external_grid[dim], use_Eparser, Efield_parser); } if (move_j) { - shiftMF(*current_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost); + shiftMF(*m_fields.get("current_cp", Direction{dim}, lev), geom[lev], num_shift, dir, lev, do_update_cost); } if (pml[lev] && pml[lev]->ok()) { const std::array& pml_B = pml[lev]->GetB_fp(); @@ -279,7 +281,7 @@ WarpX::MoveWindow (const int step, bool move_j) m_p_ext_field_params->E_external_grid[dim], use_Eparser, Efield_parser); } if (move_j) { - shiftMF(*current_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost); + shiftMF(*m_fields.get("current_cp", Direction{dim}, lev), geom[lev-1], num_shift_crse, dir, lev, do_update_cost); } if (do_pml && pml[lev]->ok()) { const std::array& pml_B = pml[lev]->GetB_cp(); diff --git a/Source/WarpX.H b/Source/WarpX.H index 413b4122ac6..e83ca3910f3 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1582,7 +1582,6 @@ private: amrex::Vector, 3 > > current_store; // Coarse patch - amrex::Vector, 3 > > current_cp; amrex::Vector, 3 > > Efield_cp; amrex::Vector, 3 > > Bfield_cp; amrex::Vector, 3 > > Efield_avg_cp; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 8773d79bb5d..a2fc9bbb73f 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -360,7 +360,6 @@ WarpX::WarpX () m_hybrid_pic_model = std::make_unique(nlevs_max); } - current_cp.resize(nlevs_max); Efield_cp.resize(nlevs_max); Bfield_cp.resize(nlevs_max); @@ -3489,9 +3488,6 @@ WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, cons case FieldType::Bfield_cp : field_pointer = Bfield_cp[lev][direction].get(); break; - case FieldType::current_cp : - field_pointer = current_cp[lev][direction].get(); - break; case FieldType::edge_lengths : field_pointer = m_edge_lengths[lev][direction].get(); break; @@ -3598,8 +3594,6 @@ WarpX::getMultiLevelField(warpx::fields::FieldType field_type) const return Efield_cp; case FieldType::Bfield_cp : return Bfield_cp; - case FieldType::current_cp : - return current_cp; default: WARPX_ABORT_WITH_MESSAGE("Invalid field type"); return Efield_fp; From 5d92a664d6dcdf6e047f0e076885f74d00d4e33b Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Wed, 11 Sep 2024 14:47:29 -0700 Subject: [PATCH 102/314] Fix FFT build --- Source/Diagnostics/WarpXIO.cpp | 24 +++++----- Source/Evolve/WarpXEvolve.cpp | 7 ++- Source/FieldSolver/WarpXPushFieldsEM.cpp | 10 ++-- .../FieldSolver/WarpXPushFieldsHybridPIC.cpp | 2 +- Source/Initialization/WarpXInitData.cpp | 12 +++-- Source/Parallelization/WarpXComm.cpp | 41 +++++++++++----- Source/Particles/MultiParticleContainer.H | 2 +- Source/Utils/WarpXMovingWindow.cpp | 4 ++ Source/WarpX.H | 12 ++--- Source/WarpX.cpp | 48 +++++-------------- 10 files changed, 83 insertions(+), 79 deletions(-) diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp index d30a47f9aaf..33261831c50 100644 --- a/Source/Diagnostics/WarpXIO.cpp +++ b/Source/Diagnostics/WarpXIO.cpp @@ -313,18 +313,18 @@ WarpX::InitFromCheckpoint () if (WarpX::fft_do_time_averaging) { - VisMF::Read(*Efield_avg_fp[lev][0], + VisMF::Read(*m_fields.get("Efield_avg_fp", Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ex_avg_fp")); - VisMF::Read(*Efield_avg_fp[lev][1], + VisMF::Read(*m_fields.get("Efield_avg_fp", Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ey_avg_fp")); - VisMF::Read(*Efield_avg_fp[lev][2], + VisMF::Read(*m_fields.get("Efield_avg_fp", Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ez_avg_fp")); - VisMF::Read(*Bfield_avg_fp[lev][0], + VisMF::Read(*m_fields.get("Bfield_avg_fp", Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Bx_avg_fp")); - VisMF::Read(*Bfield_avg_fp[lev][1], + VisMF::Read(*m_fields.get("Bfield_avg_fp", Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "By_avg_fp")); - VisMF::Read(*Bfield_avg_fp[lev][2], + VisMF::Read(*m_fields.get("Bfield_avg_fp", Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Bz_avg_fp")); } @@ -355,18 +355,18 @@ WarpX::InitFromCheckpoint () if (WarpX::fft_do_time_averaging) { - VisMF::Read(*Efield_avg_cp[lev][0], + VisMF::Read(*m_fields.get("Efield_avg_cp", Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ex_avg_cp")); - VisMF::Read(*Efield_avg_cp[lev][1], + VisMF::Read(*m_fields.get("Efield_avg_cp", Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ey_avg_cp")); - VisMF::Read(*Efield_avg_cp[lev][2], + VisMF::Read(*m_fields.get("Efield_avg_cp", Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ez_avg_cp")); - VisMF::Read(*Bfield_avg_cp[lev][0], + VisMF::Read(*m_fields.get("Bfield_avg_cp", Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Bx_avg_cp")); - VisMF::Read(*Bfield_avg_cp[lev][1], + VisMF::Read(*m_fields.get("Bfield_avg_cp", Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "By_avg_cp")); - VisMF::Read(*Bfield_avg_cp[lev][2], + VisMF::Read(*m_fields.get("Bfield_avg_cp", Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Bz_avg_cp")); } diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index f0bce2cfd76..572900e3033 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -786,7 +786,12 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) { // We summed the integral of the field over 2*dt PSATDScaleAverageFields(1._rt / (2._rt*dt[0])); - PSATDBackwardTransformEBavg(Efield_avg_fp, Bfield_avg_fp, Efield_avg_cp, Bfield_avg_cp); + PSATDBackwardTransformEBavg( + m_fields.get_mr_levels_alldirs("Efield_avg_fp", finest_level), + m_fields.get_mr_levels_alldirs("Bfield_avg_fp", finest_level), + m_fields.get_mr_levels_alldirs("Efield_avg_cp", finest_level), + m_fields.get_mr_levels_alldirs("Bfield_avg_cp", finest_level) + ); } // Evolve fields in PML diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index e61afce01b4..1117dadfcd0 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -153,10 +153,10 @@ void WarpX::PSATDBackwardTransformEB ( } void WarpX::PSATDBackwardTransformEBavg ( - const ablastr::fields::MultiLevelVectorField& E_avg_fp, - const ablastr::fields::MultiLevelVectorField& B_avg_fp, - const ablastr::fields::MultiLevelVectorField& E_avg_cp, - const ablastr::fields::MultiLevelVectorField& B_avg_cp) + ablastr::fields::MultiLevelVectorField const& E_avg_fp, + ablastr::fields::MultiLevelVectorField const& B_avg_fp, + ablastr::fields::MultiLevelVectorField const& E_avg_cp, + ablastr::fields::MultiLevelVectorField const& B_avg_cp) { const SpectralFieldIndex& Idx = spectral_solver_fp[0]->m_spectral_index; @@ -440,6 +440,8 @@ void WarpX::PSATDVayDeposition () void WarpX::PSATDSubtractCurrentPartialSumsAvg () { + using ablastr::fields::Direction; + // Subtraction of cumulative sum for Vay deposition // implemented only in 2D and 3D Cartesian geometry #if !defined (WARPX_DIM_1D_Z) && !defined (WARPX_DIM_RZ) diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index e90cc3fb7f3..06daeb7a1ee 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -176,7 +176,7 @@ void WarpX::HybridPICDepositInitialRhoAndJ () auto& rho_fp_temp = m_hybrid_pic_model->rho_fp_temp; auto& current_fp_temp = m_hybrid_pic_model->current_fp_temp; mypc->DepositCharge(amrex::GetVecOfPtrs(rho_fp_temp), 0._rt); - mypc->DepositCurrent( va2vm(current_fp_temp), dt[0], 0._rt); + mypc->DepositCurrent(va2vm(current_fp_temp), dt[0], 0._rt); SyncRho(amrex::GetVecOfPtrs(rho_fp_temp), m_fields.get_mr_levels("rho_cp", finest_level), charge_buf); SyncCurrent( m_fields.get_mr_levels_alldirs("current_fp", finest_level), diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 79174fafc37..edf7e6dda99 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -900,6 +900,8 @@ WarpX::PostRestart () void WarpX::InitLevelData (int lev, Real /*time*/) { + using ablastr::fields::Direction; + ablastr::fields::MultiLevelVectorField Efield_aux = m_fields.get_mr_levels_alldirs("Efield_aux", finest_level); ablastr::fields::MultiLevelVectorField Bfield_aux = m_fields.get_mr_levels_alldirs("Bfield_aux", finest_level); @@ -918,14 +920,14 @@ WarpX::InitLevelData (int lev, Real /*time*/) if ( is_B_ext_const && (lev <= maxlevel_extEMfield_init) ) { if (fft_do_time_averaging) { - Bfield_avg_fp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); + m_fields.get("Bfield_avg_fp", Direction{i}, lev)->setVal(m_p_ext_field_params->B_external_grid[i]); } if (lev > 0) { Bfield_aux[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); Bfield_cp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); if (fft_do_time_averaging) { - Bfield_avg_cp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); + m_fields.get("Bfield_avg_cp", Direction{i}, lev)->setVal(m_p_ext_field_params->B_external_grid[i]); } } } @@ -938,14 +940,14 @@ WarpX::InitLevelData (int lev, Real /*time*/) if ( is_E_ext_const && (lev <= maxlevel_extEMfield_init) ) { if (fft_do_time_averaging) { - Efield_avg_fp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); + m_fields.get("Efield_avg_fp", Direction{i}, lev)->setVal(m_p_ext_field_params->E_external_grid[i]); } if (lev > 0) { Efield_aux[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); Efield_cp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); if (fft_do_time_averaging) { - Efield_avg_cp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); + m_fields.get("Efield_avg_cp", Direction{i}, lev)->setVal(m_p_ext_field_params->E_external_grid[i]); } } } @@ -1352,7 +1354,7 @@ void WarpX::LoadExternalFields (int const lev) { using ablastr::fields::Direction; - + // External fields from file are currently not compatible with the moving window // In order to support the moving window, the MultiFab containing the external // fields should be updated every time the window moves. diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index e08df14420d..565f99178ae 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -97,13 +97,15 @@ WarpX::UpdateAuxilaryDataStagToNodal () #endif using ablastr::fields::Direction; + ablastr::fields::MultiLevelVectorField Efield_fp = m_fields.get_mr_levels_alldirs("Efield_fp", finest_level); + ablastr::fields::MultiLevelVectorField Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); + ablastr::fields::MultiLevelVectorField Efield_avg_fp = m_fields.get_mr_levels_alldirs("Efield_avg_fp", finest_level); + ablastr::fields::MultiLevelVectorField Bfield_avg_fp = m_fields.get_mr_levels_alldirs("Bfield_avg_fp", finest_level); ablastr::fields::MultiLevelVectorField Efield_aux = m_fields.get_mr_levels_alldirs("Efield_aux", finest_level); ablastr::fields::MultiLevelVectorField Bfield_aux = m_fields.get_mr_levels_alldirs("Bfield_aux", finest_level); - amrex::Vector,3>> const & Bmf = WarpX::fft_do_time_averaging ? - Bfield_avg_fp : Bfield_fp; - amrex::Vector,3>> const & Emf = WarpX::fft_do_time_averaging ? - Efield_avg_fp : Efield_fp; // JRA, do this one when doing Efield_avg_fp refactor to new style + ablastr::fields::MultiLevelVectorField const & Bmf = WarpX::fft_do_time_averaging ? Bfield_avg_fp : Bfield_fp; + ablastr::fields::MultiLevelVectorField const & Emf = WarpX::fft_do_time_averaging ? Efield_avg_fp : Efield_fp; // JRA, do this one when doing Efield_avg_fp refactor to new style const amrex::IntVect& Bx_stag = Bmf[0][0]->ixType().toIntVect(); const amrex::IntVect& By_stag = Bmf[0][1]->ixType().toIntVect(); @@ -380,8 +382,16 @@ WarpX::UpdateAuxilaryDataSameType () const amrex::IntVect& ng_src = guard_cells.ng_FieldGather; using ablastr::fields::Direction; + ablastr::fields::MultiLevelVectorField Efield_fp = m_fields.get_mr_levels_alldirs("Efield_fp", finest_level); + ablastr::fields::MultiLevelVectorField Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); + ablastr::fields::MultiLevelVectorField Efield_cp = m_fields.get_mr_levels_alldirs("Efield_cp", finest_level); + ablastr::fields::MultiLevelVectorField Bfield_cp = m_fields.get_mr_levels_alldirs("Bfield_cp", finest_level); ablastr::fields::MultiLevelVectorField Efield_aux = m_fields.get_mr_levels_alldirs("Efield_aux", finest_level); ablastr::fields::MultiLevelVectorField Bfield_aux = m_fields.get_mr_levels_alldirs("Bfield_aux", finest_level); + ablastr::fields::MultiLevelVectorField Efield_avg_fp = m_fields.get_mr_levels_alldirs("Efield_avg_fp", finest_level); + ablastr::fields::MultiLevelVectorField Bfield_avg_fp = m_fields.get_mr_levels_alldirs("Bfield_avg_fp", finest_level); + ablastr::fields::MultiLevelVectorField Efield_avg_cp = m_fields.get_mr_levels_alldirs("Efield_avg_cp", finest_level); + ablastr::fields::MultiLevelVectorField Bfield_avg_cp = m_fields.get_mr_levels_alldirs("Bfield_avg_cp", finest_level); // Level 0: Copy from fine to aux // Note: in some configurations, Efield_aux/Bfield_aux and Efield_fp/Bfield_fp are simply aliases to the @@ -397,9 +407,9 @@ WarpX::UpdateAuxilaryDataSameType () } else { - MultiFab::Copy(*Efield_aux[0][0], *m_fields.get("Efield_fp",Direction{0},0), 0, 0, Efield_aux[0][0]->nComp(), ng_src); - MultiFab::Copy(*Efield_aux[0][1], *m_fields.get("Efield_fp",Direction{1},0), 0, 0, Efield_aux[0][1]->nComp(), ng_src); - MultiFab::Copy(*Efield_aux[0][2], *m_fields.get("Efield_fp",Direction{2},0), 0, 0, Efield_aux[0][2]->nComp(), ng_src); + MultiFab::Copy(*Efield_aux[0][0], *Efield_fp[0][0], 0, 0, Efield_aux[0][0]->nComp(), ng_src); + MultiFab::Copy(*Efield_aux[0][1], *Efield_fp[0][1], 0, 0, Efield_aux[0][1]->nComp(), ng_src); + MultiFab::Copy(*Efield_aux[0][2], *Efield_fp[0][2], 0, 0, Efield_aux[0][2]->nComp(), ng_src); MultiFab::Copy(*Bfield_aux[0][0], *Bfield_fp[0][0], 0, 0, Bfield_aux[0][0]->nComp(), ng_src); MultiFab::Copy(*Bfield_aux[0][1], *Bfield_fp[0][1], 0, 0, Bfield_aux[0][1]->nComp(), ng_src); MultiFab::Copy(*Bfield_aux[0][2], *Bfield_fp[0][2], 0, 0, Bfield_aux[0][2]->nComp(), ng_src); @@ -806,9 +816,11 @@ WarpX::FillBoundaryE_avg (int lev, PatchType patch_type, IntVect ng) WARPX_ABORT_WITH_MESSAGE("Averaged Galilean PSATD with PML is not yet implemented"); } + ablastr::fields::MultiLevelVectorField Efield_avg_fp = m_fields.get_mr_levels_alldirs("Efield_avg_fp", finest_level); + const amrex::Periodicity& period = Geom(lev).periodicity(); if ( safe_guard_cells ){ - const Vector mf{Efield_avg_fp[lev][0].get(),Efield_avg_fp[lev][1].get(),Efield_avg_fp[lev][2].get()}; + const Vector mf{Efield_avg_fp[lev][0],Efield_avg_fp[lev][1],Efield_avg_fp[lev][2]}; ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period); } else { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( @@ -826,9 +838,11 @@ WarpX::FillBoundaryE_avg (int lev, PatchType patch_type, IntVect ng) WARPX_ABORT_WITH_MESSAGE("Averaged Galilean PSATD with PML is not yet implemented"); } + ablastr::fields::MultiLevelVectorField Efield_avg_cp = m_fields.get_mr_levels_alldirs("Efield_avg_cp", finest_level); + const amrex::Periodicity& cperiod = Geom(lev-1).periodicity(); if ( safe_guard_cells ) { - const Vector mf{Efield_avg_cp[lev][0].get(),Efield_avg_cp[lev][1].get(),Efield_avg_cp[lev][2].get()}; + const Vector mf{Efield_avg_cp[lev][0],Efield_avg_cp[lev][1],Efield_avg_cp[lev][2]}; ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, cperiod); } else { @@ -859,9 +873,12 @@ WarpX::FillBoundaryB_avg (int lev, PatchType patch_type, IntVect ng) { WARPX_ABORT_WITH_MESSAGE("Averaged Galilean PSATD with PML is not yet implemented"); } + + ablastr::fields::MultiLevelVectorField Bfield_avg_fp = m_fields.get_mr_levels_alldirs("Bfield_avg_fp", finest_level); + const amrex::Periodicity& period = Geom(lev).periodicity(); if ( safe_guard_cells ) { - const Vector mf{Bfield_avg_fp[lev][0].get(),Bfield_avg_fp[lev][1].get(),Bfield_avg_fp[lev][2].get()}; + const Vector mf{Bfield_avg_fp[lev][0],Bfield_avg_fp[lev][1],Bfield_avg_fp[lev][2]}; ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period); } else { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( @@ -879,9 +896,11 @@ WarpX::FillBoundaryB_avg (int lev, PatchType patch_type, IntVect ng) WARPX_ABORT_WITH_MESSAGE("Averaged Galilean PSATD with PML is not yet implemented"); } + ablastr::fields::MultiLevelVectorField Bfield_avg_cp = m_fields.get_mr_levels_alldirs("Bfield_avg_cp", finest_level); + const amrex::Periodicity& cperiod = Geom(lev-1).periodicity(); if ( safe_guard_cells ){ - const Vector mf{Bfield_avg_cp[lev][0].get(),Bfield_avg_cp[lev][1].get(),Bfield_avg_cp[lev][2].get()}; + const Vector mf{Bfield_avg_cp[lev][0],Bfield_avg_cp[lev][1],Bfield_avg_cp[lev][2]}; ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, cperiod); } else { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index 7453391d8d7..f15a78df6cd 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -164,7 +164,7 @@ public: * the time of the deposition. */ void - DepositCurrent (ablastr::fields::MultiLevelVectorField const & J, + DepositCurrent (ablastr::fields::MultiLevelVectorField const& J, amrex::Real dt, amrex::Real relative_time); /// diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 8e115887ba9..66557f96ea0 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -149,6 +149,10 @@ WarpX::MoveWindow (const int step, bool move_j) ablastr::fields::MultiLevelVectorField Efield_aux = m_fields.get_mr_levels_alldirs("Efield_aux", finest_level); ablastr::fields::MultiLevelVectorField Bfield_aux = m_fields.get_mr_levels_alldirs("Bfield_aux", finest_level); + ablastr::fields::MultiLevelVectorField Efield_avg_fp = m_fields.get_mr_levels_alldirs("Efield_avg_fp", finest_level); + ablastr::fields::MultiLevelVectorField Bfield_avg_fp = m_fields.get_mr_levels_alldirs("Bfield_avg_fp", finest_level); + ablastr::fields::MultiLevelVectorField Efield_avg_cp = m_fields.get_mr_levels_alldirs("Efield_avg_cp", finest_level); + ablastr::fields::MultiLevelVectorField Bfield_avg_cp = m_fields.get_mr_levels_alldirs("Bfield_avg_cp", finest_level); // Update the continuous position of the moving window, // and of the plasma injection diff --git a/Source/WarpX.H b/Source/WarpX.H index 413b4122ac6..8c69ffe6e99 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1515,8 +1515,6 @@ private: amrex::Vector, 3 > > current_fp; amrex::Vector, 3 > > Efield_fp; amrex::Vector, 3 > > Bfield_fp; - amrex::Vector, 3 > > Efield_avg_fp; - amrex::Vector, 3 > > Bfield_avg_fp; // Masks for computing dot product and global moments of fields when using grids that // have shared locations across different ranks (e.g., a Yee grid) @@ -1585,8 +1583,6 @@ private: amrex::Vector, 3 > > current_cp; amrex::Vector, 3 > > Efield_cp; amrex::Vector, 3 > > Bfield_cp; - amrex::Vector, 3 > > Efield_avg_cp; - amrex::Vector, 3 > > Bfield_avg_cp; // Copy of the coarse aux amrex::Vector, 3 > > Efield_cax; @@ -1853,10 +1849,10 @@ private: * storing the coarse patch averaged magnetic field to be transformed */ void PSATDBackwardTransformEBavg ( - const ablastr::fields::MultiLevelVectorField& E_avg_fp, - const ablastr::fields::MultiLevelVectorField& B_avg_fp, - const ablastr::fields::MultiLevelVectorField& E_avg_cp, - const ablastr::fields::MultiLevelVectorField& B_avg_cp); + ablastr::fields::MultiLevelVectorField const& E_avg_fp, + ablastr::fields::MultiLevelVectorField const& B_avg_fp, + ablastr::fields::MultiLevelVectorField const& E_avg_cp, + ablastr::fields::MultiLevelVectorField const& B_avg_cp); /** * \brief Forward FFT of J on all mesh refinement levels, diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 8773d79bb5d..37329ef7685 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -332,12 +332,6 @@ WarpX::WarpX () vector_potential_grad_buf_b_stag.resize(nlevs_max); } - if (fft_do_time_averaging) - { - Efield_avg_fp.resize(nlevs_max); - Bfield_avg_fp.resize(nlevs_max); - } - // Same as Bfield_fp/Efield_fp for reading external field data B_external_particle_field.resize(1); E_external_particle_field.resize(1); @@ -364,12 +358,6 @@ WarpX::WarpX () Efield_cp.resize(nlevs_max); Bfield_cp.resize(nlevs_max); - if (fft_do_time_averaging) - { - Efield_avg_cp.resize(nlevs_max); - Bfield_avg_cp.resize(nlevs_max); - } - Efield_cax.resize(nlevs_max); Bfield_cax.resize(nlevs_max); current_buffer_masks.resize(nlevs_max); @@ -2357,13 +2345,13 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (fft_do_time_averaging) { - AllocInitMultiFab(Bfield_avg_fp[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_avg_fp[x]", 0.0_rt); - AllocInitMultiFab(Bfield_avg_fp[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_avg_fp[y]", 0.0_rt); - AllocInitMultiFab(Bfield_avg_fp[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_avg_fp[z]", 0.0_rt); + m_fields.alloc_init( "Bfield_avg_fp", Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Bfield_avg_fp", Direction{1}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Bfield_avg_fp", Direction{2}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - AllocInitMultiFab(Efield_avg_fp[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_avg_fp[x]", 0.0_rt); - AllocInitMultiFab(Efield_avg_fp[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_avg_fp[y]", 0.0_rt); - AllocInitMultiFab(Efield_avg_fp[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_avg_fp[z]", 0.0_rt); + m_fields.alloc_init( "Efield_avg_fp", Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Efield_avg_fp", Direction{1}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Efield_avg_fp", Direction{2}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); } if (EB::enabled()) { @@ -2678,13 +2666,13 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (fft_do_time_averaging) { - AllocInitMultiFab(Bfield_avg_cp[lev][0], amrex::convert(cba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_avg_cp[x]", 0.0_rt); - AllocInitMultiFab(Bfield_avg_cp[lev][1], amrex::convert(cba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_avg_cp[y]", 0.0_rt); - AllocInitMultiFab(Bfield_avg_cp[lev][2], amrex::convert(cba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_avg_cp[z]", 0.0_rt); + m_fields.alloc_init("Bfield_avg_cp", Direction{0}, lev, amrex::convert(cba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Bfield_avg_cp", Direction{1}, lev, amrex::convert(cba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Bfield_avg_cp", Direction{2}, lev, amrex::convert(cba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - AllocInitMultiFab(Efield_avg_cp[lev][0], amrex::convert(cba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_avg_cp[x]", 0.0_rt); - AllocInitMultiFab(Efield_avg_cp[lev][1], amrex::convert(cba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_avg_cp[y]", 0.0_rt); - AllocInitMultiFab(Efield_avg_cp[lev][2], amrex::convert(cba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_avg_cp[z]", 0.0_rt); + m_fields.alloc_init("Efield_avg_cp", Direction{0}, lev, amrex::convert(cba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Efield_avg_cp", Direction{1}, lev, amrex::convert(cba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Efield_avg_cp", Direction{2}, lev, amrex::convert(cba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); } // Create the MultiFabs for the current @@ -3495,18 +3483,6 @@ WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, cons case FieldType::edge_lengths : field_pointer = m_edge_lengths[lev][direction].get(); break; - case FieldType::Efield_avg_fp : - field_pointer = Efield_avg_fp[lev][direction].get(); - break; - case FieldType::Bfield_avg_fp : - field_pointer = Bfield_avg_fp[lev][direction].get(); - break; - case FieldType::Efield_avg_cp : - field_pointer = Efield_avg_cp[lev][direction].get(); - break; - case FieldType::Bfield_avg_cp : - field_pointer = Bfield_avg_cp[lev][direction].get(); - break; default: WARPX_ABORT_WITH_MESSAGE("Invalid field type"); break; From 71fa567dc6fc7e9910b4531a6f49b75210fb1a4d Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 14:56:08 -0700 Subject: [PATCH 103/314] Remove current_fp --- .../FiniteDifferenceSolver.H | 4 ++-- .../MacroscopicEvolveE.cpp | 4 ++-- .../MagnetostaticSolver/MagnetostaticSolver.cpp | 9 ++++++--- Source/FieldSolver/WarpXPushFieldsEM.cpp | 3 ++- Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp | 9 +++++++-- Source/WarpX.H | 3 +-- Source/WarpX.cpp | 16 +++++++++------- 7 files changed, 29 insertions(+), 19 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 435f3b5ebe7..b39860c185b 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -115,7 +115,7 @@ class FiniteDifferenceSolver */ void MacroscopicEvolveE ( std::array< std::unique_ptr, 3>& Efield, std::array< std::unique_ptr, 3> const& Bfield, - std::array< std::unique_ptr, 3 > const& Jfield, + ablastr::fields::VectorField const& Jfield, std::array< std::unique_ptr, 3 > const& edge_lengths, amrex::Real dt, std::unique_ptr const& macroscopic_properties); @@ -315,7 +315,7 @@ class FiniteDifferenceSolver void MacroscopicEvolveECartesian ( std::array< std::unique_ptr< amrex::MultiFab>, 3>& Efield, std::array< std::unique_ptr< amrex::MultiFab>, 3> const& Bfield, - std::array< std::unique_ptr< amrex::MultiFab>, 3> const& Jfield, + ablastr::fields::VectorField const& Jfield, std::array< std::unique_ptr, 3 > const& edge_lengths, amrex::Real dt, std::unique_ptr const& macroscopic_properties); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp index 46e4d3efa06..8247d3e9c3c 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp @@ -38,7 +38,7 @@ using namespace amrex; void FiniteDifferenceSolver::MacroscopicEvolveE ( std::array< std::unique_ptr, 3 >& Efield, std::array< std::unique_ptr, 3 > const& Bfield, - std::array< std::unique_ptr, 3 > const& Jfield, + ablastr::fields::VectorField const& Jfield, std::array< std::unique_ptr, 3 > const& edge_lengths, amrex::Real const dt, std::unique_ptr const& macroscopic_properties) @@ -101,7 +101,7 @@ template void FiniteDifferenceSolver::MacroscopicEvolveECartesian ( std::array< std::unique_ptr, 3 >& Efield, std::array< std::unique_ptr, 3 > const& Bfield, - std::array< std::unique_ptr, 3 > const& Jfield, + ablastr::fields::VectorField const& Jfield, std::array< std::unique_ptr, 3 > const& edge_lengths, amrex::Real const dt, std::unique_ptr const& macroscopic_properties) diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp index 4d576e5fa24..1e43f0216a6 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp @@ -74,6 +74,8 @@ WarpX::ComputeMagnetostaticField() void WarpX::AddMagnetostaticFieldLabFrame() { + using ablastr::fields::Direction; + WARPX_PROFILE("WarpX::AddMagnetostaticFieldLabFrame"); // Store the boundary conditions for the field solver if they haven't been @@ -90,7 +92,7 @@ WarpX::AddMagnetostaticFieldLabFrame() // reset current_fp before depositing current density for this step for (int lev = 0; lev <= max_level; lev++) { for (int dim=0; dim < 3; dim++) { - current_fp[lev][dim]->setVal(0.); + m_fields.get("current_fp", Direction{dim}, lev)->setVal(0.); } } @@ -127,7 +129,8 @@ WarpX::AddMagnetostaticFieldLabFrame() const amrex::Real magnetostatic_absolute_tolerance = self_fields_absolute_tolerance*PhysConst::c; computeVectorPotential( - current_fp, m_fields.get_mr_levels_alldirs("vector_potential_fp_nodal", finest_level), + m_fields.get_mr_levels_alldirs("current_fp", finest_level), + m_fields.get_mr_levels_alldirs("vector_potential_fp_nodal", finest_level), self_fields_required_precision, magnetostatic_absolute_tolerance, self_fields_max_iters, self_fields_verbosity); } @@ -149,7 +152,7 @@ WarpX::AddMagnetostaticFieldLabFrame() \param[in] verbosity The verbosity setting for the MLMG solver */ void -WarpX::computeVectorPotential (const amrex::Vector,3>>& curr, +WarpX::computeVectorPotential (const ablastr::fields::MultiLevelVectorField& curr, ablastr::fields::MultiLevelVectorField const& A, Real const required_precision, Real absolute_tolerance, diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index dbda1330b9e..2f6ef0a9ac6 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -1089,7 +1089,8 @@ WarpX::MacroscopicEvolveE (int lev, PatchType patch_type, amrex::Real a_dt) { m_fdtd_solver_fp[lev]->MacroscopicEvolveE( Efield_fp[lev], Bfield_fp[lev], - current_fp[lev], m_edge_lengths[lev], + m_fields.get_alldirs("current_fp", lev), + m_edge_lengths[lev], a_dt, m_macroscopic_properties); if (do_pml && pml[lev]->ok()) { diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index 06daeb7a1ee..ab4d81d6bf0 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -21,6 +21,8 @@ using namespace amrex; void WarpX::HybridPICEvolveFields () { + using ablastr::fields::Direction; + WARPX_PROFILE("WarpX::HybridPICEvolveFields()"); // The below deposition is hard coded for a single level simulation @@ -38,7 +40,10 @@ void WarpX::HybridPICEvolveFields () if (do_fluid_species) { int const lev = 0; myfl->DepositCharge(lev, *m_fields.get("rho_fp", lev)); - myfl->DepositCurrent(lev, *current_fp[lev][0], *current_fp[lev][1], *current_fp[lev][2]); + myfl->DepositCurrent(lev, + *m_fields.get("current_fp", Direction{0}, lev), + *m_fields.get("current_fp", Direction{1}, lev), + *m_fields.get("current_fp", Direction{2}, lev)); } // Synchronize J and rho: @@ -51,7 +56,7 @@ void WarpX::HybridPICEvolveFields () // a nodal grid for (int lev = 0; lev <= finest_level; ++lev) { for (int idim = 0; idim < 3; ++idim) { - current_fp[lev][idim]->FillBoundary(Geom(lev).periodicity()); + m_fields.get("current_fp", Direction{idim}, lev)->FillBoundary(Geom(lev).periodicity()); } } diff --git a/Source/WarpX.H b/Source/WarpX.H index cca4b029db9..f449b8c1bd1 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1027,7 +1027,7 @@ public: MagnetostaticSolver::VectorPoissonBoundaryHandler m_vector_poisson_boundary_handler; void ComputeMagnetostaticField (); void AddMagnetostaticFieldLabFrame (); - void computeVectorPotential (const amrex::Vector, 3> >& curr, + void computeVectorPotential (ablastr::fields::MultiLevelVectorField const& curr, ablastr::fields::MultiLevelVectorField const& A, amrex::Real required_precision=amrex::Real(1.e-11), amrex::Real absolute_tolerance=amrex::Real(0.0), @@ -1512,7 +1512,6 @@ private: // // Fine patch - amrex::Vector, 3 > > current_fp; amrex::Vector, 3 > > Efield_fp; amrex::Vector, 3 > > Bfield_fp; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 72ebf8ddce6..61b48344085 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -3324,9 +3324,11 @@ WarpX::GatherBufferMasks (int lev) void WarpX::StoreCurrent (int lev) { + using ablastr::fields::Direction; + for (int idim = 0; idim < 3; ++idim) { if (current_store[lev][idim]) { - MultiFab::Copy(*current_store[lev][idim], *current_fp[lev][idim], + MultiFab::Copy(*current_store[lev][idim], *m_fields.get("current_fp", Direction{idim}, lev), 0, 0, 1, current_store[lev][idim]->nGrowVect()); } } @@ -3335,9 +3337,14 @@ WarpX::StoreCurrent (int lev) void WarpX::RestoreCurrent (int lev) { + using ablastr::fields::Direction; + for (int idim = 0; idim < 3; ++idim) { if (current_store[lev][idim]) { - std::swap(current_fp[lev][idim], current_store[lev][idim]); + std::swap( + *m_fields.get("current_fp", Direction{idim}, lev), + *current_store[lev][idim] + ); } } } @@ -3467,9 +3474,6 @@ WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, cons case FieldType::Bfield_fp_external : field_pointer = Bfield_fp_external[lev][direction].get(); break; - case FieldType::current_fp : - field_pointer = current_fp[lev][direction].get(); - break; case FieldType::Efield_cp : field_pointer = Efield_cp[lev][direction].get(); break; @@ -3564,8 +3568,6 @@ WarpX::getMultiLevelField(warpx::fields::FieldType field_type) const return Bfield_fp; case FieldType::Bfield_fp_external : return Bfield_fp_external; - case FieldType::current_fp : - return current_fp; case FieldType::Efield_cp : return Efield_cp; case FieldType::Bfield_cp : From 826072901f72782e1b290ca8ad53408c5dc86016 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 11 Sep 2024 15:10:32 -0700 Subject: [PATCH 104/314] Fix: FFT/PSATD RZ --- Source/BoundaryConditions/WarpXEvolvePML.cpp | 3 +++ Source/Diagnostics/BTDiagnostics.cpp | 23 ++++++++++--------- .../FiniteDifferenceSolver/ComputeDivE.cpp | 7 +++--- .../FiniteDifferenceSolver.H | 5 ++-- .../HybridPICModel/HybridPICModel.H | 2 +- .../SpectralBaseAlgorithmRZ.H | 3 ++- .../SpectralBaseAlgorithmRZ.cpp | 4 +++- .../SpectralSolver/SpectralSolverRZ.H | 4 +++- .../SpectralSolver/SpectralSolverRZ.cpp | 6 +++-- Source/Particles/MultiParticleContainer.cpp | 2 +- 10 files changed, 36 insertions(+), 23 deletions(-) diff --git a/Source/BoundaryConditions/WarpXEvolvePML.cpp b/Source/BoundaryConditions/WarpXEvolvePML.cpp index 5dc8bc2be17..572ac83ba90 100644 --- a/Source/BoundaryConditions/WarpXEvolvePML.cpp +++ b/Source/BoundaryConditions/WarpXEvolvePML.cpp @@ -16,6 +16,8 @@ #include "Utils/WarpXProfilerWrapper.H" #include "WarpX_PML_kernels.H" +#include + #ifdef AMREX_USE_SENSEI_INSITU # include #endif @@ -63,6 +65,7 @@ WarpX::DampPML (const int lev, PatchType patch_type) WARPX_PROFILE("WarpX::DampPML()"); #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev]) { + using ablastr::fields::Direction; pml_rz[lev]->ApplyDamping( m_fields.get("Efield_fp",Direction{1},lev), m_fields.get("Efield_fp",Direction{2},lev), Bfield_fp[lev][1].get(), Bfield_fp[lev][2].get(), diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 84a79c7e451..01e63d6a3b0 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -601,7 +601,8 @@ BTDiagnostics::UpdateVarnamesForRZopenPMD () { #ifdef WARPX_DIM_RZ auto & warpx = WarpX::GetInstance(); - const int ncomp_multimodefab = warpx.m_fields.get("Efield_aux", 0, 0)->nComp(); + using ablastr::fields::Direction; + const int ncomp_multimodefab = warpx.m_fields.get("Efield_aux", Direction{0}, 0)->nComp(); const int ncomp = ncomp_multimodefab; @@ -662,7 +663,7 @@ BTDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) using ablastr::fields::Direction; auto & warpx = WarpX::GetInstance(); - const int ncomp_multimodefab = warpx.getFieldPointer(FieldType::Efield_aux, 0,0)->nComp(); + const int ncomp_multimodefab = warpx.m_fields.get("Efield_aux", Direction{0}, 0)->nComp(); const int ncomp = ncomp_multimodefab; // Clear any pre-existing vector to release stored data // This ensures that when domain is load-balanced, the functors point @@ -688,23 +689,23 @@ BTDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) const auto m_cell_center_functors_at_lev_size = static_cast(m_cell_center_functors.at(lev).size()); for (int comp=0; comp(warpx.getFieldPointer(FieldType::Efield_aux, lev, 0), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Efield_aux", Direction{0}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Et" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 1), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Efield_aux", Direction{1}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Ez" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 2), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Efield_aux", Direction{2}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Br" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 0), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{0}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Bt" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 1), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{1}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Bz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 2), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{2}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "jr" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", Direction{0}, lev), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "jt" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", Direction{1}, lev), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "jz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", Direction{2}, lev), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "rho" ){ m_cell_center_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, false, -1, false, ncomp); } diff --git a/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp index a6a83d25417..3f757603845 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp @@ -126,9 +126,10 @@ void FiniteDifferenceSolver::ComputeDivECartesian ( template void FiniteDifferenceSolver::ComputeDivECylindrical ( - const std::array,3>& Efield, - amrex::MultiFab& divEfield ) { - + ablastr::fields::VectorField const & Efield, + amrex::MultiFab& divEfield +) +{ // Loop through the grids, and over the tiles within each grid #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 3b6e2aff197..e0fc344f4a4 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -234,8 +234,9 @@ class FiniteDifferenceSolver template< typename T_Algo > void ComputeDivECylindrical ( - const std::array,3>& Efield, - amrex::MultiFab& divE ); + ablastr::fields::VectorField const & Efield, + amrex::MultiFab& divE + ); template void HybridPICSolveECylindrical ( diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index dfd2c3d9653..0efe0b19e4a 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -187,7 +187,7 @@ public: std::array< amrex::ParserExecutor<4>, 3> m_J_external; bool m_external_field_has_time_dependence = false; - // Declare multifabs specifically needed for the hybrid-PIC model + // Declare multifabs specifically needed for the hybrid-PIC model (FIXME: remove me) amrex::Vector< std::unique_ptr > rho_fp_temp; amrex::Vector, 3 > > current_fp_temp; amrex::Vector, 3 > > current_fp_ampere; diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.H index 8e03a2a2559..9f6b5b09219 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.H @@ -10,6 +10,7 @@ #include "FieldSolver/SpectralSolver/SpectralKSpaceRZ.H" #include "FieldSolver/SpectralSolver/SpectralFieldDataRZ.H" +#include #include @@ -66,7 +67,7 @@ class SpectralBaseAlgorithmRZ */ void ComputeSpectralDivE ( int lev, SpectralFieldDataRZ& field_data, - const std::array,3>& Efield, + ablastr::fields::VectorField const & Efield, amrex::MultiFab& divE ); /** diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.cpp index f8ef0ef4730..3e556363a6f 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.cpp @@ -6,6 +6,8 @@ */ #include "SpectralBaseAlgorithmRZ.H" +#include + #include using namespace amrex; @@ -18,7 +20,7 @@ void SpectralBaseAlgorithmRZ::ComputeSpectralDivE ( const int lev, SpectralFieldDataRZ& field_data, - const std::array,3>& Efield, + ablastr::fields::VectorField const & Efield, amrex::MultiFab& divE ) { using amrex::operator""_rt; diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.H b/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.H index 2e94fe95da2..040acd00c51 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.H +++ b/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.H @@ -12,6 +12,7 @@ #include "SpectralAlgorithms/SpectralBaseAlgorithmRZ.H" #include "SpectralFieldDataRZ.H" +#include #include @@ -95,7 +96,8 @@ class SpectralSolverRZ * \brief Public interface to call the member function ComputeSpectralDivE * of the base class SpectralBaseAlgorithmRZ from objects of class SpectralSolverRZ */ - void ComputeSpectralDivE (int lev, const std::array,3>& Efield, + void ComputeSpectralDivE (int lev, + ablastr::fields::VectorField const & Efield, amrex::MultiFab& divE); /** diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.cpp b/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.cpp index 3529247ef56..b558f3983a3 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.cpp @@ -142,8 +142,10 @@ SpectralSolverRZ::pushSpectralFields (const bool doing_pml) { */ void SpectralSolverRZ::ComputeSpectralDivE (const int lev, - const std::array,3>& Efield, - amrex::MultiFab& divE) { + ablastr::fields::VectorField const & Efield, + amrex::MultiFab& divE +) +{ algorithm->ComputeSpectralDivE(lev, field_data, Efield, divE); } diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index fc260092fdd..560220e9eb9 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -551,7 +551,7 @@ MultiParticleContainer::DepositCurrent ( for (int lev = 0; lev < J.size(); ++lev) { WarpX::GetInstance().ApplyInverseVolumeScalingToCurrentDensity( - J[lev][0].get(), J[lev][1].get(), J[lev][2].get(), lev); + J[lev][0], J[lev][1], J[lev][2], lev); } #endif } From f45492012f34d28f7c095b88d1878833edeb68de Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 15:20:48 -0700 Subject: [PATCH 105/314] Remove current_fp --- .../FiniteDifferenceSolver.H | 8 ++++---- .../HybridPICModel/HybridPICModel.H | 12 ++++++------ .../HybridPICModel/HybridPICModel.cpp | 14 ++++++++------ .../FiniteDifferenceSolver/HybridPICSolveE.cpp | 4 ++-- .../MagnetostaticSolver/MagnetostaticSolver.cpp | 6 +++--- Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp | 15 +++++++++------ 6 files changed, 32 insertions(+), 27 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index b39860c185b..fef90474ee6 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -155,8 +155,8 @@ class FiniteDifferenceSolver * \param[in] solve_for_Faraday boolean flag for whether the E-field is solved to be used in Faraday's equation */ void HybridPICSolveE ( std::array< std::unique_ptr, 3>& Efield, - std::array< std::unique_ptr, 3>& Jfield, - std::array< std::unique_ptr, 3 > const& Jifield, + std::array< std::unique_ptr, 3 > & Jfield, + ablastr::fields::VectorField const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3> const& Bfield, amrex::MultiFab* const rhofield, @@ -345,8 +345,8 @@ class FiniteDifferenceSolver template void HybridPICSolveECartesian ( std::array< std::unique_ptr, 3>& Efield, - std::array< std::unique_ptr, 3> const& Jfield, - std::array< std::unique_ptr, 3> const& Jifield, + std::array< std::unique_ptr, 3 > const& Jfield, + ablastr::fields::VectorField const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3> const& Bfield, amrex::MultiFab* const rhofield, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index dfd2c3d9653..1c04601a730 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -87,7 +87,7 @@ public: */ void HybridPICSolveE ( amrex::Vector, 3>>& Efield, - amrex::Vector, 3>> const& Jfield, + ablastr::fields::MultiLevelVectorField const& Jfield, amrex::Vector, 3>> const& Bfield, ablastr::fields::MultiLevelScalarField const& rhofield, amrex::Vector, 3>> const& edge_lengths, @@ -95,7 +95,7 @@ public: void HybridPICSolveE ( std::array< std::unique_ptr, 3>& Efield, - std::array< std::unique_ptr, 3> const& Jfield, + ablastr::fields::VectorField const& Jfield, std::array< std::unique_ptr, 3> const& Bfield, amrex::MultiFab* const rhofield, std::array< std::unique_ptr, 3> const& edge_lengths, @@ -103,7 +103,7 @@ public: void HybridPICSolveE ( std::array< std::unique_ptr, 3>& Efield, - std::array< std::unique_ptr, 3> const& Jfield, + ablastr::fields::VectorField const& Jfield, std::array< std::unique_ptr, 3> const& Bfield, amrex::MultiFab* const rhofield, std::array< std::unique_ptr, 3> const& edge_lengths, @@ -112,7 +112,7 @@ public: void BfieldEvolveRK ( amrex::Vector, 3>>& Bfield, amrex::Vector, 3>>& Efield, - amrex::Vector, 3>> const& Jfield, + ablastr::fields::MultiLevelVectorField const& Jfield, ablastr::fields::MultiLevelScalarField const& rhofield, amrex::Vector, 3>> const& edge_lengths, amrex::Real dt, DtType a_dt_type, @@ -121,7 +121,7 @@ public: void BfieldEvolveRK ( amrex::Vector, 3>>& Bfield, amrex::Vector, 3>>& Efield, - amrex::Vector, 3>> const& Jfield, + ablastr::fields::MultiLevelVectorField const& Jfield, ablastr::fields::MultiLevelScalarField const& rhofield, amrex::Vector, 3>> const& edge_lengths, amrex::Real dt, int lev, DtType dt_type, @@ -130,7 +130,7 @@ public: void FieldPush ( amrex::Vector, 3>>& Bfield, amrex::Vector, 3>>& Efield, - amrex::Vector, 3>> const& Jfield, + ablastr::fields::MultiLevelVectorField const& Jfield, ablastr::fields::MultiLevelScalarField const& rhofield, amrex::Vector, 3>> const& edge_lengths, amrex::Real dt, DtType dt_type, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index dc536a05a86..b26ca652897 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -423,7 +423,7 @@ void HybridPICModel::CalculateCurrentAmpere ( void HybridPICModel::HybridPICSolveE ( amrex::Vector, 3>> & Efield, - amrex::Vector, 3>> const& Jfield, + ablastr::fields::MultiLevelVectorField const& Jfield, amrex::Vector, 3>> const& Bfield, ablastr::fields::MultiLevelScalarField const& rhofield, amrex::Vector, 3>> const& edge_lengths, @@ -441,7 +441,7 @@ void HybridPICModel::HybridPICSolveE ( void HybridPICModel::HybridPICSolveE ( std::array< std::unique_ptr, 3> & Efield, - std::array< std::unique_ptr, 3> const& Jfield, + ablastr::fields::VectorField const& Jfield, std::array< std::unique_ptr, 3> const& Bfield, amrex::MultiFab* const rhofield, std::array< std::unique_ptr, 3> const& edge_lengths, @@ -462,13 +462,15 @@ void HybridPICModel::HybridPICSolveE ( void HybridPICModel::HybridPICSolveE ( std::array< std::unique_ptr, 3> & Efield, - std::array< std::unique_ptr, 3> const& Jfield, + ablastr::fields::VectorField const& Jfield, std::array< std::unique_ptr, 3> const& Bfield, amrex::MultiFab* const rhofield, std::array< std::unique_ptr, 3> const& edge_lengths, const int lev, PatchType patch_type, const bool solve_for_Faraday) { + using ablastr::fields::va2vm; + auto& warpx = WarpX::GetInstance(); // Solve E field in regular cells @@ -535,7 +537,7 @@ void HybridPICModel::FillElectronPressureMF ( void HybridPICModel::BfieldEvolveRK ( amrex::Vector, 3>>& Bfield, amrex::Vector, 3>>& Efield, - amrex::Vector, 3>> const& Jfield, + ablastr::fields::MultiLevelVectorField const& Jfield, ablastr::fields::MultiLevelScalarField const& rhofield, amrex::Vector, 3>> const& edge_lengths, amrex::Real dt, DtType dt_type, @@ -554,7 +556,7 @@ void HybridPICModel::BfieldEvolveRK ( void HybridPICModel::BfieldEvolveRK ( amrex::Vector, 3>>& Bfield, amrex::Vector, 3>>& Efield, - amrex::Vector, 3>> const& Jfield, + ablastr::fields::MultiLevelVectorField const& Jfield, ablastr::fields::MultiLevelScalarField const& rhofield, amrex::Vector, 3>> const& edge_lengths, amrex::Real dt, int lev, DtType dt_type, @@ -667,7 +669,7 @@ void HybridPICModel::BfieldEvolveRK ( void HybridPICModel::FieldPush ( amrex::Vector, 3>>& Bfield, amrex::Vector, 3>>& Efield, - amrex::Vector, 3>> const& Jfield, + ablastr::fields::MultiLevelVectorField const& Jfield, ablastr::fields::MultiLevelScalarField const& rhofield, amrex::Vector, 3>> const& edge_lengths, amrex::Real dt, DtType dt_type, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp index a229163a09e..153daf7f3cf 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp @@ -353,7 +353,7 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCartesian ( void FiniteDifferenceSolver::HybridPICSolveE ( std::array< std::unique_ptr, 3 >& Efield, std::array< std::unique_ptr, 3 >& Jfield, - std::array< std::unique_ptr, 3 > const& Jifield, + ablastr::fields::VectorField const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3 > const& Bfield, amrex::MultiFab* const rhofield, @@ -706,7 +706,7 @@ template void FiniteDifferenceSolver::HybridPICSolveECartesian ( std::array< std::unique_ptr, 3 >& Efield, std::array< std::unique_ptr, 3 > const& Jfield, - std::array< std::unique_ptr, 3 > const& Jifield, + ablastr::fields::VectorField const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3 > const& Bfield, amrex::MultiFab* const rhofield, diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp index 1e43f0216a6..aeaf1e81007 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp @@ -165,9 +165,9 @@ WarpX::computeVectorPotential (const ablastr::fields::MultiLevelVectorField& cur amrex::Vector> sorted_curr; amrex::Vector> sorted_A; for (int lev = 0; lev <= finest_level; ++lev) { - sorted_curr.emplace_back(amrex::Array ({curr[lev][0].get(), - curr[lev][1].get(), - curr[lev][2].get()})); + sorted_curr.emplace_back(amrex::Array ({curr[lev][Direction{0}], + curr[lev][Direction{1}], + curr[lev][Direction{2}]})); sorted_A.emplace_back(amrex::Array ({A[lev][Direction{0}], A[lev][Direction{1}], A[lev][Direction{2}]})); diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index ab4d81d6bf0..ee4841223c3 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -22,6 +22,7 @@ using namespace amrex; void WarpX::HybridPICEvolveFields () { using ablastr::fields::Direction; + using ablastr::fields::va2vm; WARPX_PROFILE("WarpX::HybridPICEvolveFields()"); @@ -89,7 +90,7 @@ void WarpX::HybridPICEvolveFields () MultiFab::LinComb( *current_fp_temp[lev][idim], 0.5_rt, *current_fp_temp[lev][idim], 0, - 0.5_rt, *current_fp[lev][idim], 0, + 0.5_rt, *m_fields.get("current_fp", Direction{idim}, lev), 0, 0, 1, current_fp_temp[lev][idim]->nGrowVect() ); } @@ -101,7 +102,7 @@ void WarpX::HybridPICEvolveFields () for (int sub_step = 0; sub_step < sub_steps; sub_step++) { m_hybrid_pic_model->BfieldEvolveRK( - Bfield_fp, Efield_fp, current_fp_temp, amrex::GetVecOfPtrs(rho_fp_temp), + Bfield_fp, Efield_fp, va2vm(current_fp_temp), amrex::GetVecOfPtrs(rho_fp_temp), m_edge_lengths, 0.5_rt/sub_steps*dt[0], DtType::FirstHalf, guard_cells.ng_FieldSolver, WarpX::sync_nodal_points @@ -124,7 +125,9 @@ void WarpX::HybridPICEvolveFields () for (int sub_step = 0; sub_step < sub_steps; sub_step++) { m_hybrid_pic_model->BfieldEvolveRK( - Bfield_fp, Efield_fp, current_fp, amrex::GetVecOfPtrs(rho_fp_temp), + Bfield_fp, Efield_fp, + m_fields.get_mr_levels_alldirs("current_fp", finest_level), + amrex::GetVecOfPtrs(rho_fp_temp), m_edge_lengths, 0.5_rt/sub_steps*dt[0], DtType::SecondHalf, guard_cells.ng_FieldSolver, WarpX::sync_nodal_points @@ -143,7 +146,7 @@ void WarpX::HybridPICEvolveFields () MultiFab::LinComb( *current_fp_temp[lev][idim], -1._rt, *current_fp_temp[lev][idim], 0, - 2._rt, *current_fp[lev][idim], 0, + 2._rt, *m_fields.get("current_fp", Direction{idim}, lev), 0, 0, 1, current_fp_temp[lev][idim]->nGrowVect() ); } @@ -155,7 +158,7 @@ void WarpX::HybridPICEvolveFields () // Update the E field to t=n+1 using the extrapolated J_i^n+1 value m_hybrid_pic_model->CalculateCurrentAmpere(Bfield_fp, m_edge_lengths); m_hybrid_pic_model->HybridPICSolveE( - Efield_fp, current_fp_temp, Bfield_fp, m_fields.get_mr_levels("rho_fp", finest_level), m_edge_lengths, false + Efield_fp, va2vm(current_fp_temp), Bfield_fp, m_fields.get_mr_levels("rho_fp", finest_level), m_edge_lengths, false ); FillBoundaryE(guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); @@ -168,7 +171,7 @@ void WarpX::HybridPICEvolveFields () MultiFab::Copy(*rho_fp_temp[lev], *m_fields.get("rho_fp", lev), 0, 0, 1, rho_fp_temp[lev]->nGrowVect()); for (int idim = 0; idim < 3; ++idim) { - MultiFab::Copy(*current_fp_temp[lev][idim], *current_fp[lev][idim], + MultiFab::Copy(*current_fp_temp[lev][idim], *m_fields.get("current_fp", Direction{idim}, lev), 0, 0, 1, current_fp_temp[lev][idim]->nGrowVect()); } } From 8d868758e5ed74000be8929eb50d437d077bfa8a Mon Sep 17 00:00:00 2001 From: Marco Acciarri Date: Wed, 11 Sep 2024 15:23:22 -0700 Subject: [PATCH 106/314] removed Bfield_fp_external and Efield_fp_external from field type and WarpX.H --- Source/FieldSolver/Fields.H | 2 -- Source/WarpX.H | 2 -- Source/WarpX.cpp | 11 ----------- 3 files changed, 15 deletions(-) diff --git a/Source/FieldSolver/Fields.H b/Source/FieldSolver/Fields.H index e75657484fa..6d287646ae6 100644 --- a/Source/FieldSolver/Fields.H +++ b/Source/FieldSolver/Fields.H @@ -17,8 +17,6 @@ namespace warpx::fields None, Efield_fp, Bfield_fp, - Efield_fp_external, - Bfield_fp_external, current_fp, current_fp_nodal, phi_fp, diff --git a/Source/WarpX.H b/Source/WarpX.H index 8c69ffe6e99..66c8aa45c82 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1529,8 +1529,6 @@ private: amrex::Vector, 3 > > vector_potential_grad_buf_b_stag; // Same as Bfield_fp/Efield_fp for reading external field data - amrex::Vector, 3 > > Efield_fp_external; - amrex::Vector, 3 > > Bfield_fp_external; amrex::Vector, 3 > > E_external_particle_field; amrex::Vector, 3 > > B_external_particle_field; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 37329ef7685..517d6a6c5bf 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -3462,12 +3462,6 @@ WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, cons case FieldType::Bfield_fp : field_pointer = Bfield_fp[lev][direction].get(); break; - case FieldType::Efield_fp_external : - field_pointer = Efield_fp_external[lev][direction].get(); - break; - case FieldType::Bfield_fp_external : - field_pointer = Bfield_fp_external[lev][direction].get(); - break; case FieldType::current_fp : field_pointer = current_fp[lev][direction].get(); break; @@ -3512,7 +3506,6 @@ WarpX::getFieldPointerArray (const FieldType field_type, const int lev) const { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( (field_type == FieldType::Efield_fp) || (field_type == FieldType::Bfield_fp) || - (field_type == FieldType::Efield_fp_external) || (field_type == FieldType::Bfield_fp_external) || (field_type == FieldType::current_fp) || (field_type == FieldType::current_fp_nodal) || (field_type == FieldType::Efield_cp) || (field_type == FieldType::Bfield_cp) || (field_type == FieldType::current_cp), "Requested field type is not a vector."); @@ -3562,12 +3555,8 @@ WarpX::getMultiLevelField(warpx::fields::FieldType field_type) const { case FieldType::Efield_fp : return Efield_fp; - case FieldType::Efield_fp_external : - return Efield_fp_external; case FieldType::Bfield_fp : return Bfield_fp; - case FieldType::Bfield_fp_external : - return Bfield_fp_external; case FieldType::current_fp : return current_fp; case FieldType::Efield_cp : From c91ef0b60b8b3b1897676e0b5297a0f756e2cc2a Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 11 Sep 2024 15:30:04 -0700 Subject: [PATCH 107/314] `va2vm` for scalars --- Source/ablastr/fields/MultiFabRegister.H | 9 +++++++++ Source/ablastr/fields/MultiFabRegister.cpp | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index a8647ec556f..e663e69f047 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -433,6 +433,15 @@ namespace ablastr::fields const amrex::Vector, 3 > >& old_vector_on_levels ); + /** TODO: temporary, remove me + * + * @return + */ + MultiLevelScalarField + va2vm ( + const amrex::Vector >& old_scalar_on_levels + ); + } // namespace ablastr::fields #endif // ABLASTR_FIELDS_MF_REGISTER_H diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 1c2e5609dc9..2549fd67a4b 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -486,4 +486,23 @@ namespace ablastr::fields } return field_on_level; } + + MultiLevelScalarField + va2vm ( + const amrex::Vector >& old_scalar_on_levels + ) + { + int const finest_level = old_scalar_on_levels.size() - 1u; + + MultiLevelScalarField field_on_level; + field_on_level.reserve(finest_level+1); + + for (int lvl = 0; lvl <= finest_level; lvl++) + { + // insert a scalar field on a level + field_on_level.push_back(ScalarField{old_scalar_on_levels[lvl].get()}); + + } + return field_on_level; + } } // namespace ablastr::fields From 5fe95f72960415c17163ad0f4f6ac4f9fc3c0258 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 11 Sep 2024 15:33:49 -0700 Subject: [PATCH 108/314] `a2m` helper: old vector field to `VectorField` old vector is `std::array< std::unique_ptr, 3 >` --- Source/ablastr/fields/MultiFabRegister.H | 9 +++++++++ Source/ablastr/fields/MultiFabRegister.cpp | 17 +++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index e663e69f047..5964b6ca732 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -422,6 +422,15 @@ namespace ablastr::fields > m_mf_register; }; + /** TODO: temporary, remove me + * + * @return + */ + VectorField + a2m ( + const std::array< std::unique_ptr, 3 > & old_vectorfield + ); + /** TODO: temporary, remove me * * Convert get_mr_levels_alldirs type to legacy amrex::Vector, 3 > >. diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 2549fd67a4b..e1c4f2e60d3 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -461,6 +461,23 @@ namespace ablastr::fields ); } + VectorField + a2m ( + const std::array< std::unique_ptr, 3 > & old_vectorfield + ) + { + std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; + + VectorField field_on_level; + + // insert components + for (auto dir : {0, 1, 2}) + { + field_on_level[Direction{dir}] = old_vectorfield[dir].get(); + } + return field_on_level; + } + MultiLevelVectorField va2vm ( const amrex::Vector, 3 > >& old_vector_on_levels From a130cb0b93b14843c45282a61ea0100e4c0f8451 Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Wed, 11 Sep 2024 09:56:01 -0700 Subject: [PATCH 109/314] Efield_fp refactor WarpXRegrid and QED --- Source/FieldSolver/WarpX_QED_Field_Pushers.cpp | 8 +++++--- Source/Parallelization/WarpXRegrid.cpp | 7 ++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp b/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp index 9741d9b667b..4e33c5c3590 100644 --- a/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp +++ b/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp @@ -74,13 +74,15 @@ WarpX::Hybrid_QED_Push (int lev, PatchType patch_type, amrex::Real a_dt) const Real dx = dx_vec[0]; const Real dy = dx_vec[1]; const Real dz = dx_vec[2]; + + using ablastr::fields::Direction; MultiFab *Ex, *Ey, *Ez, *Bx, *By, *Bz, *Jx, *Jy, *Jz; if (patch_type == PatchType::fine) { - Ex = Efield_fp[lev][0].get(); - Ey = Efield_fp[lev][1].get(); - Ez = Efield_fp[lev][2].get(); + Ex = m_fields.get("Efield_fp",Direction{0},lev); + Ey = m_fields.get("Efield_fp",Direction{1},lev); + Ez = m_fields.get("Efield_fp",Direction{2},lev); Bx = Bfield_fp[lev][0].get(); By = Bfield_fp[lev][1].get(); Bz = Bfield_fp[lev][2].get(); diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index 4435ccbc10d..3dc82780cc5 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -170,6 +170,9 @@ WarpX::LoadBalance () void WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const DistributionMapping& dm) { + + using ablastr::fields::Direction; + bool const eb_enabled = EB::enabled(); if (ba == boxArray(lev)) { @@ -310,6 +313,8 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi void WarpX::ComputeCostsHeuristic (amrex::Vector > >& a_costs) { + using ablastr::fields::Direction; + for (int lev = 0; lev <= finest_level; ++lev) { const auto & mypc_ref = GetInstance().GetPartContainer(); @@ -328,7 +333,7 @@ WarpX::ComputeCostsHeuristic (amrex::Vector Date: Wed, 11 Sep 2024 10:57:19 -0700 Subject: [PATCH 110/314] Efield_fp refactor HybridPICModel --- .../FiniteDifferenceSolver/FiniteDifferenceSolver.H | 7 ++++--- .../HybridPICModel/HybridPICModel.H | 12 ++++++------ .../HybridPICModel/HybridPICModel.cpp | 12 ++++++------ .../FiniteDifferenceSolver/HybridPICSolveE.cpp | 6 +++--- Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp | 11 ++++++++--- 5 files changed, 27 insertions(+), 21 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index e0fc344f4a4..3dcc2e9bd2e 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -154,7 +154,8 @@ class FiniteDifferenceSolver * \param[in] hybrid_model instance of the hybrid-PIC model * \param[in] solve_for_Faraday boolean flag for whether the E-field is solved to be used in Faraday's equation */ - void HybridPICSolveE ( std::array< std::unique_ptr, 3>& Efield, + void HybridPICSolveE ( + ablastr::fields::VectorField& Efield, std::array< std::unique_ptr, 3>& Jfield, std::array< std::unique_ptr, 3 > const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, @@ -240,7 +241,7 @@ class FiniteDifferenceSolver template void HybridPICSolveECylindrical ( - std::array< std::unique_ptr, 3>& Efield, + ablastr::fields::VectorField& Efield, std::array< std::unique_ptr, 3> const& Jfield, std::array< std::unique_ptr, 3> const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, @@ -345,7 +346,7 @@ class FiniteDifferenceSolver template void HybridPICSolveECartesian ( - std::array< std::unique_ptr, 3>& Efield, + ablastr::fields::VectorField& Efield, std::array< std::unique_ptr, 3> const& Jfield, std::array< std::unique_ptr, 3> const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index 0efe0b19e4a..2bbac8b4c24 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -86,7 +86,7 @@ public: * Function to update the E-field using Ohm's law (hybrid-PIC model). */ void HybridPICSolveE ( - amrex::Vector, 3>>& Efield, + ablastr::fields::MultiLevelVectorField Efield, amrex::Vector, 3>> const& Jfield, amrex::Vector, 3>> const& Bfield, ablastr::fields::MultiLevelScalarField const& rhofield, @@ -94,7 +94,7 @@ public: bool solve_for_Faraday); void HybridPICSolveE ( - std::array< std::unique_ptr, 3>& Efield, + ablastr::fields::VectorField Efield, std::array< std::unique_ptr, 3> const& Jfield, std::array< std::unique_ptr, 3> const& Bfield, amrex::MultiFab* const rhofield, @@ -102,7 +102,7 @@ public: int lev, bool solve_for_Faraday); void HybridPICSolveE ( - std::array< std::unique_ptr, 3>& Efield, + ablastr::fields::VectorField Efield, std::array< std::unique_ptr, 3> const& Jfield, std::array< std::unique_ptr, 3> const& Bfield, amrex::MultiFab* const rhofield, @@ -111,7 +111,7 @@ public: void BfieldEvolveRK ( amrex::Vector, 3>>& Bfield, - amrex::Vector, 3>>& Efield, + ablastr::fields::MultiLevelVectorField Efield, amrex::Vector, 3>> const& Jfield, ablastr::fields::MultiLevelScalarField const& rhofield, amrex::Vector, 3>> const& edge_lengths, @@ -120,7 +120,7 @@ public: void BfieldEvolveRK ( amrex::Vector, 3>>& Bfield, - amrex::Vector, 3>>& Efield, + ablastr::fields::MultiLevelVectorField Efield, amrex::Vector, 3>> const& Jfield, ablastr::fields::MultiLevelScalarField const& rhofield, amrex::Vector, 3>> const& edge_lengths, @@ -129,7 +129,7 @@ public: void FieldPush ( amrex::Vector, 3>>& Bfield, - amrex::Vector, 3>>& Efield, + ablastr::fields::MultiLevelVectorField& Efield, amrex::Vector, 3>> const& Jfield, ablastr::fields::MultiLevelScalarField const& rhofield, amrex::Vector, 3>> const& edge_lengths, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index dc536a05a86..133fd48dd39 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -422,7 +422,7 @@ void HybridPICModel::CalculateCurrentAmpere ( } void HybridPICModel::HybridPICSolveE ( - amrex::Vector, 3>> & Efield, + ablastr::fields::MultiLevelVectorField Efield, amrex::Vector, 3>> const& Jfield, amrex::Vector, 3>> const& Bfield, ablastr::fields::MultiLevelScalarField const& rhofield, @@ -440,7 +440,7 @@ void HybridPICModel::HybridPICSolveE ( } void HybridPICModel::HybridPICSolveE ( - std::array< std::unique_ptr, 3> & Efield, + ablastr::fields::VectorField Efield, std::array< std::unique_ptr, 3> const& Jfield, std::array< std::unique_ptr, 3> const& Bfield, amrex::MultiFab* const rhofield, @@ -461,7 +461,7 @@ void HybridPICModel::HybridPICSolveE ( } void HybridPICModel::HybridPICSolveE ( - std::array< std::unique_ptr, 3> & Efield, + ablastr::fields::VectorField Efield, std::array< std::unique_ptr, 3> const& Jfield, std::array< std::unique_ptr, 3> const& Bfield, amrex::MultiFab* const rhofield, @@ -534,7 +534,7 @@ void HybridPICModel::FillElectronPressureMF ( void HybridPICModel::BfieldEvolveRK ( amrex::Vector, 3>>& Bfield, - amrex::Vector, 3>>& Efield, + ablastr::fields::MultiLevelVectorField Efield, amrex::Vector, 3>> const& Jfield, ablastr::fields::MultiLevelScalarField const& rhofield, amrex::Vector, 3>> const& edge_lengths, @@ -553,7 +553,7 @@ void HybridPICModel::BfieldEvolveRK ( void HybridPICModel::BfieldEvolveRK ( amrex::Vector, 3>>& Bfield, - amrex::Vector, 3>>& Efield, + ablastr::fields::MultiLevelVectorField Efield, amrex::Vector, 3>> const& Jfield, ablastr::fields::MultiLevelScalarField const& rhofield, amrex::Vector, 3>> const& edge_lengths, @@ -666,7 +666,7 @@ void HybridPICModel::BfieldEvolveRK ( void HybridPICModel::FieldPush ( amrex::Vector, 3>>& Bfield, - amrex::Vector, 3>>& Efield, + ablastr::fields::MultiLevelVectorField& Efield, amrex::Vector, 3>> const& Jfield, ablastr::fields::MultiLevelScalarField const& rhofield, amrex::Vector, 3>> const& edge_lengths, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp index a229163a09e..8da06855e22 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp @@ -351,7 +351,7 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCartesian ( void FiniteDifferenceSolver::HybridPICSolveE ( - std::array< std::unique_ptr, 3 >& Efield, + ablastr::fields::VectorField& Efield, std::array< std::unique_ptr, 3 >& Jfield, std::array< std::unique_ptr, 3 > const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, @@ -389,7 +389,7 @@ void FiniteDifferenceSolver::HybridPICSolveE ( #ifdef WARPX_DIM_RZ template void FiniteDifferenceSolver::HybridPICSolveECylindrical ( - std::array< std::unique_ptr, 3 >& Efield, + ablastr::fields::VectorField& Efield, std::array< std::unique_ptr, 3 > const& Jfield, std::array< std::unique_ptr, 3 > const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, @@ -704,7 +704,7 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( template void FiniteDifferenceSolver::HybridPICSolveECartesian ( - std::array< std::unique_ptr, 3 >& Efield, + ablastr::fields::VectorField& Efield, std::array< std::unique_ptr, 3 > const& Jfield, std::array< std::unique_ptr, 3 > const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index 06daeb7a1ee..f1dd8abda9d 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -96,7 +96,9 @@ void WarpX::HybridPICEvolveFields () for (int sub_step = 0; sub_step < sub_steps; sub_step++) { m_hybrid_pic_model->BfieldEvolveRK( - Bfield_fp, Efield_fp, current_fp_temp, amrex::GetVecOfPtrs(rho_fp_temp), + Bfield_fp, + m_fields.get_mr_levels_alldirs("Efield_fp", finest_level), + current_fp_temp, amrex::GetVecOfPtrs(rho_fp_temp), m_edge_lengths, 0.5_rt/sub_steps*dt[0], DtType::FirstHalf, guard_cells.ng_FieldSolver, WarpX::sync_nodal_points @@ -119,7 +121,9 @@ void WarpX::HybridPICEvolveFields () for (int sub_step = 0; sub_step < sub_steps; sub_step++) { m_hybrid_pic_model->BfieldEvolveRK( - Bfield_fp, Efield_fp, current_fp, amrex::GetVecOfPtrs(rho_fp_temp), + Bfield_fp, + m_fields.get_mr_levels_alldirs("Efield_fp", finest_level), + current_fp, amrex::GetVecOfPtrs(rho_fp_temp), m_edge_lengths, 0.5_rt/sub_steps*dt[0], DtType::SecondHalf, guard_cells.ng_FieldSolver, WarpX::sync_nodal_points @@ -150,7 +154,8 @@ void WarpX::HybridPICEvolveFields () // Update the E field to t=n+1 using the extrapolated J_i^n+1 value m_hybrid_pic_model->CalculateCurrentAmpere(Bfield_fp, m_edge_lengths); m_hybrid_pic_model->HybridPICSolveE( - Efield_fp, current_fp_temp, Bfield_fp, m_fields.get_mr_levels("rho_fp", finest_level), m_edge_lengths, false + m_fields.get_mr_levels_alldirs("Efield_fp", finest_level), + current_fp_temp, Bfield_fp, m_fields.get_mr_levels("rho_fp", finest_level), m_edge_lengths, false ); FillBoundaryE(guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); From 46631261132c65891d4efa5079962034ae2897cf Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Wed, 11 Sep 2024 11:03:05 -0700 Subject: [PATCH 111/314] Efield_fp refactor WarpXMovingWindow --- Source/Utils/WarpXMovingWindow.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 66557f96ea0..40f5a425d3a 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -139,6 +139,8 @@ WarpX::MoveWindow (const int step, bool move_j) { WARPX_PROFILE("WarpX::MoveWindow"); + using ablastr::fields::Direction; + if (step == start_moving_window_step) { amrex::Print() << Utils::TextMsg::Info("Starting moving window"); } @@ -243,7 +245,7 @@ WarpX::MoveWindow (const int step, bool move_j) } shiftMF(*Bfield_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost, m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); - shiftMF(*Efield_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost, + shiftMF(*m_fields.get("Efield_fp",Direction{0},lev), geom[lev], num_shift, dir, lev, do_update_cost, m_p_ext_field_params->E_external_grid[dim], use_Eparser, Efield_parser); if (fft_do_time_averaging) { shiftMF(*Bfield_avg_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost, @@ -465,9 +467,9 @@ WarpX::MoveWindow (const int step, bool move_j) const int lev_zero = 0; m_macroscopic_properties->InitData( Geom(lev_zero), - getField(warpx::fields::FieldType::Efield_fp, lev_zero,0).ixType().toIntVect(), - getField(warpx::fields::FieldType::Efield_fp, lev_zero,1).ixType().toIntVect(), - getField(warpx::fields::FieldType::Efield_fp, lev_zero,2).ixType().toIntVect() + m_fields.get("Efield_fp",Direction{0},lev_zero)->ixType().toIntVect(), + m_fields.get("Efield_fp",Direction{1},lev_zero)->ixType().toIntVect(), + m_fields.get("Efield_fp",Direction{2},lev_zero)->ixType().toIntVect() ); } From 07d21b35ccb361933439971a3becddabfb6b0d87 Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Wed, 11 Sep 2024 11:24:58 -0700 Subject: [PATCH 112/314] Efield_cp refactor WarpXComm and WarpXMovingWindow. --- Source/Parallelization/WarpXComm.cpp | 36 +++++++++++++++++----------- Source/Utils/WarpXMovingWindow.cpp | 2 +- Source/WarpX.cpp | 6 ++--- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 565f99178ae..3571161f550 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -312,10 +312,10 @@ WarpX::UpdateAuxilaryDataStagToNodal () const amrex::IntVect& Ex_fp_stag = m_fields.get("Efield_fp",Direction{0},lev)->ixType().toIntVect(); const amrex::IntVect& Ey_fp_stag = m_fields.get("Efield_fp",Direction{1},lev)->ixType().toIntVect(); const amrex::IntVect& Ez_fp_stag = m_fields.get("Efield_fp",Direction{2},lev)->ixType().toIntVect(); - - const amrex::IntVect& Ex_cp_stag = Efield_cp[lev][0]->ixType().toIntVect(); - const amrex::IntVect& Ey_cp_stag = Efield_cp[lev][1]->ixType().toIntVect(); - const amrex::IntVect& Ez_cp_stag = Efield_cp[lev][2]->ixType().toIntVect(); + + const amrex::IntVect& Ex_cp_stag = m_fields.get("Efield_cp",Direction{0},lev)->ixType().toIntVect(); + const amrex::IntVect& Ey_cp_stag = m_fields.get("Efield_cp",Direction{1},lev)->ixType().toIntVect(); + const amrex::IntVect& Ez_cp_stag = m_fields.get("Efield_cp",Direction{2},lev)->ixType().toIntVect(); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) @@ -328,9 +328,9 @@ WarpX::UpdateAuxilaryDataStagToNodal () Array4 const& ex_fp = m_fields.get("Efield_fp",Direction{0},lev)->const_array(mfi); Array4 const& ey_fp = m_fields.get("Efield_fp",Direction{1},lev)->const_array(mfi); Array4 const& ez_fp = m_fields.get("Efield_fp",Direction{2},lev)->const_array(mfi); - Array4 const& ex_cp = Efield_cp[lev][0]->const_array(mfi); - Array4 const& ey_cp = Efield_cp[lev][1]->const_array(mfi); - Array4 const& ez_cp = Efield_cp[lev][2]->const_array(mfi); + Array4 const& ex_cp = m_fields.get("Efield_cp",Direction{0},lev)->const_array(mfi); + Array4 const& ey_cp = m_fields.get("Efield_cp",Direction{1},lev)->const_array(mfi); + Array4 const& ez_cp = m_fields.get("Efield_cp",Direction{2},lev)->const_array(mfi); Array4 const& ex_c = Etmp[0]->const_array(mfi); Array4 const& ey_c = Etmp[1]->const_array(mfi); Array4 const& ez_c = Etmp[2]->const_array(mfi); @@ -501,9 +501,12 @@ WarpX::UpdateAuxilaryDataSameType () { if (electromagnetic_solver_id != ElectromagneticSolverAlgo::None) { - MultiFab dEx(Efield_cp[lev][0]->boxArray(), dm, Efield_cp[lev][0]->nComp(), ng); - MultiFab dEy(Efield_cp[lev][1]->boxArray(), dm, Efield_cp[lev][1]->nComp(), ng); - MultiFab dEz(Efield_cp[lev][2]->boxArray(), dm, Efield_cp[lev][2]->nComp(), ng); + MultiFab dEx(m_fields.get("Efield_cp",Direction{0},lev)->boxArray(), dm, + m_fields.get("Efield_cp",Direction{0},lev)->nComp(), ng); + MultiFab dEy(m_fields.get("Efield_cp",Direction{1},lev)->boxArray(), dm, + m_fields.get("Efield_cp",Direction{1},lev)->nComp(), ng); + MultiFab dEz(m_fields.get("Efield_cp",Direction{2},lev)->boxArray(), dm, + m_fields.get("Efield_cp",Direction{2},lev)->nComp(), ng); dEx.setVal(0.0); dEy.setVal(0.0); dEz.setVal(0.0); @@ -529,9 +532,12 @@ WarpX::UpdateAuxilaryDataSameType () MultiFab::Copy(*Efield_cax[lev][1], dEy, 0, 0, Efield_cax[lev][1]->nComp(), ng); MultiFab::Copy(*Efield_cax[lev][2], dEz, 0, 0, Efield_cax[lev][2]->nComp(), ng); } - MultiFab::Subtract(dEx, *Efield_cp[lev][0], 0, 0, Efield_cp[lev][0]->nComp(), ng); - MultiFab::Subtract(dEy, *Efield_cp[lev][1], 0, 0, Efield_cp[lev][1]->nComp(), ng); - MultiFab::Subtract(dEz, *Efield_cp[lev][2], 0, 0, Efield_cp[lev][2]->nComp(), ng); + MultiFab::Subtract(dEx, *m_fields.get("Efield_cp",Direction{0},lev), + 0, 0, m_fields.get("Efield_cp",Direction{0},lev)->nComp(), ng); + MultiFab::Subtract(dEy, *m_fields.get("Efield_cp",Direction{1},lev), + 0, 0, m_fields.get("Efield_cp",Direction{1},lev)->nComp(), ng); + MultiFab::Subtract(dEz, *m_fields.get("Efield_cp",Direction{2},lev), + 0, 0, m_fields.get("Efield_cp",Direction{2},lev)->nComp(), ng); const amrex::IntVect& refinement_ratio = refRatio(lev-1); @@ -705,7 +711,9 @@ WarpX::FillBoundaryE (const int lev, const PatchType patch_type, const amrex::In } else // coarse patch { - mf = {Efield_cp[lev][0].get(), Efield_cp[lev][1].get(), Efield_cp[lev][2].get()}; + mf = {m_fields.get("Efield_cp",Direction{0},lev), + m_fields.get("Efield_cp",Direction{1},lev), + m_fields.get("Efield_cp",Direction{2},lev)}; period = Geom(lev-1).periodicity(); } diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 40f5a425d3a..0265be21234 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -274,7 +274,7 @@ WarpX::MoveWindow (const int step, bool move_j) // coarse grid shiftMF(*Bfield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost, m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); - shiftMF(*Efield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost, + shiftMF(*m_fields.get("Efield_cp",Direction{0},lev), geom[lev], num_shift, dir, lev, do_update_cost, m_p_ext_field_params->E_external_grid[dim], use_Eparser, Efield_parser); shiftMF(*Bfield_aux[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost); shiftMF(*Efield_aux[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost); diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 517d6a6c5bf..842a0bc2d93 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2660,9 +2660,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm AllocInitMultiFab(Bfield_cp[lev][2], amrex::convert(cba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_cp[z]", 0.0_rt); // Create the MultiFabs for E - AllocInitMultiFab(Efield_cp[lev][0], amrex::convert(cba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_cp[x]", 0.0_rt); - AllocInitMultiFab(Efield_cp[lev][1], amrex::convert(cba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_cp[y]", 0.0_rt); - AllocInitMultiFab(Efield_cp[lev][2], amrex::convert(cba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_cp[z]", 0.0_rt); + m_fields.alloc_init( "Efield_cp", Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Efield_cp", Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Efield_cp", Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); if (fft_do_time_averaging) { From 07bb54f9732d426c6751fb06d4530b2506922d22 Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Wed, 11 Sep 2024 11:31:54 -0700 Subject: [PATCH 113/314] Efield_fp and Efield_cp refactor of PSATDBackwardTransformEB --- Source/FieldSolver/WarpXPushFieldsEM.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 1117dadfcd0..8dd19620706 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -800,7 +800,10 @@ WarpX::PushPSATD () PSATDPushSpectralFields(); // Inverse FFT of E, B, F, and G - PSATDBackwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); + PSATDBackwardTransformEB( m_fields.get_mr_levels_alldirs("Efield_fp",finest_level), + Bfield_fp, + m_fields.get_mr_levels_alldirs("Efield_cp",finest_level), + Bfield_cp); if (WarpX::fft_do_time_averaging) { auto Efield_avg_fp = m_fields.get_mr_levels_alldirs("Efield_avg_fp", finest_level); auto Bfield_avg_fp = m_fields.get_mr_levels_alldirs("Bfield_avg_fp", finest_level); From 7262b469a4e4ae38125d063eb838be279bc84a98 Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Wed, 11 Sep 2024 11:39:03 -0700 Subject: [PATCH 114/314] Efield_fp and Efield_cp refactor of PSATDForwardTransformEB --- Source/FieldSolver/WarpXPushFieldsEM.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 8dd19620706..2c27c297039 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -782,11 +782,10 @@ WarpX::PushPSATD () // FFT of E and B PSATDForwardTransformEB( - Efield_fp, - Bfield_fp, - Efield_cp, - Bfield_cp - ); + m_fields.get_mr_levels_alldirs("Efield_fp", finest_level), + m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level), + m_fields.get_mr_levels_alldirs("Efield_cp", finest_level), + m_fields.get_mr_levels_alldirs("Bfield_cp", finest_level) ); #ifdef WARPX_DIM_RZ if (pml_rz[0]) { pml_rz[0]->PushPSATD(0); } From bccd7520f7c2f0f128da91c63cefee58e90186fb Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Wed, 11 Sep 2024 14:51:08 -0700 Subject: [PATCH 115/314] fixed merge issue. --- Source/FieldSolver/WarpXPushFieldsEM.cpp | 55 ++++++++++++++---------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 2c27c297039..57c0418d127 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -857,15 +857,17 @@ WarpX::EvolveB (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_typ // Evolve B field in regular cells if (patch_type == PatchType::fine) { - m_fdtd_solver_fp[lev]->EvolveB(Bfield_fp[lev], Efield_fp[lev], - m_fields.get("G_fp", lev), - face_areas_lev, m_area_mod[lev], ECTRhofield[lev], Venl[lev], - m_flag_info_face[lev], m_borrowing[lev], lev, a_dt); + m_fdtd_solver_fp[lev]->EvolveB( Bfield_fp[lev], + m_fields.get_alldirs("Efield_fp",lev), + m_fields.get("G_fp", lev), + face_areas_lev, m_area_mod[lev], ECTRhofield[lev], Venl[lev], + m_flag_info_face[lev], m_borrowing[lev], lev, a_dt ); } else { - m_fdtd_solver_cp[lev]->EvolveB(Bfield_cp[lev], Efield_cp[lev], - m_fields.get("G_fp", lev), - face_areas_lev, m_area_mod[lev], ECTRhofield[lev], Venl[lev], - m_flag_info_face[lev], m_borrowing[lev], lev, a_dt); + m_fdtd_solver_cp[lev]->EvolveB( Bfield_cp[lev], + m_fields.get_alldirs("Efield_cp",lev), + m_fields.get("G_fp", lev), + face_areas_lev, m_area_mod[lev], ECTRhofield[lev], Venl[lev], + m_flag_info_face[lev], m_borrowing[lev], lev, a_dt ); } // Evolve B field in PML cells @@ -912,15 +914,17 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) // Evolve E field in regular cells auto face_areas_lev = m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev]; if (patch_type == PatchType::fine) { - m_fdtd_solver_fp[lev]->EvolveE(Efield_fp[lev], Bfield_fp[lev], - current_fp[lev], m_edge_lengths[lev], - face_areas_lev, ECTRhofield[lev], - m_fields.get("F_fp", lev), lev, a_dt ); + m_fdtd_solver_fp[lev]->EvolveE( m_fields.get_alldirs("Efield_fp",lev), + Bfield_fp[lev], + current_fp[lev], m_edge_lengths[lev], + face_areas_lev, ECTRhofield[lev], + m_fields.get("F_fp", lev), lev, a_dt ); } else { - m_fdtd_solver_cp[lev]->EvolveE(Efield_cp[lev], Bfield_cp[lev], - current_cp[lev], m_edge_lengths[lev], - face_areas_lev, ECTRhofield[lev], - m_fields.get("F_cp", lev), lev, a_dt ); + m_fdtd_solver_cp[lev]->EvolveE( m_fields.get_alldirs("Efield_cp",lev), + Bfield_cp[lev], + current_cp[lev], m_edge_lengths[lev], + face_areas_lev, ECTRhofield[lev], + m_fields.get("F_cp", lev), lev, a_dt ); } // Evolve E field in PML cells @@ -949,11 +953,13 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) #ifdef AMREX_USE_EB if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { if (patch_type == PatchType::fine) { - m_fdtd_solver_fp[lev]->EvolveECTRho(Efield_fp[lev], m_edge_lengths[lev], - face_areas_lev, ECTRhofield[lev], lev); + m_fdtd_solver_fp[lev]->EvolveECTRho( m_fields.get_alldirs("Efield_fp",lev), + m_edge_lengths[lev], + face_areas_lev, ECTRhofield[lev], lev ); } else { - m_fdtd_solver_cp[lev]->EvolveECTRho(Efield_cp[lev], m_edge_lengths[lev], - face_areas_lev, ECTRhofield[lev], lev); + m_fdtd_solver_cp[lev]->EvolveECTRho( m_fields.get_alldirs("Efield_cp",lev), + m_edge_lengths[lev], + face_areas_lev, ECTRhofield[lev], lev); } } #endif @@ -991,10 +997,12 @@ WarpX::EvolveF (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_typ // Evolve F field in regular cells if (patch_type == PatchType::fine) { - m_fdtd_solver_fp[lev]->EvolveF( m_fields.get("F_fp", lev), Efield_fp[lev], + m_fdtd_solver_fp[lev]->EvolveF( m_fields.get("F_fp", lev), + m_fields.get_alldirs("Efield_fp", lev), m_fields.get("rho_fp",lev), rhocomp, a_dt ); } else { - m_fdtd_solver_cp[lev]->EvolveF( m_fields.get("F_cp", lev), Efield_cp[lev], + m_fdtd_solver_cp[lev]->EvolveF( m_fields.get("F_cp", lev), + m_fields.get_alldirs("Efield_cp", lev), m_fields.get("rho_cp",lev), rhocomp, a_dt ); } @@ -1088,7 +1096,8 @@ WarpX::MacroscopicEvolveE (int lev, PatchType patch_type, amrex::Real a_dt) { ); m_fdtd_solver_fp[lev]->MacroscopicEvolveE( - Efield_fp[lev], Bfield_fp[lev], + m_fields.get_alldirs("Efield_fp", lev), + Bfield_fp[lev], current_fp[lev], m_edge_lengths[lev], a_dt, m_macroscopic_properties); From bd75464e3fbc02aab456175b92d6c2df962c0864 Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Wed, 11 Sep 2024 14:52:01 -0700 Subject: [PATCH 116/314] Efield_fp refactor of EvolveE and EvolveB. Implicit usage commented out to compile. WIP. --- .../FiniteDifferenceSolver/EvolveB.cpp | 6 ++-- .../FiniteDifferenceSolver/EvolveBPML.cpp | 4 +-- .../FiniteDifferenceSolver/EvolveE.cpp | 6 ++-- .../FiniteDifferenceSolver/EvolveECTRho.cpp | 4 +-- .../FiniteDifferenceSolver/EvolveF.cpp | 6 ++-- .../FiniteDifferenceSolver/EvolveFPML.cpp | 4 +-- .../FiniteDifferenceSolver.H | 36 +++++++++---------- .../MacroscopicEvolveE.cpp | 4 +-- .../ImplicitSolvers/WarpXImplicitOps.cpp | 18 +++++----- Source/Initialization/WarpXInitData.cpp | 4 +-- 10 files changed, 46 insertions(+), 46 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp index 5c6cae5c30b..3c2bd8ddf44 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp @@ -49,7 +49,7 @@ using namespace amrex; */ void FiniteDifferenceSolver::EvolveB ( [[maybe_unused]] std::array< std::unique_ptr, 3 >& Bfield, - [[maybe_unused]] std::array< std::unique_ptr, 3 > const& Efield, + [[maybe_unused]] ablastr::fields::VectorField const& Efield, [[maybe_unused]] amrex::MultiFab const * Gfield, [[maybe_unused]] ablastr::fields::VectorField const& face_areas, [[maybe_unused]] std::array< std::unique_ptr, 3 > const& area_mod, @@ -95,7 +95,7 @@ void FiniteDifferenceSolver::EvolveB ( template void FiniteDifferenceSolver::EvolveBCartesian ( std::array< std::unique_ptr, 3 >& Bfield, - std::array< std::unique_ptr, 3 > const& Efield, + ablastr::fields::VectorField const& Efield, amrex::MultiFab const * Gfield, int lev, amrex::Real const dt ) { @@ -360,7 +360,7 @@ void FiniteDifferenceSolver::EvolveBCartesianECT ( template void FiniteDifferenceSolver::EvolveBCylindrical ( std::array< std::unique_ptr, 3 >& Bfield, - std::array< std::unique_ptr, 3 > const& Efield, + ablastr::fields::VectorField const& Efield, int lev, amrex::Real const dt ) { amrex::LayoutData* cost = WarpX::getCosts(lev); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp index 0ad2c8d6802..083ed5cde41 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp @@ -42,7 +42,7 @@ using namespace amrex; */ void FiniteDifferenceSolver::EvolveBPML ( std::array< amrex::MultiFab*, 3 > Bfield, - std::array< amrex::MultiFab*, 3 > const Efield, + ablastr::fields::VectorField const Efield, amrex::Real const dt, const bool dive_cleaning) { @@ -78,7 +78,7 @@ void FiniteDifferenceSolver::EvolveBPML ( template void FiniteDifferenceSolver::EvolveBPMLCartesian ( std::array< amrex::MultiFab*, 3 > Bfield, - std::array< amrex::MultiFab*, 3 > const Efield, + ablastr::fields::VectorField const Efield, amrex::Real const dt, const bool dive_cleaning) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp index 3e7b37557a6..a7002697b71 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp @@ -50,7 +50,7 @@ using namespace ablastr::fields; * \brief Update the E field, over one timestep */ void FiniteDifferenceSolver::EvolveE ( - std::array< std::unique_ptr, 3 >& Efield, + ablastr::fields::VectorField Efield, std::array< std::unique_ptr, 3 > const& Bfield, std::array< std::unique_ptr, 3 > const& Jfield, std::array< std::unique_ptr, 3 > const& edge_lengths, @@ -93,7 +93,7 @@ void FiniteDifferenceSolver::EvolveE ( template void FiniteDifferenceSolver::EvolveECartesian ( - std::array< std::unique_ptr, 3 >& Efield, + ablastr::fields::VectorField& Efield, std::array< std::unique_ptr, 3 > const& Bfield, std::array< std::unique_ptr, 3 > const& Jfield, std::array< std::unique_ptr, 3 > const& edge_lengths, @@ -227,7 +227,7 @@ void FiniteDifferenceSolver::EvolveECartesian ( template void FiniteDifferenceSolver::EvolveECylindrical ( - std::array< std::unique_ptr, 3 >& Efield, + ablastr::fields::VectorField& Efield, std::array< std::unique_ptr, 3 > const& Bfield, std::array< std::unique_ptr, 3 > const& Jfield, std::array< std::unique_ptr, 3 > const& edge_lengths, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp index 3d6b5ca992b..0d1e6b533c1 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp @@ -48,7 +48,7 @@ using namespace ablastr::fields; * \brief Update the B field, over one timestep */ void FiniteDifferenceSolver::EvolveECTRho ( - std::array< std::unique_ptr, 3 > const& Efield, + ablastr::fields::VectorField const Efield, std::array< std::unique_ptr, 3 > const& edge_lengths, ablastr::fields::VectorField const& face_areas, std::array< std::unique_ptr, 3 >& ECTRhofield, @@ -68,7 +68,7 @@ void FiniteDifferenceSolver::EvolveECTRho ( // If we implement ECT in 1D we will need to take care of this #ifndef differently #ifndef WARPX_DIM_RZ void FiniteDifferenceSolver::EvolveRhoCartesianECT ( - std::array< std::unique_ptr, 3 > const& Efield, + ablastr::fields::VectorField const Efield, std::array< std::unique_ptr, 3 > const& edge_lengths, ablastr::fields::VectorField const& face_areas, std::array< std::unique_ptr, 3 >& ECTRhofield, const int lev ) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp index 71676aecf29..9235c73d244 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp @@ -45,7 +45,7 @@ using namespace amrex; */ void FiniteDifferenceSolver::EvolveF ( amrex::MultiFab* Ffield, - std::array< std::unique_ptr, 3 > const& Efield, + ablastr::fields::VectorField const& Efield, amrex::MultiFab* const rhofield, int const rhocomp, amrex::Real const dt ) { @@ -83,7 +83,7 @@ void FiniteDifferenceSolver::EvolveF ( template void FiniteDifferenceSolver::EvolveFCartesian ( amrex::MultiFab* Ffield, - std::array< std::unique_ptr, 3 > const& Efield, + ablastr::fields::VectorField const Efield, amrex::MultiFab* const rhofield, int const rhocomp, amrex::Real const dt ) { @@ -136,7 +136,7 @@ void FiniteDifferenceSolver::EvolveFCartesian ( template void FiniteDifferenceSolver::EvolveFCylindrical ( amrex::MultiFab* Ffield, - std::array< std::unique_ptr, 3 > const& Efield, + ablastr::fields::VectorField const& Efield, amrex::MultiFab* const rhofield, int const rhocomp, amrex::Real const dt ) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveFPML.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveFPML.cpp index 4ef056c937a..f14a42f451b 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveFPML.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveFPML.cpp @@ -41,7 +41,7 @@ using namespace amrex; */ void FiniteDifferenceSolver::EvolveFPML ( amrex::MultiFab* Ffield, - std::array< amrex::MultiFab*, 3 > const Efield, + ablastr::fields::VectorField const Efield, amrex::Real const dt ) { // Select algorithm (The choice of algorithm is a runtime option, @@ -75,7 +75,7 @@ void FiniteDifferenceSolver::EvolveFPML ( template void FiniteDifferenceSolver::EvolveFPMLCartesian ( amrex::MultiFab* Ffield, - std::array< amrex::MultiFab*, 3 > const Efield, + ablastr::fields::VectorField const Efield, amrex::Real const dt ) { // Loop through the grids, and over the tiles within each grid diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 3dcc2e9bd2e..ea2a014ca32 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -53,7 +53,7 @@ class FiniteDifferenceSolver ablastr::utils::enums::GridType grid_type ); void EvolveB ( std::array< std::unique_ptr, 3 >& Bfield, - std::array< std::unique_ptr, 3 > const& Efield, + ablastr::fields::VectorField const& Efield, amrex::MultiFab const * Gfield, ablastr::fields::VectorField const& face_areas, std::array< std::unique_ptr, 3 > const& area_mod, @@ -63,7 +63,7 @@ class FiniteDifferenceSolver std::array< std::unique_ptr >, 3 >& borrowing, int lev, amrex::Real dt ); - void EvolveE ( std::array< std::unique_ptr, 3 >& Efield, + void EvolveE ( ablastr::fields::VectorField Efield, std::array< std::unique_ptr, 3 > const& Bfield, std::array< std::unique_ptr, 3 > const& Jfield, std::array< std::unique_ptr, 3 > const& edge_lengths, @@ -73,7 +73,7 @@ class FiniteDifferenceSolver int lev, amrex::Real dt ); void EvolveF ( amrex::MultiFab* Ffield, - std::array< std::unique_ptr, 3 > const& Efield, + ablastr::fields::VectorField const& Efield, amrex::MultiFab* const rhofield, int rhocomp, amrex::Real dt ); @@ -82,7 +82,7 @@ class FiniteDifferenceSolver std::array,3> const& Bfield, amrex::Real dt); - void EvolveECTRho ( std::array< std::unique_ptr, 3 > const& Efield, + void EvolveECTRho ( ablastr::fields::VectorField const Efield, std::array< std::unique_ptr, 3 > const& edge_lengths, ablastr::fields::VectorField const& face_areas, std::array< std::unique_ptr, 3 >& ECTRhofield, @@ -113,7 +113,7 @@ class FiniteDifferenceSolver * \param[in] dt timestep of the simulation * \param[in] macroscopic_properties contains user-defined properties of the medium. */ - void MacroscopicEvolveE ( std::array< std::unique_ptr, 3>& Efield, + void MacroscopicEvolveE ( ablastr::fields::VectorField Efield, std::array< std::unique_ptr, 3> const& Bfield, std::array< std::unique_ptr, 3 > const& Jfield, std::array< std::unique_ptr, 3 > const& edge_lengths, @@ -121,11 +121,11 @@ class FiniteDifferenceSolver std::unique_ptr const& macroscopic_properties); void EvolveBPML ( std::array< amrex::MultiFab*, 3 > Bfield, - std::array< amrex::MultiFab*, 3 > Efield, + ablastr::fields::VectorField const Efield, amrex::Real dt, bool dive_cleaning); - void EvolveEPML ( std::array< amrex::MultiFab*, 3 > Efield, + void EvolveEPML ( ablastr::fields::VectorField Efield, std::array< amrex::MultiFab*, 3 > Bfield, std::array< amrex::MultiFab*, 3 > Jfield, std::array< amrex::MultiFab*, 3 > edge_lengths, @@ -134,8 +134,8 @@ class FiniteDifferenceSolver amrex::Real dt, bool pml_has_particles ); void EvolveFPML ( amrex::MultiFab* Ffield, - std::array< amrex::MultiFab*, 3 > Efield, - amrex::Real dt ); + ablastr::fields::VectorField Efield, + amrex::Real dt ); /** * \brief E-update in the hybrid PIC algorithm as described in @@ -228,7 +228,7 @@ class FiniteDifferenceSolver template< typename T_Algo > void EvolveFCylindrical ( amrex::MultiFab* Ffield, - std::array< std::unique_ptr, 3 > const& Efield, + ablastr::fields::VectorField const Efield, amrex::MultiFab* const rhofield, int rhocomp, amrex::Real dt ); @@ -264,13 +264,13 @@ class FiniteDifferenceSolver template< typename T_Algo > void EvolveBCartesian ( std::array< std::unique_ptr, 3 >& Bfield, - std::array< std::unique_ptr, 3 > const& Efield, + ablastr::fields::VectorField const& Efield, amrex::MultiFab const * Gfield, int lev, amrex::Real dt ); template< typename T_Algo > void EvolveECartesian ( - std::array< std::unique_ptr, 3 >& Efield, + ablastr::fields::VectorField& Efield, std::array< std::unique_ptr, 3 > const& Bfield, std::array< std::unique_ptr, 3 > const& Jfield, std::array< std::unique_ptr, 3 > const& edge_lengths, @@ -280,7 +280,7 @@ class FiniteDifferenceSolver template< typename T_Algo > void EvolveFCartesian ( amrex::MultiFab* Ffield, - std::array< std::unique_ptr, 3 > const& Efield, + ablastr::fields::VectorField Efield, amrex::MultiFab* const rhofield, int rhocomp, amrex::Real dt ); @@ -292,7 +292,7 @@ class FiniteDifferenceSolver amrex::Real dt); void EvolveRhoCartesianECT ( - std::array< std::unique_ptr, 3 > const& Efield, + ablastr::fields::VectorField const Efield, std::array< std::unique_ptr, 3 > const& edge_lengths, ablastr::fields::VectorField const& face_areas, std::array< std::unique_ptr, 3 >& ECTRhofield, int lev); @@ -315,7 +315,7 @@ class FiniteDifferenceSolver template< typename T_Algo, typename T_MacroAlgo > void MacroscopicEvolveECartesian ( - std::array< std::unique_ptr< amrex::MultiFab>, 3>& Efield, + ablastr::fields::VectorField Efield, std::array< std::unique_ptr< amrex::MultiFab>, 3> const& Bfield, std::array< std::unique_ptr< amrex::MultiFab>, 3> const& Jfield, std::array< std::unique_ptr, 3 > const& edge_lengths, @@ -325,13 +325,13 @@ class FiniteDifferenceSolver template< typename T_Algo > void EvolveBPMLCartesian ( std::array< amrex::MultiFab*, 3 > Bfield, - std::array< amrex::MultiFab*, 3 > Efield, + ablastr::fields::VectorField const Efield, amrex::Real dt, bool dive_cleaning); template< typename T_Algo > void EvolveEPMLCartesian ( - std::array< amrex::MultiFab*, 3 > Efield, + ablastr::fields::VectorField Efield, std::array< amrex::MultiFab*, 3 > Bfield, std::array< amrex::MultiFab*, 3 > Jfield, std::array< amrex::MultiFab*, 3 > edge_lengths, @@ -341,7 +341,7 @@ class FiniteDifferenceSolver template< typename T_Algo > void EvolveFPMLCartesian ( amrex::MultiFab* Ffield, - std::array< amrex::MultiFab*, 3 > Efield, + ablastr::fields::VectorField Efield, amrex::Real dt ); template diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp index 46e4d3efa06..cb361bdd843 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp @@ -36,7 +36,7 @@ using namespace amrex; void FiniteDifferenceSolver::MacroscopicEvolveE ( - std::array< std::unique_ptr, 3 >& Efield, + ablastr::fields::VectorField Efield, std::array< std::unique_ptr, 3 > const& Bfield, std::array< std::unique_ptr, 3 > const& Jfield, std::array< std::unique_ptr, 3 > const& edge_lengths, @@ -99,7 +99,7 @@ void FiniteDifferenceSolver::MacroscopicEvolveE ( template void FiniteDifferenceSolver::MacroscopicEvolveECartesian ( - std::array< std::unique_ptr, 3 >& Efield, + ablastr::fields::VectorField Efield, std::array< std::unique_ptr, 3 > const& Bfield, std::array< std::unique_ptr, 3 > const& Jfield, std::array< std::unique_ptr, 3 > const& edge_lengths, diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp index 2fa2c26e3cf..c923bea81a9 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp @@ -335,16 +335,16 @@ WarpX::ImplicitComputeRHSE (int lev, PatchType patch_type, amrex::Real a_dt, War // a_Erhs_vec storing only the RHS of the update equation. I.e., // c^2*dt*(curl(B^{n+theta} - mu0*J^{n+1/2}) const auto face_area_lev = m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev]; - if (patch_type == PatchType::fine) { - m_fdtd_solver_fp[lev]->EvolveE( a_Erhs_vec.getArrayVec()[lev], Bfield_fp[lev], - current_fp[lev], m_edge_lengths[lev], - face_area_lev, ECTRhofield[lev], - m_fields.get("F_fp", lev), lev, a_dt ); + if (patch_type == PatchType::fine) { // JRA FIX + //m_fdtd_solver_fp[lev]->EvolveE( a_Erhs_vec.getArrayVec()[lev], Bfield_fp[lev], + // current_fp[lev], m_edge_lengths[lev], + // face_area_lev, ECTRhofield[lev], + // m_fields.get("F_fp", lev), lev, a_dt ); } else { - m_fdtd_solver_cp[lev]->EvolveE( a_Erhs_vec.getArrayVec()[lev], Bfield_cp[lev], - current_cp[lev], m_edge_lengths[lev], - face_area_lev, ECTRhofield[lev], - m_fields.get("F_cp", lev), lev, a_dt ); + //m_fdtd_solver_cp[lev]->EvolveE( a_Erhs_vec.getArrayVec()[lev], Bfield_cp[lev], + // current_cp[lev], m_edge_lengths[lev], + // face_area_lev, ECTRhofield[lev], + // m_fields.get("F_cp", lev), lev, a_dt ); } // Compute Efield_rhs in PML cells by calling EvolveEPML diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index edf7e6dda99..e9590e9bb5a 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -1004,7 +1004,7 @@ WarpX::InitLevelData (int lev, Real /*time*/) // We initialize ECTRhofield consistently with the Efield if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { m_fdtd_solver_fp[lev]->EvolveECTRho( - Efield_fp[lev], m_edge_lengths[lev], + m_fields.get_alldirs("Efield_fp",lev), m_edge_lengths[lev], m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev], ECTRhofield[lev], lev); } @@ -1040,7 +1040,7 @@ WarpX::InitLevelData (int lev, Real /*time*/) if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { // We initialize ECTRhofield consistently with the Efield m_fdtd_solver_cp[lev]->EvolveECTRho( - Efield_cp[lev], m_edge_lengths[lev], + m_fields.get_alldirs("Efield_cp",lev), m_edge_lengths[lev], m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev], ECTRhofield[lev], lev); } From 148d6c58a83065b484791bac038d6c8432c89ce6 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 15:39:25 -0700 Subject: [PATCH 117/314] Remove current_buf --- Source/Evolve/WarpXEvolve.cpp | 2 +- .../FiniteDifferenceSolver/FiniteDifferenceSolver.H | 2 +- .../FiniteDifferenceSolver/HybridPICSolveE.cpp | 2 +- .../MagnetostaticSolver/MagnetostaticSolver.cpp | 8 +++++--- Source/FieldSolver/WarpXPushFieldsEM.cpp | 6 +++--- Source/WarpX.H | 1 - 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 572900e3033..c71990babdc 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -1138,7 +1138,7 @@ WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, m_fields.get("current_fp", Direction{1}, lev), m_fields.get("current_fp", Direction{2}, lev), lev); - if (current_buf[lev][0].get()) { + if (m_fields.has("current_buf", Direction{0}, lev)) { ApplyInverseVolumeScalingToCurrentDensity( m_fields.get("current_buf", Direction{0}, lev), m_fields.get("current_buf", Direction{1}, lev), diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 488338c6eb3..22cf0d99017 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -242,7 +242,7 @@ class FiniteDifferenceSolver void HybridPICSolveECylindrical ( std::array< std::unique_ptr, 3>& Efield, std::array< std::unique_ptr, 3> const& Jfield, - std::array< std::unique_ptr, 3> const& Jifield, + ablastr::fields::VectorField const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3> const& Bfield, amrex::MultiFab* const rhofield, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp index 153daf7f3cf..637840cccfa 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp @@ -391,7 +391,7 @@ template void FiniteDifferenceSolver::HybridPICSolveECylindrical ( std::array< std::unique_ptr, 3 >& Efield, std::array< std::unique_ptr, 3 > const& Jfield, - std::array< std::unique_ptr, 3 > const& Jifield, + ablastr::fields::VectorField const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3 > const& Bfield, amrex::MultiFab* const rhofield, diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp index aeaf1e81007..c43d1686250 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp @@ -108,9 +108,11 @@ WarpX::AddMagnetostaticFieldLabFrame() #ifdef WARPX_DIM_RZ for (int lev = 0; lev <= max_level; lev++) { - ApplyInverseVolumeScalingToCurrentDensity(current_fp[lev][0].get(), - current_fp[lev][1].get(), - current_fp[lev][2].get(), lev); + ApplyInverseVolumeScalingToCurrentDensity( + m_fields.get("current_fp", Direction{0}, lev), + m_fields.get("current_fp", Direction{1}, lev), + m_fields.get("current_fp", Direction{2}, lev), + lev ); } #endif diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 2f6ef0a9ac6..c2dc2dcb6cb 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -461,7 +461,7 @@ void WarpX::PSATDSubtractCurrentPartialSumsAvg () amrex::ignore_unused(Dy); #endif - amrex::MultiFab& Jx = *current_fp[lev][0]; + amrex::MultiFab& Jx = *m_fields.get_fields("current_fp", Direction{0}, lev); #ifdef AMREX_USE_OMP @@ -492,7 +492,7 @@ void WarpX::PSATDSubtractCurrentPartialSumsAvg () #if defined (WARPX_DIM_3D) // Subtract average of cumulative sum from Jy - amrex::MultiFab& Jy = *current_fp[lev][1]; + amrex::MultiFab& Jy = *m_fields.get_fields("current_fp", Direction{1}, lev);; for (amrex::MFIter mfi(Jy); mfi.isValid(); ++mfi) { const amrex::Box& bx = mfi.fabbox(); @@ -517,7 +517,7 @@ void WarpX::PSATDSubtractCurrentPartialSumsAvg () #endif // Subtract average of cumulative sum from Jz - amrex::MultiFab& Jz = *current_fp[lev][2]; + amrex::MultiFab& Jz = *m_fields.get_fields("current_fp", Direction{2}, lev); for (amrex::MFIter mfi(Jz); mfi.isValid(); ++mfi) { const amrex::Box& bx = mfi.fabbox(); diff --git a/Source/WarpX.H b/Source/WarpX.H index f449b8c1bd1..475594aa4c8 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1589,7 +1589,6 @@ private: amrex::Vector > gather_buffer_masks; // If charge/current deposition buffers are used - amrex::Vector, 3 > > current_buf; amrex::Vector > charge_buf; /** From fea7ed872a704313fb0c39e1f97e7fc0eadd918a Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Wed, 11 Sep 2024 15:58:41 -0700 Subject: [PATCH 118/314] Move `charge_buf` to MultiFab map --- Source/Evolve/WarpXEvolve.cpp | 15 ++++++++++----- Source/FieldSolver/ElectrostaticSolver.cpp | 10 +++++----- Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp | 2 +- Source/Parallelization/WarpXComm.cpp | 8 ++++---- Source/WarpX.H | 5 ++--- Source/WarpX.cpp | 3 +-- 6 files changed, 23 insertions(+), 20 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 572900e3033..21df7e252ef 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -909,7 +909,9 @@ WarpX::OneStep_sub1 (Real cur_time) m_fields.get_mr_levels_alldirs("current_buf", finest_level), coarse_lev); AddRhoFromFineLevelandSumBoundary( m_fields.get_mr_levels("rho_fp", finest_level), - m_fields.get_mr_levels("rho_cp", finest_level), charge_buf, coarse_lev, 0, ncomps); + m_fields.get_mr_levels("rho_cp", finest_level), + m_fields.get_mr_levels("rho_buf", finest_level), + coarse_lev, 0, ncomps); EvolveB(fine_lev, PatchType::coarse, dt[fine_lev], DtType::FirstHalf); EvolveF(fine_lev, PatchType::coarse, dt[fine_lev], DtType::FirstHalf); @@ -982,7 +984,9 @@ WarpX::OneStep_sub1 (Real cur_time) coarse_lev); AddRhoFromFineLevelandSumBoundary( m_fields.get_mr_levels("rho_fp", finest_level), - m_fields.get_mr_levels("rho_cp", finest_level), charge_buf, coarse_lev, ncomps, ncomps); + m_fields.get_mr_levels("rho_cp", finest_level), + m_fields.get_mr_levels("rho_buf", finest_level), + coarse_lev, ncomps, ncomps); EvolveE(fine_lev, PatchType::coarse, dt[fine_lev]); FillBoundaryE(fine_lev, PatchType::coarse, guard_cells.ng_FieldSolver, @@ -1126,7 +1130,8 @@ WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, m_fields.get("current_buf", Direction{0}, lev), m_fields.get("current_buf", Direction{1}, lev), m_fields.get("current_buf", Direction{2}, lev), - m_fields.get("rho_fp",lev), charge_buf[lev].get(), + m_fields.get("rho_fp",lev), + m_fields.get("rho_buf", lev), Efield_cax[lev][0].get(), Efield_cax[lev][1].get(), Efield_cax[lev][2].get(), Bfield_cax[lev][0].get(), Bfield_cax[lev][1].get(), Bfield_cax[lev][2].get(), cur_time, dt[lev], a_dt_type, skip_current, push_type); @@ -1147,8 +1152,8 @@ WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, } if (m_fields.has("rho_fp", lev)) { ApplyInverseVolumeScalingToChargeDensity(m_fields.get("rho_fp", lev), lev); - if (charge_buf[lev]) { - ApplyInverseVolumeScalingToChargeDensity(charge_buf[lev].get(), lev-1); + if (m_fields.has("rho_buf", lev)) { + ApplyInverseVolumeScalingToChargeDensity(m_fields.get("rho_buf", lev), lev-1); } } // #else diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index 9aea1e29e03..147d4636689 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -208,12 +208,12 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) } for (int lev = 0; lev <= max_level; lev++) { if (lev > 0) { - if (charge_buf[lev]) { - charge_buf[lev]->setVal(0.); + if (m_fields.has("rho_buf", lev)) { + m_fields.get("rho_buf", lev)->setVal(0.); } } } - SyncRho(amrex::GetVecOfPtrs(rho), amrex::GetVecOfPtrs(rho_coarse), charge_buf); // Apply filter, perform MPI exchange, interpolate across levels + SyncRho(amrex::GetVecOfPtrs(rho), amrex::GetVecOfPtrs(rho_coarse), m_fields.get_mr_levels("rho_buf", finest_level)); // Apply filter, perform MPI exchange, interpolate across levels // Get the particle beta vector bool const local_average = false; // Average across all MPI ranks @@ -264,8 +264,8 @@ WarpX::AddSpaceChargeFieldLabFrame () } for (int lev = 0; lev <= max_level; lev++) { if (lev > 0) { - if (charge_buf[lev]) { - charge_buf[lev]->setVal(0.); + if (m_fields.has("rho_buf", lev)) { + m_fields.get("rho_buf", lev)->setVal(0.); } } } diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index f1dd8abda9d..de2e9b2f390 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -182,7 +182,7 @@ void WarpX::HybridPICDepositInitialRhoAndJ () auto& current_fp_temp = m_hybrid_pic_model->current_fp_temp; mypc->DepositCharge(amrex::GetVecOfPtrs(rho_fp_temp), 0._rt); mypc->DepositCurrent(va2vm(current_fp_temp), dt[0], 0._rt); - SyncRho(amrex::GetVecOfPtrs(rho_fp_temp), m_fields.get_mr_levels("rho_cp", finest_level), charge_buf); + SyncRho(amrex::GetVecOfPtrs(rho_fp_temp), m_fields.get_mr_levels("rho_cp", finest_level), m_fields.get_mr_levels("rho_buf", finest_level)); SyncCurrent( m_fields.get_mr_levels_alldirs("current_fp", finest_level), m_fields.get_mr_levels_alldirs("current_cp", finest_level), diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 3571161f550..5c2f97bdd1f 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -312,7 +312,7 @@ WarpX::UpdateAuxilaryDataStagToNodal () const amrex::IntVect& Ex_fp_stag = m_fields.get("Efield_fp",Direction{0},lev)->ixType().toIntVect(); const amrex::IntVect& Ey_fp_stag = m_fields.get("Efield_fp",Direction{1},lev)->ixType().toIntVect(); const amrex::IntVect& Ez_fp_stag = m_fields.get("Efield_fp",Direction{2},lev)->ixType().toIntVect(); - + const amrex::IntVect& Ex_cp_stag = m_fields.get("Efield_cp",Direction{0},lev)->ixType().toIntVect(); const amrex::IntVect& Ey_cp_stag = m_fields.get("Efield_cp",Direction{1},lev)->ixType().toIntVect(); const amrex::IntVect& Ez_cp_stag = m_fields.get("Efield_cp",Direction{2},lev)->ixType().toIntVect(); @@ -1208,14 +1208,14 @@ WarpX::SyncRho () { SyncRho( m_fields.get_mr_levels("rho_fp", finest_level), m_fields.get_mr_levels("rho_cp", finest_level), - charge_buf); + m_fields.get_mr_levels("rho_buf", finest_level)); } void WarpX::SyncRho ( const ablastr::fields::MultiLevelScalarField& charge_fp, const ablastr::fields::MultiLevelScalarField& charge_cp, - const amrex::Vector>& charge_buffer) + ablastr::fields::MultiLevelScalarField const & charge_buffer) { WARPX_PROFILE("WarpX::SyncRho()"); @@ -1540,7 +1540,7 @@ void WarpX::ApplyFilterandSumBoundaryRho (int /*lev*/, int glev, amrex::MultiFab void WarpX::AddRhoFromFineLevelandSumBoundary ( const ablastr::fields::MultiLevelScalarField& charge_fp, const ablastr::fields::MultiLevelScalarField& charge_cp, - const amrex::Vector>& charge_buffer, + ablastr::fields::MultiLevelScalarField const & charge_buffer, const int lev, const int icomp, const int ncomp) diff --git a/Source/WarpX.H b/Source/WarpX.H index 66c8aa45c82..886cb617927 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -883,7 +883,7 @@ public: void SyncRho ( const ablastr::fields::MultiLevelScalarField& charge_fp, const ablastr::fields::MultiLevelScalarField& charge_cp, - const amrex::Vector>& charge_buffer); + ablastr::fields::MultiLevelScalarField const & charge_buffer); [[nodiscard]] amrex::Vector getnsubsteps () const {return nsubsteps;} [[nodiscard]] int getnsubsteps (int lev) const {return nsubsteps[lev];} @@ -1375,7 +1375,7 @@ private: void AddRhoFromFineLevelandSumBoundary ( const ablastr::fields::MultiLevelScalarField& charge_fp, const ablastr::fields::MultiLevelScalarField& charge_cp, - const amrex::Vector>& charge_buffer, + ablastr::fields::MultiLevelScalarField const & charge_buffer, int lev, int icomp, int ncomp); @@ -1590,7 +1590,6 @@ private: // If charge/current deposition buffers are used amrex::Vector, 3 > > current_buf; - amrex::Vector > charge_buf; /** * \brief diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 842a0bc2d93..dc8302aedd5 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -362,7 +362,6 @@ WarpX::WarpX () Bfield_cax.resize(nlevs_max); current_buffer_masks.resize(nlevs_max); gather_buffer_masks.resize(nlevs_max); - charge_buf.resize(nlevs_max); pml.resize(nlevs_max); #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) @@ -2791,7 +2790,7 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm m_fields.alloc_init("current_buf", Direction{1}, lev, amrex::convert(cba,jy_nodal_flag), dm, ncomps, ngJ, 0.0_rt); m_fields.alloc_init("current_buf", Direction{2}, lev, amrex::convert(cba,jz_nodal_flag), dm, ncomps, ngJ, 0.0_rt); if (m_fields.has("rho_cp", lev)) { - AllocInitMultiFab(charge_buf[lev], amrex::convert(cba,rho_nodal_flag),dm,2*ncomps,ngRho,lev, "charge_buf"); + m_fields.alloc_init("rho_buf", lev, amrex::convert(cba,rho_nodal_flag), dm, 2*ncomps, ngRho, 0.0_rt); } AllocInitMultiFab(current_buffer_masks[lev], ba, dm, ncomps, amrex::IntVect(1), lev, "current_buffer_masks"); // Current buffer masks have 1 ghost cell, because of the fact From 9e962795afe4d78e527a959785dbf53e730234e6 Mon Sep 17 00:00:00 2001 From: Marco Acciarri Date: Wed, 11 Sep 2024 15:59:43 -0700 Subject: [PATCH 119/314] adding current_store to m_fields register --- Source/WarpX.H | 1 - Source/WarpX.cpp | 19 ++++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Source/WarpX.H b/Source/WarpX.H index 66c8aa45c82..f9a516a8cc5 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1575,7 +1575,6 @@ private: amrex::Vector > m_distance_to_eb; // store fine patch - amrex::Vector, 3 > > current_store; // Coarse patch amrex::Vector, 3 > > current_cp; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 517d6a6c5bf..19bf9c46955 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -346,7 +346,6 @@ WarpX::WarpX () ECTRhofield.resize(nlevs_max); Venl.resize(nlevs_max); - current_store.resize(nlevs_max); if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { @@ -2467,9 +2466,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (do_subcycling && lev == 0) { - AllocInitMultiFab(current_store[lev][0], amrex::convert(ba,jx_nodal_flag),dm,ncomps,ngJ,lev, "current_store[x]"); - AllocInitMultiFab(current_store[lev][1], amrex::convert(ba,jy_nodal_flag),dm,ncomps,ngJ,lev, "current_store[y]"); - AllocInitMultiFab(current_store[lev][2], amrex::convert(ba,jz_nodal_flag),dm,ncomps,ngJ,lev, "current_store[z]"); + m_fields.alloc_init("current_store", Direction{0}, lev, amrex::convert(ba,jx_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init("current_store", Direction{1}, lev, amrex::convert(ba,jy_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init("current_store", Direction{2}, lev, amrex::convert(ba,jz_nodal_flag), dm, ncomps, ngJ, 0.0_rt); } if (do_dive_cleaning) @@ -3325,10 +3324,11 @@ WarpX::GatherBufferMasks (int lev) void WarpX::StoreCurrent (int lev) { + using ablastr::fields::Direction; for (int idim = 0; idim < 3; ++idim) { - if (current_store[lev][idim]) { - MultiFab::Copy(*current_store[lev][idim], *current_fp[lev][idim], - 0, 0, 1, current_store[lev][idim]->nGrowVect()); + if (m_fields.get("current_store",Direction{idim},lev)) { + MultiFab::Copy(*m_fields.get("current_store",Direction{idim},lev), *current_fp[lev][idim], + 0, 0, 1, m_fields.get("current_store",Direction{idim},lev)->nGrowVect()); } } } @@ -3336,9 +3336,10 @@ WarpX::StoreCurrent (int lev) void WarpX::RestoreCurrent (int lev) { + using ablastr::fields::Direction; for (int idim = 0; idim < 3; ++idim) { - if (current_store[lev][idim]) { - std::swap(current_fp[lev][idim], current_store[lev][idim]); + if (m_fields.get("current_store",Direction{idim},lev)) { + std::swap(current_fp[lev][idim], m_fields.get("current_store",Direction{idim},lev)); } } } From cefb8143412c1c79aa371a7c3a2eb7fd73746a90 Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Wed, 11 Sep 2024 23:51:45 +0200 Subject: [PATCH 120/314] fix merge conflict --- .../inputs_test_3d_embedded_boundary_picmi.py | 2 +- Python/pywarpx/fields.py | 6 ++-- Source/WarpX.H | 3 -- Source/WarpX.cpp | 30 +++++++++---------- 4 files changed, 18 insertions(+), 23 deletions(-) diff --git a/Examples/Tests/embedded_boundary_python_api/inputs_test_3d_embedded_boundary_picmi.py b/Examples/Tests/embedded_boundary_python_api/inputs_test_3d_embedded_boundary_picmi.py index 1a42c6d9622..7148cde2d3e 100755 --- a/Examples/Tests/embedded_boundary_python_api/inputs_test_3d_embedded_boundary_picmi.py +++ b/Examples/Tests/embedded_boundary_python_api/inputs_test_3d_embedded_boundary_picmi.py @@ -94,7 +94,7 @@ face_areas_y = fields.FaceAreasyWrapper() face_areas_z = fields.FaceAreaszWrapper() -print("======== Testing the wrappers of m_edge_lengths =========") +print("======== Testing the wrappers of edge_lengths =========") ly_slice_x = edge_lengths_y[nx // 2, :, :] lz_slice_x = edge_lengths_z[nx // 2, :, :] diff --git a/Python/pywarpx/fields.py b/Python/pywarpx/fields.py index 88ae8149753..5a194b25758 100644 --- a/Python/pywarpx/fields.py +++ b/Python/pywarpx/fields.py @@ -829,19 +829,19 @@ def GCPWrapper(level=0, include_ghosts=False): def EdgeLengthsxWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="m_edge_lengths[x]", level=level, include_ghosts=include_ghosts + mf_name="edge_lengths[x]", level=level, include_ghosts=include_ghosts ) def EdgeLengthsyWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="m_edge_lengths[y]", level=level, include_ghosts=include_ghosts + mf_name="edge_lengths[y]", level=level, include_ghosts=include_ghosts ) def EdgeLengthszWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="m_edge_lengths[z]", level=level, include_ghosts=include_ghosts + mf_name="edge_lengths[z]", level=level, include_ghosts=include_ghosts ) diff --git a/Source/WarpX.H b/Source/WarpX.H index 886cb617927..637c1cfffc8 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1532,9 +1532,6 @@ private: amrex::Vector, 3 > > E_external_particle_field; amrex::Vector, 3 > > B_external_particle_field; - //! EB: Lengths of the mesh edges - amrex::Vector, 3 > > m_edge_lengths; - /** EB: for every mesh face flag_info_face contains a: * * 0 if the face needs to be extended * * 1 if the face is large enough to lend area to other faces diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index dc8302aedd5..c7bf56dae0d 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -336,7 +336,6 @@ WarpX::WarpX () B_external_particle_field.resize(1); E_external_particle_field.resize(1); - m_edge_lengths.resize(nlevs_max); m_distance_to_eb.resize(nlevs_max); m_flag_info_face.resize(nlevs_max); m_flag_ext_face.resize(nlevs_max); @@ -2362,12 +2361,13 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm // EB info are needed only at the finest level if (lev == maxLevel()) { if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { - AllocInitMultiFab(m_edge_lengths[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, - guard_cells.ng_FieldSolver, lev, "m_edge_lengths[x]"); - AllocInitMultiFab(m_edge_lengths[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, - guard_cells.ng_FieldSolver, lev, "m_edge_lengths[y]"); - AllocInitMultiFab(m_edge_lengths[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, - guard_cells.ng_FieldSolver, lev, "m_edge_lengths[z]"); + //! EB: Lengths of the mesh edges + m_fields.alloc_init( "edge_lengths", Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + m_fields.alloc_init( "edge_lengths", Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + m_fields.alloc_init( "edge_lengths", Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); //! EB: Areas of the mesh faces m_fields.alloc_init( "face_areas", Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), @@ -2378,12 +2378,13 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); } if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { - AllocInitMultiFab(m_edge_lengths[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, - guard_cells.ng_FieldSolver, lev, "m_edge_lengths[x]"); - AllocInitMultiFab(m_edge_lengths[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, - guard_cells.ng_FieldSolver, lev, "m_edge_lengths[y]"); - AllocInitMultiFab(m_edge_lengths[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, - guard_cells.ng_FieldSolver, lev, "m_edge_lengths[z]"); + //! EB: Lengths of the mesh edges + m_fields.alloc_init( "edge_lengths", Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + m_fields.alloc_init( "edge_lengths", Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + m_fields.alloc_init( "edge_lengths", Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); //! EB: Areas of the mesh faces m_fields.alloc_init( "face_areas", Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), @@ -3473,9 +3474,6 @@ WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, cons case FieldType::current_cp : field_pointer = current_cp[lev][direction].get(); break; - case FieldType::edge_lengths : - field_pointer = m_edge_lengths[lev][direction].get(); - break; default: WARPX_ABORT_WITH_MESSAGE("Invalid field type"); break; From ca99dc9768cd152cb32f9a6f80c41bcc1da00749 Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Thu, 12 Sep 2024 01:02:48 +0200 Subject: [PATCH 121/314] fix merge conflict --- .../EmbeddedBoundary/WarpXFaceExtensions.cpp | 12 ++++---- Source/EmbeddedBoundary/WarpXInitEB.cpp | 6 ++-- .../FiniteDifferenceSolver/EvolveE.cpp | 4 +-- .../FiniteDifferenceSolver.H | 18 +++++------ .../HybridPICModel/HybridPICModel.H | 18 ++++++----- .../HybridPICModel/HybridPICModel.cpp | 4 +-- .../MacroscopicEvolveE.cpp | 3 +- Source/FieldSolver/WarpXPushFieldsEM.cpp | 30 ++++++++++--------- .../FieldSolver/WarpXPushFieldsHybridPIC.cpp | 16 ++++++---- 9 files changed, 61 insertions(+), 50 deletions(-) diff --git a/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp b/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp index 87592589716..9fcbdc4507b 100644 --- a/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp +++ b/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp @@ -473,9 +473,9 @@ WarpX::ComputeOneWayExtensions () auto const &S_mod = m_area_mod[maxLevel()][idim]->array(mfi); - const auto &lx = m_edge_lengths[maxLevel()][0]->array(mfi); - const auto &ly = m_edge_lengths[maxLevel()][1]->array(mfi); - const auto &lz = m_edge_lengths[maxLevel()][2]->array(mfi); + const auto &lx = m_fields.get_alldirs("edge_lengths", maxLevel())[0]->array(mfi); + const auto &ly = m_fields.get_alldirs("edge_lengths", maxLevel())[1]->array(mfi); + const auto &lz = m_fields.get_alldirs("edge_lengths", maxLevel())[2]->array(mfi); vecs_size = amrex::Scan::PrefixSum(ncells, [=] AMREX_GPU_DEVICE (int icell) { @@ -600,9 +600,9 @@ WarpX::ComputeEightWaysExtensions () int& vecs_size = borrowing.vecs_size; auto const &S_mod = m_area_mod[maxLevel()][idim]->array(mfi); - const auto &lx = m_edge_lengths[maxLevel()][0]->array(mfi); - const auto &ly = m_edge_lengths[maxLevel()][1]->array(mfi); - const auto &lz = m_edge_lengths[maxLevel()][2]->array(mfi); + const auto &lx = m_fields.get_alldirs("edge_lengths", maxLevel())[0]->array(mfi); + const auto &ly = m_fields.get_alldirs("edge_lengths", maxLevel())[1]->array(mfi); + const auto &lz = m_fields.get_alldirs("edge_lengths", maxLevel())[2]->array(mfi); vecs_size += amrex::Scan::PrefixSum(ncells, [=] AMREX_GPU_DEVICE (int icell){ diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index c1f46a7d34b..52e511d9183 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -343,9 +343,9 @@ WarpX::MarkCells(){ auto const &S = face_areas_idim_max_lev->array(mfi); auto const &flag_info_face = m_flag_info_face[maxLevel()][idim]->array(mfi); auto const &flag_ext_face = m_flag_ext_face[maxLevel()][idim]->array(mfi); - const auto &lx = m_edge_lengths[maxLevel()][0]->array(mfi); - const auto &ly = m_edge_lengths[maxLevel()][1]->array(mfi); - const auto &lz = m_edge_lengths[maxLevel()][2]->array(mfi); + const auto &lx = m_fields.get_alldirs("edge_lengths", maxLevel())[0]->array(mfi); + const auto &ly = m_fields.get_alldirs("edge_lengths", maxLevel())[1]->array(mfi); + const auto &lz = m_fields.get_alldirs("edge_lengths", maxLevel())[2]->array(mfi); auto const &mod_areas_dim = m_area_mod[maxLevel()][idim]->array(mfi); const amrex::Real dx = cell_size[0]; diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp index a7002697b71..a449d87e4b8 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp @@ -53,7 +53,7 @@ void FiniteDifferenceSolver::EvolveE ( ablastr::fields::VectorField Efield, std::array< std::unique_ptr, 3 > const& Bfield, std::array< std::unique_ptr, 3 > const& Jfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + VectorField const& edge_lengths, VectorField const& face_areas, std::array< std::unique_ptr, 3 >& ECTRhofield, amrex::MultiFab const* Ffield, @@ -96,7 +96,7 @@ void FiniteDifferenceSolver::EvolveECartesian ( ablastr::fields::VectorField& Efield, std::array< std::unique_ptr, 3 > const& Bfield, std::array< std::unique_ptr, 3 > const& Jfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + VectorField const& edge_lengths, amrex::MultiFab const* Ffield, int lev, amrex::Real const dt ) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index ea2a014ca32..94ef5d488fe 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -66,7 +66,7 @@ class FiniteDifferenceSolver void EvolveE ( ablastr::fields::VectorField Efield, std::array< std::unique_ptr, 3 > const& Bfield, std::array< std::unique_ptr, 3 > const& Jfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, ablastr::fields::VectorField const& face_areas, std::array< std::unique_ptr, 3 >& ECTRhofield, amrex::MultiFab const* Ffield, @@ -83,8 +83,8 @@ class FiniteDifferenceSolver amrex::Real dt); void EvolveECTRho ( ablastr::fields::VectorField const Efield, - std::array< std::unique_ptr, 3 > const& edge_lengths, - ablastr::fields::VectorField const& face_areas, + ablastr::fields::VectorField const& edge_lengths, + ablastr::fields::VectorField const& face_areas, std::array< std::unique_ptr, 3 >& ECTRhofield, int lev ); @@ -116,7 +116,7 @@ class FiniteDifferenceSolver void MacroscopicEvolveE ( ablastr::fields::VectorField Efield, std::array< std::unique_ptr, 3> const& Bfield, std::array< std::unique_ptr, 3 > const& Jfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, amrex::Real dt, std::unique_ptr const& macroscopic_properties); @@ -220,7 +220,7 @@ class FiniteDifferenceSolver std::array< std::unique_ptr, 3 >& Efield, std::array< std::unique_ptr, 3 > const& Bfield, std::array< std::unique_ptr, 3 > const& Jfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, amrex::MultiFab const* Ffield, int lev, amrex::Real dt ); @@ -248,7 +248,7 @@ class FiniteDifferenceSolver std::array< std::unique_ptr, 3> const& Bfield, amrex::MultiFab* const rhofield, std::unique_ptr const& Pefield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, int lev, HybridPICModel const* hybrid_model, bool solve_for_Faraday ); @@ -256,7 +256,7 @@ class FiniteDifferenceSolver void CalculateCurrentAmpereCylindrical ( std::array< std::unique_ptr, 3 >& Jfield, std::array< std::unique_ptr, 3 > const& Bfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, int lev ); @@ -273,7 +273,7 @@ class FiniteDifferenceSolver ablastr::fields::VectorField& Efield, std::array< std::unique_ptr, 3 > const& Bfield, std::array< std::unique_ptr, 3 > const& Jfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, amrex::MultiFab const* Ffield, int lev, amrex::Real dt ); @@ -293,7 +293,7 @@ class FiniteDifferenceSolver void EvolveRhoCartesianECT ( ablastr::fields::VectorField const Efield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, ablastr::fields::VectorField const& face_areas, std::array< std::unique_ptr, 3 >& ECTRhofield, int lev); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index 2bbac8b4c24..9fbd0d06a75 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -19,6 +19,8 @@ #include "Utils/WarpXConst.H" #include "Utils/WarpXProfilerWrapper.H" +#include + #include #include @@ -55,10 +57,10 @@ public: * of time and therefore this should be re-evaluated at every step. */ void GetCurrentExternal ( - amrex::Vector, 3>> const& edge_lengths + ablastr::fields::MultiLevelVectorField const& edge_lengths ); void GetCurrentExternal ( - std::array< std::unique_ptr, 3> const& edge_lengths, + ablastr::fields::MultiLevelVectorField const& edge_lengths, int lev ); @@ -90,7 +92,7 @@ public: amrex::Vector, 3>> const& Jfield, amrex::Vector, 3>> const& Bfield, ablastr::fields::MultiLevelScalarField const& rhofield, - amrex::Vector, 3>> const& edge_lengths, + ablastr::fields::MultiLevelVectorField const& edge_lengths, bool solve_for_Faraday); void HybridPICSolveE ( @@ -98,7 +100,7 @@ public: std::array< std::unique_ptr, 3> const& Jfield, std::array< std::unique_ptr, 3> const& Bfield, amrex::MultiFab* const rhofield, - std::array< std::unique_ptr, 3> const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, int lev, bool solve_for_Faraday); void HybridPICSolveE ( @@ -106,7 +108,7 @@ public: std::array< std::unique_ptr, 3> const& Jfield, std::array< std::unique_ptr, 3> const& Bfield, amrex::MultiFab* const rhofield, - std::array< std::unique_ptr, 3> const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, int lev, PatchType patch_type, bool solve_for_Faraday); void BfieldEvolveRK ( @@ -114,7 +116,7 @@ public: ablastr::fields::MultiLevelVectorField Efield, amrex::Vector, 3>> const& Jfield, ablastr::fields::MultiLevelScalarField const& rhofield, - amrex::Vector, 3>> const& edge_lengths, + ablastr::fields::MultiLevelVectorField const& edge_lengths, amrex::Real dt, DtType a_dt_type, amrex::IntVect ng, std::optional nodal_sync); @@ -123,7 +125,7 @@ public: ablastr::fields::MultiLevelVectorField Efield, amrex::Vector, 3>> const& Jfield, ablastr::fields::MultiLevelScalarField const& rhofield, - amrex::Vector, 3>> const& edge_lengths, + ablastr::fields::MultiLevelVectorField const& edge_lengths, amrex::Real dt, int lev, DtType dt_type, amrex::IntVect ng, std::optional nodal_sync); @@ -132,7 +134,7 @@ public: ablastr::fields::MultiLevelVectorField& Efield, amrex::Vector, 3>> const& Jfield, ablastr::fields::MultiLevelScalarField const& rhofield, - amrex::Vector, 3>> const& edge_lengths, + ablastr::fields::MultiLevelVectorField const& edge_lengths, amrex::Real dt, DtType dt_type, amrex::IntVect ng, std::optional nodal_sync); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 133fd48dd39..a7547ba9354 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -251,7 +251,7 @@ void HybridPICModel::InitData () } void HybridPICModel::GetCurrentExternal ( - amrex::Vector, 3>> const& edge_lengths) + ablastr::fields::MultiLevelVectorField const& edge_lengths) { if (!m_external_field_has_time_dependence) { return; } @@ -264,7 +264,7 @@ void HybridPICModel::GetCurrentExternal ( void HybridPICModel::GetCurrentExternal ( - std::array< std::unique_ptr, 3> const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, int lev) { // This logic matches closely to WarpX::InitializeExternalFieldsOnGridUsingParser diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp index cb361bdd843..e50c3674fb0 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp @@ -34,12 +34,13 @@ #include using namespace amrex; +using namespace ablastr::fields; void FiniteDifferenceSolver::MacroscopicEvolveE ( ablastr::fields::VectorField Efield, std::array< std::unique_ptr, 3 > const& Bfield, std::array< std::unique_ptr, 3 > const& Jfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + VectorField const& edge_lengths, amrex::Real const dt, std::unique_ptr const& macroscopic_properties) { diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 57c0418d127..31a4501476a 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -912,19 +912,20 @@ void WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) { // Evolve E field in regular cells - auto face_areas_lev = m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev]; + const auto face_areas_lev = m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev]; + const auto edge_lengths_lev = m_fields.get_mr_levels_alldirs("edge_lengths", finest_level)[lev]; if (patch_type == PatchType::fine) { - m_fdtd_solver_fp[lev]->EvolveE( m_fields.get_alldirs("Efield_fp",lev), - Bfield_fp[lev], - current_fp[lev], m_edge_lengths[lev], - face_areas_lev, ECTRhofield[lev], - m_fields.get("F_fp", lev), lev, a_dt ); + m_fdtd_solver_fp[lev]->EvolveE(m_fields.get_alldirs("Efield_fp",lev), + Bfield_fp[lev], + current_fp[lev], edge_lengths_lev, + face_areas_lev, ECTRhofield[lev], + m_fields.get("F_fp", lev), lev, a_dt ); } else { - m_fdtd_solver_cp[lev]->EvolveE( m_fields.get_alldirs("Efield_cp",lev), - Bfield_cp[lev], - current_cp[lev], m_edge_lengths[lev], - face_areas_lev, ECTRhofield[lev], - m_fields.get("F_cp", lev), lev, a_dt ); + m_fdtd_solver_cp[lev]->EvolveE(m_fields.get_alldirs("Efield_cp",lev), + Bfield_cp[lev], + current_cp[lev], edge_lengths_lev, + face_areas_lev, ECTRhofield[lev], + m_fields.get("F_cp", lev), lev, a_dt ); } // Evolve E field in PML cells @@ -954,11 +955,11 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { if (patch_type == PatchType::fine) { m_fdtd_solver_fp[lev]->EvolveECTRho( m_fields.get_alldirs("Efield_fp",lev), - m_edge_lengths[lev], + edge_lengths_lev, face_areas_lev, ECTRhofield[lev], lev ); } else { m_fdtd_solver_cp[lev]->EvolveECTRho( m_fields.get_alldirs("Efield_cp",lev), - m_edge_lengths[lev], + edge_lengths_lev, face_areas_lev, ECTRhofield[lev], lev); } } @@ -1098,7 +1099,8 @@ WarpX::MacroscopicEvolveE (int lev, PatchType patch_type, amrex::Real a_dt) { m_fdtd_solver_fp[lev]->MacroscopicEvolveE( m_fields.get_alldirs("Efield_fp", lev), Bfield_fp[lev], - current_fp[lev], m_edge_lengths[lev], + current_fp[lev], + m_fields.get_alldirs("edge_lengths", lev), a_dt, m_macroscopic_properties); if (do_pml && pml[lev]->ok()) { diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index de2e9b2f390..3ad75b079c1 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -59,7 +59,8 @@ void WarpX::HybridPICEvolveFields () const int sub_steps = m_hybrid_pic_model->m_substeps; // Get the external current - m_hybrid_pic_model->GetCurrentExternal(m_edge_lengths); + m_hybrid_pic_model->GetCurrentExternal( + m_fields.get_mr_levels_alldirs("edge_lengths", finest_level)); // Reference hybrid-PIC multifabs auto& rho_fp_temp = m_hybrid_pic_model->rho_fp_temp; @@ -99,7 +100,8 @@ void WarpX::HybridPICEvolveFields () Bfield_fp, m_fields.get_mr_levels_alldirs("Efield_fp", finest_level), current_fp_temp, amrex::GetVecOfPtrs(rho_fp_temp), - m_edge_lengths, 0.5_rt/sub_steps*dt[0], + m_fields.get_mr_levels_alldirs("edge_lenghts", finest_level), + 0.5_rt/sub_steps*dt[0], DtType::FirstHalf, guard_cells.ng_FieldSolver, WarpX::sync_nodal_points ); @@ -124,7 +126,8 @@ void WarpX::HybridPICEvolveFields () Bfield_fp, m_fields.get_mr_levels_alldirs("Efield_fp", finest_level), current_fp, amrex::GetVecOfPtrs(rho_fp_temp), - m_edge_lengths, 0.5_rt/sub_steps*dt[0], + m_fields.get_mr_levels_alldirs("edge_lenghts", finest_level), + 0.5_rt/sub_steps*dt[0], DtType::SecondHalf, guard_cells.ng_FieldSolver, WarpX::sync_nodal_points ); @@ -152,10 +155,13 @@ void WarpX::HybridPICEvolveFields () m_hybrid_pic_model->CalculateElectronPressure(); // Update the E field to t=n+1 using the extrapolated J_i^n+1 value - m_hybrid_pic_model->CalculateCurrentAmpere(Bfield_fp, m_edge_lengths); + m_hybrid_pic_model->CalculateCurrentAmpere( + Bfield_fp, + m_fields.get_mr_levels_alldirs("edge_lengths", finest_level)); m_hybrid_pic_model->HybridPICSolveE( m_fields.get_mr_levels_alldirs("Efield_fp", finest_level), - current_fp_temp, Bfield_fp, m_fields.get_mr_levels("rho_fp", finest_level), m_edge_lengths, false + current_fp_temp, Bfield_fp, m_fields.get_mr_levels("rho_fp", finest_level), + m_fields.get_mr_levels_alldirs("edge_lengths", finest_level), false ); FillBoundaryE(guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); From f28b2ef2e8d90cc9d16706821d453f822e617e7d Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Thu, 12 Sep 2024 01:05:47 +0200 Subject: [PATCH 122/314] fix merge conflict --- .../FiniteDifferenceSolver/EvolveECTRho.cpp | 4 ++-- .../FiniteDifferenceSolver/FiniteDifferenceSolver.H | 6 +++--- .../HybridPICModel/HybridPICModel.H | 4 ++-- .../HybridPICModel/HybridPICModel.cpp | 2 +- .../FiniteDifferenceSolver/HybridPICSolveE.cpp | 4 ++-- .../FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp | 12 ++++++++---- 6 files changed, 18 insertions(+), 14 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp index 0d1e6b533c1..fc4bebc13d5 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp @@ -49,7 +49,7 @@ using namespace ablastr::fields; */ void FiniteDifferenceSolver::EvolveECTRho ( ablastr::fields::VectorField const Efield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, ablastr::fields::VectorField const& face_areas, std::array< std::unique_ptr, 3 >& ECTRhofield, const int lev) { @@ -69,7 +69,7 @@ void FiniteDifferenceSolver::EvolveECTRho ( #ifndef WARPX_DIM_RZ void FiniteDifferenceSolver::EvolveRhoCartesianECT ( ablastr::fields::VectorField const Efield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, ablastr::fields::VectorField const& face_areas, std::array< std::unique_ptr, 3 >& ECTRhofield, const int lev ) { #ifdef AMREX_USE_EB diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 94ef5d488fe..f95f7e58695 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -178,7 +178,7 @@ class FiniteDifferenceSolver void CalculateCurrentAmpere ( std::array< std::unique_ptr, 3>& Jfield, std::array< std::unique_ptr, 3> const& Bfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, int lev ); private: @@ -318,7 +318,7 @@ class FiniteDifferenceSolver ablastr::fields::VectorField Efield, std::array< std::unique_ptr< amrex::MultiFab>, 3> const& Bfield, std::array< std::unique_ptr< amrex::MultiFab>, 3> const& Jfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, amrex::Real dt, std::unique_ptr const& macroscopic_properties); @@ -361,7 +361,7 @@ class FiniteDifferenceSolver void CalculateCurrentAmpereCartesian ( std::array< std::unique_ptr, 3 >& Jfield, std::array< std::unique_ptr, 3 > const& Bfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, int lev ); #endif diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index 9fbd0d06a75..11d6fb7e5a9 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -75,11 +75,11 @@ public: */ void CalculateCurrentAmpere ( amrex::Vector, 3>> const& Bfield, - amrex::Vector, 3>> const& edge_lengths + ablastr::fields::MultiLevelVectorField const& edge_lengths ); void CalculateCurrentAmpere ( std::array< std::unique_ptr, 3> const& Bfield, - std::array< std::unique_ptr, 3> const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, int lev ); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index a7547ba9354..a8ee446cc24 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -393,7 +393,7 @@ void HybridPICModel::GetCurrentExternal ( void HybridPICModel::CalculateCurrentAmpere ( amrex::Vector, 3>> const& Bfield, - amrex::Vector, 3>> const& edge_lengths) + ablastr::fields::MultiLevelVectorField const& edge_lengths) { auto& warpx = WarpX::GetInstance(); for (int lev = 0; lev <= warpx.finestLevel(); ++lev) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp index 8da06855e22..cce3a6dad50 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp @@ -26,7 +26,7 @@ using namespace amrex; void FiniteDifferenceSolver::CalculateCurrentAmpere ( std::array< std::unique_ptr, 3>& Jfield, std::array< std::unique_ptr, 3> const& Bfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, int lev ) { // Select algorithm (The choice of algorithm is a runtime option, @@ -244,7 +244,7 @@ template void FiniteDifferenceSolver::CalculateCurrentAmpereCartesian ( std::array< std::unique_ptr, 3 >& Jfield, std::array< std::unique_ptr, 3 > const& Bfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, int lev ) { diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp index c923bea81a9..ff92fe387b1 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp @@ -337,13 +337,17 @@ WarpX::ImplicitComputeRHSE (int lev, PatchType patch_type, amrex::Real a_dt, War const auto face_area_lev = m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev]; if (patch_type == PatchType::fine) { // JRA FIX //m_fdtd_solver_fp[lev]->EvolveE( a_Erhs_vec.getArrayVec()[lev], Bfield_fp[lev], - // current_fp[lev], m_edge_lengths[lev], - // face_area_lev, ECTRhofield[lev], + // current_fp[lev], + // m_fields.get_alldirs("edge_lengths", lev), + // m_fields.get_alldirs("face_areas", lev), + // ECTRhofield[lev], // m_fields.get("F_fp", lev), lev, a_dt ); } else { //m_fdtd_solver_cp[lev]->EvolveE( a_Erhs_vec.getArrayVec()[lev], Bfield_cp[lev], - // current_cp[lev], m_edge_lengths[lev], - // face_area_lev, ECTRhofield[lev], + // current_cp[lev], + // m_fields.get_alldirs("edge_lengths", lev), + // m_fields.get_alldirs("face_areas", lev), + // ECTRhofield[lev], // m_fields.get("F_cp", lev), lev, a_dt ); } From 5a3e7eac21e799c615919e0d47a6c3f0c02500c9 Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Thu, 12 Sep 2024 01:06:32 +0200 Subject: [PATCH 123/314] fix merge conflict --- .../FiniteDifferenceSolver.H | 2 +- .../HybridPICModel/HybridPICModel.H | 2 +- .../HybridPICModel/HybridPICModel.cpp | 16 ++++---- .../HybridPICSolveE.cpp | 2 +- .../MacroscopicEvolveE.cpp | 2 +- Source/Initialization/WarpXInitData.cpp | 40 ++++++++++--------- Source/WarpX.H | 4 +- 7 files changed, 36 insertions(+), 32 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index f95f7e58695..a012cda7d1c 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -162,7 +162,7 @@ class FiniteDifferenceSolver std::array< std::unique_ptr, 3> const& Bfield, amrex::MultiFab* const rhofield, std::unique_ptr const& Pefield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, int lev, HybridPICModel const* hybrid_model, bool solve_for_Faraday ); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index 11d6fb7e5a9..53d1bf02151 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -60,7 +60,7 @@ public: ablastr::fields::MultiLevelVectorField const& edge_lengths ); void GetCurrentExternal ( - ablastr::fields::MultiLevelVectorField const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, int lev ); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index a8ee446cc24..1cbe6470bbe 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -246,7 +246,7 @@ void HybridPICModel::InitData () }; } #endif - GetCurrentExternal(edge_lengths, lev); + (edge_lengths, lev); } } @@ -404,7 +404,7 @@ void HybridPICModel::CalculateCurrentAmpere ( void HybridPICModel::CalculateCurrentAmpere ( std::array< std::unique_ptr, 3> const& Bfield, - std::array< std::unique_ptr, 3> const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, const int lev) { WARPX_PROFILE("WarpX::CalculateCurrentAmpere()"); @@ -426,7 +426,7 @@ void HybridPICModel::HybridPICSolveE ( amrex::Vector, 3>> const& Jfield, amrex::Vector, 3>> const& Bfield, ablastr::fields::MultiLevelScalarField const& rhofield, - amrex::Vector, 3>> const& edge_lengths, + ablastr::fields::MultiLevelVectorField const& edge_lengths, const bool solve_for_Faraday) { auto& warpx = WarpX::GetInstance(); @@ -444,7 +444,7 @@ void HybridPICModel::HybridPICSolveE ( std::array< std::unique_ptr, 3> const& Jfield, std::array< std::unique_ptr, 3> const& Bfield, amrex::MultiFab* const rhofield, - std::array< std::unique_ptr, 3> const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, const int lev, const bool solve_for_Faraday) { WARPX_PROFILE("WarpX::HybridPICSolveE()"); @@ -465,7 +465,7 @@ void HybridPICModel::HybridPICSolveE ( std::array< std::unique_ptr, 3> const& Jfield, std::array< std::unique_ptr, 3> const& Bfield, amrex::MultiFab* const rhofield, - std::array< std::unique_ptr, 3> const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, const int lev, PatchType patch_type, const bool solve_for_Faraday) { @@ -537,7 +537,7 @@ void HybridPICModel::BfieldEvolveRK ( ablastr::fields::MultiLevelVectorField Efield, amrex::Vector, 3>> const& Jfield, ablastr::fields::MultiLevelScalarField const& rhofield, - amrex::Vector, 3>> const& edge_lengths, + ablastr::fields::MultiLevelVectorField const& edge_lengths, amrex::Real dt, DtType dt_type, IntVect ng, std::optional nodal_sync ) { @@ -556,7 +556,7 @@ void HybridPICModel::BfieldEvolveRK ( ablastr::fields::MultiLevelVectorField Efield, amrex::Vector, 3>> const& Jfield, ablastr::fields::MultiLevelScalarField const& rhofield, - amrex::Vector, 3>> const& edge_lengths, + ablastr::fields::MultiLevelVectorField const& edge_lengths, amrex::Real dt, int lev, DtType dt_type, IntVect ng, std::optional nodal_sync ) { @@ -669,7 +669,7 @@ void HybridPICModel::FieldPush ( ablastr::fields::MultiLevelVectorField& Efield, amrex::Vector, 3>> const& Jfield, ablastr::fields::MultiLevelScalarField const& rhofield, - amrex::Vector, 3>> const& edge_lengths, + ablastr::fields::MultiLevelVectorField const& edge_lengths, amrex::Real dt, DtType dt_type, IntVect ng, std::optional nodal_sync ) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp index cce3a6dad50..de5a770643b 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp @@ -358,7 +358,7 @@ void FiniteDifferenceSolver::HybridPICSolveE ( std::array< std::unique_ptr, 3 > const& Bfield, amrex::MultiFab* const rhofield, std::unique_ptr const& Pefield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, int lev, HybridPICModel const* hybrid_model, const bool solve_for_Faraday) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp index e50c3674fb0..4db0b00f8f5 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp @@ -103,7 +103,7 @@ void FiniteDifferenceSolver::MacroscopicEvolveECartesian ( ablastr::fields::VectorField Efield, std::array< std::unique_ptr, 3 > const& Bfield, std::array< std::unique_ptr, 3 > const& Jfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, amrex::Real const dt, std::unique_ptr const& macroscopic_properties) { diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index e9590e9bb5a..ef0ea3ed030 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -973,8 +973,8 @@ WarpX::InitLevelData (int lev, Real /*time*/) m_p_ext_field_params->Bxfield_parser->compile<3>(), m_p_ext_field_params->Byfield_parser->compile<3>(), m_p_ext_field_params->Bzfield_parser->compile<3>(), - m_edge_lengths[lev], - m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev], + m_fields.get_alldirs("edge_lengths", lev), + m_fields.get_alldirs("face_areas", lev), 'B', lev, PatchType::fine); @@ -985,7 +985,7 @@ WarpX::InitLevelData (int lev, Real /*time*/) m_p_ext_field_params->Bxfield_parser->compile<3>(), m_p_ext_field_params->Byfield_parser->compile<3>(), m_p_ext_field_params->Bzfield_parser->compile<3>(), - m_edge_lengths[lev], + m_fields.get_alldirs("edge_lengths", lev), m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev], 'B', lev, PatchType::coarse); @@ -1004,7 +1004,8 @@ WarpX::InitLevelData (int lev, Real /*time*/) // We initialize ECTRhofield consistently with the Efield if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { m_fdtd_solver_fp[lev]->EvolveECTRho( - m_fields.get_alldirs("Efield_fp",lev), m_edge_lengths[lev], + m_fields.get_alldirs("Efield_fp",lev), + m_fields.get_alldirs("edge_lengths", lev), m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev], ECTRhofield[lev], lev); } @@ -1019,8 +1020,8 @@ WarpX::InitLevelData (int lev, Real /*time*/) m_p_ext_field_params->Exfield_parser->compile<3>(), m_p_ext_field_params->Eyfield_parser->compile<3>(), m_p_ext_field_params->Ezfield_parser->compile<3>(), - m_edge_lengths[lev], - m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev], + m_fields.get_alldirs("edge_lengths", lev), + m_fields.get_alldirs("face_areas", lev), 'E', lev, PatchType::fine); @@ -1031,8 +1032,8 @@ WarpX::InitLevelData (int lev, Real /*time*/) m_p_ext_field_params->Exfield_parser->compile<3>(), m_p_ext_field_params->Eyfield_parser->compile<3>(), m_p_ext_field_params->Ezfield_parser->compile<3>(), - m_edge_lengths[lev], - m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev], + m_fields.get_alldirs("edge_lengths", lev), + m_fields.get_alldirs("face_areas", lev), 'E', lev, PatchType::coarse); #ifdef AMREX_USE_EB @@ -1040,7 +1041,8 @@ WarpX::InitLevelData (int lev, Real /*time*/) if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { // We initialize ECTRhofield consistently with the Efield m_fdtd_solver_cp[lev]->EvolveECTRho( - m_fields.get_alldirs("Efield_cp",lev), m_edge_lengths[lev], + m_fields.get_alldirs("Efield_cp",lev), + m_fields.get_alldirs("edge_lengths", lev), m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev], ECTRhofield[lev], lev); } @@ -1066,7 +1068,7 @@ WarpX::InitializeExternalFieldsOnGridUsingParser ( MultiFab *mfx, MultiFab *mfy, MultiFab *mfz, ParserExecutor<3> const& xfield_parser, ParserExecutor<3> const& yfield_parser, ParserExecutor<3> const& zfield_parser, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, ablastr::fields::VectorField const& face_areas, [[maybe_unused]] const char field, const int lev, PatchType patch_type) @@ -1282,13 +1284,15 @@ void WarpX::InitializeEBGridData (int lev) if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD ) { auto const eb_fact = fieldEBFactory(lev); + ComputeEdgeLengths( + m_fields.get_alldirs("edge_lengths", lev), eb_fact); + ScaleEdges( + m_fields.get_alldirs("edge_lengths", lev), CellSize(lev)); - ComputeEdgeLengths(m_edge_lengths[lev], eb_fact); - ScaleEdges(m_edge_lengths[lev], CellSize(lev)); - - auto face_areas_lev = m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev]; - ComputeFaceAreas(face_areas_lev, eb_fact); - ScaleAreas(face_areas_lev, CellSize(lev)); + ComputeFaceAreas( + m_fields.get_alldirs("face_areas", lev), eb_fact); + ScaleAreas( + m_fields.get_alldirs("face_areas", lev), CellSize(lev)); if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { MarkCells(); @@ -1406,8 +1410,8 @@ WarpX::LoadExternalFields (int const lev) m_p_ext_field_params->Exfield_parser->compile<3>(), m_p_ext_field_params->Eyfield_parser->compile<3>(), m_p_ext_field_params->Ezfield_parser->compile<3>(), - m_edge_lengths[lev], - m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev], + m_fields.get_alldirs("edge_lengths", lev), + m_fields.get_alldirs("face_areas", lev), 'E', lev, PatchType::fine); } diff --git a/Source/WarpX.H b/Source/WarpX.H index 637c1cfffc8..8ff557e2cb2 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1062,8 +1062,8 @@ public: amrex::ParserExecutor<3> const& xfield_parser, amrex::ParserExecutor<3> const& yfield_parser, amrex::ParserExecutor<3> const& zfield_parser, - std::array< std::unique_ptr, 3 > const& edge_lengths, - std::array const& face_areas, + ablastr::fields::VectorField const& edge_lengths, + ablastr::fields::VectorField const& face_areas, [[maybe_unused]] char field, int lev, PatchType patch_type); From bcfa040338a2050c86141e7ee1987864e100e043 Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Thu, 12 Sep 2024 00:34:37 +0200 Subject: [PATCH 124/314] continue work to use field record for edge_lengths --- Source/BoundaryConditions/PML.cpp | 4 ++-- Source/EmbeddedBoundary/WarpXInitEB.cpp | 4 ++-- .../HybridPICSolveE.cpp | 2 +- Source/Initialization/WarpXInitData.cpp | 19 +++++++++---------- Source/WarpX.H | 4 ++-- 5 files changed, 16 insertions(+), 17 deletions(-) diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index f154e4323e6..b1efb425e94 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -736,8 +736,8 @@ PML::PML (const int lev, const BoxArray& grid_ba, auto const eb_fact = fieldEBFactory(); - WarpX::ComputeEdgeLengths(pml_edge_lengths, eb_fact); - WarpX::ScaleEdges(pml_edge_lengths, WarpX::CellSize(lev)); + WarpX::ComputeEdgeLengths(ablastr::fields::va2vm(pml_edge_lengths), eb_fact); + WarpX::ScaleEdges(ablastr::fields::va2vm(pml_edge_lengths), WarpX::CellSize(lev)); } } diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index 52e511d9183..8252b26c894 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -119,7 +119,7 @@ WarpX::InitEB () #ifdef AMREX_USE_EB void -WarpX::ComputeEdgeLengths (std::array< std::unique_ptr, 3 >& edge_lengths, +WarpX::ComputeEdgeLengths (ablastr::fields::VectorField& edge_lengths, const amrex::EBFArrayBoxFactory& eb_fact) { BL_PROFILE("ComputeEdgeLengths"); @@ -247,7 +247,7 @@ WarpX::ComputeFaceAreas (VectorField& face_areas, void -WarpX::ScaleEdges (std::array< std::unique_ptr, 3 >& edge_lengths, +WarpX::ScaleEdges (ablastr::fields::VectorField& edge_lengths, const std::array& cell_size) { BL_PROFILE("ScaleEdges"); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp index de5a770643b..c7124e94f2d 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp @@ -711,7 +711,7 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( std::array< std::unique_ptr, 3 > const& Bfield, amrex::MultiFab* const rhofield, std::unique_ptr const& Pefield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField edge_lengths, int lev, HybridPICModel const* hybrid_model, const bool solve_for_Faraday ) { diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index ef0ea3ed030..734df2b8eac 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -1284,15 +1284,14 @@ void WarpX::InitializeEBGridData (int lev) if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD ) { auto const eb_fact = fieldEBFactory(lev); - ComputeEdgeLengths( - m_fields.get_alldirs("edge_lengths", lev), eb_fact); - ScaleEdges( - m_fields.get_alldirs("edge_lengths", lev), CellSize(lev)); - ComputeFaceAreas( - m_fields.get_alldirs("face_areas", lev), eb_fact); - ScaleAreas( - m_fields.get_alldirs("face_areas", lev), CellSize(lev)); + auto edge_lengths_lev = m_fields.get_alldirs("edge_lengths", lev); + ComputeEdgeLengths(edge_lengths_lev, eb_fact); + ScaleEdges(edge_lengths_lev, CellSize(lev)); + + auto face_areas_lev = m_fields.get_alldirs("face_areas", lev); + ComputeFaceAreas(face_areas_lev, eb_fact); + ScaleAreas(face_areas_lev, CellSize(lev)); if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { MarkCells(); @@ -1382,8 +1381,8 @@ WarpX::LoadExternalFields (int const lev) m_p_ext_field_params->Bxfield_parser->compile<3>(), m_p_ext_field_params->Byfield_parser->compile<3>(), m_p_ext_field_params->Bzfield_parser->compile<3>(), - m_edge_lengths[lev], - m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev], + m_fields.get_alldirs("edge_lengths", lev), + m_fields.get_alldirs("face_areas", lev), 'B', lev, PatchType::fine); } diff --git a/Source/WarpX.H b/Source/WarpX.H index 8ff557e2cb2..9f27380541e 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1143,7 +1143,7 @@ public: * \brief Compute the length of the mesh edges. Here the length is a value in [0, 1]. * An edge of length 0 is fully covered. */ - static void ComputeEdgeLengths (std::array< std::unique_ptr, 3 >& edge_lengths, + static void ComputeEdgeLengths (ablastr::fields::VectorField& edge_lengths, const amrex::EBFArrayBoxFactory& eb_fact); /** * \brief Compute the area of the mesh faces. Here the area is a value in [0, 1]. @@ -1155,7 +1155,7 @@ public: /** * \brief Scale the edges lengths by the mesh width to obtain the real lengths. */ - static void ScaleEdges (std::array< std::unique_ptr, 3 >& edge_lengths, + static void ScaleEdges (ablastr::fields::VectorField& edge_lengths, const std::array& cell_size); /** * \brief Scale the edges areas by the mesh width to obtain the real areas. From db976cff82ee269e4776b02d7fbe21cc4087da73 Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Thu, 12 Sep 2024 01:06:59 +0200 Subject: [PATCH 125/314] fix merge conflict --- Source/BoundaryConditions/PML.cpp | 5 +++-- Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp | 2 +- .../FiniteDifferenceSolver/FiniteDifferenceSolver.H | 2 +- .../HybridPICModel/HybridPICModel.cpp | 2 +- .../FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp | 6 +++--- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index b1efb425e94..eba0886f1a9 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -736,8 +736,9 @@ PML::PML (const int lev, const BoxArray& grid_ba, auto const eb_fact = fieldEBFactory(); - WarpX::ComputeEdgeLengths(ablastr::fields::va2vm(pml_edge_lengths), eb_fact); - WarpX::ScaleEdges(ablastr::fields::va2vm(pml_edge_lengths), WarpX::CellSize(lev)); + auto t_pml_edge_lengths = ablastr::fields::a2m(pml_edge_lengths); + WarpX::ComputeEdgeLengths(t_pml_edge_lengths, eb_fact); + WarpX::ScaleEdges(t_pml_edge_lengths, WarpX::CellSize(lev)); } } diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp index a449d87e4b8..0b6f9de6019 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp @@ -230,7 +230,7 @@ void FiniteDifferenceSolver::EvolveECylindrical ( ablastr::fields::VectorField& Efield, std::array< std::unique_ptr, 3 > const& Bfield, std::array< std::unique_ptr, 3 > const& Jfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, amrex::MultiFab const* Ffield, int lev, amrex::Real const dt ) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index a012cda7d1c..404185e5487 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -353,7 +353,7 @@ class FiniteDifferenceSolver std::array< std::unique_ptr, 3> const& Bfield, amrex::MultiFab* const rhofield, std::unique_ptr const& Pefield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, int lev, HybridPICModel const* hybrid_model, bool solve_for_Faraday ); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 1cbe6470bbe..fe95b26e4e3 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -246,7 +246,7 @@ void HybridPICModel::InitData () }; } #endif - (edge_lengths, lev); + GetCurrentExternal(ablastr::fields::a2m(edge_lengths), lev); } } diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp index c7124e94f2d..e51883ccb16 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp @@ -61,7 +61,7 @@ template void FiniteDifferenceSolver::CalculateCurrentAmpereCylindrical ( std::array< std::unique_ptr, 3 >& Jfield, std::array< std::unique_ptr, 3 > const& Bfield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, int lev ) { @@ -396,7 +396,7 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( std::array< std::unique_ptr, 3 > const& Bfield, amrex::MultiFab* const rhofield, std::unique_ptr const& Pefield, - std::array< std::unique_ptr, 3 > const& edge_lengths, + ablastr::fields::VectorField const& edge_lengths, int lev, HybridPICModel const* hybrid_model, const bool solve_for_Faraday ) { @@ -711,7 +711,7 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( std::array< std::unique_ptr, 3 > const& Bfield, amrex::MultiFab* const rhofield, std::unique_ptr const& Pefield, - ablastr::fields::VectorField edge_lengths, + ablastr::fields::VectorField const& edge_lengths, int lev, HybridPICModel const* hybrid_model, const bool solve_for_Faraday ) { From c101fd71a06c5854afb8235919091556478cc28c Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 16:11:26 -0700 Subject: [PATCH 126/314] Fix conflicts --- .../HybridPICModel/HybridPICModel.H | 10 +++++----- .../HybridPICModel/HybridPICModel.cpp | 6 +++--- Source/FieldSolver/WarpXPushFieldsEM.cpp | 3 ++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index c6e98e767b5..fc85297d8b4 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -86,7 +86,7 @@ public: * Function to update the E-field using Ohm's law (hybrid-PIC model). */ void HybridPICSolveE ( - ablastr::fields::MultiLevelVectorField const& Efield, + ablastr::fields::MultiLevelVectorField Efield, ablastr::fields::MultiLevelVectorField const& Jfield, amrex::Vector, 3>> const& Bfield, ablastr::fields::MultiLevelScalarField const& rhofield, @@ -94,7 +94,7 @@ public: bool solve_for_Faraday); void HybridPICSolveE ( - ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField Efield, ablastr::fields::VectorField const& Jfield, std::array< std::unique_ptr, 3> const& Bfield, amrex::MultiFab* const rhofield, @@ -102,7 +102,7 @@ public: int lev, bool solve_for_Faraday); void HybridPICSolveE ( - ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField Efield, ablastr::fields::VectorField const& Jfield, std::array< std::unique_ptr, 3> const& Bfield, amrex::MultiFab* const rhofield, @@ -111,7 +111,7 @@ public: void BfieldEvolveRK ( amrex::Vector, 3>>& Bfield, - ablastr::fields::MultiLevelVectorField const& Efield, + ablastr::fields::MultiLevelVectorField Efield, ablastr::fields::MultiLevelVectorField const& Jfield, ablastr::fields::MultiLevelScalarField const& rhofield, amrex::Vector, 3>> const& edge_lengths, @@ -120,7 +120,7 @@ public: void BfieldEvolveRK ( amrex::Vector, 3>>& Bfield, - ablastr::fields::MultiLevelVectorField const& Efield, + ablastr::fields::MultiLevelVectorField Efield, ablastr::fields::MultiLevelVectorField const& Jfield, ablastr::fields::MultiLevelScalarField const& rhofield, amrex::Vector, 3>> const& edge_lengths, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index a13f18c9130..b437b79331c 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -422,7 +422,7 @@ void HybridPICModel::CalculateCurrentAmpere ( } void HybridPICModel::HybridPICSolveE ( - ablastr::fields::MultiLevelVectorField const& Efield, + ablastr::fields::MultiLevelVectorField Efield, ablastr::fields::MultiLevelVectorField const& Jfield, amrex::Vector, 3>> const& Bfield, ablastr::fields::MultiLevelScalarField const& rhofield, @@ -440,7 +440,7 @@ void HybridPICModel::HybridPICSolveE ( } void HybridPICModel::HybridPICSolveE ( - ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField Efield, ablastr::fields::VectorField const& Jfield, std::array< std::unique_ptr, 3> const& Bfield, amrex::MultiFab* const rhofield, @@ -461,7 +461,7 @@ void HybridPICModel::HybridPICSolveE ( } void HybridPICModel::HybridPICSolveE ( - ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField Efield, ablastr::fields::VectorField const& Jfield, std::array< std::unique_ptr, 3> const& Bfield, amrex::MultiFab* const rhofield, diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index df2dc9d08b5..f1477cde014 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -1098,7 +1098,8 @@ WarpX::MacroscopicEvolveE (int lev, PatchType patch_type, amrex::Real a_dt) { ); m_fdtd_solver_fp[lev]->MacroscopicEvolveE( - Efield_fp[lev], Bfield_fp[lev], + m_fields.get_alldirs("Efield_fp", lev), + Bfield_fp[lev], m_fields.get_alldirs("current_fp", lev), m_edge_lengths[lev], a_dt, m_macroscopic_properties); From 5534916bd42e249222c64c85b3ff2b7cc167140f Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Thu, 12 Sep 2024 01:16:35 +0200 Subject: [PATCH 127/314] fix issue --- Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp index ff92fe387b1..9822ca59272 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp @@ -334,7 +334,6 @@ WarpX::ImplicitComputeRHSE (int lev, PatchType patch_type, amrex::Real a_dt, War // a_Erhs_vec is set to zero above, calling EvolveE below results in // a_Erhs_vec storing only the RHS of the update equation. I.e., // c^2*dt*(curl(B^{n+theta} - mu0*J^{n+1/2}) - const auto face_area_lev = m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev]; if (patch_type == PatchType::fine) { // JRA FIX //m_fdtd_solver_fp[lev]->EvolveE( a_Erhs_vec.getArrayVec()[lev], Bfield_fp[lev], // current_fp[lev], From 716d958ec6ccb84508119946884563670dc54b9d Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 11 Sep 2024 15:46:01 -0700 Subject: [PATCH 128/314] `Efield_cax` & `Bfield_cax` --- Source/Evolve/WarpXEvolve.cpp | 8 ++++++-- Source/Parallelization/WarpXComm.cpp | 4 ++++ Source/WarpX.H | 2 -- Source/WarpX.cpp | 29 ++++++++++++++-------------- 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 21df7e252ef..7350369de27 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -1132,8 +1132,12 @@ WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, m_fields.get("current_buf", Direction{2}, lev), m_fields.get("rho_fp",lev), m_fields.get("rho_buf", lev), - Efield_cax[lev][0].get(), Efield_cax[lev][1].get(), Efield_cax[lev][2].get(), - Bfield_cax[lev][0].get(), Bfield_cax[lev][1].get(), Bfield_cax[lev][2].get(), + m_fields.get("Efield_cax", Direction{0}, lev), + m_fields.get("Efield_cax", Direction{1}, lev), + m_fields.get("Efield_cax", Direction{2}, lev), + m_fields.get("Bfield_cax", Direction{0}, lev), + m_fields.get("Bfield_cax", Direction{1}, lev), + m_fields.get("Bfield_cax", Direction{2}, lev), cur_time, dt[lev], a_dt_type, skip_current, push_type); if (! skip_current) { #ifdef WARPX_DIM_RZ diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 5c2f97bdd1f..2a51013a028 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -103,6 +103,8 @@ WarpX::UpdateAuxilaryDataStagToNodal () ablastr::fields::MultiLevelVectorField Bfield_avg_fp = m_fields.get_mr_levels_alldirs("Bfield_avg_fp", finest_level); ablastr::fields::MultiLevelVectorField Efield_aux = m_fields.get_mr_levels_alldirs("Efield_aux", finest_level); ablastr::fields::MultiLevelVectorField Bfield_aux = m_fields.get_mr_levels_alldirs("Bfield_aux", finest_level); + ablastr::fields::MultiLevelVectorField Efield_cax = m_fields.get_mr_levels_alldirs("Efield_cax", finest_level); + ablastr::fields::MultiLevelVectorField Bfield_cax = m_fields.get_mr_levels_alldirs("Bfield_cax", finest_level); ablastr::fields::MultiLevelVectorField const & Bmf = WarpX::fft_do_time_averaging ? Bfield_avg_fp : Bfield_fp; ablastr::fields::MultiLevelVectorField const & Emf = WarpX::fft_do_time_averaging ? Efield_avg_fp : Efield_fp; // JRA, do this one when doing Efield_avg_fp refactor to new style @@ -388,6 +390,8 @@ WarpX::UpdateAuxilaryDataSameType () ablastr::fields::MultiLevelVectorField Bfield_cp = m_fields.get_mr_levels_alldirs("Bfield_cp", finest_level); ablastr::fields::MultiLevelVectorField Efield_aux = m_fields.get_mr_levels_alldirs("Efield_aux", finest_level); ablastr::fields::MultiLevelVectorField Bfield_aux = m_fields.get_mr_levels_alldirs("Bfield_aux", finest_level); + ablastr::fields::MultiLevelVectorField Efield_cax = m_fields.get_mr_levels_alldirs("Efield_cax", finest_level); + ablastr::fields::MultiLevelVectorField Bfield_cax = m_fields.get_mr_levels_alldirs("Bfield_cax", finest_level); ablastr::fields::MultiLevelVectorField Efield_avg_fp = m_fields.get_mr_levels_alldirs("Efield_avg_fp", finest_level); ablastr::fields::MultiLevelVectorField Bfield_avg_fp = m_fields.get_mr_levels_alldirs("Bfield_avg_fp", finest_level); ablastr::fields::MultiLevelVectorField Efield_avg_cp = m_fields.get_mr_levels_alldirs("Efield_avg_cp", finest_level); diff --git a/Source/WarpX.H b/Source/WarpX.H index 9f27380541e..08ad6365e11 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1580,8 +1580,6 @@ private: amrex::Vector, 3 > > Bfield_cp; // Copy of the coarse aux - amrex::Vector, 3 > > Efield_cax; - amrex::Vector, 3 > > Bfield_cax; amrex::Vector > current_buffer_masks; amrex::Vector > gather_buffer_masks; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index c7bf56dae0d..70c434d4f9d 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -357,8 +357,6 @@ WarpX::WarpX () Efield_cp.resize(nlevs_max); Bfield_cp.resize(nlevs_max); - Efield_cax.resize(nlevs_max); - Bfield_cax.resize(nlevs_max); current_buffer_masks.resize(nlevs_max); gather_buffer_masks.resize(nlevs_max); @@ -2763,22 +2761,25 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (n_field_gather_buffer > 0 || mypc->nSpeciesGatherFromMainGrid() > 0) { if (aux_is_nodal) { BoxArray const& cnba = amrex::convert(cba,IntVect::TheNodeVector()); - AllocInitMultiFab(Bfield_cax[lev][0], cnba,dm,ncomps,ngEB,lev, "Bfield_cax[x]", 0.0_rt); - AllocInitMultiFab(Bfield_cax[lev][1], cnba,dm,ncomps,ngEB,lev, "Bfield_cax[y]", 0.0_rt); - AllocInitMultiFab(Bfield_cax[lev][2], cnba,dm,ncomps,ngEB,lev, "Bfield_cax[z]", 0.0_rt); - AllocInitMultiFab(Efield_cax[lev][0], cnba,dm,ncomps,ngEB,lev, "Efield_cax[x]", 0.0_rt); - AllocInitMultiFab(Efield_cax[lev][1], cnba,dm,ncomps,ngEB,lev, "Efield_cax[y]", 0.0_rt); - AllocInitMultiFab(Efield_cax[lev][2], cnba,dm,ncomps,ngEB,lev, "Efield_cax[z]", 0.0_rt); + // Create the MultiFabs for B + m_fields.alloc_init("Bfield_cax", Direction{0}, lev, cnba, dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Bfield_cax", Direction{1}, lev, cnba, dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Bfield_cax", Direction{2}, lev, cnba, dm, ncomps, ngEB, 0.0_rt); + + // Create the MultiFabs for E + m_fields.alloc_init("Efield_cax", Direction{0}, lev, cnba, dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Efield_cax", Direction{1}, lev, cnba, dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Efield_cax", Direction{2}, lev, cnba, dm, ncomps, ngEB, 0.0_rt); } else { // Create the MultiFabs for B - AllocInitMultiFab(Bfield_cax[lev][0], amrex::convert(cba,Bx_nodal_flag),dm,ncomps,ngEB,lev, "Bfield_cax[x]", 0.0_rt); - AllocInitMultiFab(Bfield_cax[lev][1], amrex::convert(cba,By_nodal_flag),dm,ncomps,ngEB,lev, "Bfield_cax[y]", 0.0_rt); - AllocInitMultiFab(Bfield_cax[lev][2], amrex::convert(cba,Bz_nodal_flag),dm,ncomps,ngEB,lev, "Bfield_cax[z]", 0.0_rt); + m_fields.alloc_init("Bfield_cax", Direction{0}, lev, amrex::convert(cba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Bfield_cax", Direction{1}, lev, amrex::convert(cba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Bfield_cax", Direction{2}, lev, amrex::convert(cba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); // Create the MultiFabs for E - AllocInitMultiFab(Efield_cax[lev][0], amrex::convert(cba,Ex_nodal_flag),dm,ncomps,ngEB,lev, "Efield_cax[x]", 0.0_rt); - AllocInitMultiFab(Efield_cax[lev][1], amrex::convert(cba,Ey_nodal_flag),dm,ncomps,ngEB,lev, "Efield_cax[y]", 0.0_rt); - AllocInitMultiFab(Efield_cax[lev][2], amrex::convert(cba,Ez_nodal_flag),dm,ncomps,ngEB,lev, "Efield_cax[z]", 0.0_rt); + m_fields.alloc_init("Efield_cax", Direction{0}, lev, amrex::convert(cba,Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Efield_cax", Direction{1}, lev, amrex::convert(cba,Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Efield_cax", Direction{2}, lev, amrex::convert(cba,Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); } AllocInitMultiFab(gather_buffer_masks[lev], ba, dm, ncomps, amrex::IntVect(1), lev, "gather_buffer_masks"); From a928cb1c3d933384f708029703f9d1137a618b12 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 11 Sep 2024 16:33:33 -0700 Subject: [PATCH 129/314] Some RZ Updates (EvolveE/B root needs update first) --- Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp | 4 ++-- Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp | 2 +- Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp | 2 +- .../FiniteDifferenceSolver/FiniteDifferenceSolver.H | 10 +++++----- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp index 3c2bd8ddf44..7809c5769b3 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp @@ -65,7 +65,7 @@ void FiniteDifferenceSolver::EvolveB ( #ifdef WARPX_DIM_RZ if ((m_fdtd_algo == ElectromagneticSolverAlgo::Yee)|| (m_fdtd_algo == ElectromagneticSolverAlgo::HybridPIC)){ - EvolveBCylindrical ( Bfield, Efield, lev, dt ); + EvolveBCylindrical ( ablastr::fields::va2vm(Bfield), ablastr::fields::va2vm(Efield), lev, dt ); #else if (m_grid_type == GridType::Collocated) { @@ -359,7 +359,7 @@ void FiniteDifferenceSolver::EvolveBCartesianECT ( template void FiniteDifferenceSolver::EvolveBCylindrical ( - std::array< std::unique_ptr, 3 >& Bfield, + ablastr::fields::VectorField Bfield, ablastr::fields::VectorField const& Efield, int lev, amrex::Real const dt ) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp index 0b6f9de6019..d38b727c923 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp @@ -227,7 +227,7 @@ void FiniteDifferenceSolver::EvolveECartesian ( template void FiniteDifferenceSolver::EvolveECylindrical ( - ablastr::fields::VectorField& Efield, + ablastr::fields::VectorField Efield, std::array< std::unique_ptr, 3 > const& Bfield, std::array< std::unique_ptr, 3 > const& Jfield, ablastr::fields::VectorField const& edge_lengths, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp index 9235c73d244..c7f836e47ec 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp @@ -136,7 +136,7 @@ void FiniteDifferenceSolver::EvolveFCartesian ( template void FiniteDifferenceSolver::EvolveFCylindrical ( amrex::MultiFab* Ffield, - ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const & Efield, amrex::MultiFab* const rhofield, int const rhocomp, amrex::Real const dt ) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 404185e5487..5670e1c8700 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -210,15 +210,15 @@ class FiniteDifferenceSolver #ifdef WARPX_DIM_RZ template< typename T_Algo > void EvolveBCylindrical ( - std::array< std::unique_ptr, 3 >& Bfield, - std::array< std::unique_ptr, 3 > const& Efield, + ablastr::fields::VectorField Bfield, + ablastr::fields::VectorField const & Efield, int lev, amrex::Real dt ); template< typename T_Algo > void EvolveECylindrical ( - std::array< std::unique_ptr, 3 >& Efield, - std::array< std::unique_ptr, 3 > const& Bfield, + ablastr::fields::VectorField Efield, + ablastr::fields::VectorField const& Bfield, std::array< std::unique_ptr, 3 > const& Jfield, ablastr::fields::VectorField const& edge_lengths, amrex::MultiFab const* Ffield, @@ -228,7 +228,7 @@ class FiniteDifferenceSolver template< typename T_Algo > void EvolveFCylindrical ( amrex::MultiFab* Ffield, - ablastr::fields::VectorField const Efield, + ablastr::fields::VectorField const & Efield, amrex::MultiFab* const rhofield, int rhocomp, amrex::Real dt ); From 02da32d210084cb9a67910bf71da8d7022e53039 Mon Sep 17 00:00:00 2001 From: Marco Acciarri Date: Wed, 11 Sep 2024 16:50:52 -0700 Subject: [PATCH 130/314] added current_store to m_fields --- Source/WarpX.cpp | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index d5bd49c386d..a0a8f774266 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -3325,18 +3325,11 @@ void WarpX::StoreCurrent (int lev) { using ablastr::fields::Direction; -<<<<<<< HEAD for (int idim = 0; idim < 3; ++idim) { - if (m_fields.get("current_store",Direction{idim},lev)) { - MultiFab::Copy(*m_fields.get("current_store",Direction{idim},lev), *current_fp[lev][idim], - 0, 0, 1, m_fields.get("current_store",Direction{idim},lev)->nGrowVect()); -======= - - for (int idim = 0; idim < 3; ++idim) { - if (current_store[lev][idim]) { - MultiFab::Copy(*current_store[lev][idim], *m_fields.get("current_fp", Direction{idim}, lev), - 0, 0, 1, current_store[lev][idim]->nGrowVect()); ->>>>>>> 13525fccc0592c1d45b029cf653d5696da69ee32 + if (m_fields.get("current_store", Direction{idim},lev)) { + MultiFab::Copy(*m_fields.get("current_store", Direction{idim}, lev), + *m_fields.get("current_fp", Direction{idim}, lev), + 0, 0, 1, m_fields.get("current_store", Direction{idim}, lev)->nGrowVect()); } } } @@ -3345,19 +3338,13 @@ void WarpX::RestoreCurrent (int lev) { using ablastr::fields::Direction; -<<<<<<< HEAD - for (int idim = 0; idim < 3; ++idim) { - if (m_fields.get("current_store",Direction{idim},lev)) { - std::swap(current_fp[lev][idim], m_fields.get("current_store",Direction{idim},lev)); -======= for (int idim = 0; idim < 3; ++idim) { - if (current_store[lev][idim]) { + if (m_fields.get("current_store", Direction{idim}, lev)) { std::swap( *m_fields.get("current_fp", Direction{idim}, lev), - *current_store[lev][idim] + *m_fields.get("current_store", Direction{idim}, lev) ); ->>>>>>> 13525fccc0592c1d45b029cf653d5696da69ee32 } } } From 7e2d96fce1013378ed70f694d3db8ef0f61abb56 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 17:06:20 -0700 Subject: [PATCH 131/314] Fix EB compilation --- Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp | 4 ++-- .../FiniteDifferenceSolver/FiniteDifferenceSolver.H | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp index 7809c5769b3..3c2bd8ddf44 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp @@ -65,7 +65,7 @@ void FiniteDifferenceSolver::EvolveB ( #ifdef WARPX_DIM_RZ if ((m_fdtd_algo == ElectromagneticSolverAlgo::Yee)|| (m_fdtd_algo == ElectromagneticSolverAlgo::HybridPIC)){ - EvolveBCylindrical ( ablastr::fields::va2vm(Bfield), ablastr::fields::va2vm(Efield), lev, dt ); + EvolveBCylindrical ( Bfield, Efield, lev, dt ); #else if (m_grid_type == GridType::Collocated) { @@ -359,7 +359,7 @@ void FiniteDifferenceSolver::EvolveBCartesianECT ( template void FiniteDifferenceSolver::EvolveBCylindrical ( - ablastr::fields::VectorField Bfield, + std::array< std::unique_ptr, 3 >& Bfield, ablastr::fields::VectorField const& Efield, int lev, amrex::Real const dt ) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 98e9d1f972a..0db5d0c2eb1 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -209,7 +209,7 @@ class FiniteDifferenceSolver #ifdef WARPX_DIM_RZ template< typename T_Algo > void EvolveBCylindrical ( - ablastr::fields::VectorField Bfield, + std::array< std::unique_ptr, 3 >& Bfield, ablastr::fields::VectorField const & Efield, int lev, amrex::Real dt ); @@ -217,8 +217,8 @@ class FiniteDifferenceSolver template< typename T_Algo > void EvolveECylindrical ( ablastr::fields::VectorField Efield, - ablastr::fields::VectorField const& Bfield, - std::array< std::unique_ptr, 3 > const& Jfield, + std::array< std::unique_ptr, 3 > const& Bfield, + ablastr::fields::VectorField const& Jfield, ablastr::fields::VectorField const& edge_lengths, amrex::MultiFab const* Ffield, int lev, From b527a2590031b4a2e6606345ad94970f77c6c16e Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Thu, 12 Sep 2024 01:59:42 +0200 Subject: [PATCH 132/314] add external E/B particle fields to new field register --- Source/Initialization/WarpXInitData.cpp | 48 ++++++++++++++++++------- Source/Parallelization/WarpXComm.cpp | 15 ++++---- Source/WarpX.H | 4 --- Source/WarpX.cpp | 32 ++++++++--------- 4 files changed, 61 insertions(+), 38 deletions(-) diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 734df2b8eac..30a8dc5aae1 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -1441,13 +1441,25 @@ WarpX::LoadExternalFields (int const lev) #if defined(WARPX_DIM_RZ) WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, "External field reading is not implemented for more than one RZ mode (see #3829)"); - ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][0].get(), "B", "r"); - ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][1].get(), "B", "t"); - ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][2].get(), "B", "z"); + ReadExternalFieldFromFile(external_fields_path, + m_fields.get("B_external_particle_field", Direction{0}, lev), + "B", "r"); + ReadExternalFieldFromFile(external_fields_path, + m_fields.get("B_external_particle_field", Direction{1}, lev), + "B", "t"); + ReadExternalFieldFromFile(external_fields_path, + m_fields.get("B_external_particle_field", Direction{2}, lev), + "B", "z"); #else - ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][0].get(), "B", "x"); - ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][1].get(), "B", "y"); - ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][2].get(), "B", "z"); + ReadExternalFieldFromFile(external_fields_path, + m_fields.get("B_external_particle_field", Direction{0}, lev), + "B", "x"); + ReadExternalFieldFromFile(external_fields_path, + m_fields.get("B_external_particle_field", Direction{1}, lev), + "B", "y"); + ReadExternalFieldFromFile(external_fields_path, + m_fields.get("B_external_particle_field", Direction{2}, lev), + "B", "z"); #endif } if (mypc->m_E_ext_particle_s == "read_from_file") { @@ -1457,13 +1469,25 @@ WarpX::LoadExternalFields (int const lev) #if defined(WARPX_DIM_RZ) WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, "External field reading is not implemented for more than one RZ mode (see #3829)"); - ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][0].get(), "E", "r"); - ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][1].get(), "E", "t"); - ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][2].get(), "E", "z"); + ReadExternalFieldFromFile(external_fields_path, + m_fields.get("E_external_particle_field", Direction{0}, lev), + "E", "r"); + ReadExternalFieldFromFile(external_fields_path, + m_fields.get("E_external_particle_field", Direction{1}, lev), + "E", "t"); + ReadExternalFieldFromFile(external_fields_path, + m_fields.get("E_external_particle_field", Direction{2}, lev), + "E", "z"); #else - ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][0].get(), "E", "x"); - ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][1].get(), "E", "y"); - ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][2].get(), "E", "z"); + ReadExternalFieldFromFile(external_fields_path, + m_fields.get("E_external_particle_field", Direction{0}, lev), + "E", "x"); + ReadExternalFieldFromFile(external_fields_path, + m_fields.get("E_external_particle_field", Direction{1}, lev), + "E", "y"); + ReadExternalFieldFromFile(external_fields_path, + m_fields.get("E_external_particle_field", Direction{2}, lev), + "E", "z"); #endif } } diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 2a51013a028..e62068bea68 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -71,15 +71,18 @@ WarpX::UpdateAuxilaryData () auto Bfield_aux_lvl_1 = m_fields.get("Bfield_aux", Direction{1}, lev); auto Bfield_aux_lvl_2 = m_fields.get("Bfield_aux", Direction{2}, lev); + const auto& E_ext_lev = m_fields.get_alldirs("E_external_particle_field", lev); + const auto& B_ext_lev = m_fields.get_alldirs("B_external_particle_field", lev); + if (mypc->m_E_ext_particle_s == "read_from_file") { - amrex::MultiFab::Add(*Bfield_aux_lvl_0, *E_external_particle_field[lev][0], 0, 0, E_external_particle_field[lev][0]->nComp(), guard_cells.ng_FieldGather); - amrex::MultiFab::Add(*Bfield_aux_lvl_1, *E_external_particle_field[lev][1], 0, 0, E_external_particle_field[lev][1]->nComp(), guard_cells.ng_FieldGather); - amrex::MultiFab::Add(*Bfield_aux_lvl_2, *E_external_particle_field[lev][2], 0, 0, E_external_particle_field[lev][2]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Bfield_aux_lvl_0, *E_ext_lev[0], 0, 0, E_ext_lev[0]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Bfield_aux_lvl_1, *E_ext_lev[1], 0, 0, E_ext_lev[1]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Bfield_aux_lvl_2, *E_ext_lev[2], 0, 0, E_ext_lev[2]->nComp(), guard_cells.ng_FieldGather); } if (mypc->m_B_ext_particle_s == "read_from_file") { - amrex::MultiFab::Add(*Bfield_aux_lvl_0, *B_external_particle_field[lev][0], 0, 0, B_external_particle_field[lev][0]->nComp(), guard_cells.ng_FieldGather); - amrex::MultiFab::Add(*Bfield_aux_lvl_1, *B_external_particle_field[lev][1], 0, 0, B_external_particle_field[lev][0]->nComp(), guard_cells.ng_FieldGather); - amrex::MultiFab::Add(*Bfield_aux_lvl_2, *B_external_particle_field[lev][2], 0, 0, B_external_particle_field[lev][0]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Bfield_aux_lvl_0, *B_ext_lev[0], 0, 0, B_ext_lev[0]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Bfield_aux_lvl_1, *B_ext_lev[1], 0, 0, B_ext_lev[1]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Bfield_aux_lvl_2, *B_ext_lev[2], 0, 0, B_ext_lev[2]->nComp(), guard_cells.ng_FieldGather); } } diff --git a/Source/WarpX.H b/Source/WarpX.H index ecd8050e7fb..06696d49b78 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1527,10 +1527,6 @@ private: amrex::Vector, 3 > > vector_potential_grad_buf_e_stag; amrex::Vector, 3 > > vector_potential_grad_buf_b_stag; - // Same as Bfield_fp/Efield_fp for reading external field data - amrex::Vector, 3 > > E_external_particle_field; - amrex::Vector, 3 > > B_external_particle_field; - /** EB: for every mesh face flag_info_face contains a: * * 0 if the face needs to be extended * * 1 if the face is large enough to lend area to other faces diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index b27d6f513f8..8e904c56a28 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -332,10 +332,6 @@ WarpX::WarpX () vector_potential_grad_buf_b_stag.resize(nlevs_max); } - // Same as Bfield_fp/Efield_fp for reading external field data - B_external_particle_field.resize(1); - E_external_particle_field.resize(1); - m_distance_to_eb.resize(nlevs_max); m_flag_info_face.resize(nlevs_max); m_flag_ext_face.resize(nlevs_max); @@ -2613,12 +2609,14 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm auto Bfield_aux_levl_0 = m_fields.get("Bfield_aux", Direction{0}, lev); auto Bfield_aux_levl_1 = m_fields.get("Bfield_aux", Direction{1}, lev); auto Bfield_aux_levl_2 = m_fields.get("Bfield_aux", Direction{2}, lev); - AllocInitMultiFab(B_external_particle_field[lev][0], amrex::convert(ba, Bfield_aux_levl_0->ixType()), - dm, ncomps, ngEB, lev, "B_external_particle_field[x]", 0.0_rt); - AllocInitMultiFab(B_external_particle_field[lev][1], amrex::convert(ba, Bfield_aux_levl_1->ixType()), - dm, ncomps, ngEB, lev, "B_external_particle_field[y]", 0.0_rt); - AllocInitMultiFab(B_external_particle_field[lev][2], amrex::convert(ba, Bfield_aux_levl_2->ixType()), - dm, ncomps, ngEB, lev, "B_external_particle_field[z]", 0.0_rt); + + // Same as Bfield_fp for reading external field data + m_fields.alloc_init( "B_external_particle_field", Direction{0}, lev, amrex::convert(ba, Bfield_aux_levl_0->ixType()), + dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "B_external_particle_field", Direction{1}, lev, amrex::convert(ba, Bfield_aux_levl_1->ixType()), + dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "B_external_particle_field", Direction{2}, lev, amrex::convert(ba, Bfield_aux_levl_2->ixType()), + dm, ncomps, ngEB, 0.0_rt); } if (m_p_ext_field_params->E_ext_grid_type != ExternalFieldType::default_zero && m_p_ext_field_params->E_ext_grid_type != ExternalFieldType::constant) { // These fields will be added directly to the grid, i.e. to fp, and need to match the index type @@ -2634,12 +2632,14 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm auto Efield_aux_levl_0 = m_fields.get("Efield_aux", Direction{0}, lev); auto Efield_aux_levl_1 = m_fields.get("Efield_aux", Direction{1}, lev); auto Efield_aux_levl_2 = m_fields.get("Efield_aux", Direction{2}, lev); - AllocInitMultiFab(E_external_particle_field[lev][0], amrex::convert(ba, Efield_aux_levl_0->ixType()), - dm, ncomps, ngEB, lev, "E_external_particle_field[x]", 0.0_rt); - AllocInitMultiFab(E_external_particle_field[lev][1], amrex::convert(ba, Efield_aux_levl_1->ixType()), - dm, ncomps, ngEB, lev, "E_external_particle_field[y]", 0.0_rt); - AllocInitMultiFab(E_external_particle_field[lev][2], amrex::convert(ba, Efield_aux_levl_2->ixType()), - dm, ncomps, ngEB, lev, "E_external_particle_field[z]", 0.0_rt); + + // Same as Efield_fp for reading external field data + m_fields.alloc_init( "E_external_particle_field", Direction{0}, lev, amrex::convert(ba, Efield_aux_levl_0->ixType()), + dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "E_external_particle_field", Direction{1}, lev, amrex::convert(ba, Efield_aux_levl_1->ixType()), + dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "E_external_particle_field", Direction{2}, lev, amrex::convert(ba, Efield_aux_levl_2->ixType()), + dm, ncomps, ngEB, 0.0_rt); } // From a434195eb1202cfb59787c44bf17a89420adcb85 Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Thu, 12 Sep 2024 02:06:44 +0200 Subject: [PATCH 133/314] cleanup --- Source/FieldSolver/WarpXPushFieldsEM.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 6622fb64c1f..72289ea471a 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -912,21 +912,21 @@ void WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) { // Evolve E field in regular cells - const auto face_areas_lev = m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev]; - const auto edge_lengths_lev = m_fields.get_mr_levels_alldirs("edge_lengths", finest_level)[lev]; if (patch_type == PatchType::fine) { m_fdtd_solver_fp[lev]->EvolveE( m_fields.get_alldirs("Efield_fp",lev), Bfield_fp[lev], m_fields.get_alldirs("current_fp", lev), m_fields.get_alldirs("edge_lengths", lev), - face_areas_lev, ECTRhofield[lev], + m_fields.get_alldirs("face_areas", lev), + ECTRhofield[lev], m_fields.get("F_fp", lev), lev, a_dt ); } else { m_fdtd_solver_cp[lev]->EvolveE( m_fields.get_alldirs("Efield_cp",lev), Bfield_cp[lev], m_fields.get_alldirs("current_cp", lev), m_fields.get_alldirs("edge_lengths", lev), - face_areas_lev, ECTRhofield[lev], + m_fields.get_alldirs("face_areas", lev), + ECTRhofield[lev], m_fields.get("F_cp", lev), lev, a_dt ); } From 1a65e781be48ddccad20f94b8bf1507be928fe31 Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Thu, 12 Sep 2024 02:07:54 +0200 Subject: [PATCH 134/314] cleanup --- Source/FieldSolver/WarpXPushFieldsEM.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 72289ea471a..3c66c756334 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -957,12 +957,14 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { if (patch_type == PatchType::fine) { m_fdtd_solver_fp[lev]->EvolveECTRho( m_fields.get_alldirs("Efield_fp",lev), - edge_lengths_lev, - face_areas_lev, ECTRhofield[lev], lev ); + m_fields.get_alldirs("edge_lengths", lev), + m_fields.get_alldirs("face_areas", lev), + ECTRhofield[lev], lev ); } else { m_fdtd_solver_cp[lev]->EvolveECTRho( m_fields.get_alldirs("Efield_cp",lev), - edge_lengths_lev, - face_areas_lev, ECTRhofield[lev], lev); + m_fields.get_alldirs("edge_lengths", lev), + m_fields.get_alldirs("face_areas", lev), + ECTRhofield[lev], lev); } } #endif From 93304d2761927ed499bfcf99af7f78c4c80252f5 Mon Sep 17 00:00:00 2001 From: David Grote Date: Wed, 11 Sep 2024 12:16:48 -0700 Subject: [PATCH 135/314] Change 1 --- Source/BoundaryConditions/PML.cpp | 6 +++--- Source/BoundaryConditions/PML_RZ.cpp | 4 ++-- Source/BoundaryConditions/WarpXEvolvePML.cpp | 3 ++- .../WarpXFieldBoundaries.cpp | 17 ++++++++++------- .../FlushFormats/FlushFormatCheckpoint.cpp | 6 +++--- .../FlushFormats/FlushFormatPlotfile.cpp | 10 ++++++---- Source/Diagnostics/WarpXIO.cpp | 8 ++++---- .../EmbeddedBoundary/WarpXFaceExtensions.cpp | 14 +++++++------- Source/EmbeddedBoundary/WarpXInitEB.cpp | 2 ++ Source/FieldSolver/ElectrostaticSolver.cpp | 13 ++++++++----- .../ApplySilverMuellerBoundary.cpp | 14 +++++++------- .../FiniteDifferenceSolver/EvolveE.cpp | 12 ++++++------ .../FiniteDifferenceSolver.H | 10 +++++----- .../HybridPICModel/HybridPICModel.cpp | 6 +++--- .../ImplicitSolvers/WarpXImplicitOps.cpp | 17 ++++++++++------- Source/FieldSolver/WarpXPushFieldsEM.cpp | 4 ++-- Source/Utils/WarpXMovingWindow.cpp | 2 +- Source/WarpX.H | 4 ++-- Source/WarpX.cpp | 19 +++++++++---------- 19 files changed, 92 insertions(+), 79 deletions(-) diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index eba0886f1a9..16502a1ee33 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -709,9 +709,9 @@ PML::PML (const int lev, const BoxArray& grid_ba, warpx.m_fields.alloc_init("pml_E_fp", Direction{1}, lev, ba_Ey, dm, ncompe, nge, 0.0_rt); warpx.m_fields.alloc_init("pml_E_fp", Direction{2}, lev, ba_Ez, dm, ncompe, nge, 0.0_rt); - const amrex::BoxArray ba_Bx = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Bfield_fp, 0,0).ixType().toIntVect()); - const amrex::BoxArray ba_By = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Bfield_fp, 0,1).ixType().toIntVect()); - const amrex::BoxArray ba_Bz = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Bfield_fp, 0,2).ixType().toIntVect()); + const amrex::BoxArray ba_Bx = amrex::convert(ba, warpx.m_fields.get("Bfield_XX",Direction{0},0)->ixType().toIntVect()); + const amrex::BoxArray ba_By = amrex::convert(ba, warpx.m_fields.get("Bfield_XX",Direction{1},0)->ixType().toIntVect()); + const amrex::BoxArray ba_Bz = amrex::convert(ba, warpx.m_fields.get("Bfield_XX",Direction{2},0)->ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_B_fp[0], ba_Bx, dm, ncompb, ngb, lev, "pml_B_fp[x]", 0.0_rt); WarpX::AllocInitMultiFab(pml_B_fp[1], ba_By, dm, ncompb, ngb, lev, "pml_B_fp[y]", 0.0_rt); WarpX::AllocInitMultiFab(pml_B_fp[2], ba_Bz, dm, ncompb, ngb, lev, "pml_B_fp[z]", 0.0_rt); diff --git a/Source/BoundaryConditions/PML_RZ.cpp b/Source/BoundaryConditions/PML_RZ.cpp index 8c07a775e14..860220c3771 100644 --- a/Source/BoundaryConditions/PML_RZ.cpp +++ b/Source/BoundaryConditions/PML_RZ.cpp @@ -52,8 +52,8 @@ PML_RZ::PML_RZ (const int lev, const amrex::BoxArray& grid_ba, const amrex::Dist warpx.m_fields.alloc_init("pml_E_fp", Direction{0}, lev, ba_Er, grid_dm, Er_fp.nComp(), Er_fp.nGrowVect(), 0.0_rt); warpx.m_fields.alloc_init("pml_E_fp", Direction{1}, lev, ba_Et, grid_dm, Et_fp.nComp(), Et_fp.nGrowVect(), 0.0_rt); - const amrex::MultiFab & Br_fp = WarpX::GetInstance().getField(FieldType::Bfield_fp, lev,0); - const amrex::MultiFab & Bt_fp = WarpX::GetInstance().getField(FieldType::Bfield_fp, lev,1); + const amrex::MultiFab & Br_fp = *warpx.m_fields.get("Bfield_XX",Direction{0},lev); + const amrex::MultiFab & Bt_fp = *warpx.m_fields.get("Bfield_XX",Direction{1},lev); const amrex::BoxArray ba_Br = amrex::convert(grid_ba, Br_fp.ixType().toIntVect()); const amrex::BoxArray ba_Bt = amrex::convert(grid_ba, Bt_fp.ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_B_fp[0], ba_Br, grid_dm, Br_fp.nComp(), Br_fp.nGrowVect(), lev, "pml_B_fp[0]", 0.0_rt); diff --git a/Source/BoundaryConditions/WarpXEvolvePML.cpp b/Source/BoundaryConditions/WarpXEvolvePML.cpp index 572ac83ba90..9311b962e6f 100644 --- a/Source/BoundaryConditions/WarpXEvolvePML.cpp +++ b/Source/BoundaryConditions/WarpXEvolvePML.cpp @@ -68,7 +68,8 @@ WarpX::DampPML (const int lev, PatchType patch_type) using ablastr::fields::Direction; pml_rz[lev]->ApplyDamping( m_fields.get("Efield_fp",Direction{1},lev), m_fields.get("Efield_fp",Direction{2},lev), - Bfield_fp[lev][1].get(), Bfield_fp[lev][2].get(), + m_fields.get("Bfield_XX",Direction{1},lev), + m_fields.get("Bfield_XX",Direction{2},lev), dt[lev]); } #endif diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index 8caa0f0091a..5f2dc2e5f49 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -107,12 +107,14 @@ void WarpX::ApplyEfieldBoundary(const int lev, PatchType patch_type) void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_dt_type) { + using ablastr::fields::Direction; + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { if (patch_type == PatchType::fine) { PEC::ApplyPECtoBfield( { - getFieldPointer(FieldType::Bfield_fp, lev, 0), - getFieldPointer(FieldType::Bfield_fp, lev, 1), - getFieldPointer(FieldType::Bfield_fp, lev, 2) }, + m_fields.get("Bfield_XX",Direction{0},lev), + m_fields.get("Bfield_XX",Direction{1},lev), + m_fields.get("Bfield_XX",Direction{2},lev) }, field_boundary_lo, field_boundary_hi, get_ng_fieldgather(), Geom(lev), lev, patch_type, ref_ratio); @@ -134,8 +136,9 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d if (a_dt_type == DtType::FirstHalf) { if(::isAnyBoundary(field_boundary_lo, field_boundary_hi)){ auto Efield_fp_new = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); // JRA, new to prevent shadow + auto Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_XX",max_level); m_fdtd_solver_fp[0]->ApplySilverMuellerBoundary( - Efield_fp_new[lev], Bfield_fp[lev], + Efield_fp_new[lev], Bfield_fp_new[lev], Geom(lev).Domain(), dt[lev], field_boundary_lo, field_boundary_hi); } @@ -144,9 +147,9 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d #ifdef WARPX_DIM_RZ if (patch_type == PatchType::fine) { - ApplyFieldBoundaryOnAxis(getFieldPointer(FieldType::Bfield_fp, lev, 0), - getFieldPointer(FieldType::Bfield_fp, lev, 1), - getFieldPointer(FieldType::Bfield_fp, lev, 2), lev); + ApplyFieldBoundaryOnAxis(m_fields.get("Bfield_XX",Direction{0},lev), + m_fields.get("Bfield_XX",Direction{1},lev), + m_fields.get("Bfield_XX",Direction{2},lev), lev); } else { ApplyFieldBoundaryOnAxis(getFieldPointer(FieldType::Bfield_cp, lev, 0), getFieldPointer(FieldType::Bfield_cp, lev, 1), diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index 0d3a6cad839..249085e8e68 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -74,11 +74,11 @@ FlushFormatCheckpoint::WriteToFile ( amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ey_fp")); VisMF::Write(*warpx.m_fields.get("Efield_fp", Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ez_fp")); - VisMF::Write(warpx.getField(FieldType::Bfield_fp, lev, 0), + VisMF::Write(*warpx.m_fields.get("Bfield_XX", Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bx_fp")); - VisMF::Write(warpx.getField(FieldType::Bfield_fp, lev, 1), + VisMF::Write(*warpx.m_fields.get("Bfield_XX", Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "By_fp")); - VisMF::Write(warpx.getField(FieldType::Bfield_fp, lev, 2), + VisMF::Write(*warpx.m_fields.get("Bfield_XX", Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bz_fp")); if (WarpX::fft_do_time_averaging) diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index 3af9b6c91ec..b816e23253f 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -588,11 +588,11 @@ FlushFormatPlotfile::WriteAllRawFields( default_level_prefix, "jy_fp", lev,plot_raw_fields_guards ); WriteRawMF( *warpx.m_fields.get("current_fp",Direction{2}, lev), dm, raw_pltname, default_level_prefix, "jz_fp", lev,plot_raw_fields_guards ); - WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 0), dm, raw_pltname, + WriteRawMF( *warpx.m_fields.get("Bfield_XX", Direction{0}, lev), dm, raw_pltname, default_level_prefix, "Bx_fp", lev, plot_raw_fields_guards ); - WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 1), dm, raw_pltname, + WriteRawMF( *warpx.m_fields.get("Bfield_XX", Direction{1}, lev), dm, raw_pltname, default_level_prefix, "By_fp", lev, plot_raw_fields_guards ); - WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 2), dm, raw_pltname, + WriteRawMF( *warpx.m_fields.get("Bfield_XX", Direction{2}, lev), dm, raw_pltname, default_level_prefix, "Bz_fp", lev, plot_raw_fields_guards ); if (warpx.m_fields.has("F_fp", lev)) { @@ -642,7 +642,9 @@ FlushFormatPlotfile::WriteAllRawFields( dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards); WriteCoarseVector( "B", warpx.getFieldPointer(FieldType::Bfield_cp, lev, 0), warpx.getFieldPointer(FieldType::Bfield_cp, lev, 1), warpx.getFieldPointer(FieldType::Bfield_cp, lev, 2), - warpx.getFieldPointer(FieldType::Bfield_fp, lev, 0), warpx.getFieldPointer(FieldType::Bfield_fp, lev, 1), warpx.getFieldPointer(FieldType::Bfield_fp, lev, 2), + warpx.m_fields.get("Bfield_XX", Direction{0}, lev), + warpx.m_fields.get("Bfield_XX", Direction{1}, lev), + warpx.m_fields.get("Bfield_XX", Direction{2}, lev), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards); WriteCoarseVector( "j", warpx.m_fields.get("current_cp", Direction{0}, lev), warpx.m_fields.get("current_cp", Direction{1}, lev), warpx.m_fields.get("current_cp", Direction{2}, lev), diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp index b3b0867eb8f..a99cdfd241c 100644 --- a/Source/Diagnostics/WarpXIO.cpp +++ b/Source/Diagnostics/WarpXIO.cpp @@ -283,7 +283,7 @@ WarpX::InitFromCheckpoint () for (int i = 0; i < 3; ++i) { m_fields.get("current_fp",Direction{i},lev)->setVal(0.0); m_fields.get("Efield_fp",Direction{i},lev)->setVal(0.0); - Bfield_fp[lev][i]->setVal(0.0); + m_fields.get("Bfield_XX",Direction{i},lev)->setVal(0.0); } if (lev > 0) { @@ -304,11 +304,11 @@ WarpX::InitFromCheckpoint () VisMF::Read(*m_fields.get("Efield_fp", Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ez_fp")); - VisMF::Read(*Bfield_fp[lev][0], + VisMF::Read(*m_fields.get("Bfield_XX", Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Bx_fp")); - VisMF::Read(*Bfield_fp[lev][1], + VisMF::Read(*m_fields.get("Bfield_XX", Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "By_fp")); - VisMF::Read(*Bfield_fp[lev][2], + VisMF::Read(*m_fields.get("Bfield_XX", Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Bz_fp")); if (WarpX::fft_do_time_averaging) diff --git a/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp b/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp index 9fcbdc4507b..7972bd551db 100644 --- a/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp +++ b/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp @@ -286,7 +286,7 @@ WarpX::ComputeFaceExtensions () void WarpX::InitBorrowing() { int idim = 0; - for (amrex::MFIter mfi(*Bfield_fp[maxLevel()][idim]); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get("Bfield_XX", Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { amrex::Box const &box = mfi.validbox(); auto &borrowing_x = (*m_borrowing[maxLevel()][idim])[mfi]; borrowing_x.inds_pointer.resize(box); @@ -302,7 +302,7 @@ WarpX::InitBorrowing() { } idim = 1; - for (amrex::MFIter mfi(*Bfield_fp[maxLevel()][idim]); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get("Bfield_XX", Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { amrex::Box const &box = mfi.validbox(); auto &borrowing_y = (*m_borrowing[maxLevel()][idim])[mfi]; borrowing_y.inds_pointer.resize(box); @@ -315,7 +315,7 @@ WarpX::InitBorrowing() { } idim = 2; - for (amrex::MFIter mfi(*Bfield_fp[maxLevel()][idim]); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get("Bfield_XX", Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { amrex::Box const &box = mfi.validbox(); auto &borrowing_z = (*m_borrowing[maxLevel()][idim])[mfi]; borrowing_z.inds_pointer.resize(box); @@ -456,7 +456,7 @@ WarpX::ComputeOneWayExtensions () WARPX_ABORT_WITH_MESSAGE( "ComputeOneWayExtensions: Only implemented in 2D3V and 3D3V"); #endif - for (amrex::MFIter mfi(*Bfield_fp[maxLevel()][idim]); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get("Bfield_XX", Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { amrex::Box const &box = mfi.validbox(); auto const &S = m_fields.get("face_areas", Direction{idim}, maxLevel())->array(mfi); @@ -583,7 +583,7 @@ WarpX::ComputeEightWaysExtensions () WARPX_ABORT_WITH_MESSAGE( "ComputeEightWaysExtensions: Only implemented in 2D3V and 3D3V"); #endif - for (amrex::MFIter mfi(*Bfield_fp[maxLevel()][idim]); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get("Bfield_XX", Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { amrex::Box const &box = mfi.validbox(); @@ -734,7 +734,7 @@ WarpX::ApplyBCKCorrection (const int idim) const amrex::Real dy = cell_size[1]; const amrex::Real dz = cell_size[2]; - for (amrex::MFIter mfi(*Bfield_fp[maxLevel()][idim], amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get("Bfield_XX", Direction{idim}, maxLevel()), amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { const amrex::Box &box = mfi.tilebox(); const amrex::Array4 &flag_ext_face = m_flag_ext_face[maxLevel()][idim]->array(mfi); @@ -762,7 +762,7 @@ void WarpX::ShrinkBorrowing () { for(int idim = 0; idim < AMREX_SPACEDIM; idim++) { - for (amrex::MFIter mfi(*Bfield_fp[maxLevel()][idim]); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get("Bfield_XX", Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { auto &borrowing = (*m_borrowing[maxLevel()][idim])[mfi]; borrowing.inds.resize(borrowing.vecs_size); borrowing.neigh_faces.resize(borrowing.vecs_size); diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index 8252b26c894..5ca89f9957b 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -317,6 +317,8 @@ WarpX::ScaleAreas(ablastr::fields::VectorField& face_areas, void WarpX::MarkCells(){ + using ablastr::fields::Direction; + #ifndef WARPX_DIM_RZ auto const &cell_size = CellSize(maxLevel()); diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index 147d4636689..67df4fea640 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -69,7 +69,7 @@ WarpX::ComputeSpaceChargeField (bool const reset_fields) for (int lev = 0; lev <= max_level; lev++) { for (int comp=0; comp<3; comp++) { m_fields.get("Efield_fp",Direction{comp},lev)->setVal(0); - Bfield_fp[lev][comp]->setVal(0); + m_fields.get("Bfield_XX",Direction{comp},lev)->setVal(0); } } } @@ -142,8 +142,9 @@ WarpX::AddBoundaryField () // Compute the corresponding electric and magnetic field, from the potential phi. auto Efield_fp_new = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); // JRA, new to prevent shadow + auto Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_XX",max_level); computeE( Efield_fp_new, phi, beta ); - computeB( Bfield_fp, phi, beta ); + computeB( Bfield_fp_new, phi, beta ); // de-allocate temporary for (int lev = 0; lev <= max_level; lev++) { @@ -230,8 +231,9 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) // Compute the corresponding electric and magnetic field, from the potential phi auto Efield_fp_new = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); // JRA, new to prevent shadow + auto Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_XX",max_level); computeE( Efield_fp_new, phi, beta ); - computeB( Bfield_fp, phi, beta ); + computeB( Bfield_fp_new, phi, beta ); // de-allocate temporary for (int lev = 0; lev <= max_level; lev++) { @@ -308,13 +310,14 @@ WarpX::AddSpaceChargeFieldLabFrame () // Compute the electric field. Note that if an EB is used the electric // field will be calculated in the computePhi call. auto Efield_fp_new = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); // JRA, new to prevent shadow + auto Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_XX",max_level); if (!EB::enabled()) { computeE( Efield_fp_new, phi_fp, beta ); } else { if (IsPythonCallbackInstalled("poissonsolver")) { computeE(Efield_fp_new, phi_fp, beta); } } // Compute the magnetic field - computeB( Bfield_fp, phi_fp, beta ); + computeB( Bfield_fp_new, phi_fp, beta ); } /* Compute the potential `phi` by solving the Poisson equation with `rho` as @@ -701,7 +704,7 @@ WarpX::computeE (ablastr::fields::MultiLevelVectorField& E, \param[in] beta Represents the velocity of the source of `phi` */ void -WarpX::computeB (amrex::Vector, 3> >& B, +WarpX::computeB (ablastr::fields::MultiLevelVectorField& B, const ablastr::fields::MultiLevelScalarField& phi, std::array const beta ) const { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp b/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp index 5020a062bd7..9ae7c6fb6fd 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp @@ -36,7 +36,7 @@ using namespace amrex; */ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( ablastr::fields::VectorField& Efield, - std::array< std::unique_ptr, 3 >& Bfield, + ablastr::fields::VectorField& Bfield, amrex::Box domain_box, amrex::Real const dt, amrex::Vector field_boundary_lo, @@ -86,9 +86,9 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( Array4 const& Er = Efield[Direction{0}]->array(mfi); Array4 const& Et = Efield[Direction{1}]->array(mfi); Array4 const& Ez = Efield[Direction{2}]->array(mfi); - Array4 const& Br = Bfield[0]->array(mfi); - Array4 const& Bt = Bfield[1]->array(mfi); - Array4 const& Bz = Bfield[2]->array(mfi); + Array4 const& Br = Bfield[Direction{0}]->array(mfi); + Array4 const& Bt = Bfield[Direction{1}]->array(mfi); + Array4 const& Bz = Bfield[Direction{2}]->array(mfi); // Extract tileboxes for which to loop Box tbr = mfi.tilebox(Bfield[0]->ixType().toIntVect()); @@ -213,10 +213,10 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( #ifndef WARPX_DIM_1D_Z Array4 const& Ez = Efield[Direction{2}]->array(mfi); #endif - Array4 const& Bx = Bfield[0]->array(mfi); - Array4 const& By = Bfield[1]->array(mfi); + Array4 const& Bx = Bfield[Direction{0}]->array(mfi); + Array4 const& By = Bfield[Direction{1}]->array(mfi); #ifndef WARPX_DIM_1D_Z - Array4 const& Bz = Bfield[2]->array(mfi); + Array4 const& Bz = Bfield[Direction{2}]->array(mfi); #endif // Extract the tileboxes for which to loop diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp index 474f50c6626..d833c7dbebd 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp @@ -50,8 +50,8 @@ using namespace ablastr::fields; * \brief Update the E field, over one timestep */ void FiniteDifferenceSolver::EvolveE ( - ablastr::fields::VectorField Efield, - std::array< std::unique_ptr, 3 > const& Bfield, + ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& Jfield, VectorField const& edge_lengths, VectorField const& face_areas, @@ -93,8 +93,8 @@ void FiniteDifferenceSolver::EvolveE ( template void FiniteDifferenceSolver::EvolveECartesian ( - ablastr::fields::VectorField& Efield, - std::array< std::unique_ptr, 3 > const& Bfield, + ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& Jfield, VectorField const& edge_lengths, amrex::MultiFab const* Ffield, @@ -227,8 +227,8 @@ void FiniteDifferenceSolver::EvolveECartesian ( template void FiniteDifferenceSolver::EvolveECylindrical ( - ablastr::fields::VectorField Efield, - std::array< std::unique_ptr, 3 > const& Bfield, + ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& Jfield, ablastr::fields::VectorField const& edge_lengths, amrex::MultiFab const* Ffield, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 0db5d0c2eb1..902d751a838 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -63,8 +63,8 @@ class FiniteDifferenceSolver std::array< std::unique_ptr >, 3 >& borrowing, int lev, amrex::Real dt ); - void EvolveE ( ablastr::fields::VectorField Efield, - std::array< std::unique_ptr, 3 > const& Bfield, + void EvolveE ( ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& Jfield, ablastr::fields::VectorField const& edge_lengths, ablastr::fields::VectorField const& face_areas, @@ -90,7 +90,7 @@ class FiniteDifferenceSolver void ApplySilverMuellerBoundary ( ablastr::fields::VectorField & Efield, - std::array< std::unique_ptr, 3 >& Bfield, + ablastr::fields::VectorField & Bfield, amrex::Box domain_box, amrex::Real dt, amrex::Vector field_boundary_lo, @@ -269,8 +269,8 @@ class FiniteDifferenceSolver template< typename T_Algo > void EvolveECartesian ( - ablastr::fields::VectorField& Efield, - std::array< std::unique_ptr, 3 > const& Bfield, + ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& Jfield, ablastr::fields::VectorField const& edge_lengths, amrex::MultiFab const* Ffield, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index d5253deb362..464363fb56b 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -159,9 +159,9 @@ void HybridPICModel::InitData () amrex::IntVect Jx_stag = warpx.getField(FieldType::current_fp, 0,0).ixType().toIntVect(); amrex::IntVect Jy_stag = warpx.getField(FieldType::current_fp, 0,1).ixType().toIntVect(); amrex::IntVect Jz_stag = warpx.getField(FieldType::current_fp, 0,2).ixType().toIntVect(); - amrex::IntVect Bx_stag = warpx.getField(FieldType::Bfield_fp, 0,0).ixType().toIntVect(); - amrex::IntVect By_stag = warpx.getField(FieldType::Bfield_fp, 0,1).ixType().toIntVect(); - amrex::IntVect Bz_stag = warpx.getField(FieldType::Bfield_fp, 0,2).ixType().toIntVect(); + amrex::IntVect Bx_stag = warpx.m_fields.get("Bfield_XX",Direction{0},0)->ixType().toIntVect(); + amrex::IntVect By_stag = warpx.m_fields.get("Bfield_XX",Direction{1},0)->ixType().toIntVect(); + amrex::IntVect Bz_stag = warpx.m_fields.get("Bfield_XX",Direction{2},0)->ixType().toIntVect(); amrex::IntVect Ex_stag = warpx.m_fields.get("Efield_fp",Direction{0},0)->ixType().toIntVect(); amrex::IntVect Ey_stag = warpx.m_fields.get("Efield_fp",Direction{1},0)->ixType().toIntVect(); amrex::IntVect Ez_stag = warpx.m_fields.get("Efield_fp",Direction{2},0)->ixType().toIntVect(); diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp index 9d75f754616..b94850aa91f 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp @@ -85,9 +85,10 @@ void WarpX::UpdateMagneticFieldAndApplyBCs( const amrex::Vector, 3 > >& a_Bn, amrex::Real a_thetadt ) { - amrex::MultiFab::Copy(*Bfield_fp[0][0], *a_Bn[0][0], 0, 0, ncomps, a_Bn[0][0]->nGrowVect()); - amrex::MultiFab::Copy(*Bfield_fp[0][1], *a_Bn[0][1], 0, 0, ncomps, a_Bn[0][1]->nGrowVect()); - amrex::MultiFab::Copy(*Bfield_fp[0][2], *a_Bn[0][2], 0, 0, ncomps, a_Bn[0][2]->nGrowVect()); + using ablastr::fields::Direction; + amrex::MultiFab::Copy(*m_fields.get("Bfield_XX", Direction{0}, 0), *a_Bn[0][0], 0, 0, ncomps, a_Bn[0][0]->nGrowVect()); + amrex::MultiFab::Copy(*m_fields.get("Bfield_XX", Direction{1}, 0), *a_Bn[0][1], 0, 0, ncomps, a_Bn[0][1]->nGrowVect()); + amrex::MultiFab::Copy(*m_fields.get("Bfield_XX", Direction{2}, 0), *a_Bn[0][2], 0, 0, ncomps, a_Bn[0][2]->nGrowVect()); EvolveB(a_thetadt, DtType::Full); ApplyMagneticFieldBCs(); } @@ -96,7 +97,7 @@ void WarpX::FinishMagneticFieldAndApplyBCs( const amrex::Vector, 3 > >& a_Bn, amrex::Real a_theta ) { - FinishImplicitField(Bfield_fp, a_Bn, a_theta); + FinishImplicitField(m_fields.get_mr_levels_alldirs("Bfield_XX", 0), a_Bn, a_theta); ApplyMagneticFieldBCs(); } @@ -248,7 +249,7 @@ WarpX::FinishImplicitParticleUpdate () } void -WarpX::FinishImplicitField( amrex::Vector, 3 > >& Field_fp, +WarpX::FinishImplicitField( const ablastr::fields::MultiLevelVectorField& Field_fp, const amrex::Vector, 3 > >& Field_n, amrex::Real theta ) { @@ -335,14 +336,16 @@ WarpX::ImplicitComputeRHSE (int lev, PatchType patch_type, amrex::Real a_dt, War // a_Erhs_vec storing only the RHS of the update equation. I.e., // c^2*dt*(curl(B^{n+theta} - mu0*J^{n+1/2}) if (patch_type == PatchType::fine) { // JRA FIX - //m_fdtd_solver_fp[lev]->EvolveE( a_Erhs_vec.getArrayVec()[lev], Bfield_fp[lev], + //m_fdtd_solver_fp[lev]->EvolveE( a_Erhs_vec.getArrayVec()[lev], + // m_fields.get_alldirs("Bfield_fp", lev), // m_fields.get_alldirs("current_fp", lev), // m_fields.get_alldirs("edge_lengths", lev), // m_fields.get_alldirs("face_areas", lev), // ECTRhofield[lev], // m_fields.get("F_fp", lev), lev, a_dt ); } else { - //m_fdtd_solver_cp[lev]->EvolveE( a_Erhs_vec.getArrayVec()[lev], Bfield_cp[lev], + //m_fdtd_solver_cp[lev]->EvolveE( a_Erhs_vec.getArrayVec()[lev], + // m_fields.get_alldirs("Bfield_cp", lev), // m_fields.get_alldirs("current_cp", lev), // m_fields.get_alldirs("edge_lengths", lev), // m_fields.get_alldirs("face_areas", lev), diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 3c66c756334..89d2dfcf153 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -914,7 +914,7 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) // Evolve E field in regular cells if (patch_type == PatchType::fine) { m_fdtd_solver_fp[lev]->EvolveE( m_fields.get_alldirs("Efield_fp",lev), - Bfield_fp[lev], + m_fields.get_alldirs("Bfield_fp", lev), m_fields.get_alldirs("current_fp", lev), m_fields.get_alldirs("edge_lengths", lev), m_fields.get_alldirs("face_areas", lev), @@ -922,7 +922,7 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) m_fields.get("F_fp", lev), lev, a_dt ); } else { m_fdtd_solver_cp[lev]->EvolveE( m_fields.get_alldirs("Efield_cp",lev), - Bfield_cp[lev], + m_fields.get_alldirs("Bfield_cp", lev), m_fields.get_alldirs("current_cp", lev), m_fields.get_alldirs("edge_lengths", lev), m_fields.get_alldirs("face_areas", lev), diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 79515ef2586..21aea0e81bb 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -245,7 +245,7 @@ WarpX::MoveWindow (const int step, bool move_j) if (dim == 1) { Efield_parser = m_p_ext_field_params->Eyfield_parser->compile<3>(); } if (dim == 2) { Efield_parser = m_p_ext_field_params->Ezfield_parser->compile<3>(); } } - shiftMF(*Bfield_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost, + shiftMF(*m_fields.get("Bfield_XX", Direction{dim}, lev), geom[lev], num_shift, dir, lev, do_update_cost, m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); shiftMF(*m_fields.get("Efield_fp",Direction{0},lev), geom[lev], num_shift, dir, lev, do_update_cost, m_p_ext_field_params->E_external_grid[dim], use_Eparser, Efield_parser); diff --git a/Source/WarpX.H b/Source/WarpX.H index 06696d49b78..0046137c786 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -132,7 +132,7 @@ public: void ApplyMagneticFieldBCs (); void FinishMagneticFieldAndApplyBCs ( const amrex::Vector, 3 > >& a_Bn, amrex::Real a_theta ); - void FinishImplicitField ( amrex::Vector, 3 > >& Field_fp, + void FinishImplicitField ( const ablastr::fields::MultiLevelVectorField& Field_fp, const amrex::Vector, 3 > >& Field_n, amrex::Real theta ); void ImplicitComputeRHSE ( amrex::Real dt, WarpXSolverVec& a_Erhs_vec); @@ -1017,7 +1017,7 @@ public: void computeE (ablastr::fields::MultiLevelVectorField& E, const ablastr::fields::MultiLevelScalarField& phi, std::array beta = {{0,0,0}} ) const; - void computeB (amrex::Vector, 3> >& B, + void computeB (ablastr::fields::MultiLevelVectorField& B, const ablastr::fields::MultiLevelScalarField& phi, std::array beta = {{0,0,0}} ) const; void computePhiTriDiagonal (const ablastr::fields::MultiLevelScalarField& rho, diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 8e904c56a28..a9084f94388 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2262,9 +2262,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm // const std::array dx = CellSize(lev); - AllocInitMultiFab(Bfield_fp[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp[x]", 0.0_rt); - AllocInitMultiFab(Bfield_fp[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp[y]", 0.0_rt); - AllocInitMultiFab(Bfield_fp[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp[z]", 0.0_rt); + m_fields.alloc_init( "Bfield_XX", Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Bfield_XX", Direction{1}, lev, amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Bfield_XX", Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); m_fields.alloc_init( "Efield_fp", Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); m_fields.alloc_init( "Efield_fp", Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); @@ -2568,9 +2568,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm m_fields.alloc_init("Bfield_aux", Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); } else { // In this case, the aux grid is simply an alias of the fp grid (most common case in WarpX) - m_fields.alias_init("Bfield_aux", "Bfield_avg_fp", Direction{0}, lev, 0.0_rt); - m_fields.alias_init("Bfield_aux", "Bfield_avg_fp", Direction{1}, lev, 0.0_rt); - m_fields.alias_init("Bfield_aux", "Bfield_avg_fp", Direction{2}, lev, 0.0_rt); + m_fields.alias_init("Bfield_aux", "Bfield_fp", Direction{0}, lev, 0.0_rt); + m_fields.alias_init("Bfield_aux", "Bfield_fp", Direction{1}, lev, 0.0_rt); + m_fields.alias_init("Bfield_aux", "Bfield_fp", Direction{2}, lev, 0.0_rt); } if (mypc->m_E_ext_particle_s == "read_from_file") { m_fields.alloc_init("Efield_aux", Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); @@ -2596,13 +2596,12 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm // The external fields that are read from file if (m_p_ext_field_params->B_ext_grid_type != ExternalFieldType::default_zero && m_p_ext_field_params->B_ext_grid_type != ExternalFieldType::constant) { // These fields will be added directly to the grid, i.e. to fp, and need to match the index type - m_fields.alloc_init( "Bfield_fp_external", Direction{0}, lev, amrex::convert(ba, Bfield_fp[lev][0]->ixType()), + m_fields.alloc_init( "Bfield_fp_external", Direction{0}, lev, amrex::convert(ba, m_fields.get("Bfield_fp",Direction{0},lev)->ixType()), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Bfield_fp_external", Direction{1}, lev, amrex::convert(ba, Bfield_fp[lev][1]->ixType()), + m_fields.alloc_init( "Bfield_fp_external", Direction{1}, lev, amrex::convert(ba, m_fields.get("Bfield_fp",Direction{1},lev)->ixType()), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Bfield_fp_external", Direction{2}, lev, amrex::convert(ba, Bfield_fp[lev][2]->ixType()), + m_fields.alloc_init( "Bfield_fp_external", Direction{2}, lev, amrex::convert(ba, m_fields.get("Bfield_fp",Direction{2},lev)->ixType()), dm, ncomps, ngEB, 0.0_rt); - } if (mypc->m_B_ext_particle_s == "read_from_file") { // These fields will be added to the fields that the particles see, and need to match the index type From e96d03488225ba5158af779553bbded3b8448914 Mon Sep 17 00:00:00 2001 From: David Grote Date: Wed, 11 Sep 2024 13:29:00 -0700 Subject: [PATCH 136/314] Chage 2 --- Source/EmbeddedBoundary/WarpXInitEB.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index 5ca89f9957b..c576b2fc25c 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -335,7 +335,7 @@ WarpX::MarkCells(){ WARPX_ABORT_WITH_MESSAGE( "MarkCells: Only implemented in 2D3V and 3D3V"); #endif - for (amrex::MFIter mfi(*Bfield_fp[maxLevel()][idim]); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get("Bfield_XX",Direction{idim},maxLevel())); mfi.isValid(); ++mfi) { auto* face_areas_idim_max_lev = m_fields.get("face_areas", Direction{idim}, maxLevel()); From 2d9a600bdc418d7c029eeca0a3b7d7f5fc889aeb Mon Sep 17 00:00:00 2001 From: David Grote Date: Wed, 11 Sep 2024 17:13:51 -0700 Subject: [PATCH 137/314] Change 3 --- Source/BoundaryConditions/PML.cpp | 6 ++-- Source/BoundaryConditions/PML_RZ.cpp | 4 +-- Source/BoundaryConditions/WarpXEvolvePML.cpp | 4 +-- .../WarpXFieldBoundaries.cpp | 14 ++++---- .../FlushFormats/FlushFormatCheckpoint.cpp | 6 ++-- .../FlushFormats/FlushFormatPlotfile.cpp | 12 +++---- Source/Diagnostics/WarpXIO.cpp | 8 ++--- .../EmbeddedBoundary/WarpXFaceExtensions.cpp | 14 ++++---- Source/EmbeddedBoundary/WarpXInitEB.cpp | 2 +- Source/FieldSolver/ElectrostaticSolver.cpp | 8 ++--- .../FiniteDifferenceSolver/EvolveB.cpp | 8 ++--- .../FiniteDifferenceSolver/EvolveG.cpp | 4 +-- .../FiniteDifferenceSolver.H | 34 +++++++++---------- .../HybridPICModel/HybridPICModel.H | 28 +++++++-------- .../HybridPICModel/HybridPICModel.cpp | 34 +++++++++---------- .../HybridPICSolveE.cpp | 12 +++---- .../MacroscopicEvolveE.cpp | 8 ++--- .../ImplicitSolvers/WarpXImplicitOps.cpp | 8 ++--- .../MagnetostaticSolver/MagnetostaticSolver.H | 4 +-- .../MagnetostaticSolver.cpp | 5 +-- Source/FieldSolver/WarpXPushFieldsEM.cpp | 21 +++++++----- .../FieldSolver/WarpXPushFieldsHybridPIC.cpp | 12 ++++--- .../FieldSolver/WarpX_QED_Field_Pushers.cpp | 6 ++-- Source/Initialization/WarpXInitData.cpp | 7 ++-- Source/Parallelization/WarpXComm.cpp | 22 ++++++------ Source/Parallelization/WarpXRegrid.cpp | 3 +- Source/Utils/WarpXMovingWindow.cpp | 2 +- Source/WarpX.H | 2 +- Source/WarpX.cpp | 15 ++++---- 29 files changed, 164 insertions(+), 149 deletions(-) diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index 16502a1ee33..b9a9d0edaa0 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -709,9 +709,9 @@ PML::PML (const int lev, const BoxArray& grid_ba, warpx.m_fields.alloc_init("pml_E_fp", Direction{1}, lev, ba_Ey, dm, ncompe, nge, 0.0_rt); warpx.m_fields.alloc_init("pml_E_fp", Direction{2}, lev, ba_Ez, dm, ncompe, nge, 0.0_rt); - const amrex::BoxArray ba_Bx = amrex::convert(ba, warpx.m_fields.get("Bfield_XX",Direction{0},0)->ixType().toIntVect()); - const amrex::BoxArray ba_By = amrex::convert(ba, warpx.m_fields.get("Bfield_XX",Direction{1},0)->ixType().toIntVect()); - const amrex::BoxArray ba_Bz = amrex::convert(ba, warpx.m_fields.get("Bfield_XX",Direction{2},0)->ixType().toIntVect()); + const amrex::BoxArray ba_Bx = amrex::convert(ba, warpx.m_fields.get("Bfield_fp",Direction{0},0)->ixType().toIntVect()); + const amrex::BoxArray ba_By = amrex::convert(ba, warpx.m_fields.get("Bfield_fp",Direction{1},0)->ixType().toIntVect()); + const amrex::BoxArray ba_Bz = amrex::convert(ba, warpx.m_fields.get("Bfield_fp",Direction{2},0)->ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_B_fp[0], ba_Bx, dm, ncompb, ngb, lev, "pml_B_fp[x]", 0.0_rt); WarpX::AllocInitMultiFab(pml_B_fp[1], ba_By, dm, ncompb, ngb, lev, "pml_B_fp[y]", 0.0_rt); WarpX::AllocInitMultiFab(pml_B_fp[2], ba_Bz, dm, ncompb, ngb, lev, "pml_B_fp[z]", 0.0_rt); diff --git a/Source/BoundaryConditions/PML_RZ.cpp b/Source/BoundaryConditions/PML_RZ.cpp index 860220c3771..10bcb990586 100644 --- a/Source/BoundaryConditions/PML_RZ.cpp +++ b/Source/BoundaryConditions/PML_RZ.cpp @@ -52,8 +52,8 @@ PML_RZ::PML_RZ (const int lev, const amrex::BoxArray& grid_ba, const amrex::Dist warpx.m_fields.alloc_init("pml_E_fp", Direction{0}, lev, ba_Er, grid_dm, Er_fp.nComp(), Er_fp.nGrowVect(), 0.0_rt); warpx.m_fields.alloc_init("pml_E_fp", Direction{1}, lev, ba_Et, grid_dm, Et_fp.nComp(), Et_fp.nGrowVect(), 0.0_rt); - const amrex::MultiFab & Br_fp = *warpx.m_fields.get("Bfield_XX",Direction{0},lev); - const amrex::MultiFab & Bt_fp = *warpx.m_fields.get("Bfield_XX",Direction{1},lev); + const amrex::MultiFab & Br_fp = *warpx.m_fields.get("Bfield_fp",Direction{0},lev); + const amrex::MultiFab & Bt_fp = *warpx.m_fields.get("Bfield_fp",Direction{1},lev); const amrex::BoxArray ba_Br = amrex::convert(grid_ba, Br_fp.ixType().toIntVect()); const amrex::BoxArray ba_Bt = amrex::convert(grid_ba, Bt_fp.ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_B_fp[0], ba_Br, grid_dm, Br_fp.nComp(), Br_fp.nGrowVect(), lev, "pml_B_fp[0]", 0.0_rt); diff --git a/Source/BoundaryConditions/WarpXEvolvePML.cpp b/Source/BoundaryConditions/WarpXEvolvePML.cpp index 9311b962e6f..0010b3e4544 100644 --- a/Source/BoundaryConditions/WarpXEvolvePML.cpp +++ b/Source/BoundaryConditions/WarpXEvolvePML.cpp @@ -68,8 +68,8 @@ WarpX::DampPML (const int lev, PatchType patch_type) using ablastr::fields::Direction; pml_rz[lev]->ApplyDamping( m_fields.get("Efield_fp",Direction{1},lev), m_fields.get("Efield_fp",Direction{2},lev), - m_fields.get("Bfield_XX",Direction{1},lev), - m_fields.get("Bfield_XX",Direction{2},lev), + m_fields.get("Bfield_fp",Direction{1},lev), + m_fields.get("Bfield_fp",Direction{2},lev), dt[lev]); } #endif diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index 5f2dc2e5f49..bc2fb8cad1a 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -112,9 +112,9 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { if (patch_type == PatchType::fine) { PEC::ApplyPECtoBfield( { - m_fields.get("Bfield_XX",Direction{0},lev), - m_fields.get("Bfield_XX",Direction{1},lev), - m_fields.get("Bfield_XX",Direction{2},lev) }, + m_fields.get("Bfield_fp",Direction{0},lev), + m_fields.get("Bfield_fp",Direction{1},lev), + m_fields.get("Bfield_fp",Direction{2},lev) }, field_boundary_lo, field_boundary_hi, get_ng_fieldgather(), Geom(lev), lev, patch_type, ref_ratio); @@ -136,7 +136,7 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d if (a_dt_type == DtType::FirstHalf) { if(::isAnyBoundary(field_boundary_lo, field_boundary_hi)){ auto Efield_fp_new = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); // JRA, new to prevent shadow - auto Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_XX",max_level); + auto Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_fp",max_level); m_fdtd_solver_fp[0]->ApplySilverMuellerBoundary( Efield_fp_new[lev], Bfield_fp_new[lev], Geom(lev).Domain(), dt[lev], @@ -147,9 +147,9 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d #ifdef WARPX_DIM_RZ if (patch_type == PatchType::fine) { - ApplyFieldBoundaryOnAxis(m_fields.get("Bfield_XX",Direction{0},lev), - m_fields.get("Bfield_XX",Direction{1},lev), - m_fields.get("Bfield_XX",Direction{2},lev), lev); + ApplyFieldBoundaryOnAxis(m_fields.get("Bfield_fp",Direction{0},lev), + m_fields.get("Bfield_fp",Direction{1},lev), + m_fields.get("Bfield_fp",Direction{2},lev), lev); } else { ApplyFieldBoundaryOnAxis(getFieldPointer(FieldType::Bfield_cp, lev, 0), getFieldPointer(FieldType::Bfield_cp, lev, 1), diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index 249085e8e68..d9d3551dfd8 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -74,11 +74,11 @@ FlushFormatCheckpoint::WriteToFile ( amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ey_fp")); VisMF::Write(*warpx.m_fields.get("Efield_fp", Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ez_fp")); - VisMF::Write(*warpx.m_fields.get("Bfield_XX", Direction{0}, lev), + VisMF::Write(*warpx.m_fields.get("Bfield_fp", Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bx_fp")); - VisMF::Write(*warpx.m_fields.get("Bfield_XX", Direction{1}, lev), + VisMF::Write(*warpx.m_fields.get("Bfield_fp", Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "By_fp")); - VisMF::Write(*warpx.m_fields.get("Bfield_XX", Direction{2}, lev), + VisMF::Write(*warpx.m_fields.get("Bfield_fp", Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bz_fp")); if (WarpX::fft_do_time_averaging) diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index b816e23253f..55904f4d28d 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -588,11 +588,11 @@ FlushFormatPlotfile::WriteAllRawFields( default_level_prefix, "jy_fp", lev,plot_raw_fields_guards ); WriteRawMF( *warpx.m_fields.get("current_fp",Direction{2}, lev), dm, raw_pltname, default_level_prefix, "jz_fp", lev,plot_raw_fields_guards ); - WriteRawMF( *warpx.m_fields.get("Bfield_XX", Direction{0}, lev), dm, raw_pltname, + WriteRawMF( *warpx.m_fields.get("Bfield_fp", Direction{0}, lev), dm, raw_pltname, default_level_prefix, "Bx_fp", lev, plot_raw_fields_guards ); - WriteRawMF( *warpx.m_fields.get("Bfield_XX", Direction{1}, lev), dm, raw_pltname, + WriteRawMF( *warpx.m_fields.get("Bfield_fp", Direction{1}, lev), dm, raw_pltname, default_level_prefix, "By_fp", lev, plot_raw_fields_guards ); - WriteRawMF( *warpx.m_fields.get("Bfield_XX", Direction{2}, lev), dm, raw_pltname, + WriteRawMF( *warpx.m_fields.get("Bfield_fp", Direction{2}, lev), dm, raw_pltname, default_level_prefix, "Bz_fp", lev, plot_raw_fields_guards ); if (warpx.m_fields.has("F_fp", lev)) { @@ -642,9 +642,9 @@ FlushFormatPlotfile::WriteAllRawFields( dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards); WriteCoarseVector( "B", warpx.getFieldPointer(FieldType::Bfield_cp, lev, 0), warpx.getFieldPointer(FieldType::Bfield_cp, lev, 1), warpx.getFieldPointer(FieldType::Bfield_cp, lev, 2), - warpx.m_fields.get("Bfield_XX", Direction{0}, lev), - warpx.m_fields.get("Bfield_XX", Direction{1}, lev), - warpx.m_fields.get("Bfield_XX", Direction{2}, lev), + warpx.m_fields.get("Bfield_fp", Direction{0}, lev), + warpx.m_fields.get("Bfield_fp", Direction{1}, lev), + warpx.m_fields.get("Bfield_fp", Direction{2}, lev), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards); WriteCoarseVector( "j", warpx.m_fields.get("current_cp", Direction{0}, lev), warpx.m_fields.get("current_cp", Direction{1}, lev), warpx.m_fields.get("current_cp", Direction{2}, lev), diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp index a99cdfd241c..6c9d9c3caee 100644 --- a/Source/Diagnostics/WarpXIO.cpp +++ b/Source/Diagnostics/WarpXIO.cpp @@ -283,7 +283,7 @@ WarpX::InitFromCheckpoint () for (int i = 0; i < 3; ++i) { m_fields.get("current_fp",Direction{i},lev)->setVal(0.0); m_fields.get("Efield_fp",Direction{i},lev)->setVal(0.0); - m_fields.get("Bfield_XX",Direction{i},lev)->setVal(0.0); + m_fields.get("Bfield_fp",Direction{i},lev)->setVal(0.0); } if (lev > 0) { @@ -304,11 +304,11 @@ WarpX::InitFromCheckpoint () VisMF::Read(*m_fields.get("Efield_fp", Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ez_fp")); - VisMF::Read(*m_fields.get("Bfield_XX", Direction{0}, lev), + VisMF::Read(*m_fields.get("Bfield_fp", Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Bx_fp")); - VisMF::Read(*m_fields.get("Bfield_XX", Direction{1}, lev), + VisMF::Read(*m_fields.get("Bfield_fp", Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "By_fp")); - VisMF::Read(*m_fields.get("Bfield_XX", Direction{2}, lev), + VisMF::Read(*m_fields.get("Bfield_fp", Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Bz_fp")); if (WarpX::fft_do_time_averaging) diff --git a/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp b/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp index 7972bd551db..e42caaed6dd 100644 --- a/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp +++ b/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp @@ -286,7 +286,7 @@ WarpX::ComputeFaceExtensions () void WarpX::InitBorrowing() { int idim = 0; - for (amrex::MFIter mfi(*m_fields.get("Bfield_XX", Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get("Bfield_fp", Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { amrex::Box const &box = mfi.validbox(); auto &borrowing_x = (*m_borrowing[maxLevel()][idim])[mfi]; borrowing_x.inds_pointer.resize(box); @@ -302,7 +302,7 @@ WarpX::InitBorrowing() { } idim = 1; - for (amrex::MFIter mfi(*m_fields.get("Bfield_XX", Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get("Bfield_fp", Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { amrex::Box const &box = mfi.validbox(); auto &borrowing_y = (*m_borrowing[maxLevel()][idim])[mfi]; borrowing_y.inds_pointer.resize(box); @@ -315,7 +315,7 @@ WarpX::InitBorrowing() { } idim = 2; - for (amrex::MFIter mfi(*m_fields.get("Bfield_XX", Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get("Bfield_fp", Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { amrex::Box const &box = mfi.validbox(); auto &borrowing_z = (*m_borrowing[maxLevel()][idim])[mfi]; borrowing_z.inds_pointer.resize(box); @@ -456,7 +456,7 @@ WarpX::ComputeOneWayExtensions () WARPX_ABORT_WITH_MESSAGE( "ComputeOneWayExtensions: Only implemented in 2D3V and 3D3V"); #endif - for (amrex::MFIter mfi(*m_fields.get("Bfield_XX", Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get("Bfield_fp", Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { amrex::Box const &box = mfi.validbox(); auto const &S = m_fields.get("face_areas", Direction{idim}, maxLevel())->array(mfi); @@ -583,7 +583,7 @@ WarpX::ComputeEightWaysExtensions () WARPX_ABORT_WITH_MESSAGE( "ComputeEightWaysExtensions: Only implemented in 2D3V and 3D3V"); #endif - for (amrex::MFIter mfi(*m_fields.get("Bfield_XX", Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get("Bfield_fp", Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { amrex::Box const &box = mfi.validbox(); @@ -734,7 +734,7 @@ WarpX::ApplyBCKCorrection (const int idim) const amrex::Real dy = cell_size[1]; const amrex::Real dz = cell_size[2]; - for (amrex::MFIter mfi(*m_fields.get("Bfield_XX", Direction{idim}, maxLevel()), amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get("Bfield_fp", Direction{idim}, maxLevel()), amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { const amrex::Box &box = mfi.tilebox(); const amrex::Array4 &flag_ext_face = m_flag_ext_face[maxLevel()][idim]->array(mfi); @@ -762,7 +762,7 @@ void WarpX::ShrinkBorrowing () { for(int idim = 0; idim < AMREX_SPACEDIM; idim++) { - for (amrex::MFIter mfi(*m_fields.get("Bfield_XX", Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get("Bfield_fp", Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { auto &borrowing = (*m_borrowing[maxLevel()][idim])[mfi]; borrowing.inds.resize(borrowing.vecs_size); borrowing.neigh_faces.resize(borrowing.vecs_size); diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index c576b2fc25c..4cea931b300 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -335,7 +335,7 @@ WarpX::MarkCells(){ WARPX_ABORT_WITH_MESSAGE( "MarkCells: Only implemented in 2D3V and 3D3V"); #endif - for (amrex::MFIter mfi(*m_fields.get("Bfield_XX",Direction{idim},maxLevel())); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get("Bfield_fp",Direction{idim},maxLevel())); mfi.isValid(); ++mfi) { auto* face_areas_idim_max_lev = m_fields.get("face_areas", Direction{idim}, maxLevel()); diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index 67df4fea640..0fdf6bb73f4 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -69,7 +69,7 @@ WarpX::ComputeSpaceChargeField (bool const reset_fields) for (int lev = 0; lev <= max_level; lev++) { for (int comp=0; comp<3; comp++) { m_fields.get("Efield_fp",Direction{comp},lev)->setVal(0); - m_fields.get("Bfield_XX",Direction{comp},lev)->setVal(0); + m_fields.get("Bfield_fp",Direction{comp},lev)->setVal(0); } } } @@ -142,7 +142,7 @@ WarpX::AddBoundaryField () // Compute the corresponding electric and magnetic field, from the potential phi. auto Efield_fp_new = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); // JRA, new to prevent shadow - auto Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_XX",max_level); + auto Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_fp",max_level); computeE( Efield_fp_new, phi, beta ); computeB( Bfield_fp_new, phi, beta ); @@ -231,7 +231,7 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) // Compute the corresponding electric and magnetic field, from the potential phi auto Efield_fp_new = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); // JRA, new to prevent shadow - auto Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_XX",max_level); + auto Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_fp",max_level); computeE( Efield_fp_new, phi, beta ); computeB( Bfield_fp_new, phi, beta ); @@ -310,7 +310,7 @@ WarpX::AddSpaceChargeFieldLabFrame () // Compute the electric field. Note that if an EB is used the electric // field will be calculated in the computePhi call. auto Efield_fp_new = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); // JRA, new to prevent shadow - auto Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_XX",max_level); + auto Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_fp",max_level); if (!EB::enabled()) { computeE( Efield_fp_new, phi_fp, beta ); } else { if (IsPythonCallbackInstalled("poissonsolver")) { computeE(Efield_fp_new, phi_fp, beta); } diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp index 3c2bd8ddf44..ef842bffb3d 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp @@ -48,7 +48,7 @@ using namespace amrex; * \brief Update the B field, over one timestep */ void FiniteDifferenceSolver::EvolveB ( - [[maybe_unused]] std::array< std::unique_ptr, 3 >& Bfield, + [[maybe_unused]] ablastr::fields::VectorField const& Bfield, [[maybe_unused]] ablastr::fields::VectorField const& Efield, [[maybe_unused]] amrex::MultiFab const * Gfield, [[maybe_unused]] ablastr::fields::VectorField const& face_areas, @@ -94,7 +94,7 @@ void FiniteDifferenceSolver::EvolveB ( template void FiniteDifferenceSolver::EvolveBCartesian ( - std::array< std::unique_ptr, 3 >& Bfield, + ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& Efield, amrex::MultiFab const * Gfield, int lev, amrex::Real const dt ) { @@ -193,7 +193,7 @@ void FiniteDifferenceSolver::EvolveBCartesian ( void FiniteDifferenceSolver::EvolveBCartesianECT ( - std::array< std::unique_ptr, 3 >& Bfield, + ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& face_areas, std::array< std::unique_ptr, 3 > const& area_mod, std::array< std::unique_ptr, 3 >& ECTRhofield, @@ -359,7 +359,7 @@ void FiniteDifferenceSolver::EvolveBCartesianECT ( template void FiniteDifferenceSolver::EvolveBCylindrical ( - std::array< std::unique_ptr, 3 >& Bfield, + ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& Efield, int lev, amrex::Real const dt ) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveG.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveG.cpp index 730b6067bc5..759644201bc 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveG.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveG.cpp @@ -39,7 +39,7 @@ using namespace amrex; void FiniteDifferenceSolver::EvolveG ( amrex::MultiFab* Gfield, - std::array,3> const& Bfield, + ablastr::fields::VectorField const& Bfield, amrex::Real const dt) { #ifdef WARPX_DIM_RZ @@ -71,7 +71,7 @@ void FiniteDifferenceSolver::EvolveG ( template void FiniteDifferenceSolver::EvolveGCartesian ( amrex::MultiFab* Gfield, - std::array,3> const& Bfield, + ablastr::fields::VectorField const& Bfield, amrex::Real const dt) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 902d751a838..ea24248299f 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -52,7 +52,7 @@ class FiniteDifferenceSolver std::array cell_size, ablastr::utils::enums::GridType grid_type ); - void EvolveB ( std::array< std::unique_ptr, 3 >& Bfield, + void EvolveB ( ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& Efield, amrex::MultiFab const * Gfield, ablastr::fields::VectorField const& face_areas, @@ -79,7 +79,7 @@ class FiniteDifferenceSolver amrex::Real dt ); void EvolveG (amrex::MultiFab* Gfield, - std::array,3> const& Bfield, + ablastr::fields::VectorField const& Bfield, amrex::Real dt); void EvolveECTRho ( ablastr::fields::VectorField const Efield, @@ -114,7 +114,7 @@ class FiniteDifferenceSolver * \param[in] macroscopic_properties contains user-defined properties of the medium. */ void MacroscopicEvolveE ( ablastr::fields::VectorField Efield, - std::array< std::unique_ptr, 3> const& Bfield, + ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& Jfield, ablastr::fields::VectorField const& edge_lengths, amrex::Real dt, @@ -158,7 +158,7 @@ class FiniteDifferenceSolver std::array< std::unique_ptr, 3 > & Jfield, ablastr::fields::VectorField const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, - std::array< std::unique_ptr, 3> const& Bfield, + ablastr::fields::VectorField const& Bfield, amrex::MultiFab* const rhofield, std::unique_ptr const& Pefield, ablastr::fields::VectorField const& edge_lengths, @@ -176,7 +176,7 @@ class FiniteDifferenceSolver */ void CalculateCurrentAmpere ( std::array< std::unique_ptr, 3>& Jfield, - std::array< std::unique_ptr, 3> const& Bfield, + ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& edge_lengths, int lev ); @@ -209,15 +209,15 @@ class FiniteDifferenceSolver #ifdef WARPX_DIM_RZ template< typename T_Algo > void EvolveBCylindrical ( - std::array< std::unique_ptr, 3 >& Bfield, + ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const & Efield, int lev, amrex::Real dt ); template< typename T_Algo > void EvolveECylindrical ( - ablastr::fields::VectorField Efield, - std::array< std::unique_ptr, 3 > const& Bfield, + ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& Jfield, ablastr::fields::VectorField const& edge_lengths, amrex::MultiFab const* Ffield, @@ -244,7 +244,7 @@ class FiniteDifferenceSolver std::array< std::unique_ptr, 3> const& Jfield, ablastr::fields::VectorField const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, - std::array< std::unique_ptr, 3> const& Bfield, + ablastr::fields::VectorField const& Bfield, amrex::MultiFab* const rhofield, std::unique_ptr const& Pefield, ablastr::fields::VectorField const& edge_lengths, @@ -254,7 +254,7 @@ class FiniteDifferenceSolver template void CalculateCurrentAmpereCylindrical ( std::array< std::unique_ptr, 3 >& Jfield, - std::array< std::unique_ptr, 3 > const& Bfield, + ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& edge_lengths, int lev ); @@ -262,7 +262,7 @@ class FiniteDifferenceSolver #else template< typename T_Algo > void EvolveBCartesian ( - std::array< std::unique_ptr, 3 >& Bfield, + ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& Efield, amrex::MultiFab const * Gfield, int lev, amrex::Real dt ); @@ -287,7 +287,7 @@ class FiniteDifferenceSolver template< typename T_Algo > void EvolveGCartesian ( amrex::MultiFab* Gfield, - std::array,3> const& Bfield, + ablastr::fields::VectorField const& Bfield, amrex::Real dt); void EvolveRhoCartesianECT ( @@ -297,7 +297,7 @@ class FiniteDifferenceSolver std::array< std::unique_ptr, 3 >& ECTRhofield, int lev); void EvolveBCartesianECT ( - std::array< std::unique_ptr, 3 >& Bfield, + ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& face_areas, std::array< std::unique_ptr, 3 > const& area_mod, std::array< std::unique_ptr, 3 >& ECTRhofield, @@ -314,8 +314,8 @@ class FiniteDifferenceSolver template< typename T_Algo, typename T_MacroAlgo > void MacroscopicEvolveECartesian ( - ablastr::fields::VectorField Efield, - std::array< std::unique_ptr< amrex::MultiFab>, 3> const& Bfield, + ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& Jfield, ablastr::fields::VectorField const& edge_lengths, amrex::Real dt, @@ -349,7 +349,7 @@ class FiniteDifferenceSolver std::array< std::unique_ptr, 3 > const& Jfield, ablastr::fields::VectorField const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, - std::array< std::unique_ptr, 3> const& Bfield, + ablastr::fields::VectorField const& Bfield, amrex::MultiFab* const rhofield, std::unique_ptr const& Pefield, ablastr::fields::VectorField const& edge_lengths, @@ -359,7 +359,7 @@ class FiniteDifferenceSolver template void CalculateCurrentAmpereCartesian ( std::array< std::unique_ptr, 3 >& Jfield, - std::array< std::unique_ptr, 3 > const& Bfield, + ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& edge_lengths, int lev ); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index eb49cd88d56..70dc2eecb33 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -74,11 +74,11 @@ public: * \param[in] edge_lengths Length of cell edges taking embedded boundaries into account */ void CalculateCurrentAmpere ( - amrex::Vector, 3>> const& Bfield, + ablastr::fields::MultiLevelVectorField const& Bfield, ablastr::fields::MultiLevelVectorField const& edge_lengths ); void CalculateCurrentAmpere ( - std::array< std::unique_ptr, 3> const& Bfield, + ablastr::fields::MultiLevelVectorField const& Bfield, ablastr::fields::VectorField const& edge_lengths, int lev ); @@ -88,32 +88,32 @@ public: * Function to update the E-field using Ohm's law (hybrid-PIC model). */ void HybridPICSolveE ( - ablastr::fields::MultiLevelVectorField Efield, + ablastr::fields::MultiLevelVectorField const& Efield, ablastr::fields::MultiLevelVectorField const& Jfield, - amrex::Vector, 3>> const& Bfield, + ablastr::fields::MultiLevelVectorField const& Bfield, ablastr::fields::MultiLevelScalarField const& rhofield, ablastr::fields::MultiLevelVectorField const& edge_lengths, bool solve_for_Faraday); void HybridPICSolveE ( - ablastr::fields::VectorField Efield, + ablastr::fields::VectorField const& Efield, ablastr::fields::VectorField const& Jfield, - std::array< std::unique_ptr, 3> const& Bfield, + ablastr::fields::VectorField const& Bfield, amrex::MultiFab* const rhofield, ablastr::fields::VectorField const& edge_lengths, int lev, bool solve_for_Faraday); void HybridPICSolveE ( - ablastr::fields::VectorField Efield, + ablastr::fields::VectorField const& Efield, ablastr::fields::VectorField const& Jfield, - std::array< std::unique_ptr, 3> const& Bfield, + ablastr::fields::VectorField const& Bfield, amrex::MultiFab* const rhofield, ablastr::fields::VectorField const& edge_lengths, int lev, PatchType patch_type, bool solve_for_Faraday); void BfieldEvolveRK ( - amrex::Vector, 3>>& Bfield, - ablastr::fields::MultiLevelVectorField Efield, + ablastr::fields::MultiLevelVectorField const& Bfield, + ablastr::fields::MultiLevelVectorField const& Efield, ablastr::fields::MultiLevelVectorField const& Jfield, ablastr::fields::MultiLevelScalarField const& rhofield, ablastr::fields::MultiLevelVectorField const& edge_lengths, @@ -121,8 +121,8 @@ public: amrex::IntVect ng, std::optional nodal_sync); void BfieldEvolveRK ( - amrex::Vector, 3>>& Bfield, - ablastr::fields::MultiLevelVectorField Efield, + ablastr::fields::MultiLevelVectorField const& Bfield, + ablastr::fields::MultiLevelVectorField const& Efield, ablastr::fields::MultiLevelVectorField const& Jfield, ablastr::fields::MultiLevelScalarField const& rhofield, ablastr::fields::MultiLevelVectorField const& edge_lengths, @@ -130,8 +130,8 @@ public: amrex::IntVect ng, std::optional nodal_sync); void FieldPush ( - amrex::Vector, 3>>& Bfield, - ablastr::fields::MultiLevelVectorField& Efield, + ablastr::fields::MultiLevelVectorField const& Bfield, + ablastr::fields::MultiLevelVectorField const& Efield, ablastr::fields::MultiLevelVectorField const& Jfield, ablastr::fields::MultiLevelScalarField const& rhofield, ablastr::fields::MultiLevelVectorField const& edge_lengths, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 464363fb56b..4cbf1f8376e 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -159,9 +159,9 @@ void HybridPICModel::InitData () amrex::IntVect Jx_stag = warpx.getField(FieldType::current_fp, 0,0).ixType().toIntVect(); amrex::IntVect Jy_stag = warpx.getField(FieldType::current_fp, 0,1).ixType().toIntVect(); amrex::IntVect Jz_stag = warpx.getField(FieldType::current_fp, 0,2).ixType().toIntVect(); - amrex::IntVect Bx_stag = warpx.m_fields.get("Bfield_XX",Direction{0},0)->ixType().toIntVect(); - amrex::IntVect By_stag = warpx.m_fields.get("Bfield_XX",Direction{1},0)->ixType().toIntVect(); - amrex::IntVect Bz_stag = warpx.m_fields.get("Bfield_XX",Direction{2},0)->ixType().toIntVect(); + amrex::IntVect Bx_stag = warpx.m_fields.get("Bfield_fp",Direction{0},0)->ixType().toIntVect(); + amrex::IntVect By_stag = warpx.m_fields.get("Bfield_fp",Direction{1},0)->ixType().toIntVect(); + amrex::IntVect Bz_stag = warpx.m_fields.get("Bfield_fp",Direction{2},0)->ixType().toIntVect(); amrex::IntVect Ex_stag = warpx.m_fields.get("Efield_fp",Direction{0},0)->ixType().toIntVect(); amrex::IntVect Ey_stag = warpx.m_fields.get("Efield_fp",Direction{1},0)->ixType().toIntVect(); amrex::IntVect Ez_stag = warpx.m_fields.get("Efield_fp",Direction{2},0)->ixType().toIntVect(); @@ -392,7 +392,7 @@ void HybridPICModel::GetCurrentExternal ( } void HybridPICModel::CalculateCurrentAmpere ( - amrex::Vector, 3>> const& Bfield, + ablastr::fields::MultiLevelVectorField const& Bfield, ablastr::fields::MultiLevelVectorField const& edge_lengths) { auto& warpx = WarpX::GetInstance(); @@ -403,7 +403,7 @@ void HybridPICModel::CalculateCurrentAmpere ( } void HybridPICModel::CalculateCurrentAmpere ( - std::array< std::unique_ptr, 3> const& Bfield, + ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& edge_lengths, const int lev) { @@ -422,9 +422,9 @@ void HybridPICModel::CalculateCurrentAmpere ( } void HybridPICModel::HybridPICSolveE ( - ablastr::fields::MultiLevelVectorField Efield, + ablastr::fields::MultiLevelVectorField const& Efield, ablastr::fields::MultiLevelVectorField const& Jfield, - amrex::Vector, 3>> const& Bfield, + ablastr::fields::MultiLevelVectorField const& Bfield, ablastr::fields::MultiLevelScalarField const& rhofield, ablastr::fields::MultiLevelVectorField const& edge_lengths, const bool solve_for_Faraday) @@ -440,9 +440,9 @@ void HybridPICModel::HybridPICSolveE ( } void HybridPICModel::HybridPICSolveE ( - ablastr::fields::VectorField Efield, + ablastr::fields::VectorField const& Efield, ablastr::fields::VectorField const& Jfield, - std::array< std::unique_ptr, 3> const& Bfield, + ablastr::fields::VectorField const& Bfield, amrex::MultiFab* const rhofield, ablastr::fields::VectorField const& edge_lengths, const int lev, const bool solve_for_Faraday) @@ -461,9 +461,9 @@ void HybridPICModel::HybridPICSolveE ( } void HybridPICModel::HybridPICSolveE ( - ablastr::fields::VectorField Efield, + ablastr::fields::VectorField const& Efield, ablastr::fields::VectorField const& Jfield, - std::array< std::unique_ptr, 3> const& Bfield, + ablastr::fields::VectorField const& Bfield, amrex::MultiFab* const rhofield, ablastr::fields::VectorField const& edge_lengths, const int lev, PatchType patch_type, @@ -535,8 +535,8 @@ void HybridPICModel::FillElectronPressureMF ( } void HybridPICModel::BfieldEvolveRK ( - amrex::Vector, 3>>& Bfield, - ablastr::fields::MultiLevelVectorField Efield, + ablastr::fields::MultiLevelVectorField const& Bfield, + ablastr::fields::MultiLevelVectorField const&Efield, ablastr::fields::MultiLevelVectorField const& Jfield, ablastr::fields::MultiLevelScalarField const& rhofield, ablastr::fields::MultiLevelVectorField const& edge_lengths, @@ -554,8 +554,8 @@ void HybridPICModel::BfieldEvolveRK ( } void HybridPICModel::BfieldEvolveRK ( - amrex::Vector, 3>>& Bfield, - ablastr::fields::MultiLevelVectorField Efield, + ablastr::fields::MultiLevelVectorField const& Bfield, + ablastr::fields::MultiLevelVectorField const& Efield, ablastr::fields::MultiLevelVectorField const& Jfield, ablastr::fields::MultiLevelScalarField const& rhofield, ablastr::fields::MultiLevelVectorField const& edge_lengths, @@ -667,8 +667,8 @@ void HybridPICModel::BfieldEvolveRK ( } void HybridPICModel::FieldPush ( - amrex::Vector, 3>>& Bfield, - ablastr::fields::MultiLevelVectorField& Efield, + ablastr::fields::MultiLevelVectorField const& Bfield, + ablastr::fields::MultiLevelVectorField const& Efield, ablastr::fields::MultiLevelVectorField const& Jfield, ablastr::fields::MultiLevelScalarField const& rhofield, ablastr::fields::MultiLevelVectorField const& edge_lengths, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp index e66c8cdd53a..f30506c11dd 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp @@ -25,7 +25,7 @@ using namespace amrex; void FiniteDifferenceSolver::CalculateCurrentAmpere ( std::array< std::unique_ptr, 3>& Jfield, - std::array< std::unique_ptr, 3> const& Bfield, + ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& edge_lengths, int lev ) { @@ -60,7 +60,7 @@ void FiniteDifferenceSolver::CalculateCurrentAmpere ( template void FiniteDifferenceSolver::CalculateCurrentAmpereCylindrical ( std::array< std::unique_ptr, 3 >& Jfield, - std::array< std::unique_ptr, 3 > const& Bfield, + ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& edge_lengths, int lev ) @@ -243,7 +243,7 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCylindrical ( template void FiniteDifferenceSolver::CalculateCurrentAmpereCartesian ( std::array< std::unique_ptr, 3 >& Jfield, - std::array< std::unique_ptr, 3 > const& Bfield, + ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& edge_lengths, int lev ) @@ -355,7 +355,7 @@ void FiniteDifferenceSolver::HybridPICSolveE ( std::array< std::unique_ptr, 3 >& Jfield, ablastr::fields::VectorField const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, - std::array< std::unique_ptr, 3 > const& Bfield, + ablastr::fields::VectorField const& Bfield, amrex::MultiFab* const rhofield, std::unique_ptr const& Pefield, ablastr::fields::VectorField const& edge_lengths, @@ -393,7 +393,7 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( std::array< std::unique_ptr, 3 > const& Jfield, ablastr::fields::VectorField const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, - std::array< std::unique_ptr, 3 > const& Bfield, + ablastr::fields::VectorField const& Bfield, amrex::MultiFab* const rhofield, std::unique_ptr const& Pefield, ablastr::fields::VectorField const& edge_lengths, @@ -708,7 +708,7 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( std::array< std::unique_ptr, 3 > const& Jfield, ablastr::fields::VectorField const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, - std::array< std::unique_ptr, 3 > const& Bfield, + ablastr::fields::VectorField const& Bfield, amrex::MultiFab* const rhofield, std::unique_ptr const& Pefield, ablastr::fields::VectorField const& edge_lengths, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp index 6e823fa8a9f..708728c4e5b 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp @@ -37,8 +37,8 @@ using namespace amrex; using namespace ablastr::fields; void FiniteDifferenceSolver::MacroscopicEvolveE ( - ablastr::fields::VectorField Efield, - std::array< std::unique_ptr, 3 > const& Bfield, + ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& Jfield, VectorField const& edge_lengths, amrex::Real const dt, @@ -100,8 +100,8 @@ void FiniteDifferenceSolver::MacroscopicEvolveE ( template void FiniteDifferenceSolver::MacroscopicEvolveECartesian ( - ablastr::fields::VectorField Efield, - std::array< std::unique_ptr, 3 > const& Bfield, + ablastr::fields::VectorField const& Efield, + ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& Jfield, ablastr::fields::VectorField const& edge_lengths, amrex::Real const dt, diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp index b94850aa91f..33d6d2dc912 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp @@ -86,9 +86,9 @@ WarpX::UpdateMagneticFieldAndApplyBCs( const amrex::VectornGrowVect()); - amrex::MultiFab::Copy(*m_fields.get("Bfield_XX", Direction{1}, 0), *a_Bn[0][1], 0, 0, ncomps, a_Bn[0][1]->nGrowVect()); - amrex::MultiFab::Copy(*m_fields.get("Bfield_XX", Direction{2}, 0), *a_Bn[0][2], 0, 0, ncomps, a_Bn[0][2]->nGrowVect()); + amrex::MultiFab::Copy(*m_fields.get("Bfield_fp", Direction{0}, 0), *a_Bn[0][0], 0, 0, ncomps, a_Bn[0][0]->nGrowVect()); + amrex::MultiFab::Copy(*m_fields.get("Bfield_fp", Direction{1}, 0), *a_Bn[0][1], 0, 0, ncomps, a_Bn[0][1]->nGrowVect()); + amrex::MultiFab::Copy(*m_fields.get("Bfield_fp", Direction{2}, 0), *a_Bn[0][2], 0, 0, ncomps, a_Bn[0][2]->nGrowVect()); EvolveB(a_thetadt, DtType::Full); ApplyMagneticFieldBCs(); } @@ -97,7 +97,7 @@ void WarpX::FinishMagneticFieldAndApplyBCs( const amrex::Vector, 3 > >& a_Bn, amrex::Real a_theta ) { - FinishImplicitField(m_fields.get_mr_levels_alldirs("Bfield_XX", 0), a_Bn, a_theta); + FinishImplicitField(m_fields.get_mr_levels_alldirs("Bfield_fp", 0), a_Bn, a_theta); ApplyMagneticFieldBCs(); } diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H index a8bbc954e29..1df9efea296 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H @@ -34,12 +34,12 @@ namespace MagnetostaticSolver { */ class EBCalcBfromVectorPotentialPerLevel { private: - const amrex::Vector, 3>>& m_b_field; + const ablastr::fields::MultiLevelVectorField& m_b_field; const amrex::Vector, 3>>& m_grad_buf_e_stag; const amrex::Vector, 3>>& m_grad_buf_b_stag; public: - EBCalcBfromVectorPotentialPerLevel(const amrex::Vector, 3>>& b_field, + EBCalcBfromVectorPotentialPerLevel(const ablastr::fields::MultiLevelVectorField& b_field, const amrex::Vector, 3>>& grad_buf_e_stag, const amrex::Vector, 3>>& grad_buf_b_stag) : m_b_field(b_field), diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp index c43d1686250..8864b35d6ee 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp @@ -159,7 +159,7 @@ WarpX::computeVectorPotential (const ablastr::fields::MultiLevelVectorField& cur Real const required_precision, Real absolute_tolerance, int const max_iters, - int const verbosity) const + int const verbosity) // const // This breaks non-const m_fields.get_mr_levels_alldirs { using ablastr::fields::Direction; @@ -176,7 +176,8 @@ WarpX::computeVectorPotential (const ablastr::fields::MultiLevelVectorField& cur } #if defined(AMREX_USE_EB) - const std::optional post_A_calculation({Bfield_fp, + ablastr::fields::MultiLevelVectorField Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); + const std::optional post_A_calculation({Bfield_fp_new, vector_potential_grad_buf_e_stag, vector_potential_grad_buf_b_stag}); diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 89d2dfcf153..e9068c5345c 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -853,20 +853,20 @@ WarpX::EvolveB (int lev, amrex::Real a_dt, DtType a_dt_type) void WarpX::EvolveB (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_type) { - auto face_areas_lev = m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev]; - // Evolve B field in regular cells if (patch_type == PatchType::fine) { - m_fdtd_solver_fp[lev]->EvolveB( Bfield_fp[lev], + m_fdtd_solver_fp[lev]->EvolveB( m_fields.get_alldirs("Bfield_fp",lev), m_fields.get_alldirs("Efield_fp",lev), m_fields.get("G_fp", lev), - face_areas_lev, m_area_mod[lev], ECTRhofield[lev], Venl[lev], + m_fields.get("face_areas", lev), + m_area_mod[lev], ECTRhofield[lev], Venl[lev], m_flag_info_face[lev], m_borrowing[lev], lev, a_dt ); } else { - m_fdtd_solver_cp[lev]->EvolveB( Bfield_cp[lev], + m_fdtd_solver_cp[lev]->EvolveB( m_fields.get_alldirs("Bfield_cp",lev), m_fields.get_alldirs("Efield_cp",lev), m_fields.get("G_fp", lev), - face_areas_lev, m_area_mod[lev], ECTRhofield[lev], Venl[lev], + m_fields.get("face_areas", lev), + m_area_mod[lev], ECTRhofield[lev], Venl[lev], m_flag_info_face[lev], m_borrowing[lev], lev, a_dt ); } @@ -1057,15 +1057,17 @@ WarpX::EvolveG (int lev, PatchType patch_type, amrex::Real a_dt, DtType /*a_dt_t // Evolve G field in regular cells if (patch_type == PatchType::fine) { + ablastr::fields::MultiLevelVectorField const& Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); m_fdtd_solver_fp[lev]->EvolveG( m_fields.get("G_fp", lev), - Bfield_fp[lev], a_dt); + Bfield_fp_new[lev], a_dt); } else // coarse patch { + ablastr::fields::MultiLevelVectorField const& Bfield_cp_new = m_fields.get_mr_levels_alldirs("Bfield_cp", finest_level); m_fdtd_solver_cp[lev]->EvolveG( m_fields.get("G_cp", lev), - Bfield_cp[lev], a_dt); + Bfield_cp_new[lev], a_dt); } // TODO Evolution in PML cells will go here @@ -1100,9 +1102,10 @@ WarpX::MacroscopicEvolveE (int lev, PatchType patch_type, amrex::Real a_dt) { "Macroscopic EvolveE is not implemented for lev>0, yet." ); + ablastr::fields::MultiLevelVectorField const& Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); m_fdtd_solver_fp[lev]->MacroscopicEvolveE( m_fields.get_alldirs("Efield_fp", lev), - Bfield_fp[lev], + m_fields.get_alldirs("Bfield_fp", lev), m_fields.get_alldirs("current_fp", lev), m_fields.get_alldirs("edge_lengths", lev), a_dt, m_macroscopic_properties); diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index a096999cf5b..3de96b57018 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -97,13 +97,15 @@ void WarpX::HybridPICEvolveFields () } } + ablastr::fields::MultiLevelVectorField const& Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); + // Push the B field from t=n to t=n+1/2 using the current and density // at t=n, while updating the E field along with B using the electron // momentum equation for (int sub_step = 0; sub_step < sub_steps; sub_step++) { m_hybrid_pic_model->BfieldEvolveRK( - Bfield_fp, + m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level), m_fields.get_mr_levels_alldirs("Efield_fp", finest_level), va2vm(current_fp_temp), amrex::GetVecOfPtrs(rho_fp_temp), m_fields.get_mr_levels_alldirs("edge_lenghts", finest_level), @@ -129,7 +131,7 @@ void WarpX::HybridPICEvolveFields () for (int sub_step = 0; sub_step < sub_steps; sub_step++) { m_hybrid_pic_model->BfieldEvolveRK( - Bfield_fp, + m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level), m_fields.get_mr_levels_alldirs("Efield_fp", finest_level), m_fields.get_mr_levels_alldirs("current_fp", finest_level), amrex::GetVecOfPtrs(rho_fp_temp), @@ -163,11 +165,13 @@ void WarpX::HybridPICEvolveFields () // Update the E field to t=n+1 using the extrapolated J_i^n+1 value m_hybrid_pic_model->CalculateCurrentAmpere( - Bfield_fp, + m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level), m_fields.get_mr_levels_alldirs("edge_lengths", finest_level)); m_hybrid_pic_model->HybridPICSolveE( m_fields.get_mr_levels_alldirs("Efield_fp", finest_level), - va2vm(current_fp_temp), Bfield_fp, m_fields.get_mr_levels("rho_fp", finest_level), + va2vm(current_fp_temp), + m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level), + m_fields.get_mr_levels("rho_fp", finest_level), m_fields.get_mr_levels_alldirs("edge_lengths", finest_level), false ); FillBoundaryE(guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); diff --git a/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp b/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp index 877e3fd9d68..07483aa283f 100644 --- a/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp +++ b/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp @@ -85,9 +85,9 @@ WarpX::Hybrid_QED_Push (int lev, PatchType patch_type, amrex::Real a_dt) Ex = m_fields.get("Efield_fp",Direction{0},lev); Ey = m_fields.get("Efield_fp",Direction{1},lev); Ez = m_fields.get("Efield_fp",Direction{2},lev); - Bx = Bfield_fp[lev][0].get(); - By = Bfield_fp[lev][1].get(); - Bz = Bfield_fp[lev][2].get(); + Bx = m_fields.get("Bfield_fp",Direction{0},lev); + By = m_fields.get("Bfield_fp",Direction{1},lev); + Bz = m_fields.get("Bfield_fp",Direction{2},lev); Jx = m_fields.get("current_fp", Direction{0}, lev); Jy = m_fields.get("current_fp", Direction{1}, lev); Jz = m_fields.get("current_fp", Direction{2}, lev); diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 30a8dc5aae1..02e36d1eed7 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -643,10 +643,11 @@ WarpX::AddExternalFields (int const lev) { } } if (m_p_ext_field_params->B_ext_grid_type != ExternalFieldType::default_zero) { + ablastr::fields::MultiLevelVectorField const& Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::constant) { - Bfield_fp[lev][0]->plus(m_p_ext_field_params->B_external_grid[0], guard_cells.ng_alloc_EB.min()); - Bfield_fp[lev][1]->plus(m_p_ext_field_params->B_external_grid[1], guard_cells.ng_alloc_EB.min()); - Bfield_fp[lev][2]->plus(m_p_ext_field_params->B_external_grid[2], guard_cells.ng_alloc_EB.min()); + Bfield_fp_new[lev][0]->plus(m_p_ext_field_params->B_external_grid[0], guard_cells.ng_alloc_EB.min()); + Bfield_fp_new[lev][1]->plus(m_p_ext_field_params->B_external_grid[1], guard_cells.ng_alloc_EB.min()); + Bfield_fp_new[lev][2]->plus(m_p_ext_field_params->B_external_grid[2], guard_cells.ng_alloc_EB.min()); } else { amrex::MultiFab::Add(*Bfield_fp[lev][0], *m_fields.get("Bfield_fp_external",Direction{0},lev), 0, 0, 1, guard_cells.ng_alloc_EB); diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index e62068bea68..2878c799e74 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -59,7 +59,9 @@ WarpX::UpdateAuxilaryData () auto Bfield_aux_lvl0_0 = m_fields.get("Bfield_aux", Direction{0}, 0); - if (Bfield_aux_lvl0_0->ixType() == Bfield_fp[0][0]->ixType()) { + ablastr::fields::MultiLevelVectorField const& Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); + + if (Bfield_aux_lvl0_0->ixType() == Bfield_fp_new[0][0]->ixType()) { UpdateAuxilaryDataSameType(); } else { UpdateAuxilaryDataStagToNodal(); @@ -100,17 +102,17 @@ WarpX::UpdateAuxilaryDataStagToNodal () #endif using ablastr::fields::Direction; - ablastr::fields::MultiLevelVectorField Efield_fp = m_fields.get_mr_levels_alldirs("Efield_fp", finest_level); - ablastr::fields::MultiLevelVectorField Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); - ablastr::fields::MultiLevelVectorField Efield_avg_fp = m_fields.get_mr_levels_alldirs("Efield_avg_fp", finest_level); - ablastr::fields::MultiLevelVectorField Bfield_avg_fp = m_fields.get_mr_levels_alldirs("Bfield_avg_fp", finest_level); - ablastr::fields::MultiLevelVectorField Efield_aux = m_fields.get_mr_levels_alldirs("Efield_aux", finest_level); - ablastr::fields::MultiLevelVectorField Bfield_aux = m_fields.get_mr_levels_alldirs("Bfield_aux", finest_level); - ablastr::fields::MultiLevelVectorField Efield_cax = m_fields.get_mr_levels_alldirs("Efield_cax", finest_level); - ablastr::fields::MultiLevelVectorField Bfield_cax = m_fields.get_mr_levels_alldirs("Bfield_cax", finest_level); + ablastr::fields::MultiLevelVectorField const& Efield_fp = m_fields.get_mr_levels_alldirs("Efield_fp", finest_level); + ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); + ablastr::fields::MultiLevelVectorField const& Efield_avg_fp = m_fields.get_mr_levels_alldirs("Efield_avg_fp", finest_level); + ablastr::fields::MultiLevelVectorField const& Bfield_avg_fp = m_fields.get_mr_levels_alldirs("Bfield_avg_fp", finest_level); + ablastr::fields::MultiLevelVectorField const& Efield_aux = m_fields.get_mr_levels_alldirs("Efield_aux", finest_level); + ablastr::fields::MultiLevelVectorField const& Bfield_aux = m_fields.get_mr_levels_alldirs("Bfield_aux", finest_level); + ablastr::fields::MultiLevelVectorField const& Efield_cax = m_fields.get_mr_levels_alldirs("Efield_cax", finest_level); + ablastr::fields::MultiLevelVectorField const& Bfield_cax = m_fields.get_mr_levels_alldirs("Bfield_cax", finest_level); ablastr::fields::MultiLevelVectorField const & Bmf = WarpX::fft_do_time_averaging ? Bfield_avg_fp : Bfield_fp; - ablastr::fields::MultiLevelVectorField const & Emf = WarpX::fft_do_time_averaging ? Efield_avg_fp : Efield_fp; // JRA, do this one when doing Efield_avg_fp refactor to new style + ablastr::fields::MultiLevelVectorField const & Emf = WarpX::fft_do_time_averaging ? Efield_avg_fp : Efield_fp; const amrex::IntVect& Bx_stag = Bmf[0][0]->ixType().toIntVect(); const amrex::IntVect& By_stag = Bmf[0][1]->ixType().toIntVect(); diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index 3dc82780cc5..6771f9b07d8 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -181,12 +181,13 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi m_fields.remake_level(lev, dm); // Fine patch + ablastr::fields::MultiLevelVectorField const& Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); for (int idim=0; idim < 3; ++idim) { if (eb_enabled) { if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { - m_borrowing[lev][idim] = std::make_unique>(amrex::convert(ba, Bfield_fp[lev][idim]->ixType().toIntVect()), dm); + m_borrowing[lev][idim] = std::make_unique>(amrex::convert(ba, Bfield_fp_new[lev][idim]->ixType().toIntVect()), dm); } } } diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 21aea0e81bb..095849bc74e 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -245,7 +245,7 @@ WarpX::MoveWindow (const int step, bool move_j) if (dim == 1) { Efield_parser = m_p_ext_field_params->Eyfield_parser->compile<3>(); } if (dim == 2) { Efield_parser = m_p_ext_field_params->Ezfield_parser->compile<3>(); } } - shiftMF(*m_fields.get("Bfield_XX", Direction{dim}, lev), geom[lev], num_shift, dir, lev, do_update_cost, + shiftMF(*m_fields.get("Bfield_fp", Direction{dim}, lev), geom[lev], num_shift, dir, lev, do_update_cost, m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); shiftMF(*m_fields.get("Efield_fp",Direction{0},lev), geom[lev], num_shift, dir, lev, do_update_cost, m_p_ext_field_params->E_external_grid[dim], use_Eparser, Efield_parser); diff --git a/Source/WarpX.H b/Source/WarpX.H index 0046137c786..dc5dc325d16 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1032,7 +1032,7 @@ public: amrex::Real required_precision=amrex::Real(1.e-11), amrex::Real absolute_tolerance=amrex::Real(0.0), int max_iters=200, - int verbosity=2) const; + int verbosity=2); // const; void setVectorPotentialBC (ablastr::fields::MultiLevelVectorField const& A) const; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index a9084f94388..97d21ff13eb 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2262,9 +2262,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm // const std::array dx = CellSize(lev); - m_fields.alloc_init( "Bfield_XX", Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Bfield_XX", Direction{1}, lev, amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Bfield_XX", Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Bfield_fp", Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Bfield_fp", Direction{1}, lev, amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Bfield_fp", Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); m_fields.alloc_init( "Efield_fp", Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); m_fields.alloc_init( "Efield_fp", Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); @@ -2596,11 +2596,14 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm // The external fields that are read from file if (m_p_ext_field_params->B_ext_grid_type != ExternalFieldType::default_zero && m_p_ext_field_params->B_ext_grid_type != ExternalFieldType::constant) { // These fields will be added directly to the grid, i.e. to fp, and need to match the index type - m_fields.alloc_init( "Bfield_fp_external", Direction{0}, lev, amrex::convert(ba, m_fields.get("Bfield_fp",Direction{0},lev)->ixType()), + m_fields.alloc_init( "Bfield_fp_external", Direction{0}, lev, + amrex::convert(ba, m_fields.get("Bfield_fp",Direction{0},lev)->ixType()), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Bfield_fp_external", Direction{1}, lev, amrex::convert(ba, m_fields.get("Bfield_fp",Direction{1},lev)->ixType()), + m_fields.alloc_init( "Bfield_fp_external", Direction{1}, lev, + amrex::convert(ba, m_fields.get("Bfield_fp",Direction{1},lev)->ixType()), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Bfield_fp_external", Direction{2}, lev, amrex::convert(ba, m_fields.get("Bfield_fp",Direction{2},lev)->ixType()), + m_fields.alloc_init( "Bfield_fp_external", Direction{2}, lev, + amrex::convert(ba, m_fields.get("Bfield_fp",Direction{2},lev)->ixType()), dm, ncomps, ngEB, 0.0_rt); } if (mypc->m_B_ext_particle_s == "read_from_file") { From f9dfc8da1e2df8b5b7269ccfde0d9da415856990 Mon Sep 17 00:00:00 2001 From: David Grote Date: Wed, 11 Sep 2024 15:46:06 -0700 Subject: [PATCH 138/314] Change 4 --- .../BoundaryConditions/WarpXFieldBoundaries.cpp | 4 ++-- Source/FieldSolver/ElectrostaticSolver.cpp | 12 ++++++------ .../MagnetostaticSolver/MagnetostaticSolver.cpp | 4 ++-- Source/FieldSolver/WarpXPushFieldsEM.cpp | 6 +++--- Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp | 2 +- Source/Initialization/WarpXInitData.cpp | 8 ++++---- Source/Parallelization/WarpXComm.cpp | 16 +++++++++++----- Source/Parallelization/WarpXRegrid.cpp | 4 ++-- Source/WarpX.H | 1 - Source/WarpX.cpp | 7 +------ 10 files changed, 32 insertions(+), 32 deletions(-) diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index bc2fb8cad1a..5b4c3506a13 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -136,9 +136,9 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d if (a_dt_type == DtType::FirstHalf) { if(::isAnyBoundary(field_boundary_lo, field_boundary_hi)){ auto Efield_fp_new = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); // JRA, new to prevent shadow - auto Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_fp",max_level); + auto Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp",max_level); m_fdtd_solver_fp[0]->ApplySilverMuellerBoundary( - Efield_fp_new[lev], Bfield_fp_new[lev], + Efield_fp_new[lev], Bfield_fp[lev], Geom(lev).Domain(), dt[lev], field_boundary_lo, field_boundary_hi); } diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index 0fdf6bb73f4..ae9166d2ac8 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -142,9 +142,9 @@ WarpX::AddBoundaryField () // Compute the corresponding electric and magnetic field, from the potential phi. auto Efield_fp_new = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); // JRA, new to prevent shadow - auto Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_fp",max_level); + auto Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp",max_level); computeE( Efield_fp_new, phi, beta ); - computeB( Bfield_fp_new, phi, beta ); + computeB( Bfield_fp, phi, beta ); // de-allocate temporary for (int lev = 0; lev <= max_level; lev++) { @@ -231,9 +231,9 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) // Compute the corresponding electric and magnetic field, from the potential phi auto Efield_fp_new = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); // JRA, new to prevent shadow - auto Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_fp",max_level); + auto Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp",max_level); computeE( Efield_fp_new, phi, beta ); - computeB( Bfield_fp_new, phi, beta ); + computeB( Bfield_fp, phi, beta ); // de-allocate temporary for (int lev = 0; lev <= max_level; lev++) { @@ -310,14 +310,14 @@ WarpX::AddSpaceChargeFieldLabFrame () // Compute the electric field. Note that if an EB is used the electric // field will be calculated in the computePhi call. auto Efield_fp_new = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); // JRA, new to prevent shadow - auto Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_fp",max_level); + auto Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp",max_level); if (!EB::enabled()) { computeE( Efield_fp_new, phi_fp, beta ); } else { if (IsPythonCallbackInstalled("poissonsolver")) { computeE(Efield_fp_new, phi_fp, beta); } } // Compute the magnetic field - computeB( Bfield_fp_new, phi_fp, beta ); + computeB( Bfield_fp, phi_fp, beta ); } /* Compute the potential `phi` by solving the Poisson equation with `rho` as diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp index 8864b35d6ee..5b952e84ba8 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp @@ -176,8 +176,8 @@ WarpX::computeVectorPotential (const ablastr::fields::MultiLevelVectorField& cur } #if defined(AMREX_USE_EB) - ablastr::fields::MultiLevelVectorField Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); - const std::optional post_A_calculation({Bfield_fp_new, + ablastr::fields::MultiLevelVectorField Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); + const std::optional post_A_calculation({Bfield_fp, vector_potential_grad_buf_e_stag, vector_potential_grad_buf_b_stag}); diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index e9068c5345c..6ade3bfda99 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -1057,10 +1057,10 @@ WarpX::EvolveG (int lev, PatchType patch_type, amrex::Real a_dt, DtType /*a_dt_t // Evolve G field in regular cells if (patch_type == PatchType::fine) { - ablastr::fields::MultiLevelVectorField const& Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); + ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); m_fdtd_solver_fp[lev]->EvolveG( m_fields.get("G_fp", lev), - Bfield_fp_new[lev], a_dt); + Bfield_fp[lev], a_dt); } else // coarse patch { @@ -1102,7 +1102,7 @@ WarpX::MacroscopicEvolveE (int lev, PatchType patch_type, amrex::Real a_dt) { "Macroscopic EvolveE is not implemented for lev>0, yet." ); - ablastr::fields::MultiLevelVectorField const& Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); + ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); m_fdtd_solver_fp[lev]->MacroscopicEvolveE( m_fields.get_alldirs("Efield_fp", lev), m_fields.get_alldirs("Bfield_fp", lev), diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index 3de96b57018..1728c0c837f 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -97,7 +97,7 @@ void WarpX::HybridPICEvolveFields () } } - ablastr::fields::MultiLevelVectorField const& Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); + ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); // Push the B field from t=n to t=n+1/2 using the current and density // at t=n, while updating the E field along with B using the electron diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 02e36d1eed7..f821555b9c9 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -643,11 +643,11 @@ WarpX::AddExternalFields (int const lev) { } } if (m_p_ext_field_params->B_ext_grid_type != ExternalFieldType::default_zero) { - ablastr::fields::MultiLevelVectorField const& Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); + ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::constant) { - Bfield_fp_new[lev][0]->plus(m_p_ext_field_params->B_external_grid[0], guard_cells.ng_alloc_EB.min()); - Bfield_fp_new[lev][1]->plus(m_p_ext_field_params->B_external_grid[1], guard_cells.ng_alloc_EB.min()); - Bfield_fp_new[lev][2]->plus(m_p_ext_field_params->B_external_grid[2], guard_cells.ng_alloc_EB.min()); + Bfield_fp[lev][0]->plus(m_p_ext_field_params->B_external_grid[0], guard_cells.ng_alloc_EB.min()); + Bfield_fp[lev][1]->plus(m_p_ext_field_params->B_external_grid[1], guard_cells.ng_alloc_EB.min()); + Bfield_fp[lev][2]->plus(m_p_ext_field_params->B_external_grid[2], guard_cells.ng_alloc_EB.min()); } else { amrex::MultiFab::Add(*Bfield_fp[lev][0], *m_fields.get("Bfield_fp_external",Direction{0},lev), 0, 0, 1, guard_cells.ng_alloc_EB); diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 2878c799e74..fe770165f84 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -59,9 +59,9 @@ WarpX::UpdateAuxilaryData () auto Bfield_aux_lvl0_0 = m_fields.get("Bfield_aux", Direction{0}, 0); - ablastr::fields::MultiLevelVectorField const& Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); + ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); - if (Bfield_aux_lvl0_0->ixType() == Bfield_fp_new[0][0]->ixType()) { + if (Bfield_aux_lvl0_0->ixType() == Bfield_fp[0][0]->ixType()) { UpdateAuxilaryDataSameType(); } else { UpdateAuxilaryDataStagToNodal(); @@ -402,6 +402,8 @@ WarpX::UpdateAuxilaryDataSameType () ablastr::fields::MultiLevelVectorField Efield_avg_cp = m_fields.get_mr_levels_alldirs("Efield_avg_cp", finest_level); ablastr::fields::MultiLevelVectorField Bfield_avg_cp = m_fields.get_mr_levels_alldirs("Bfield_avg_cp", finest_level); + ablastr::fields::MultiLevelVectorField Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); + // Level 0: Copy from fine to aux // Note: in some configurations, Efield_aux/Bfield_aux and Efield_fp/Bfield_fp are simply aliases to the // same MultiFab object. MultiFab::Copy operation automatically detects this and does nothing in this case. @@ -774,12 +776,14 @@ WarpX::FillBoundaryB (const int lev, const PatchType patch_type, const amrex::In if (patch_type == PatchType::fine) { - mf = {Bfield_fp[lev][0].get(), Bfield_fp[lev][1].get(), Bfield_fp[lev][2].get()}; + ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); + mf = {Bfield_fp[lev][0], Bfield_fp[lev][1], Bfield_fp[lev][2]}; period = Geom(lev).periodicity(); } else // coarse patch { - mf = {Bfield_cp[lev][0].get(), Bfield_cp[lev][1].get(), Bfield_cp[lev][2].get()}; + ablastr::fields::MultiLevelVectorField const& Bfield_cp_new = m_fields.get_mr_levels_alldirs("Bfield_cp", finest_level); + mf = {Bfield_cp_new[lev][0], Bfield_cp_new[lev][1], Bfield_cp_new[lev][2]}; period = Geom(lev-1).periodicity(); } @@ -884,6 +888,8 @@ WarpX::FillBoundaryB_avg (int lev, IntVect ng) void WarpX::FillBoundaryB_avg (int lev, PatchType patch_type, IntVect ng) { + using ablastr::fields::Direction; + if (patch_type == PatchType::fine) { if (do_pml && pml[lev]->ok()) @@ -899,7 +905,7 @@ WarpX::FillBoundaryB_avg (int lev, PatchType patch_type, IntVect ng) ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period); } else { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - ng.allLE(Bfield_fp[lev][0]->nGrowVect()), + ng.allLE(m_fields.get("Bfield_fp",Direction{0},lev)->nGrowVect()), "Error: in FillBoundaryB, requested more guard cells than allocated"); ablastr::utils::communication::FillBoundary(*Bfield_avg_fp[lev][0], ng, WarpX::do_single_precision_comms, period); ablastr::utils::communication::FillBoundary(*Bfield_avg_fp[lev][1], ng, WarpX::do_single_precision_comms, period); diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index 6771f9b07d8..b4deed06b4b 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -181,13 +181,13 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi m_fields.remake_level(lev, dm); // Fine patch - ablastr::fields::MultiLevelVectorField const& Bfield_fp_new = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); + ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); for (int idim=0; idim < 3; ++idim) { if (eb_enabled) { if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { - m_borrowing[lev][idim] = std::make_unique>(amrex::convert(ba, Bfield_fp_new[lev][idim]->ixType().toIntVect()), dm); + m_borrowing[lev][idim] = std::make_unique>(amrex::convert(ba, Bfield_fp[lev][idim]->ixType().toIntVect()), dm); } } } diff --git a/Source/WarpX.H b/Source/WarpX.H index dc5dc325d16..84c49216354 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1513,7 +1513,6 @@ private: // Fine patch amrex::Vector, 3 > > Efield_fp; - amrex::Vector, 3 > > Bfield_fp; // Masks for computing dot product and global moments of fields when using grids that // have shared locations across different ranks (e.g., a Yee grid) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 97d21ff13eb..6690e125bd2 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -318,7 +318,7 @@ WarpX::WarpX () } Efield_fp.resize(nlevs_max); - Bfield_fp.resize(nlevs_max); + /* Bfield_fp.resize(nlevs_max); */ Efield_dotMask.resize(nlevs_max); Bfield_dotMask.resize(nlevs_max); @@ -3468,9 +3468,6 @@ WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, cons case FieldType::Efield_fp : field_pointer = Efield_fp[lev][direction].get(); break; - case FieldType::Bfield_fp : - field_pointer = Bfield_fp[lev][direction].get(); - break; case FieldType::Efield_cp : field_pointer = Efield_cp[lev][direction].get(); break; @@ -3555,8 +3552,6 @@ WarpX::getMultiLevelField(warpx::fields::FieldType field_type) const { case FieldType::Efield_fp : return Efield_fp; - case FieldType::Bfield_fp : - return Bfield_fp; case FieldType::Efield_cp : return Efield_cp; case FieldType::Bfield_cp : From 822390e94a0f4f6d1ae441ac72941bb41645c935 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 17:15:44 -0700 Subject: [PATCH 139/314] Remove current_fp --- .../HybridPICModel/HybridPICModel.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index d5253deb362..778b247c677 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -156,9 +156,9 @@ void HybridPICModel::InitData () using ablastr::fields::Direction; // Get the grid staggering of the fields involved in calculating E - amrex::IntVect Jx_stag = warpx.getField(FieldType::current_fp, 0,0).ixType().toIntVect(); - amrex::IntVect Jy_stag = warpx.getField(FieldType::current_fp, 0,1).ixType().toIntVect(); - amrex::IntVect Jz_stag = warpx.getField(FieldType::current_fp, 0,2).ixType().toIntVect(); + amrex::IntVect Jx_stag = warpx.m_fields.get("current_fp",Direction{0},0)->ixType().toIntVect(); + amrex::IntVect Jy_stag = warpx.m_fields.get("current_fp",Direction{1},0)->ixType().toIntVect(); + amrex::IntVect Jz_stag = warpx.m_fields.get("current_fp",Direction{2},0)->ixType().toIntVect(); amrex::IntVect Bx_stag = warpx.getField(FieldType::Bfield_fp, 0,0).ixType().toIntVect(); amrex::IntVect By_stag = warpx.getField(FieldType::Bfield_fp, 0,1).ixType().toIntVect(); amrex::IntVect Bz_stag = warpx.getField(FieldType::Bfield_fp, 0,2).ixType().toIntVect(); From 9633ea8e9995a93c978346926d195744ae0fe002 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 17:22:32 -0700 Subject: [PATCH 140/314] Fix compilation --- .../FiniteDifferenceSolver/FiniteDifferenceSolver.H | 3 ++- Source/FieldSolver/WarpXPushFieldsEM.cpp | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index ea24248299f..7d7e12da50d 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -113,7 +113,8 @@ class FiniteDifferenceSolver * \param[in] dt timestep of the simulation * \param[in] macroscopic_properties contains user-defined properties of the medium. */ - void MacroscopicEvolveE ( ablastr::fields::VectorField Efield, + void MacroscopicEvolveE ( + ablastr::fields::VectorField const& Efield, ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& Jfield, ablastr::fields::VectorField const& edge_lengths, diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 6ade3bfda99..231e4196bf2 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -858,14 +858,14 @@ WarpX::EvolveB (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_typ m_fdtd_solver_fp[lev]->EvolveB( m_fields.get_alldirs("Bfield_fp",lev), m_fields.get_alldirs("Efield_fp",lev), m_fields.get("G_fp", lev), - m_fields.get("face_areas", lev), + m_fields.get_alldirs("face_areas", lev), m_area_mod[lev], ECTRhofield[lev], Venl[lev], m_flag_info_face[lev], m_borrowing[lev], lev, a_dt ); } else { m_fdtd_solver_cp[lev]->EvolveB( m_fields.get_alldirs("Bfield_cp",lev), m_fields.get_alldirs("Efield_cp",lev), m_fields.get("G_fp", lev), - m_fields.get("face_areas", lev), + m_fields.get_alldirs("face_areas", lev), m_area_mod[lev], ECTRhofield[lev], Venl[lev], m_flag_info_face[lev], m_borrowing[lev], lev, a_dt ); } From cfaaa64559a34da6f8a9aa447d2eb3560115f5d2 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 17:24:35 -0700 Subject: [PATCH 141/314] Fix Bfield --- .../FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index 70dc2eecb33..02f5b27db17 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -78,7 +78,7 @@ public: ablastr::fields::MultiLevelVectorField const& edge_lengths ); void CalculateCurrentAmpere ( - ablastr::fields::MultiLevelVectorField const& Bfield, + ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& edge_lengths, int lev ); From c838e87508b83eef83865865dd7d1ab8ef0ac976 Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Wed, 11 Sep 2024 16:43:09 -0700 Subject: [PATCH 142/314] Efield_fp refactor for implicit solvers. --- .../ImplicitSolvers/WarpXImplicitOps.cpp | 31 +++++++++---------- .../ImplicitSolvers/WarpXSolverVec.H | 20 ++++++------ .../ImplicitSolvers/WarpXSolverVec.cpp | 29 ++++++++++++----- 3 files changed, 47 insertions(+), 33 deletions(-) diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp index 33d6d2dc912..cbf6ba24f97 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp @@ -73,7 +73,8 @@ WarpX::SetElectricFieldAndApplyBCs ( const WarpXSolverVec& a_E ) a_E.getArrayVecType()==warpx::fields::FieldType::Efield_fp, "WarpX::SetElectricFieldAndApplyBCs() must be called with Efield_fp type"); - const amrex::Vector, 3 > >& Evec = a_E.getArrayVec(); + ablastr::fields::MultiLevelVectorField Efield_fp = m_fields.get_mr_levels_alldirs("Efield_fp",finest_level); + const ablastr::fields::MultiLevelVectorField& Evec = a_E.getArrayVec(); amrex::MultiFab::Copy(*Efield_fp[0][0], *Evec[0][0], 0, 0, ncomps, Evec[0][0]->nGrowVect()); amrex::MultiFab::Copy(*Efield_fp[0][1], *Evec[0][1], 0, 0, ncomps, Evec[0][1]->nGrowVect()); amrex::MultiFab::Copy(*Efield_fp[0][2], *Evec[0][2], 0, 0, ncomps, Evec[0][2]->nGrowVect()); @@ -335,22 +336,20 @@ WarpX::ImplicitComputeRHSE (int lev, PatchType patch_type, amrex::Real a_dt, War // a_Erhs_vec is set to zero above, calling EvolveE below results in // a_Erhs_vec storing only the RHS of the update equation. I.e., // c^2*dt*(curl(B^{n+theta} - mu0*J^{n+1/2}) - if (patch_type == PatchType::fine) { // JRA FIX - //m_fdtd_solver_fp[lev]->EvolveE( a_Erhs_vec.getArrayVec()[lev], - // m_fields.get_alldirs("Bfield_fp", lev), - // m_fields.get_alldirs("current_fp", lev), - // m_fields.get_alldirs("edge_lengths", lev), - // m_fields.get_alldirs("face_areas", lev), - // ECTRhofield[lev], - // m_fields.get("F_fp", lev), lev, a_dt ); + if (patch_type == PatchType::fine) { + m_fdtd_solver_fp[lev]->EvolveE( a_Erhs_vec.getArrayVec()[lev], Bfield_fp[lev], + m_fields.get_alldirs("current_fp", lev), + m_fields.get_alldirs("edge_lengths", lev), + m_fields.get_alldirs("face_areas", lev), + ECTRhofield[lev], + m_fields.get("F_fp", lev), lev, a_dt ); } else { - //m_fdtd_solver_cp[lev]->EvolveE( a_Erhs_vec.getArrayVec()[lev], - // m_fields.get_alldirs("Bfield_cp", lev), - // m_fields.get_alldirs("current_cp", lev), - // m_fields.get_alldirs("edge_lengths", lev), - // m_fields.get_alldirs("face_areas", lev), - // ECTRhofield[lev], - // m_fields.get("F_cp", lev), lev, a_dt ); + m_fdtd_solver_cp[lev]->EvolveE( a_Erhs_vec.getArrayVec()[lev], Bfield_cp[lev], + m_fields.get_alldirs("current_cp", lev), + m_fields.get_alldirs("edge_lengths", lev), + m_fields.get_alldirs("face_areas", lev), + ECTRhofield[lev], + m_fields.get("F_cp", lev), lev, a_dt ); } // Compute Efield_rhs in PML cells by calling EvolveEPML diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H index f884f5fa623..5bf0a75691c 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H +++ b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H @@ -12,6 +12,7 @@ #include #include +#include #include #include @@ -59,7 +60,7 @@ public: WarpXSolverVec(const WarpXSolverVec&) = delete; - ~WarpXSolverVec() = default; + ~WarpXSolverVec(); using value_type = amrex::Real; using RT = value_type; @@ -94,13 +95,13 @@ public: for (int lev = 0; lev < m_num_amr_levels; ++lev) { if (m_array_type != warpx::fields::FieldType::None) { for (int n = 0; n < 3; ++n) { - const std::unique_ptr& this_field = a_solver_vec.getArrayVec()[lev][n]; + const amrex::MultiFab* this_field = a_solver_vec.getArrayVec()[lev][n]; amrex::MultiFab::Copy( *m_array_vec[lev][n], *this_field, 0, 0, m_ncomp, amrex::IntVect::TheZeroVector() ); } } if (m_scalar_type != warpx::fields::FieldType::None) { - const std::unique_ptr& this_scalar = a_solver_vec.getScalarVec()[lev]; + const amrex::MultiFab* this_scalar = a_solver_vec.getScalarVec()[lev]; amrex::MultiFab::Copy( *m_scalar_vec[lev], *this_scalar, 0, 0, m_ncomp, amrex::IntVect::TheZeroVector() ); } @@ -270,22 +271,23 @@ public: return std::sqrt(norm); } - [[nodiscard]] const amrex::Vector,3>>& getArrayVec() const {return m_array_vec;} - amrex::Vector,3>>& getArrayVec() {return m_array_vec;} + [[nodiscard]] const ablastr::fields::MultiLevelVectorField& getArrayVec() const {return m_array_vec;} + ablastr::fields::MultiLevelVectorField& getArrayVec() {return m_array_vec;} - [[nodiscard]] const amrex::Vector>& getScalarVec() const {return m_scalar_vec;} - amrex::Vector>& getScalarVec() {return m_scalar_vec;} + [[nodiscard]] const ablastr::fields::MultiLevelScalarField& getScalarVec() const {return m_scalar_vec;} + ablastr::fields::MultiLevelScalarField& getScalarVec() {return m_scalar_vec;} // solver vector types are type warpx::fields::FieldType [[nodiscard]] warpx::fields::FieldType getArrayVecType () const { return m_array_type; } [[nodiscard]] warpx::fields::FieldType getScalarVecType () const { return m_scalar_type; } + private: bool m_is_defined = false; - amrex::Vector,3>> m_array_vec; - amrex::Vector> m_scalar_vec; + ablastr::fields::MultiLevelVectorField m_array_vec; + ablastr::fields::MultiLevelScalarField m_scalar_vec; warpx::fields::FieldType m_array_type = warpx::fields::FieldType::None; warpx::fields::FieldType m_scalar_type = warpx::fields::FieldType::None; diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp index f2a88d82d42..feaba87ee02 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp @@ -9,6 +9,17 @@ using namespace warpx::fields; +WarpXSolverVec::~WarpXSolverVec () +{ + for (auto & lvl : m_array_vec) + { + for (int i =0; i<3; ++i) + { + delete lvl[i]; + } + } +} + void WarpXSolverVec::Define ( WarpX* a_WarpX, FieldType a_array_type, FieldType a_scalar_type ) @@ -36,14 +47,15 @@ void WarpXSolverVec::Define ( WarpX* a_WarpX, isFieldArray(m_array_type), "WarpXSolverVec::Define() called with array_type not an array field"); + //m_array_vec.reserve(m_num_amr_levels); for (int lev = 0; lev < m_num_amr_levels; ++lev) { using arr_mf_type = std::array; const arr_mf_type this_array = m_WarpX->getFieldPointerArray(m_array_type, lev); for (int n = 0; n < 3; n++) { - m_array_vec[lev][n] = std::make_unique( this_array[n]->boxArray(), - this_array[n]->DistributionMap(), - this_array[n]->nComp(), - amrex::IntVect::TheZeroVector() ); + m_array_vec[lev][n] = new amrex::MultiFab( this_array[n]->boxArray(), + this_array[n]->DistributionMap(), + this_array[n]->nComp(), + amrex::IntVect::TheZeroVector() ); } } @@ -56,12 +68,13 @@ void WarpXSolverVec::Define ( WarpX* a_WarpX, !isFieldArray(m_scalar_type), "WarpXSolverVec::Define() called with scalar_type not a scalar field "); + //m_scalar_vec.reserve(m_num_amr_levels); for (int lev = 0; lev < m_num_amr_levels; ++lev) { const amrex::MultiFab* this_mf = m_WarpX->getFieldPointer(m_scalar_type,lev,0); - m_scalar_vec[lev] = std::make_unique( this_mf->boxArray(), - this_mf->DistributionMap(), - this_mf->nComp(), - amrex::IntVect::TheZeroVector() ); + m_scalar_vec[lev] = new amrex::MultiFab( this_mf->boxArray(), + this_mf->DistributionMap(), + this_mf->nComp(), + amrex::IntVect::TheZeroVector() ); } } From da5b2655ef101bf25b18cfdeaa6c86572e34a005 Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Wed, 11 Sep 2024 16:54:49 -0700 Subject: [PATCH 143/314] Efield_fp refactor complete --- Source/Initialization/WarpXInitData.cpp | 1 + Source/WarpX.H | 4 ++++ Source/WarpX.cpp | 13 +++++-------- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index f821555b9c9..58e90692b97 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -631,6 +631,7 @@ WarpX::AddExternalFields (int const lev) { // FIXME: RZ multimode has more than one component for all these if (m_p_ext_field_params->E_ext_grid_type != ExternalFieldType::default_zero) { + ablastr::fields::MultiLevelVectorField Efield_fp = m_fields.get_mr_levels_alldirs("Efield_fp",finest_level); if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::constant) { Efield_fp[lev][0]->plus(m_p_ext_field_params->E_external_grid[0], guard_cells.ng_alloc_EB.min()); Efield_fp[lev][1]->plus(m_p_ext_field_params->E_external_grid[1], guard_cells.ng_alloc_EB.min()); diff --git a/Source/WarpX.H b/Source/WarpX.H index 84c49216354..1554af1c340 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1512,7 +1512,11 @@ private: // // Fine patch +<<<<<<< HEAD amrex::Vector, 3 > > Efield_fp; +======= + amrex::Vector, 3 > > Bfield_fp; +>>>>>>> 98b2b8714 (Efield_fp refactor complete) // Masks for computing dot product and global moments of fields when using grids that // have shared locations across different ranks (e.g., a Yee grid) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 6690e125bd2..a8a7d5a8050 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -317,9 +317,6 @@ WarpX::WarpX () myfl = std::make_unique(nlevs_max); } - Efield_fp.resize(nlevs_max); - /* Bfield_fp.resize(nlevs_max); */ - Efield_dotMask.resize(nlevs_max); Bfield_dotMask.resize(nlevs_max); Afield_dotMask.resize(nlevs_max); @@ -3465,8 +3462,8 @@ WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, cons switch(field_type) { - case FieldType::Efield_fp : - field_pointer = Efield_fp[lev][direction].get(); + case FieldType::Bfield_fp : + field_pointer = Bfield_fp[lev][direction].get(); break; case FieldType::Efield_cp : field_pointer = Efield_cp[lev][direction].get(); @@ -3550,15 +3547,15 @@ WarpX::getMultiLevelField(warpx::fields::FieldType field_type) const { switch(field_type) { - case FieldType::Efield_fp : - return Efield_fp; + case FieldType::Bfield_fp : + return Bfield_fp; case FieldType::Efield_cp : return Efield_cp; case FieldType::Bfield_cp : return Bfield_cp; default: WARPX_ABORT_WITH_MESSAGE("Invalid field type"); - return Efield_fp; + return Bfield_fp; } } From c5f3829a1a2d473e1111831e0f48db29d60bd7b3 Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Wed, 11 Sep 2024 17:23:14 -0700 Subject: [PATCH 144/314] removed Bfield_fp from WarpX.cpp --- Source/WarpX.H | 7 ------- Source/WarpX.cpp | 7 +------ 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/Source/WarpX.H b/Source/WarpX.H index 1554af1c340..a590e61ded5 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1511,13 +1511,6 @@ private: // Fields: First array for level, second for direction // - // Fine patch -<<<<<<< HEAD - amrex::Vector, 3 > > Efield_fp; -======= - amrex::Vector, 3 > > Bfield_fp; ->>>>>>> 98b2b8714 (Efield_fp refactor complete) - // Masks for computing dot product and global moments of fields when using grids that // have shared locations across different ranks (e.g., a Yee grid) mutable amrex::Vector,3 > > Efield_dotMask; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index a8a7d5a8050..1b92a75babf 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -3462,9 +3462,6 @@ WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, cons switch(field_type) { - case FieldType::Bfield_fp : - field_pointer = Bfield_fp[lev][direction].get(); - break; case FieldType::Efield_cp : field_pointer = Efield_cp[lev][direction].get(); break; @@ -3547,15 +3544,13 @@ WarpX::getMultiLevelField(warpx::fields::FieldType field_type) const { switch(field_type) { - case FieldType::Bfield_fp : - return Bfield_fp; case FieldType::Efield_cp : return Efield_cp; case FieldType::Bfield_cp : return Bfield_cp; default: WARPX_ABORT_WITH_MESSAGE("Invalid field type"); - return Bfield_fp; + return Efield_cp; } } From ec9e34a2af74ca4571142cf7a621fc82d051a6e3 Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Wed, 11 Sep 2024 17:30:27 -0700 Subject: [PATCH 145/314] update Bfield_fp in WarpXImplicitOps --- Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp index cbf6ba24f97..24d3d2e0987 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp @@ -337,14 +337,16 @@ WarpX::ImplicitComputeRHSE (int lev, PatchType patch_type, amrex::Real a_dt, War // a_Erhs_vec storing only the RHS of the update equation. I.e., // c^2*dt*(curl(B^{n+theta} - mu0*J^{n+1/2}) if (patch_type == PatchType::fine) { - m_fdtd_solver_fp[lev]->EvolveE( a_Erhs_vec.getArrayVec()[lev], Bfield_fp[lev], + m_fdtd_solver_fp[lev]->EvolveE( a_Erhs_vec.getArrayVec()[lev], + m_fields.get_alldirs("Bfield_fp", lev), m_fields.get_alldirs("current_fp", lev), m_fields.get_alldirs("edge_lengths", lev), m_fields.get_alldirs("face_areas", lev), ECTRhofield[lev], m_fields.get("F_fp", lev), lev, a_dt ); } else { - m_fdtd_solver_cp[lev]->EvolveE( a_Erhs_vec.getArrayVec()[lev], Bfield_cp[lev], + m_fdtd_solver_cp[lev]->EvolveE( a_Erhs_vec.getArrayVec()[lev], + m_fields.get_alldirs("Bfield_fp", lev), m_fields.get_alldirs("current_cp", lev), m_fields.get_alldirs("edge_lengths", lev), m_fields.get_alldirs("face_areas", lev), From 912abbfeee3879da43e22ed82d577903b1fe2926 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 17:30:31 -0700 Subject: [PATCH 146/314] Fix compilation --- .../FiniteDifferenceSolver/FiniteDifferenceSolver.H | 4 ++-- Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp | 4 ++-- Source/Parallelization/WarpXComm.cpp | 2 -- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 7d7e12da50d..a90777f3c23 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -155,7 +155,7 @@ class FiniteDifferenceSolver * \param[in] hybrid_model instance of the hybrid-PIC model * \param[in] solve_for_Faraday boolean flag for whether the E-field is solved to be used in Faraday's equation */ - void HybridPICSolveE ( ablastr::fields::VectorField& Efield, + void HybridPICSolveE ( ablastr::fields::VectorField const& Efield, std::array< std::unique_ptr, 3 > & Jfield, ablastr::fields::VectorField const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, @@ -241,7 +241,7 @@ class FiniteDifferenceSolver template void HybridPICSolveECylindrical ( - ablastr::fields::VectorField& Efield, + ablastr::fields::VectorField const& Efield, std::array< std::unique_ptr, 3> const& Jfield, ablastr::fields::VectorField const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp index f30506c11dd..ea93d928338 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp @@ -351,7 +351,7 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCartesian ( void FiniteDifferenceSolver::HybridPICSolveE ( - ablastr::fields::VectorField& Efield, + ablastr::fields::VectorField const& Efield, std::array< std::unique_ptr, 3 >& Jfield, ablastr::fields::VectorField const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, @@ -389,7 +389,7 @@ void FiniteDifferenceSolver::HybridPICSolveE ( #ifdef WARPX_DIM_RZ template void FiniteDifferenceSolver::HybridPICSolveECylindrical ( - ablastr::fields::VectorField& Efield, + ablastr::fields::VectorField const& Efield, std::array< std::unique_ptr, 3 > const& Jfield, ablastr::fields::VectorField const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index fe770165f84..90adb6c75d5 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -402,8 +402,6 @@ WarpX::UpdateAuxilaryDataSameType () ablastr::fields::MultiLevelVectorField Efield_avg_cp = m_fields.get_mr_levels_alldirs("Efield_avg_cp", finest_level); ablastr::fields::MultiLevelVectorField Bfield_avg_cp = m_fields.get_mr_levels_alldirs("Bfield_avg_cp", finest_level); - ablastr::fields::MultiLevelVectorField Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); - // Level 0: Copy from fine to aux // Note: in some configurations, Efield_aux/Bfield_aux and Efield_fp/Bfield_fp are simply aliases to the // same MultiFab object. MultiFab::Copy operation automatically detects this and does nothing in this case. From 396f0b048033f58eea25a5fdd528c088428ab4e2 Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Wed, 11 Sep 2024 17:42:02 -0700 Subject: [PATCH 147/314] change ref to const ref. --- .../FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H | 2 +- Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index a90777f3c23..ac36ed8afa5 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -346,7 +346,7 @@ class FiniteDifferenceSolver template void HybridPICSolveECartesian ( - ablastr::fields::VectorField& Efield, + ablastr::fields::VectorField const& Efield, std::array< std::unique_ptr, 3 > const& Jfield, ablastr::fields::VectorField const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp index ea93d928338..0d08e0f767f 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp @@ -704,7 +704,7 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( template void FiniteDifferenceSolver::HybridPICSolveECartesian ( - ablastr::fields::VectorField& Efield, + ablastr::fields::VectorField const& Efield, std::array< std::unique_ptr, 3 > const& Jfield, ablastr::fields::VectorField const& Jifield, std::array< std::unique_ptr, 3 > const& Jextfield, From dd4a0a3475091506ce6859a2f4f159c634cd53c5 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 17:49:35 -0700 Subject: [PATCH 148/314] Remove Efield_cp, Bfield_cp --- Source/Diagnostics/WarpXIO.cpp | 16 ++++---- Source/Initialization/WarpXInitData.cpp | 17 ++++----- Source/Utils/WarpXMovingWindow.cpp | 4 +- Source/WarpX.H | 6 --- Source/WarpX.cpp | 49 ++----------------------- 5 files changed, 22 insertions(+), 70 deletions(-) diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp index 6c9d9c3caee..b932620c7b8 100644 --- a/Source/Diagnostics/WarpXIO.cpp +++ b/Source/Diagnostics/WarpXIO.cpp @@ -292,8 +292,8 @@ WarpX::InitFromCheckpoint () m_fields.get("Bfield_aux", Direction{i}, lev)->setVal(0.0); m_fields.get("current_cp",Direction{i},lev)->setVal(0.0); - Efield_cp[lev][i]->setVal(0.0); - Bfield_cp[lev][i]->setVal(0.0); + m_fields.get("Efield_cp",Direction{i},lev)->setVal(0.0); + m_fields.get("Bfield_cp",Direction{i},lev)->setVal(0.0); } } @@ -339,18 +339,18 @@ WarpX::InitFromCheckpoint () if (lev > 0) { - VisMF::Read(*Efield_cp[lev][0], + VisMF::Read(*m_fields.get("Efield_fp", Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ex_cp")); - VisMF::Read(*Efield_cp[lev][1], + VisMF::Read(*m_fields.get("Efield_fp", Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ey_cp")); - VisMF::Read(*Efield_cp[lev][2], + VisMF::Read(*m_fields.get("Efield_fp", Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ez_cp")); - VisMF::Read(*Bfield_cp[lev][0], + VisMF::Read(*m_fields.get("Bfield_fp", Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Bx_cp")); - VisMF::Read(*Bfield_cp[lev][1], + VisMF::Read(*m_fields.get("Bfield_fp", Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "By_cp")); - VisMF::Read(*Bfield_cp[lev][2], + VisMF::Read(*m_fields.get("Bfield_fp", Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Bz_cp")); if (WarpX::fft_do_time_averaging) diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 58e90692b97..4409c50fbc7 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -927,7 +927,7 @@ WarpX::InitLevelData (int lev, Real /*time*/) if (lev > 0) { Bfield_aux[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); - Bfield_cp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); + m_fields.get("Bfield_cp", Direction{i}, lev)->setVal(m_p_ext_field_params->B_external_grid[i]); if (fft_do_time_averaging) { m_fields.get("Bfield_avg_cp", Direction{i}, lev)->setVal(m_p_ext_field_params->B_external_grid[i]); } @@ -944,10 +944,9 @@ WarpX::InitLevelData (int lev, Real /*time*/) if (fft_do_time_averaging) { m_fields.get("Efield_avg_fp", Direction{i}, lev)->setVal(m_p_ext_field_params->E_external_grid[i]); } - if (lev > 0) { Efield_aux[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); - Efield_cp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); + m_fields.get("Efield_avg_cp", Direction{i}, lev)->setVal(m_p_ext_field_params->E_external_grid[i]); if (fft_do_time_averaging) { m_fields.get("Efield_avg_cp", Direction{i}, lev)->setVal(m_p_ext_field_params->E_external_grid[i]); } @@ -981,9 +980,9 @@ WarpX::InitLevelData (int lev, Real /*time*/) lev, PatchType::fine); InitializeExternalFieldsOnGridUsingParser( - Bfield_cp[lev][0].get(), - Bfield_cp[lev][1].get(), - Bfield_cp[lev][2].get(), + m_fields.get_alldirs("Bfield_cp", Direction{0}, lev), + m_fields.get_alldirs("Bfield_cp", Direction{1}, lev), + m_fields.get_alldirs("Bfield_cp", Direction{2}, lev), m_p_ext_field_params->Bxfield_parser->compile<3>(), m_p_ext_field_params->Byfield_parser->compile<3>(), m_p_ext_field_params->Bzfield_parser->compile<3>(), @@ -1028,9 +1027,9 @@ WarpX::InitLevelData (int lev, Real /*time*/) lev, PatchType::fine); InitializeExternalFieldsOnGridUsingParser( - Efield_cp[lev][0].get(), - Efield_cp[lev][1].get(), - Efield_cp[lev][2].get(), + m_fields.get_alldirs("Efield_cp", Direction{0}, lev), + m_fields.get_alldirs("Efield_cp", Direction{1}, lev), + m_fields.get_alldirs("Efield_cp", Direction{2}, lev), m_p_ext_field_params->Exfield_parser->compile<3>(), m_p_ext_field_params->Eyfield_parser->compile<3>(), m_p_ext_field_params->Ezfield_parser->compile<3>(), diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 095849bc74e..e82d4bd0e88 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -274,9 +274,9 @@ WarpX::MoveWindow (const int step, bool move_j) #endif if (lev > 0) { // coarse grid - shiftMF(*Bfield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost, + shiftMF(*m_fields.get("Bfield_cp",Direction{0},lev), geom[lev-1], num_shift_crse, dir, lev, do_update_cost, m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); - shiftMF(*m_fields.get("Efield_cp",Direction{0},lev), geom[lev], num_shift, dir, lev, do_update_cost, + shiftMF(*m_fields.get("Efield_cp",Direction{0},lev), geom[lev-1], num_shift, dir, lev, do_update_cost, m_p_ext_field_params->E_external_grid[dim], use_Eparser, Efield_parser); shiftMF(*Bfield_aux[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost); shiftMF(*Efield_aux[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost); diff --git a/Source/WarpX.H b/Source/WarpX.H index 07ace4ece5d..afb3ef0105b 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1562,12 +1562,6 @@ private: //EB level set amrex::Vector > m_distance_to_eb; - // store fine patch - - // Coarse patch - amrex::Vector, 3 > > Efield_cp; - amrex::Vector, 3 > > Bfield_cp; - // Copy of the coarse aux amrex::Vector > current_buffer_masks; amrex::Vector > gather_buffer_masks; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 2897adeb3b6..0c0ff7a9155 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -345,9 +345,6 @@ WarpX::WarpX () m_hybrid_pic_model = std::make_unique(nlevs_max); } - Efield_cp.resize(nlevs_max); - Bfield_cp.resize(nlevs_max); - current_buffer_masks.resize(nlevs_max); gather_buffer_masks.resize(nlevs_max); @@ -2650,9 +2647,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm const std::array cdx = CellSize(lev-1); // Create the MultiFabs for B - AllocInitMultiFab(Bfield_cp[lev][0], amrex::convert(cba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_cp[x]", 0.0_rt); - AllocInitMultiFab(Bfield_cp[lev][1], amrex::convert(cba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_cp[y]", 0.0_rt); - AllocInitMultiFab(Bfield_cp[lev][2], amrex::convert(cba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_cp[z]", 0.0_rt); + m_fields.alloc_init( "Bfield_cp", Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Bfield_cp", Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Bfield_cp", Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); // Create the MultiFabs for E m_fields.alloc_init( "Efield_cp", Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); @@ -3326,7 +3323,7 @@ WarpX::StoreCurrent (int lev) using ablastr::fields::Direction; for (int idim = 0; idim < 3; ++idim) { if (m_fields.get("current_store", Direction{idim},lev)) { - MultiFab::Copy(*m_fields.get("current_store", Direction{idim}, lev), + MultiFab::Copy(*m_fields.get("current_store", Direction{idim}, lev), *m_fields.get("current_fp", Direction{idim}, lev), 0, 0, 1, m_fields.get("current_store", Direction{idim}, lev)->nGrowVect()); } @@ -3452,29 +3449,6 @@ WarpX::AllocInitMultiFabFromModel ( multifab_map[name_with_suffix] = mf.get(); } -amrex::MultiFab* -WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, const int direction) const -{ - // This function does *not* check if the returned field pointer is != nullptr - - amrex::MultiFab* field_pointer = nullptr; - - switch(field_type) - { - case FieldType::Efield_cp : - field_pointer = Efield_cp[lev][direction].get(); - break; - case FieldType::Bfield_cp : - field_pointer = Bfield_cp[lev][direction].get(); - break; - default: - WARPX_ABORT_WITH_MESSAGE("Invalid field type"); - break; - } - - return field_pointer; -} - bool WarpX::isFieldInitialized (const FieldType field_type, const int lev, const int direction) const { @@ -3538,21 +3512,6 @@ WarpX::MakeDistributionMap (int lev, amrex::BoxArray const& ba) } } -const amrex::Vector,3>>& -WarpX::getMultiLevelField(warpx::fields::FieldType field_type) const -{ - switch(field_type) - { - case FieldType::Efield_cp : - return Efield_cp; - case FieldType::Bfield_cp : - return Bfield_cp; - default: - WARPX_ABORT_WITH_MESSAGE("Invalid field type"); - return Efield_cp; - } -} - const amrex::iMultiFab* WarpX::getFieldDotMaskPointer ( FieldType field_type, int lev, int dir ) const { From 5b9068014afcccbc29f89462a4bd225930c18c34 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 11 Sep 2024 17:55:15 -0700 Subject: [PATCH 149/314] Remove Unused `_slice` MultiFabs. Old diags or so, unused vars. --- Source/WarpX.H | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/Source/WarpX.H b/Source/WarpX.H index 07ace4ece5d..6ff7466d4d2 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1699,17 +1699,11 @@ private: guardCellManager guard_cells; - //Slice Parameters + // Slice Parameters int slice_max_grid_size; int slice_plot_int = -1; amrex::RealBox slice_realbox; amrex::IntVect slice_cr_ratio; - amrex::Vector< std::unique_ptr > F_slice; - amrex::Vector< std::unique_ptr > G_slice; - amrex::Vector< std::unique_ptr > rho_slice; - amrex::Vector, 3 > > current_slice; - amrex::Vector, 3 > > Efield_slice; - amrex::Vector, 3 > > Bfield_slice; bool fft_periodic_single_box = false; int nox_fft = 16; From b56bedf6be565fd5517c3cf1723ef5defe36b146 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 17:57:48 -0700 Subject: [PATCH 150/314] Remove Efield_cp, Bfield_cp --- Source/Initialization/WarpXInitData.cpp | 12 ++++++------ Source/Parallelization/WarpXComm.cpp | 2 ++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 4409c50fbc7..6273b5fb6fc 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -980,9 +980,9 @@ WarpX::InitLevelData (int lev, Real /*time*/) lev, PatchType::fine); InitializeExternalFieldsOnGridUsingParser( - m_fields.get_alldirs("Bfield_cp", Direction{0}, lev), - m_fields.get_alldirs("Bfield_cp", Direction{1}, lev), - m_fields.get_alldirs("Bfield_cp", Direction{2}, lev), + m_fields.get("Bfield_cp", Direction{0}, lev), + m_fields.get("Bfield_cp", Direction{1}, lev), + m_fields.get("Bfield_cp", Direction{2}, lev), m_p_ext_field_params->Bxfield_parser->compile<3>(), m_p_ext_field_params->Byfield_parser->compile<3>(), m_p_ext_field_params->Bzfield_parser->compile<3>(), @@ -1027,9 +1027,9 @@ WarpX::InitLevelData (int lev, Real /*time*/) lev, PatchType::fine); InitializeExternalFieldsOnGridUsingParser( - m_fields.get_alldirs("Efield_cp", Direction{0}, lev), - m_fields.get_alldirs("Efield_cp", Direction{1}, lev), - m_fields.get_alldirs("Efield_cp", Direction{2}, lev), + m_fields.get("Efield_cp", Direction{0}, lev), + m_fields.get("Efield_cp", Direction{1}, lev), + m_fields.get("Efield_cp", Direction{2}, lev), m_p_ext_field_params->Exfield_parser->compile<3>(), m_p_ext_field_params->Eyfield_parser->compile<3>(), m_p_ext_field_params->Ezfield_parser->compile<3>(), diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 90adb6c75d5..c46699ea82a 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -104,6 +104,8 @@ WarpX::UpdateAuxilaryDataStagToNodal () ablastr::fields::MultiLevelVectorField const& Efield_fp = m_fields.get_mr_levels_alldirs("Efield_fp", finest_level); ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); + ablastr::fields::MultiLevelVectorField const& Efield_cp = m_fields.get_mr_levels_alldirs("Efield_cp", finest_level); + ablastr::fields::MultiLevelVectorField const& Bfield_cp = m_fields.get_mr_levels_alldirs("Bfield_cp", finest_level); ablastr::fields::MultiLevelVectorField const& Efield_avg_fp = m_fields.get_mr_levels_alldirs("Efield_avg_fp", finest_level); ablastr::fields::MultiLevelVectorField const& Bfield_avg_fp = m_fields.get_mr_levels_alldirs("Bfield_avg_fp", finest_level); ablastr::fields::MultiLevelVectorField const& Efield_aux = m_fields.get_mr_levels_alldirs("Efield_aux", finest_level); From e8408682aeb7888f43be27bca293f4444982eaeb Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 17:59:48 -0700 Subject: [PATCH 151/314] Remove getFieldPointerUnchecked --- Source/WarpX.H | 14 -------------- Source/WarpX.cpp | 16 ---------------- 2 files changed, 30 deletions(-) diff --git a/Source/WarpX.H b/Source/WarpX.H index afb3ef0105b..d00753cad77 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1566,20 +1566,6 @@ private: amrex::Vector > current_buffer_masks; amrex::Vector > gather_buffer_masks; - /** - * \brief - * Get a pointer to the field data. Does not check if the pointer - * is not nullptr. - * - * \param field_type[in] the field type - * \param lev[in] the mesh refinement level - * \param direction[in] the field component (0 by default) - * - * \return the pointer to an amrex::MultiFab containing the field data - */ - [[nodiscard]] amrex::MultiFab* - getFieldPointerUnchecked (warpx::fields::FieldType field_type, int lev, int direction = 0) const; - // PML int do_pml = 0; int do_silver_mueller = 0; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 0c0ff7a9155..40d5a7be2d2 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -3449,22 +3449,6 @@ WarpX::AllocInitMultiFabFromModel ( multifab_map[name_with_suffix] = mf.get(); } -bool -WarpX::isFieldInitialized (const FieldType field_type, const int lev, const int direction) const -{ - const bool is_field_init = (getFieldPointerUnchecked(field_type, lev, direction) != nullptr); - return is_field_init; -} - -amrex::MultiFab* -WarpX::getFieldPointer (const FieldType field_type, const int lev, const int direction) const -{ - auto* const field_pointer = getFieldPointerUnchecked(field_type, lev, direction); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - field_pointer != nullptr, "Requested field is not initialized!"); - return field_pointer; -} - std::array WarpX::getFieldPointerArray (const FieldType field_type, const int lev) const { From 5e8200157c88ded66b1f52ef37a71812b19b7d40 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 18:00:42 -0700 Subject: [PATCH 152/314] Remove isFieldInitialized --- Source/WarpX.H | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/Source/WarpX.H b/Source/WarpX.H index d00753cad77..c9b5664f751 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -477,19 +477,6 @@ public: static std::map multifab_map; static std::map imultifab_map; - /** - * \brief - * Check if a field is initialized. - * - * \param field_type[in] the field type - * \param lev[in] the mesh refinement level - * \param direction[in] the field component (0 by default) - * - * \return true if the field is initialized, false otherwise - */ - [[nodiscard]] bool - isFieldInitialized (warpx::fields::FieldType field_type, int lev, int direction = 0) const; - /** * \brief * Get a pointer to the field data. From 4290c0cecb0e74dee8298de2eac3184898f08b5a Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 11 Sep 2024 18:11:43 -0700 Subject: [PATCH 153/314] Magnetostatic Solver `Vector_potential_grad_buf_e/b_stag` --- .../MagnetostaticSolver/MagnetostaticSolver.H | 18 +++--- .../MagnetostaticSolver.cpp | 61 ++++++++++--------- Source/WarpX.H | 5 -- Source/WarpX.cpp | 37 ++++------- 4 files changed, 54 insertions(+), 67 deletions(-) diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H index 1df9efea296..c07551c165c 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H @@ -7,6 +7,8 @@ #ifndef WARPX_MAGNETOSTATICSOLVER_H_ #define WARPX_MAGNETOSTATICSOLVER_H_ +#include + #include #include #include @@ -34,23 +36,23 @@ namespace MagnetostaticSolver { */ class EBCalcBfromVectorPotentialPerLevel { private: - const ablastr::fields::MultiLevelVectorField& m_b_field; - const amrex::Vector, 3>>& m_grad_buf_e_stag; - const amrex::Vector, 3>>& m_grad_buf_b_stag; + ablastr::fields::MultiLevelVectorField m_b_field; + ablastr::fields::MultiLevelVectorField m_grad_buf_e_stag; + ablastr::fields::MultiLevelVectorField m_grad_buf_b_stag; public: - EBCalcBfromVectorPotentialPerLevel(const ablastr::fields::MultiLevelVectorField& b_field, - const amrex::Vector, 3>>& grad_buf_e_stag, - const amrex::Vector, 3>>& grad_buf_b_stag) + EBCalcBfromVectorPotentialPerLevel (ablastr::fields::MultiLevelVectorField const & b_field, + ablastr::fields::MultiLevelVectorField const & grad_buf_e_stag, + ablastr::fields::MultiLevelVectorField const & grad_buf_b_stag) : m_b_field(b_field), m_grad_buf_e_stag(grad_buf_e_stag), m_grad_buf_b_stag(grad_buf_b_stag) {} - void operator()(amrex::Array,3> & mlmg, int lev); + void operator() (amrex::Array, 3> & mlmg, int lev); // Function to perform interpolation from cell edges to cell faces - void doInterp(const std::unique_ptr &src, const std::unique_ptr &dst); + void doInterp (amrex::MultiFab & src, amrex::MultiFab & dst); }; } // namespace MagnetostaticSolver diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp index 5b952e84ba8..3487a7f3dd6 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp @@ -57,8 +57,6 @@ using namespace amrex; -//using ablastr::fields::Direction; // FIXME - void WarpX::ComputeMagnetostaticField() { @@ -154,7 +152,7 @@ WarpX::AddMagnetostaticFieldLabFrame() \param[in] verbosity The verbosity setting for the MLMG solver */ void -WarpX::computeVectorPotential (const ablastr::fields::MultiLevelVectorField& curr, +WarpX::computeVectorPotential (ablastr::fields::MultiLevelVectorField const& curr, ablastr::fields::MultiLevelVectorField const& A, Real const required_precision, Real absolute_tolerance, @@ -177,9 +175,12 @@ WarpX::computeVectorPotential (const ablastr::fields::MultiLevelVectorField& cur #if defined(AMREX_USE_EB) ablastr::fields::MultiLevelVectorField Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); - const std::optional post_A_calculation({Bfield_fp, - vector_potential_grad_buf_e_stag, - vector_potential_grad_buf_b_stag}); + const std::optional post_A_calculation( + { + Bfield_fp, + m_fields.get_mr_levels_alldirs("vector_potential_grad_buf_e_stag", finest_level), + m_fields.get_mr_levels_alldirs("vector_potential_grad_buf_b_stag", finest_level) + }); amrex::Vector factories; for (int lev = 0; lev <= finest_level; ++lev) { @@ -370,8 +371,8 @@ void MagnetostaticSolver::VectorPoissonBoundaryHandler::defineVectorPotentialBCs bcs_set = true; } -void MagnetostaticSolver::EBCalcBfromVectorPotentialPerLevel::doInterp(const std::unique_ptr &src, - const std::unique_ptr &dst) +void MagnetostaticSolver::EBCalcBfromVectorPotentialPerLevel::doInterp (amrex::MultiFab & src, + amrex::MultiFab & dst) { WarpX &warpx = WarpX::GetInstance(); @@ -387,20 +388,20 @@ void MagnetostaticSolver::EBCalcBfromVectorPotentialPerLevel::doInterp(const std amrex::Real const * stencil_coeffs_z = warpx.device_field_centering_stencil_coeffs_z.data(); // Synchronize the ghost cells, do halo exchange - ablastr::utils::communication::FillBoundary(*src, - src->nGrowVect(), + ablastr::utils::communication::FillBoundary(src, + src.nGrowVect(), WarpX::do_single_precision_comms); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*dst, TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(dst, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - IntVect const src_stag = src->ixType().toIntVect(); - IntVect const dst_stag = dst->ixType().toIntVect(); + IntVect const src_stag = src.ixType().toIntVect(); + IntVect const dst_stag = dst.ixType().toIntVect(); - Array4 const& src_arr = src->const_array(mfi); - Array4 const& dst_arr = dst->array(mfi); + Array4 const& src_arr = src.const_array(mfi); + Array4 const& dst_arr = dst.array(mfi); const Box bx = mfi.tilebox(); @@ -422,9 +423,9 @@ void MagnetostaticSolver::EBCalcBfromVectorPotentialPerLevel::operator()(amrex:: const amrex::Array buf_ptr = { #if defined(WARPX_DIM_3D) - m_grad_buf_e_stag[lev][0].get(), - m_grad_buf_e_stag[lev][1].get(), - m_grad_buf_e_stag[lev][2].get() + m_grad_buf_e_stag[lev][0], + m_grad_buf_e_stag[lev][1], + m_grad_buf_e_stag[lev][2] #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) m_grad_buf_e_stag[lev][0].get(), m_grad_buf_e_stag[lev][2].get() @@ -435,13 +436,13 @@ void MagnetostaticSolver::EBCalcBfromVectorPotentialPerLevel::operator()(amrex:: mlmg[0]->getGradSolution({buf_ptr}); // Interpolate dAx/dz to By grid buffer, then add to By - this->doInterp(m_grad_buf_e_stag[lev][2], - m_grad_buf_b_stag[lev][1]); + this->doInterp(*m_grad_buf_e_stag[lev][2], + *m_grad_buf_b_stag[lev][1]); MultiFab::Add(*(m_b_field[lev][1]), *(m_grad_buf_b_stag[lev][1]), 0, 0, 1, 0 ); // Interpolate dAx/dy to Bz grid buffer, then subtract from Bz - this->doInterp(m_grad_buf_e_stag[lev][1], - m_grad_buf_b_stag[lev][2]); + this->doInterp(*m_grad_buf_e_stag[lev][1], + *m_grad_buf_b_stag[lev][2]); m_grad_buf_b_stag[lev][2]->mult(-1._rt); MultiFab::Add(*(m_b_field[lev][2]), *(m_grad_buf_b_stag[lev][2]), 0, 0, 1, 0 ); @@ -449,13 +450,13 @@ void MagnetostaticSolver::EBCalcBfromVectorPotentialPerLevel::operator()(amrex:: mlmg[1]->getGradSolution({buf_ptr}); // Interpolate dAy/dx to Bz grid buffer, then add to Bz - this->doInterp(m_grad_buf_e_stag[lev][0], - m_grad_buf_b_stag[lev][2]); + this->doInterp(*m_grad_buf_e_stag[lev][0], + *m_grad_buf_b_stag[lev][2]); MultiFab::Add(*(m_b_field[lev][2]), *(m_grad_buf_b_stag[lev][2]), 0, 0, 1, 0 ); // Interpolate dAy/dz to Bx grid buffer, then subtract from Bx - this->doInterp(m_grad_buf_e_stag[lev][2], - m_grad_buf_b_stag[lev][0]); + this->doInterp(*m_grad_buf_e_stag[lev][2], + *m_grad_buf_b_stag[lev][0]); m_grad_buf_b_stag[lev][0]->mult(-1._rt); MultiFab::Add(*(m_b_field[lev][0]), *(m_grad_buf_b_stag[lev][0]), 0, 0, 1, 0 ); @@ -463,13 +464,13 @@ void MagnetostaticSolver::EBCalcBfromVectorPotentialPerLevel::operator()(amrex:: mlmg[2]->getGradSolution({buf_ptr}); // Interpolate dAz/dy to Bx grid buffer, then add to Bx - this->doInterp(m_grad_buf_e_stag[lev][1], - m_grad_buf_b_stag[lev][0]); + this->doInterp(*m_grad_buf_e_stag[lev][1], + *m_grad_buf_b_stag[lev][0]); MultiFab::Add(*(m_b_field[lev][0]), *(m_grad_buf_b_stag[lev][0]), 0, 0, 1, 0 ); // Interpolate dAz/dx to By grid buffer, then subtract from By - this->doInterp(m_grad_buf_e_stag[lev][0], - m_grad_buf_b_stag[lev][1]); + this->doInterp(*m_grad_buf_e_stag[lev][0], + *m_grad_buf_b_stag[lev][1]); m_grad_buf_b_stag[lev][1]->mult(-1._rt); MultiFab::Add(*(m_b_field[lev][1]), *(m_grad_buf_b_stag[lev][1]), 0, 0, 1, 0 ); } diff --git a/Source/WarpX.H b/Source/WarpX.H index 6ff7466d4d2..41c5e60a9bd 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1518,11 +1518,6 @@ private: mutable amrex::Vector,3 > > Afield_dotMask; mutable amrex::Vector< std::unique_ptr > phi_dotMask; - // Memory buffers for computing magnetostatic fields - // Vector Potential A and previous step. Time buffer needed for computing dA/dt to first order - amrex::Vector, 3 > > vector_potential_grad_buf_e_stag; - amrex::Vector, 3 > > vector_potential_grad_buf_b_stag; - /** EB: for every mesh face flag_info_face contains a: * * 0 if the face needs to be extended * * 1 if the face is large enough to lend area to other faces diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 2897adeb3b6..90cff70cbd7 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -322,13 +322,6 @@ WarpX::WarpX () Afield_dotMask.resize(nlevs_max); phi_dotMask.resize(nlevs_max); - // Only allocate vector potential arrays when using the Magnetostatic Solver - if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) - { - vector_potential_grad_buf_e_stag.resize(nlevs_max); - vector_potential_grad_buf_b_stag.resize(nlevs_max); - } - m_distance_to_eb.resize(nlevs_max); m_flag_info_face.resize(nlevs_max); m_flag_ext_face.resize(nlevs_max); @@ -2287,23 +2280,19 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) { - m_fields.alloc_init( "vector_potential_fp_nodal", Direction{0}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngRho, 0.0_rt); - m_fields.alloc_init( "vector_potential_fp_nodal", Direction{1}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngRho, 0.0_rt); - m_fields.alloc_init( "vector_potential_fp_nodal", Direction{2}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngRho, 0.0_rt); - - AllocInitMultiFab(vector_potential_grad_buf_e_stag[lev][0], amrex::convert(ba, Ex_nodal_flag), - dm, ncomps, ngEB, lev, "vector_potential_grad_buf_e_stag[x]", 0.0_rt); - AllocInitMultiFab(vector_potential_grad_buf_e_stag[lev][1], amrex::convert(ba, Ey_nodal_flag), - dm, ncomps, ngEB, lev, "vector_potential_grad_buf_e_stag[y]", 0.0_rt); - AllocInitMultiFab(vector_potential_grad_buf_e_stag[lev][2], amrex::convert(ba, Ez_nodal_flag), - dm, ncomps, ngEB, lev, "vector_potential_grad_buf_e_stag[z]", 0.0_rt); - - AllocInitMultiFab(vector_potential_grad_buf_b_stag[lev][0], amrex::convert(ba, Bx_nodal_flag), - dm, ncomps, ngEB, lev, "vector_potential_grad_buf_b_stag[x]", 0.0_rt); - AllocInitMultiFab(vector_potential_grad_buf_b_stag[lev][1], amrex::convert(ba, By_nodal_flag), - dm, ncomps, ngEB, lev, "vector_potential_grad_buf_b_stag[y]", 0.0_rt); - AllocInitMultiFab(vector_potential_grad_buf_b_stag[lev][2], amrex::convert(ba, Bz_nodal_flag), - dm, ncomps, ngEB, lev, "vector_potential_grad_buf_b_stag[z]", 0.0_rt); + m_fields.alloc_init("vector_potential_fp_nodal", Direction{0}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngRho, 0.0_rt); + m_fields.alloc_init("vector_potential_fp_nodal", Direction{1}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngRho, 0.0_rt); + m_fields.alloc_init("vector_potential_fp_nodal", Direction{2}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngRho, 0.0_rt); + + // Memory buffers for computing magnetostatic fields + // Vector Potential A and previous step. Time buffer needed for computing dA/dt to first order + m_fields.alloc_init("vector_potential_grad_buf_e_stag", Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("vector_potential_grad_buf_e_stag", Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("vector_potential_grad_buf_e_stag", Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + + m_fields.alloc_init("vector_potential_grad_buf_b_stag", Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("vector_potential_grad_buf_b_stag", Direction{1}, lev, amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("vector_potential_grad_buf_b_stag", Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); } // Allocate extra multifabs needed by the kinetic-fluid hybrid algorithm. From 40f7fe6f700e09625df55301976ee241857ac45a Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Thu, 12 Sep 2024 03:22:39 +0200 Subject: [PATCH 154/314] add area_mod to new field register --- .../EmbeddedBoundary/WarpXFaceExtensions.cpp | 17 +++++++++-------- Source/EmbeddedBoundary/WarpXInitEB.cpp | 8 ++++---- .../FiniteDifferenceSolver/EvolveB.cpp | 4 ++-- .../FiniteDifferenceSolver.H | 4 ++-- Source/FieldSolver/WarpXPushFieldsEM.cpp | 6 ++++-- Source/WarpX.H | 5 +---- Source/WarpX.cpp | 18 +++++++++++------- 7 files changed, 33 insertions(+), 29 deletions(-) diff --git a/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp b/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp index e42caaed6dd..b52ac88d289 100644 --- a/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp +++ b/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp @@ -471,11 +471,11 @@ WarpX::ComputeOneWayExtensions () amrex::Real* borrowing_area = borrowing.area.data(); int& vecs_size = borrowing.vecs_size; - auto const &S_mod = m_area_mod[maxLevel()][idim]->array(mfi); + auto const &S_mod = m_fields.get("area_mod", Direction{idim}, maxLevel())->array(mfi); - const auto &lx = m_fields.get_alldirs("edge_lengths", maxLevel())[0]->array(mfi); - const auto &ly = m_fields.get_alldirs("edge_lengths", maxLevel())[1]->array(mfi); - const auto &lz = m_fields.get_alldirs("edge_lengths", maxLevel())[2]->array(mfi); + const auto &lx = m_fields.get("edge_lengths", Direction{0}, maxLevel())->array(mfi); + const auto &ly = m_fields.get("edge_lengths", Direction{1}, maxLevel())->array(mfi); + const auto &lz = m_fields.get("edge_lengths", Direction{2}, maxLevel())->array(mfi); vecs_size = amrex::Scan::PrefixSum(ncells, [=] AMREX_GPU_DEVICE (int icell) { @@ -599,10 +599,11 @@ WarpX::ComputeEightWaysExtensions () amrex::Real* borrowing_area = borrowing.area.data(); int& vecs_size = borrowing.vecs_size; - auto const &S_mod = m_area_mod[maxLevel()][idim]->array(mfi); - const auto &lx = m_fields.get_alldirs("edge_lengths", maxLevel())[0]->array(mfi); - const auto &ly = m_fields.get_alldirs("edge_lengths", maxLevel())[1]->array(mfi); - const auto &lz = m_fields.get_alldirs("edge_lengths", maxLevel())[2]->array(mfi); + auto const &S_mod = m_fields.get("area_mod", Direction{idim}, maxLevel())->array(mfi); + + const auto &lx = m_fields.get("edge_lengths", Direction{0}, maxLevel())->array(mfi); + const auto &ly = m_fields.get("edge_lengths", Direction{1}, maxLevel())->array(mfi); + const auto &lz = m_fields.get("edge_lengths", Direction{2}, maxLevel())->array(mfi); vecs_size += amrex::Scan::PrefixSum(ncells, [=] AMREX_GPU_DEVICE (int icell){ diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index 4cea931b300..e8c14d5451c 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -345,10 +345,10 @@ WarpX::MarkCells(){ auto const &S = face_areas_idim_max_lev->array(mfi); auto const &flag_info_face = m_flag_info_face[maxLevel()][idim]->array(mfi); auto const &flag_ext_face = m_flag_ext_face[maxLevel()][idim]->array(mfi); - const auto &lx = m_fields.get_alldirs("edge_lengths", maxLevel())[0]->array(mfi); - const auto &ly = m_fields.get_alldirs("edge_lengths", maxLevel())[1]->array(mfi); - const auto &lz = m_fields.get_alldirs("edge_lengths", maxLevel())[2]->array(mfi); - auto const &mod_areas_dim = m_area_mod[maxLevel()][idim]->array(mfi); + const auto &lx = m_fields.get("edge_lengths", Direction{0}, maxLevel())->array(mfi); + const auto &ly = m_fields.get("edge_lengths", Direction{1}, maxLevel())->array(mfi); + const auto &lz = m_fields.get("edge_lengths", Direction{2}, maxLevel())->array(mfi); + auto const &mod_areas_dim = m_fields.get("area_mod", Direction{idim}, maxLevel())->array(mfi); const amrex::Real dx = cell_size[0]; const amrex::Real dy = cell_size[1]; diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp index ef842bffb3d..31248caacaf 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp @@ -52,7 +52,7 @@ void FiniteDifferenceSolver::EvolveB ( [[maybe_unused]] ablastr::fields::VectorField const& Efield, [[maybe_unused]] amrex::MultiFab const * Gfield, [[maybe_unused]] ablastr::fields::VectorField const& face_areas, - [[maybe_unused]] std::array< std::unique_ptr, 3 > const& area_mod, + [[maybe_unused]] ablastr::fields::VectorField const& area_mod, [[maybe_unused]] std::array< std::unique_ptr, 3 >& ECTRhofield, [[maybe_unused]] std::array< std::unique_ptr, 3 >& Venl, [[maybe_unused]] std::array< std::unique_ptr, 3 >& flag_info_cell, @@ -195,7 +195,7 @@ void FiniteDifferenceSolver::EvolveBCartesian ( void FiniteDifferenceSolver::EvolveBCartesianECT ( ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& face_areas, - std::array< std::unique_ptr, 3 > const& area_mod, + ablastr::fields::VectorField const& area_mod, std::array< std::unique_ptr, 3 >& ECTRhofield, std::array< std::unique_ptr, 3 >& Venl, std::array< std::unique_ptr, 3 >& flag_info_cell, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index ac36ed8afa5..1f80886ea62 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -56,7 +56,7 @@ class FiniteDifferenceSolver ablastr::fields::VectorField const& Efield, amrex::MultiFab const * Gfield, ablastr::fields::VectorField const& face_areas, - std::array< std::unique_ptr, 3 > const& area_mod, + ablastr::fields::VectorField const& area_mod, std::array< std::unique_ptr, 3 >& ECTRhofield, std::array< std::unique_ptr, 3 >& Venl, std::array< std::unique_ptr, 3 >& flag_info_cell, @@ -300,7 +300,7 @@ class FiniteDifferenceSolver void EvolveBCartesianECT ( ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& face_areas, - std::array< std::unique_ptr, 3 > const& area_mod, + ablastr::fields::VectorField const& area_mod, std::array< std::unique_ptr, 3 >& ECTRhofield, std::array< std::unique_ptr, 3 >& Venl, std::array< std::unique_ptr, 3 >& flag_info_cell, diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 231e4196bf2..17c017b0950 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -859,14 +859,16 @@ WarpX::EvolveB (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_typ m_fields.get_alldirs("Efield_fp",lev), m_fields.get("G_fp", lev), m_fields.get_alldirs("face_areas", lev), - m_area_mod[lev], ECTRhofield[lev], Venl[lev], + m_fields.get_alldirs("area_mod", lev), + ECTRhofield[lev], Venl[lev], m_flag_info_face[lev], m_borrowing[lev], lev, a_dt ); } else { m_fdtd_solver_cp[lev]->EvolveB( m_fields.get_alldirs("Bfield_cp",lev), m_fields.get_alldirs("Efield_cp",lev), m_fields.get("G_fp", lev), m_fields.get_alldirs("face_areas", lev), - m_area_mod[lev], ECTRhofield[lev], Venl[lev], + m_fields.get_alldirs("area_mod", lev), + ECTRhofield[lev], Venl[lev], m_flag_info_face[lev], m_borrowing[lev], lev, a_dt ); } diff --git a/Source/WarpX.H b/Source/WarpX.H index 41c5e60a9bd..7356d5c6080 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1532,10 +1532,7 @@ private: * and in WarpX::ComputeEightWaysExtensions * This is only used for the ECT solver.*/ amrex::Vector, 3 > > m_flag_ext_face; - /** EB: m_area_mod contains the modified areas of the mesh faces, i.e. if a face is enlarged it - * contains the area of the enlarged face - * This is only used for the ECT solver.*/ - amrex::Vector, 3 > > m_area_mod; + /** EB: m_borrowing contains the info about the enlarged cells, i.e. for every enlarged cell it * contains the info of which neighbors are being intruded (and the amount of borrowed area). * This is only used for the ECT solver.*/ diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 90cff70cbd7..6d2a430305c 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -326,7 +326,6 @@ WarpX::WarpX () m_flag_info_face.resize(nlevs_max); m_flag_ext_face.resize(nlevs_max); m_borrowing.resize(nlevs_max); - m_area_mod.resize(nlevs_max); ECTRhofield.resize(nlevs_max); Venl.resize(nlevs_max); @@ -2384,12 +2383,17 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm guard_cells.ng_FieldSolver, lev, "m_flag_ext_face[y]"); AllocInitMultiFab(m_flag_ext_face[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_flag_ext_face[z]"); - AllocInitMultiFab(m_area_mod[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, - guard_cells.ng_FieldSolver, lev, "m_area_mod[x]"); - AllocInitMultiFab(m_area_mod[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, - guard_cells.ng_FieldSolver, lev, "m_area_mod[y]"); - AllocInitMultiFab(m_area_mod[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, - guard_cells.ng_FieldSolver, lev, "m_area_mod[z]"); + + /** EB: area_mod contains the modified areas of the mesh faces, i.e. if a face is enlarged it + * contains the area of the enlarged face + * This is only used for the ECT solver.*/ + m_fields.alloc_init( "area_mod", Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + m_fields.alloc_init( "area_mod", Direction{1}, lev, amrex::convert(ba, By_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + m_fields.alloc_init( "area_mod", Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + m_borrowing[lev][0] = std::make_unique>( amrex::convert(ba, Bx_nodal_flag), dm); m_borrowing[lev][1] = std::make_unique>( From 88da1e28044c59e913732a5b3e7860b74cf636fe Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 18:22:57 -0700 Subject: [PATCH 155/314] Re-introduce getFieldPointerArray --- .../WarpXFieldBoundaries.cpp | 24 ++++++++--------- .../FlushFormats/FlushFormatPlotfile.cpp | 14 +++++++--- Source/WarpX.H | 14 ++++++++++ Source/WarpX.cpp | 27 +++++++++++++++++++ 4 files changed, 63 insertions(+), 16 deletions(-) diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index 5b4c3506a13..e56af6492a8 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -73,9 +73,9 @@ void WarpX::ApplyEfieldBoundary(const int lev, PatchType patch_type) } } else { PEC::ApplyPECtoEfield( - {getFieldPointer(FieldType::Efield_cp, lev, 0), - getFieldPointer(FieldType::Efield_cp, lev, 1), - getFieldPointer(FieldType::Efield_cp, lev, 2)}, + {m_fields.get("Efield_cp",Direction{0},lev), + m_fields.get("Efield_cp",Direction{1},lev), + m_fields.get("Efield_cp",Direction{2},lev)}, field_boundary_lo, field_boundary_hi, get_ng_fieldgather(), Geom(lev), lev, patch_type, ref_ratio); @@ -98,9 +98,9 @@ void WarpX::ApplyEfieldBoundary(const int lev, PatchType patch_type) m_fields.get("Efield_fp",Direction{1},lev), m_fields.get("Efield_fp",Direction{2},lev), lev); } else { - ApplyFieldBoundaryOnAxis(getFieldPointer(FieldType::Efield_cp, lev, 0), - getFieldPointer(FieldType::Efield_cp, lev, 1), - getFieldPointer(FieldType::Efield_cp, lev, 2), lev); + ApplyFieldBoundaryOnAxis(m_fields.get("Efield_cp",Direction{0},lev), + m_fields.get("Efield_cp",Direction{1},lev), + m_fields.get("Efield_cp",Direction{2},lev), lev); } #endif } @@ -120,9 +120,9 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d lev, patch_type, ref_ratio); } else { PEC::ApplyPECtoBfield( { - getFieldPointer(FieldType::Bfield_cp, lev, 0), - getFieldPointer(FieldType::Bfield_cp, lev, 1), - getFieldPointer(FieldType::Bfield_cp, lev, 2)}, + m_fields.get("Bfield_cp",Direction{0},lev), + m_fields.get("Bfield_cp",Direction{1},lev), + m_fields.get("Bfield_cp",Direction{2},lev) }, field_boundary_lo, field_boundary_hi, get_ng_fieldgather(), Geom(lev), lev, patch_type, ref_ratio); @@ -151,9 +151,9 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d m_fields.get("Bfield_fp",Direction{1},lev), m_fields.get("Bfield_fp",Direction{2},lev), lev); } else { - ApplyFieldBoundaryOnAxis(getFieldPointer(FieldType::Bfield_cp, lev, 0), - getFieldPointer(FieldType::Bfield_cp, lev, 1), - getFieldPointer(FieldType::Bfield_cp, lev, 2), lev); + ApplyFieldBoundaryOnAxis(m_fields.get("Bfield_cp",Direction{0},lev), + m_fields.get("Bfield_cp",Direction{1},lev), + m_fields.get("Bfield_cp",Direction{2},lev), lev); } #endif } diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index 55904f4d28d..9dc8d85d9d6 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -637,12 +637,18 @@ FlushFormatPlotfile::WriteAllRawFields( // Coarse path if (lev > 0) { WriteCoarseVector( "E", - warpx.getFieldPointer(FieldType::Efield_cp, lev, 0), warpx.getFieldPointer(FieldType::Efield_cp, lev, 1), warpx.getFieldPointer(FieldType::Efield_cp, lev, 2), - warpx.getFieldPointer(FieldType::Efield_fp, lev, 0), warpx.getFieldPointer(FieldType::Efield_fp, lev, 1), warpx.getFieldPointer(FieldType::Efield_fp, lev, 2), + warpx.m_fields.get("Efield_cp", Direction{0}, lev), + warpx.m_fields.get("Efield_cp", Direction{1}, lev), + warpx.m_fields.get("Efield_cp", Direction{2}, lev), + warpx.m_fields.get("Efield_fp", Direction{0}, lev), + warpx.m_fields.get("Efield_fp", Direction{1}, lev), + warpx.m_fields.get("Efield_fp", Direction{2}, lev), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards); WriteCoarseVector( "B", - warpx.getFieldPointer(FieldType::Bfield_cp, lev, 0), warpx.getFieldPointer(FieldType::Bfield_cp, lev, 1), warpx.getFieldPointer(FieldType::Bfield_cp, lev, 2), - warpx.m_fields.get("Bfield_fp", Direction{0}, lev), + warpx.m_fields.get("Bfield_cp", Direction{0}, lev), + warpx.m_fields.get("Bfield_cp", Direction{1}, lev), + warpx.m_fields.get("Bfield_cp", Direction{2}, lev), + warpx.m_fields.get("Bfield_cp", Direction{0}, lev), warpx.m_fields.get("Bfield_fp", Direction{1}, lev), warpx.m_fields.get("Bfield_fp", Direction{2}, lev), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards); diff --git a/Source/WarpX.H b/Source/WarpX.H index c9b5664f751..a57fd14c2bd 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1553,6 +1553,20 @@ private: amrex::Vector > current_buffer_masks; amrex::Vector > gather_buffer_masks; + /** + * \brief + * Get a pointer to the field data. Does not check if the pointer + * is not nullptr. + * + * \param field_type[in] the field type + * \param lev[in] the mesh refinement level + * \param direction[in] the field component (0 by default) + * + * \return the pointer to an amrex::MultiFab containing the field data + */ + [[nodiscard]] amrex::MultiFab* + getFieldPointerUnchecked (warpx::fields::FieldType field_type, int lev, int direction = 0) const; + // PML int do_pml = 0; int do_silver_mueller = 0; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 40d5a7be2d2..e260f4a9786 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -3449,6 +3449,33 @@ WarpX::AllocInitMultiFabFromModel ( multifab_map[name_with_suffix] = mf.get(); } + +amrex::MultiFab* +WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, const int direction) const +{ + // This function does *not* check if the returned field pointer is != nullptr + + amrex::MultiFab* field_pointer = nullptr; + + switch(field_type) + { + default: + WARPX_ABORT_WITH_MESSAGE("Invalid field type"); + break; + } + + return field_pointer; +} + +amrex::MultiFab* +WarpX::getFieldPointer (const FieldType field_type, const int lev, const int direction) const +{ + auto* const field_pointer = getFieldPointerUnchecked(field_type, lev, direction); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + field_pointer != nullptr, "Requested field is not initialized!"); + return field_pointer; +} + std::array WarpX::getFieldPointerArray (const FieldType field_type, const int lev) const { From bc72939abd94660184c308eab00a2a160810e446 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 11 Sep 2024 18:30:04 -0700 Subject: [PATCH 156/314] Fix Bugs: `mf_name(name, dir, level)` Init/Alias --- Source/ablastr/fields/MultiFabRegister.cpp | 48 ++++++++++++++++------ 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index e1c4f2e60d3..c4249595126 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -27,10 +27,13 @@ namespace ablastr::fields bool redistribute_on_remake ) { - name = mf_name(name, level); + // checks + if (has(name, level)) { + throw std::runtime_error("MultiFabRegister::alloc_init failed because " + name + " already exists."); + } - // Checks - // TODO: does the key already exist? error + // fully qualified name + name = mf_name(name, level); // allocate const auto tag = amrex::MFInfo().SetTag(name); @@ -76,10 +79,17 @@ namespace ablastr::fields bool redistribute_on_remake ) { - name = mf_name(name, level); + // checks + if (has(name, dir, level)) { + throw std::runtime_error( + "MultiFabRegister::alloc_init failed because " + + mf_name(name, dir, level) + + " already exists." + ); + } - // Checks - // TODO: does the key already exist? error + // fully qualified name + name = mf_name(name, dir, level); // allocate const auto tag = amrex::MFInfo().SetTag(name); @@ -133,12 +143,19 @@ namespace ablastr::fields std::optional initial_value ) { + // checks + if (has(new_name, level)) { + throw std::runtime_error( + "MultiFabRegister::alias_init failed because " + + mf_name(new_name, level) + + " already exists." + ); + } + + // fully qualified name new_name = mf_name(new_name, level); alias_name = mf_name(alias_name, level); - // Checks - // TODO: does the key already exist? error - MultiFabOwner & alias = m_mf_register[alias_name]; amrex::MultiFab & mf_alias = alias.m_mf; @@ -180,12 +197,19 @@ namespace ablastr::fields std::optional initial_value ) { + // checks + if (has(new_name, dir, level)) { + throw std::runtime_error( + "MultiFabRegister::alias_init failed because " + + mf_name(new_name, dir, level) + + " already exists." + ); + } + + // fully qualified name new_name = mf_name(new_name, dir, level); alias_name = mf_name(alias_name, dir, level); - // Checks - // TODO: does the key already exist? error - MultiFabOwner & alias = m_mf_register[alias_name]; amrex::MultiFab & mf_alias = alias.m_mf; From a9e05af12d5bc57f97a3993aeeab668d6d382aaa Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 18:34:28 -0700 Subject: [PATCH 157/314] Fix RZ error --- .../FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp index 3487a7f3dd6..c5df9a17e39 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp @@ -427,8 +427,8 @@ void MagnetostaticSolver::EBCalcBfromVectorPotentialPerLevel::operator()(amrex:: m_grad_buf_e_stag[lev][1], m_grad_buf_e_stag[lev][2] #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - m_grad_buf_e_stag[lev][0].get(), - m_grad_buf_e_stag[lev][2].get() + m_grad_buf_e_stag[lev][0], + m_grad_buf_e_stag[lev][2] #endif }; From 17bc4576f69b7de0a41034162bf7b56cf75f4553 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 18:35:41 -0700 Subject: [PATCH 158/314] Cleanup --- Source/WarpX.H | 19 ------------------- Source/WarpX.cpp | 19 ------------------- 2 files changed, 38 deletions(-) diff --git a/Source/WarpX.H b/Source/WarpX.H index 2480600ff9d..ab9d22ab805 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -452,25 +452,6 @@ public: const std::string& name, std::optional initial_value); - /** - * \brief - * Allocate the MultiFab so that is like the specified MultiFab (same ba and dm) - * and optionally initialize it. This also adds the MultiFab - * to the map of MultiFabs (used to ease the access to MultiFabs from the Python - * interface - * - * \param mf[out] The MultiFab unique pointer to be allocated - * \param mf_model[in] The MultiFab to model - * \param name[in] The name of the MultiFab to use in the map - * \param initial_value[in] The optional initial value - */ - static void AllocInitMultiFabFromModel ( - std::unique_ptr& mf, - amrex::MultiFab& mf_model, - int level, - const std::string& name, - std::optional initial_value = {}); - // Maps of all of the MultiFabs and iMultiFabs used (this can include MFs from other classes) // This is a convenience for the Python interface, allowing all MultiFabs // to be easily referenced from Python. diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 345d67da4ee..42ba12b1e99 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -3424,25 +3424,6 @@ WarpX::AliasInitMultiFab ( multifab_map[name_with_suffix] = mf.get(); } -void -WarpX::AllocInitMultiFabFromModel ( - std::unique_ptr& mf, - amrex::MultiFab& mf_model, - const int level, - const std::string& name, - std::optional initial_value) -{ - const auto name_with_suffix = TagWithLevelSuffix(name, level); - const auto tag = amrex::MFInfo().SetTag(name_with_suffix); - mf = std::make_unique(mf_model.boxArray(), mf_model.DistributionMap(), - mf_model.nComp(), mf_model.nGrowVect(), tag); - if (initial_value) { - mf->setVal(*initial_value); - } - multifab_map[name_with_suffix] = mf.get(); -} - - amrex::MultiFab* WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, const int direction) const { From 12596628a717456ee6774a53add31e236da27ae8 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 22:01:08 -0700 Subject: [PATCH 159/314] Fix SegFault for Efield_fp --- Source/WarpX.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 42ba12b1e99..9d3e7f747cc 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2564,9 +2564,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm m_fields.alloc_init("Efield_aux", Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); } else { // In this case, the aux grid is simply an alias of the fp grid (most common case in WarpX) - m_fields.alias_init("Efield_aux", "Efield_avg_fp", Direction{0}, lev, 0.0_rt); - m_fields.alias_init("Efield_aux", "Efield_avg_fp", Direction{1}, lev, 0.0_rt); - m_fields.alias_init("Efield_aux", "Efield_avg_fp", Direction{2}, lev, 0.0_rt); + m_fields.alias_init("Efield_aux", "Efield_fp", Direction{0}, lev, 0.0_rt); + m_fields.alias_init("Efield_aux", "Efield_fp", Direction{1}, lev, 0.0_rt); + m_fields.alias_init("Efield_aux", "Efield_fp", Direction{2}, lev, 0.0_rt); } } } else { From 7dc206ca71f316196ec06b7e812176714599f4b2 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 22:02:11 -0700 Subject: [PATCH 160/314] Use ignore_unused --- Source/WarpX.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 9d3e7f747cc..ffc3a3476cf 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -3431,6 +3431,8 @@ WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, cons amrex::MultiFab* field_pointer = nullptr; + amrex::ignore_unused(lev, direction); + switch(field_type) { default: From 7a22fd54a0d08231620e1aff14f989056e43f605 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Sep 2024 22:11:53 -0700 Subject: [PATCH 161/314] Avoid calling get for fields that do not exist --- Source/Parallelization/WarpXComm.cpp | 74 +++++++++++++--------------- 1 file changed, 34 insertions(+), 40 deletions(-) diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index c46699ea82a..9d2cac9ccfb 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -73,15 +73,14 @@ WarpX::UpdateAuxilaryData () auto Bfield_aux_lvl_1 = m_fields.get("Bfield_aux", Direction{1}, lev); auto Bfield_aux_lvl_2 = m_fields.get("Bfield_aux", Direction{2}, lev); - const auto& E_ext_lev = m_fields.get_alldirs("E_external_particle_field", lev); - const auto& B_ext_lev = m_fields.get_alldirs("B_external_particle_field", lev); - if (mypc->m_E_ext_particle_s == "read_from_file") { + const auto& E_ext_lev = m_fields.get_alldirs("E_external_particle_field", lev); amrex::MultiFab::Add(*Bfield_aux_lvl_0, *E_ext_lev[0], 0, 0, E_ext_lev[0]->nComp(), guard_cells.ng_FieldGather); amrex::MultiFab::Add(*Bfield_aux_lvl_1, *E_ext_lev[1], 0, 0, E_ext_lev[1]->nComp(), guard_cells.ng_FieldGather); amrex::MultiFab::Add(*Bfield_aux_lvl_2, *E_ext_lev[2], 0, 0, E_ext_lev[2]->nComp(), guard_cells.ng_FieldGather); } if (mypc->m_B_ext_particle_s == "read_from_file") { + const auto& B_ext_lev = m_fields.get_alldirs("B_external_particle_field", lev); amrex::MultiFab::Add(*Bfield_aux_lvl_0, *B_ext_lev[0], 0, 0, B_ext_lev[0]->nComp(), guard_cells.ng_FieldGather); amrex::MultiFab::Add(*Bfield_aux_lvl_1, *B_ext_lev[1], 0, 0, B_ext_lev[1]->nComp(), guard_cells.ng_FieldGather); amrex::MultiFab::Add(*Bfield_aux_lvl_2, *B_ext_lev[2], 0, 0, B_ext_lev[2]->nComp(), guard_cells.ng_FieldGather); @@ -196,10 +195,10 @@ WarpX::UpdateAuxilaryDataStagToNodal () { if (electromagnetic_solver_id != ElectromagneticSolverAlgo::None) { Array,3> Btmp; - if (Bfield_cax[lev][0]) { + if (m_fields.has("Bfield_cax", Direction{0}, lev)) { for (int i = 0; i < 3; ++i) { Btmp[i] = std::make_unique( - *Bfield_cax[lev][i], amrex::make_alias, 0, 1); + *m_fields.get("Bfield_cax", Direction{i}, lev), amrex::make_alias, 0, 1); } } else { const IntVect ngtmp = Bfield_aux[lev-1][0]->nGrowVect(); @@ -290,10 +289,10 @@ WarpX::UpdateAuxilaryDataStagToNodal () { if (electromagnetic_solver_id != ElectromagneticSolverAlgo::None) { Array,3> Etmp; - if (Efield_cax[lev][0]) { + if (m_fields.has("Efield_cax", Direction{0}, lev)) { for (int i = 0; i < 3; ++i) { Etmp[i] = std::make_unique( - *Efield_cax[lev][i], amrex::make_alias, 0, 1); + *m_fields.get("Efield_cax", Direction{i}, lev), amrex::make_alias, 0, 1); } } else { const IntVect ngtmp = Efield_aux[lev-1][0]->nGrowVect(); @@ -393,28 +392,20 @@ WarpX::UpdateAuxilaryDataSameType () using ablastr::fields::Direction; ablastr::fields::MultiLevelVectorField Efield_fp = m_fields.get_mr_levels_alldirs("Efield_fp", finest_level); ablastr::fields::MultiLevelVectorField Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); - ablastr::fields::MultiLevelVectorField Efield_cp = m_fields.get_mr_levels_alldirs("Efield_cp", finest_level); - ablastr::fields::MultiLevelVectorField Bfield_cp = m_fields.get_mr_levels_alldirs("Bfield_cp", finest_level); ablastr::fields::MultiLevelVectorField Efield_aux = m_fields.get_mr_levels_alldirs("Efield_aux", finest_level); ablastr::fields::MultiLevelVectorField Bfield_aux = m_fields.get_mr_levels_alldirs("Bfield_aux", finest_level); - ablastr::fields::MultiLevelVectorField Efield_cax = m_fields.get_mr_levels_alldirs("Efield_cax", finest_level); - ablastr::fields::MultiLevelVectorField Bfield_cax = m_fields.get_mr_levels_alldirs("Bfield_cax", finest_level); - ablastr::fields::MultiLevelVectorField Efield_avg_fp = m_fields.get_mr_levels_alldirs("Efield_avg_fp", finest_level); - ablastr::fields::MultiLevelVectorField Bfield_avg_fp = m_fields.get_mr_levels_alldirs("Bfield_avg_fp", finest_level); - ablastr::fields::MultiLevelVectorField Efield_avg_cp = m_fields.get_mr_levels_alldirs("Efield_avg_cp", finest_level); - ablastr::fields::MultiLevelVectorField Bfield_avg_cp = m_fields.get_mr_levels_alldirs("Bfield_avg_cp", finest_level); // Level 0: Copy from fine to aux // Note: in some configurations, Efield_aux/Bfield_aux and Efield_fp/Bfield_fp are simply aliases to the // same MultiFab object. MultiFab::Copy operation automatically detects this and does nothing in this case. if (WarpX::fft_do_time_averaging) { - MultiFab::Copy(*Efield_aux[0][0], *Efield_avg_fp[0][0], 0, 0, Efield_aux[0][0]->nComp(), ng_src); - MultiFab::Copy(*Efield_aux[0][1], *Efield_avg_fp[0][1], 0, 0, Efield_aux[0][1]->nComp(), ng_src); - MultiFab::Copy(*Efield_aux[0][2], *Efield_avg_fp[0][2], 0, 0, Efield_aux[0][2]->nComp(), ng_src); - MultiFab::Copy(*Bfield_aux[0][0], *Bfield_avg_fp[0][0], 0, 0, Bfield_aux[0][0]->nComp(), ng_src); - MultiFab::Copy(*Bfield_aux[0][1], *Bfield_avg_fp[0][1], 0, 0, Bfield_aux[0][1]->nComp(), ng_src); - MultiFab::Copy(*Bfield_aux[0][2], *Bfield_avg_fp[0][2], 0, 0, Bfield_aux[0][2]->nComp(), ng_src); + MultiFab::Copy(*Efield_aux[0][0], *m_fields.get("Efield_avg_fp", Direction{0}, 0), 0, 0, Efield_aux[0][0]->nComp(), ng_src); + MultiFab::Copy(*Efield_aux[0][1], *m_fields.get("Efield_avg_fp", Direction{1}, 0), 0, 0, Efield_aux[0][1]->nComp(), ng_src); + MultiFab::Copy(*Efield_aux[0][2], *m_fields.get("Efield_avg_fp", Direction{2}, 0), 0, 0, Efield_aux[0][2]->nComp(), ng_src); + MultiFab::Copy(*Bfield_aux[0][0], *m_fields.get("Bfield_avg_fp", Direction{0}, 0), 0, 0, Bfield_aux[0][0]->nComp(), ng_src); + MultiFab::Copy(*Bfield_aux[0][1], *m_fields.get("Bfield_avg_fp", Direction{1}, 0), 0, 0, Bfield_aux[0][1]->nComp(), ng_src); + MultiFab::Copy(*Bfield_aux[0][2], *m_fields.get("Bfield_avg_fp", Direction{2}, 0), 0, 0, Bfield_aux[0][2]->nComp(), ng_src); } else { @@ -428,19 +419,19 @@ WarpX::UpdateAuxilaryDataSameType () for (int lev = 1; lev <= finest_level; ++lev) { const amrex::Periodicity& crse_period = Geom(lev-1).periodicity(); - const IntVect& ng = Bfield_cp[lev][0]->nGrowVect(); - const DistributionMapping& dm = Bfield_cp[lev][0]->DistributionMap(); + const IntVect& ng = m_fields.get("Bfield_cp", Direction{0}, lev)->nGrowVect(); + const DistributionMapping& dm = m_fields.get("Bfield_cp", Direction{0}, lev)->DistributionMap(); // B field { if (electromagnetic_solver_id != ElectromagneticSolverAlgo::None) { - MultiFab dBx(Bfield_cp[lev][0]->boxArray(), dm, Bfield_cp[lev][0]->nComp(), ng); - MultiFab dBy(Bfield_cp[lev][1]->boxArray(), dm, Bfield_cp[lev][1]->nComp(), ng); - MultiFab dBz(Bfield_cp[lev][2]->boxArray(), dm, Bfield_cp[lev][2]->nComp(), ng); - dBx.setVal(0.0); - dBy.setVal(0.0); - dBz.setVal(0.0); + MultiFab dBx(m_fields.get("Bfield_cp",Direction{0},lev)->boxArray(), dm, + m_fields.get("Bfield_cp",Direction{0},lev)->nComp(), ng); + MultiFab dBy(m_fields.get("Bfield_cp",Direction{1},lev)->boxArray(), dm, + m_fields.get("Bfield_cp",Direction{1},lev)->nComp(), ng); + MultiFab dBz(m_fields.get("Bfield_cp",Direction{2},lev)->boxArray(), dm, + m_fields.get("Bfield_cp",Direction{2},lev)->nComp(), ng); // Copy Bfield_aux to the dB MultiFabs, using up to ng_src (=ng_FieldGather) guard // cells from Bfield_aux and filling up to ng (=nGrow) guard cells in the dB MultiFabs @@ -455,15 +446,18 @@ WarpX::UpdateAuxilaryDataSameType () Bfield_aux[lev - 1][2]->nComp(), ng_src, ng, WarpX::do_single_precision_comms, crse_period); - if (Bfield_cax[lev][0]) + if (m_fields.has("Bfield_cax", Direction{0}, lev)) { - MultiFab::Copy(*Bfield_cax[lev][0], dBx, 0, 0, Bfield_cax[lev][0]->nComp(), ng); - MultiFab::Copy(*Bfield_cax[lev][1], dBy, 0, 0, Bfield_cax[lev][1]->nComp(), ng); - MultiFab::Copy(*Bfield_cax[lev][2], dBz, 0, 0, Bfield_cax[lev][2]->nComp(), ng); + MultiFab::Copy(*m_fields.get("Bfield_cax", Direction{0}, lev), dBx, 0, 0, m_fields.get("Bfield_cax", Direction{0}, lev)->nComp(), ng); + MultiFab::Copy(*m_fields.get("Bfield_cax", Direction{1}, lev), dBy, 0, 0, m_fields.get("Bfield_cax", Direction{1}, lev)->nComp(), ng); + MultiFab::Copy(*m_fields.get("Bfield_cax", Direction{2}, lev), dBz, 0, 0, m_fields.get("Bfield_cax", Direction{2}, lev)->nComp(), ng); } - MultiFab::Subtract(dBx, *Bfield_cp[lev][0], 0, 0, Bfield_cp[lev][0]->nComp(), ng); - MultiFab::Subtract(dBy, *Bfield_cp[lev][1], 0, 0, Bfield_cp[lev][1]->nComp(), ng); - MultiFab::Subtract(dBz, *Bfield_cp[lev][2], 0, 0, Bfield_cp[lev][2]->nComp(), ng); + MultiFab::Subtract(dBx, *m_fields.get("Bfield_cp",Direction{0},lev), + 0, 0, m_fields.get("Bfield_cp",Direction{0},lev)->nComp(), ng); + MultiFab::Subtract(dBy, *m_fields.get("Bfield_cp",Direction{1},lev), + 0, 0, m_fields.get("Bfield_cp",Direction{1},lev)->nComp(), ng); + MultiFab::Subtract(dBz, *m_fields.get("Bfield_cp",Direction{2},lev), + 0, 0, m_fields.get("Bfield_cp",Direction{2},lev)->nComp(), ng); const amrex::IntVect& refinement_ratio = refRatio(lev-1); @@ -537,11 +531,11 @@ WarpX::UpdateAuxilaryDataSameType () WarpX::do_single_precision_comms, crse_period); - if (Efield_cax[lev][0]) + if (m_fields.has("Efield_cax", Direction{0}, lev)) { - MultiFab::Copy(*Efield_cax[lev][0], dEx, 0, 0, Efield_cax[lev][0]->nComp(), ng); - MultiFab::Copy(*Efield_cax[lev][1], dEy, 0, 0, Efield_cax[lev][1]->nComp(), ng); - MultiFab::Copy(*Efield_cax[lev][2], dEz, 0, 0, Efield_cax[lev][2]->nComp(), ng); + MultiFab::Copy(*m_fields.get("Efield_cax", Direction{0}, lev), dEx, 0, 0, m_fields.get("Efield_cax", Direction{0}, lev)->nComp(), ng); + MultiFab::Copy(*m_fields.get("Efield_cax", Direction{1}, lev), dEy, 0, 0, m_fields.get("Efield_cax", Direction{1}, lev)->nComp(), ng); + MultiFab::Copy(*m_fields.get("Efield_cax", Direction{2}, lev), dEz, 0, 0, m_fields.get("Efield_cax", Direction{2}, lev)->nComp(), ng); } MultiFab::Subtract(dEx, *m_fields.get("Efield_cp",Direction{0},lev), 0, 0, m_fields.get("Efield_cp",Direction{0},lev)->nComp(), ng); From c076b0c66078953d046374b7f2603327eb5348c5 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 12 Sep 2024 06:15:11 -0700 Subject: [PATCH 162/314] Temporary solution: return nullpointer when field is not registered --- Source/ablastr/fields/MultiFabRegister.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index c4249595126..3b8c383748a 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -317,7 +317,7 @@ namespace ablastr::fields name = mf_name(name, level); if (m_mf_register.count(name) == 0) { - throw std::runtime_error("MultiFabRegister::get name does not exist in register: " + name); + return nullptr; } amrex::MultiFab & mf = m_mf_register[name].m_mf; @@ -334,7 +334,7 @@ namespace ablastr::fields name = mf_name(name, dir, level); if (m_mf_register.count(name) == 0) { - throw std::runtime_error("MultiFabRegister::get name does not exist in register: " + name); + return nullptr; } amrex::MultiFab & mf = m_mf_register[name].m_mf; From e0dc73d1a8224aa337aa4bb5106760a32527e87f Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 12 Sep 2024 06:16:41 -0700 Subject: [PATCH 163/314] Fix bug when outputting the fields --- Source/Diagnostics/FullDiagnostics.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index 6fd6e2fd463..ce64a1effbc 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -181,9 +181,9 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) // Make sure all multifabs have the same number of components for (int dim=0; dim<3; dim++){ AMREX_ALWAYS_ASSERT( - warpx.m_fields.get("Efield_aux", Direction{lev}, dim)->nComp() == ncomp_multimodefab ); + warpx.m_fields.get("Efield_aux", Direction{dim}, lev)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( - warpx.m_fields.get("Bfield_aux", Direction{lev}, dim)->nComp() == ncomp_multimodefab ); + warpx.m_fields.get("Bfield_aux", Direction{dim}, lev)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( warpx.m_fields.get("current_fp", Direction{dim}, lev)->nComp() == ncomp_multimodefab ); } From c015421b826dc84d03d15333140195b86d8c05a3 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 12 Sep 2024 06:25:19 -0700 Subject: [PATCH 164/314] Correct typo in output --- Source/Diagnostics/FullDiagnostics.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index ce64a1effbc..493e996ae9f 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -741,7 +741,7 @@ FullDiagnostics::InitializeFieldFunctors (int lev) } else if ( m_varnames[comp] == "Bx" ){ m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{0}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "By" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Efield_aux", Direction{1}, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{1}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "jx" ){ m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, true, deposit_current); deposit_current = false; From e6fbb4f8fcc4972b4a23570e445d6adc34277a60 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 12 Sep 2024 06:40:25 -0700 Subject: [PATCH 165/314] Fix bug for vay deposition --- Source/WarpX.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index ffc3a3476cf..f7b74b14090 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2269,9 +2269,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Vay) { - m_fields.alloc_init( "current_fp_vay", Direction{0}, lev, amrex::convert(ba, jx_nodal_flag), dm, ncomps, ngJ, 0.0_rt); - m_fields.alloc_init( "current_fp_vay", Direction{1}, lev, amrex::convert(ba, jy_nodal_flag), dm, ncomps, ngJ, 0.0_rt); - m_fields.alloc_init( "current_fp_vay", Direction{2}, lev, amrex::convert(ba, jz_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init( "current_fp_vay", Direction{0}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init( "current_fp_vay", Direction{1}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init( "current_fp_vay", Direction{2}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngJ, 0.0_rt); } if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) From 9f88e83f288d7b77bfece175a012c3ad769f0c51 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Sep 2024 13:45:51 +0000 Subject: [PATCH 166/314] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp b/Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp index d153aed3db3..e3d419dbae9 100644 --- a/Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp +++ b/Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp @@ -345,7 +345,7 @@ WarpX::ProjectionCleanDivB() { } warpx::initialization::ProjectionDivCleaner dc("Bfield_fp_external"); - + dc.setSourceFromBfield(); dc.solve(); From 771ec1a573be1c38e295ba7e7afb629f0d336997 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 12 Sep 2024 08:58:13 -0700 Subject: [PATCH 167/314] `MultiFabRegister::alias_init`: Check Alias Exists --- Source/ablastr/fields/MultiFabRegister.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 3b8c383748a..961711c23dc 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -151,6 +151,13 @@ namespace ablastr::fields " already exists." ); } + if (!has(alias_name, level)) { + throw std::runtime_error( + "MultiFabRegister::alias_init failed because " + + mf_name(alias_name, level) + + " does not exist." + ); + } // fully qualified name new_name = mf_name(new_name, level); @@ -205,6 +212,13 @@ namespace ablastr::fields " already exists." ); } + if (!has(alias_name, dir, level)) { + throw std::runtime_error( + "MultiFabRegister::alias_init failed because " + + mf_name(alias_name, dir, level) + + " does not exist." + ); + } // fully qualified name new_name = mf_name(new_name, dir, level); @@ -317,6 +331,7 @@ namespace ablastr::fields name = mf_name(name, level); if (m_mf_register.count(name) == 0) { + // FIXME: temporary, throw a std::runtime_error return nullptr; } amrex::MultiFab & mf = m_mf_register[name].m_mf; @@ -334,6 +349,7 @@ namespace ablastr::fields name = mf_name(name, dir, level); if (m_mf_register.count(name) == 0) { + // FIXME: temporary, throw a std::runtime_error return nullptr; } amrex::MultiFab & mf = m_mf_register[name].m_mf; From 6233de4e1758a0b7e6affadd4463cbacbacbe3e0 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 12 Sep 2024 09:30:17 -0700 Subject: [PATCH 168/314] Fix typo in moving window --- Source/Utils/WarpXMovingWindow.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index e82d4bd0e88..5488c2238fd 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -256,7 +256,7 @@ WarpX::MoveWindow (const int step, bool move_j) m_p_ext_field_params-> E_external_grid[dim], use_Eparser, Efield_parser); } if (move_j) { - shiftMF(*m_fields.get("current_cp", Direction{dim}, lev), geom[lev], num_shift, dir, lev, do_update_cost); + shiftMF(*m_fields.get("current_fp", Direction{dim}, lev), geom[lev], num_shift, dir, lev, do_update_cost); } if (pml[lev] && pml[lev]->ok()) { const std::array& pml_B = pml[lev]->GetB_fp(); From e198bd8aba0fc5fcac68b5df140fb91d8056b6aa Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Thu, 12 Sep 2024 19:03:03 +0200 Subject: [PATCH 169/314] use multifabregister for ECTRhofield --- .../FiniteDifferenceSolver/EvolveB.cpp | 4 ++-- .../FiniteDifferenceSolver/EvolveE.cpp | 2 +- .../FiniteDifferenceSolver/EvolveECTRho.cpp | 4 ++-- .../FiniteDifferenceSolver.H | 10 +++++----- .../ImplicitSolvers/WarpXImplicitOps.cpp | 4 ++-- Source/FieldSolver/WarpXPushFieldsEM.cpp | 16 +++++++++------ Source/Initialization/WarpXInitData.cpp | 6 ++++-- Source/WarpX.H | 8 -------- Source/WarpX.cpp | 20 ++++++++++++------- 9 files changed, 39 insertions(+), 35 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp index 31248caacaf..aad79ebf2ae 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp @@ -53,7 +53,7 @@ void FiniteDifferenceSolver::EvolveB ( [[maybe_unused]] amrex::MultiFab const * Gfield, [[maybe_unused]] ablastr::fields::VectorField const& face_areas, [[maybe_unused]] ablastr::fields::VectorField const& area_mod, - [[maybe_unused]] std::array< std::unique_ptr, 3 >& ECTRhofield, + [[maybe_unused]] ablastr::fields::VectorField const& ECTRhofield, [[maybe_unused]] std::array< std::unique_ptr, 3 >& Venl, [[maybe_unused]] std::array< std::unique_ptr, 3 >& flag_info_cell, [[maybe_unused]] std::array< std::unique_ptr >, 3 >& borrowing, @@ -196,7 +196,7 @@ void FiniteDifferenceSolver::EvolveBCartesianECT ( ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& face_areas, ablastr::fields::VectorField const& area_mod, - std::array< std::unique_ptr, 3 >& ECTRhofield, + ablastr::fields::VectorField const& ECTRhofield, std::array< std::unique_ptr, 3 >& Venl, std::array< std::unique_ptr, 3 >& flag_info_cell, std::array< std::unique_ptr >, 3 >& borrowing, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp index d833c7dbebd..ff24a289915 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp @@ -55,7 +55,7 @@ void FiniteDifferenceSolver::EvolveE ( ablastr::fields::VectorField const& Jfield, VectorField const& edge_lengths, VectorField const& face_areas, - std::array< std::unique_ptr, 3 >& ECTRhofield, + VectorField const& ECTRhofield, amrex::MultiFab const* Ffield, int lev, amrex::Real const dt ) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp index fc4bebc13d5..b2102a93130 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp @@ -51,7 +51,7 @@ void FiniteDifferenceSolver::EvolveECTRho ( ablastr::fields::VectorField const Efield, ablastr::fields::VectorField const& edge_lengths, ablastr::fields::VectorField const& face_areas, - std::array< std::unique_ptr, 3 >& ECTRhofield, + ablastr::fields::VectorField const& ECTRhofield, const int lev) { #if !defined(WARPX_DIM_RZ) and defined(AMREX_USE_EB) @@ -71,7 +71,7 @@ void FiniteDifferenceSolver::EvolveRhoCartesianECT ( ablastr::fields::VectorField const Efield, ablastr::fields::VectorField const& edge_lengths, ablastr::fields::VectorField const& face_areas, - std::array< std::unique_ptr, 3 >& ECTRhofield, const int lev ) { + ablastr::fields::VectorField const& ECTRhofield, const int lev ) { #ifdef AMREX_USE_EB #if !(defined(WARPX_DIM_3D) || defined(WARPX_DIM_XZ)) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 1f80886ea62..7f4a15ffa0d 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -57,7 +57,7 @@ class FiniteDifferenceSolver amrex::MultiFab const * Gfield, ablastr::fields::VectorField const& face_areas, ablastr::fields::VectorField const& area_mod, - std::array< std::unique_ptr, 3 >& ECTRhofield, + ablastr::fields::VectorField const& ECTRhofield, std::array< std::unique_ptr, 3 >& Venl, std::array< std::unique_ptr, 3 >& flag_info_cell, std::array< std::unique_ptr >, 3 >& borrowing, @@ -68,7 +68,7 @@ class FiniteDifferenceSolver ablastr::fields::VectorField const& Jfield, ablastr::fields::VectorField const& edge_lengths, ablastr::fields::VectorField const& face_areas, - std::array< std::unique_ptr, 3 >& ECTRhofield, + ablastr::fields::VectorField const& ECTRhofield, amrex::MultiFab const* Ffield, int lev, amrex::Real dt ); @@ -85,7 +85,7 @@ class FiniteDifferenceSolver void EvolveECTRho ( ablastr::fields::VectorField const Efield, ablastr::fields::VectorField const& edge_lengths, ablastr::fields::VectorField const& face_areas, - std::array< std::unique_ptr, 3 >& ECTRhofield, + ablastr::fields::VectorField const& ECTRhofield, int lev ); void ApplySilverMuellerBoundary ( @@ -295,13 +295,13 @@ class FiniteDifferenceSolver ablastr::fields::VectorField const Efield, ablastr::fields::VectorField const& edge_lengths, ablastr::fields::VectorField const& face_areas, - std::array< std::unique_ptr, 3 >& ECTRhofield, int lev); + ablastr::fields::VectorField const& ECTRhofield, int lev); void EvolveBCartesianECT ( ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& face_areas, ablastr::fields::VectorField const& area_mod, - std::array< std::unique_ptr, 3 >& ECTRhofield, + ablastr::fields::VectorField const& ECTRhofield, std::array< std::unique_ptr, 3 >& Venl, std::array< std::unique_ptr, 3 >& flag_info_cell, std::array< std::unique_ptr >, 3 >& borrowing, diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp index 24d3d2e0987..2e859c01877 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp @@ -342,7 +342,7 @@ WarpX::ImplicitComputeRHSE (int lev, PatchType patch_type, amrex::Real a_dt, War m_fields.get_alldirs("current_fp", lev), m_fields.get_alldirs("edge_lengths", lev), m_fields.get_alldirs("face_areas", lev), - ECTRhofield[lev], + m_fields.get_alldirs("ECTRhofield", lev), m_fields.get("F_fp", lev), lev, a_dt ); } else { m_fdtd_solver_cp[lev]->EvolveE( a_Erhs_vec.getArrayVec()[lev], @@ -350,7 +350,7 @@ WarpX::ImplicitComputeRHSE (int lev, PatchType patch_type, amrex::Real a_dt, War m_fields.get_alldirs("current_cp", lev), m_fields.get_alldirs("edge_lengths", lev), m_fields.get_alldirs("face_areas", lev), - ECTRhofield[lev], + m_fields.get_alldirs("ECTRhofield", lev), m_fields.get("F_cp", lev), lev, a_dt ); } diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 17c017b0950..19220e2cdc7 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -860,7 +860,8 @@ WarpX::EvolveB (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_typ m_fields.get("G_fp", lev), m_fields.get_alldirs("face_areas", lev), m_fields.get_alldirs("area_mod", lev), - ECTRhofield[lev], Venl[lev], + m_fields.get_alldirs("ECTRhofield", lev), + Venl[lev], m_flag_info_face[lev], m_borrowing[lev], lev, a_dt ); } else { m_fdtd_solver_cp[lev]->EvolveB( m_fields.get_alldirs("Bfield_cp",lev), @@ -868,7 +869,8 @@ WarpX::EvolveB (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_typ m_fields.get("G_fp", lev), m_fields.get_alldirs("face_areas", lev), m_fields.get_alldirs("area_mod", lev), - ECTRhofield[lev], Venl[lev], + m_fields.get_alldirs("ECTRhofield", lev), + Venl[lev], m_flag_info_face[lev], m_borrowing[lev], lev, a_dt ); } @@ -920,7 +922,7 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) m_fields.get_alldirs("current_fp", lev), m_fields.get_alldirs("edge_lengths", lev), m_fields.get_alldirs("face_areas", lev), - ECTRhofield[lev], + m_fields.get_alldirs("ECTRhofield", lev), m_fields.get("F_fp", lev), lev, a_dt ); } else { m_fdtd_solver_cp[lev]->EvolveE( m_fields.get_alldirs("Efield_cp",lev), @@ -928,7 +930,7 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) m_fields.get_alldirs("current_cp", lev), m_fields.get_alldirs("edge_lengths", lev), m_fields.get_alldirs("face_areas", lev), - ECTRhofield[lev], + m_fields.get_alldirs("ECTRhofield", lev), m_fields.get("F_cp", lev), lev, a_dt ); } @@ -961,12 +963,14 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) m_fdtd_solver_fp[lev]->EvolveECTRho( m_fields.get_alldirs("Efield_fp",lev), m_fields.get_alldirs("edge_lengths", lev), m_fields.get_alldirs("face_areas", lev), - ECTRhofield[lev], lev ); + m_fields.get_alldirs("ECTRhofield", lev), + lev ); } else { m_fdtd_solver_cp[lev]->EvolveECTRho( m_fields.get_alldirs("Efield_cp",lev), m_fields.get_alldirs("edge_lengths", lev), m_fields.get_alldirs("face_areas", lev), - ECTRhofield[lev], lev); + m_fields.get_alldirs("ECTRhofield", lev), + lev); } } #endif diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 6273b5fb6fc..59957ee312e 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -1008,7 +1008,8 @@ WarpX::InitLevelData (int lev, Real /*time*/) m_fields.get_alldirs("Efield_fp",lev), m_fields.get_alldirs("edge_lengths", lev), m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev], - ECTRhofield[lev], lev); + m_fields.get_alldirs("ECTRhofield", lev), + lev); } } #endif @@ -1045,7 +1046,8 @@ WarpX::InitLevelData (int lev, Real /*time*/) m_fields.get_alldirs("Efield_cp",lev), m_fields.get_alldirs("edge_lengths", lev), m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev], - ECTRhofield[lev], lev); + m_fields.get_alldirs("ECTRhofield", lev), + lev); } } #endif diff --git a/Source/WarpX.H b/Source/WarpX.H index ab9d22ab805..c48bc234ab1 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1506,14 +1506,6 @@ private: * This is only used for the ECT solver.*/ amrex::Vector >, 3 > > m_borrowing; - /** ECTRhofield is needed only by the ect - * solver and it contains the electromotive force density for every mesh face. - * The name ECTRhofield has been used to comply with the notation of the paper - * https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=4463918 (page 9, equation 4 - * and below). - * Although it's called rho it has nothing to do with the charge density! - * This is only used for the ECT solver.*/ - amrex::Vector, 3 > > ECTRhofield; /** Venl contains the electromotive force for every mesh face, i.e. every entry is * the corresponding entry in ECTRhofield multiplied by the total area (possibly with enlargement) * This is only used for the ECT solver.*/ diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index f7b74b14090..af67e414ceb 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -327,7 +327,6 @@ WarpX::WarpX () m_flag_ext_face.resize(nlevs_max); m_borrowing.resize(nlevs_max); - ECTRhofield.resize(nlevs_max); Venl.resize(nlevs_max); @@ -2404,12 +2403,19 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm AllocInitMultiFab(Venl[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "Venl[z]"); - AllocInitMultiFab(ECTRhofield[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, - guard_cells.ng_FieldSolver, lev, "ECTRhofield[x]", 0.0_rt); - AllocInitMultiFab(ECTRhofield[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, - guard_cells.ng_FieldSolver, lev, "ECTRhofield[y]", 0.0_rt); - AllocInitMultiFab(ECTRhofield[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, - guard_cells.ng_FieldSolver, lev, "ECTRhofield[z]", 0.0_rt); + /** ECTRhofield is needed only by the ect + * solver and it contains the electromotive force density for every mesh face. + * The name ECTRhofield has been used to comply with the notation of the paper + * https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=4463918 (page 9, equation 4 + * and below). + * Although it's called rho it has nothing to do with the charge density! + * This is only used for the ECT solver.*/ + m_fields.alloc_init( "ECTRhofield", Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + m_fields.alloc_init( "ECTRhofield", Direction{1}, lev, amrex::convert(ba, By_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + m_fields.alloc_init( "ECTRhofield", Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); } } } From f243375577ea30b509f6e589e3f94b9a514106c3 Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Thu, 12 Sep 2024 10:07:09 -0700 Subject: [PATCH 170/314] Intermediate refactoring of ImplicitSolvers. getFieldPointerArray() removed from WarpX. --- .../ImplicitSolvers/SemiImplicitEM.cpp | 2 +- .../ImplicitSolvers/ThetaImplicitEM.cpp | 2 +- .../ImplicitSolvers/WarpXSolverVec.H | 15 +++++-- .../ImplicitSolvers/WarpXSolverVec.cpp | 45 ++++++++++++++----- Source/WarpX.H | 12 ----- Source/WarpX.cpp | 15 ------- 6 files changed, 46 insertions(+), 45 deletions(-) diff --git a/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp index cfd18354878..e87324aa760 100644 --- a/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp +++ b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp @@ -20,7 +20,7 @@ void SemiImplicitEM::Define ( WarpX* a_WarpX ) m_WarpX = a_WarpX; // Define E and Eold vectors - m_E.Define( m_WarpX, FieldType::Efield_fp ); + m_E.Define( m_WarpX, "Efield_fp" ); m_Eold.Define( m_E ); // Parse implicit solver parameters diff --git a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp index 4c86389797f..5169b57b56d 100644 --- a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp +++ b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp @@ -21,7 +21,7 @@ void ThetaImplicitEM::Define ( WarpX* const a_WarpX ) m_WarpX = a_WarpX; // Define E and Eold vectors - m_E.Define( m_WarpX, FieldType::Efield_fp ); + m_E.Define( m_WarpX, "Efield_fp" ); m_Eold.Define( m_E ); // Define Bold MultiFab diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H index 5bf0a75691c..74e45ba631c 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H +++ b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H @@ -68,16 +68,16 @@ public: [[nodiscard]] inline bool IsDefined () const { return m_is_defined; } void Define ( WarpX* a_WarpX, - warpx::fields::FieldType a_array_type, - warpx::fields::FieldType a_scalar_type = warpx::fields::FieldType::None ); + std::string a_vector_type_name, + std::string a_scalar_type_name = "none" ); inline void Define ( const WarpXSolverVec& a_solver_vec ) { assertIsDefined( a_solver_vec ); Define( WarpXSolverVec::m_WarpX, - a_solver_vec.getArrayVecType(), - a_solver_vec.getScalarVecType() ); + a_solver_vec.getVectorType(), + a_solver_vec.getScalarType() ); } [[nodiscard]] RT dotProduct( const WarpXSolverVec& a_X ) const; @@ -281,6 +281,10 @@ public: [[nodiscard]] warpx::fields::FieldType getArrayVecType () const { return m_array_type; } [[nodiscard]] warpx::fields::FieldType getScalarVecType () const { return m_scalar_type; } + // solver vector type names + [[nodiscard]] std::string getVectorType () const { return m_vector_type_name; } + [[nodiscard]] std::string getScalarType () const { return m_scalar_type_name; } + private: @@ -292,6 +296,9 @@ private: warpx::fields::FieldType m_array_type = warpx::fields::FieldType::None; warpx::fields::FieldType m_scalar_type = warpx::fields::FieldType::None; + std::string m_vector_type_name = "none"; + std::string m_scalar_type_name = "none"; + static constexpr int m_ncomp = 1; static constexpr int m_num_amr_levels = 1; diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp index feaba87ee02..ee146d0ae29 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp @@ -20,9 +20,9 @@ WarpXSolverVec::~WarpXSolverVec () } } -void WarpXSolverVec::Define ( WarpX* a_WarpX, - FieldType a_array_type, - FieldType a_scalar_type ) +void WarpXSolverVec::Define ( WarpX* a_WarpX, + std::string a_vector_type_name, + std::string a_scalar_type_name ) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( !IsDefined(), @@ -34,8 +34,33 @@ void WarpXSolverVec::Define ( WarpX* a_WarpX, m_warpx_ptr_defined = true; } - m_array_type = a_array_type; - m_scalar_type = a_scalar_type; + m_vector_type_name = a_vector_type_name; + m_scalar_type_name = a_scalar_type_name; + + if (m_vector_type_name=="Efield_fp") { + m_array_type = FieldType::Efield_fp; + } + else if (m_vector_type_name=="Bfield_fp") { + m_array_type = FieldType::Efield_fp; + } + else if (m_vector_type_name=="vector_potential_fp") { + m_array_type = FieldType::vector_potential_fp; + } + else if (m_vector_type_name!="none") { + WARPX_ABORT_WITH_MESSAGE(a_vector_type_name + +"is not a valid option for array type used in Definining" + +"a WarpXSolverVec. Valid array types are: Efield_fp, Bfield_fp," + +"and vector_potential_fp"); + } + + if (m_scalar_type_name=="phi_fp") { + m_scalar_type = FieldType::phi_fp; + } + else if (m_scalar_type_name!="none") { + WARPX_ABORT_WITH_MESSAGE(a_scalar_type_name + +"is not a valid option for scalar type used in Definining" + +"a WarpXSolverVec. Valid scalar types are: phi_fp"); + } m_array_vec.resize(m_num_amr_levels); m_scalar_vec.resize(m_num_amr_levels); @@ -47,10 +72,8 @@ void WarpXSolverVec::Define ( WarpX* a_WarpX, isFieldArray(m_array_type), "WarpXSolverVec::Define() called with array_type not an array field"); - //m_array_vec.reserve(m_num_amr_levels); for (int lev = 0; lev < m_num_amr_levels; ++lev) { - using arr_mf_type = std::array; - const arr_mf_type this_array = m_WarpX->getFieldPointerArray(m_array_type, lev); + const ablastr::fields::VectorField this_array = m_WarpX->m_fields.get_alldirs(m_vector_type_name,lev); for (int n = 0; n < 3; n++) { m_array_vec[lev][n] = new amrex::MultiFab( this_array[n]->boxArray(), this_array[n]->DistributionMap(), @@ -68,9 +91,8 @@ void WarpXSolverVec::Define ( WarpX* a_WarpX, !isFieldArray(m_scalar_type), "WarpXSolverVec::Define() called with scalar_type not a scalar field "); - //m_scalar_vec.reserve(m_num_amr_levels); for (int lev = 0; lev < m_num_amr_levels; ++lev) { - const amrex::MultiFab* this_mf = m_WarpX->getFieldPointer(m_scalar_type,lev,0); + const amrex::MultiFab* this_mf = m_WarpX->m_fields.get(m_scalar_type_name,lev); m_scalar_vec[lev] = new amrex::MultiFab( this_mf->boxArray(), this_mf->DistributionMap(), this_mf->nComp(), @@ -100,8 +122,7 @@ void WarpXSolverVec::Copy ( FieldType a_array_type, for (int lev = 0; lev < m_num_amr_levels; ++lev) { if (m_array_type != FieldType::None) { - using arr_mf_type = std::array; - const arr_mf_type this_array = m_WarpX->getFieldPointerArray(m_array_type, lev); + const ablastr::fields::VectorField this_array = m_WarpX->m_fields.get_alldirs(m_vector_type_name,lev); for (int n = 0; n < 3; ++n) { amrex::MultiFab::Copy( *m_array_vec[lev][n], *this_array[n], 0, 0, m_ncomp, amrex::IntVect::TheZeroVector() ); diff --git a/Source/WarpX.H b/Source/WarpX.H index c48bc234ab1..35d3a0ef758 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -471,18 +471,6 @@ public: [[nodiscard]] amrex::MultiFab* getFieldPointer (warpx::fields::FieldType field_type, int lev, int direction = 0) const; - /** - * \brief - * For vector fields, get an array of three pointers to the field data. - * - * \param field_type[in] the field type - * \param lev[in] the mesh refinement level - * - * \return an array of three pointers amrex::MultiFab* containing the field data - */ - [[nodiscard]] std::array - getFieldPointerArray (warpx::fields::FieldType field_type, int lev) const; - /** * \brief * Get a constant reference to the field data. diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index af67e414ceb..c112b45487c 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -3458,21 +3458,6 @@ WarpX::getFieldPointer (const FieldType field_type, const int lev, const int dir return field_pointer; } -std::array -WarpX::getFieldPointerArray (const FieldType field_type, const int lev) const -{ - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - (field_type == FieldType::Efield_fp) || (field_type == FieldType::Bfield_fp) || - (field_type == FieldType::current_fp) || (field_type == FieldType::current_fp_nodal) || - (field_type == FieldType::Efield_cp) || (field_type == FieldType::Bfield_cp) || - (field_type == FieldType::current_cp), "Requested field type is not a vector."); - - return std::array{ - getFieldPointer(field_type, lev, 0), - getFieldPointer(field_type, lev, 1), - getFieldPointer(field_type, lev, 2)}; -} - const amrex::MultiFab& WarpX::getField(FieldType field_type, const int lev, const int direction) const { From bf6b5d36f5a0916f82dbd994e57d30ca7ea3f4be Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Thu, 12 Sep 2024 10:19:43 -0700 Subject: [PATCH 171/314] Add Python bindings --- Source/Python/MultiFabRegister.cpp | 104 +++++++++++++++++++++++++---- 1 file changed, 91 insertions(+), 13 deletions(-) diff --git a/Source/Python/MultiFabRegister.cpp b/Source/Python/MultiFabRegister.cpp index fd9d1201771..b7c531c905f 100644 --- a/Source/Python/MultiFabRegister.cpp +++ b/Source/Python/MultiFabRegister.cpp @@ -7,6 +7,9 @@ #include +#include +#include +#include void init_MultiFabRegister (py::module & m) { @@ -14,7 +17,54 @@ void init_MultiFabRegister (py::module & m) py::class_(m, "MultiFabRegister") .def("alloc_init", - &MultiFabRegister::alloc_init + py::overload_cast< + std::string, + int, + amrex::BoxArray const &, + amrex::DistributionMapping const &, + int, + amrex::IntVect const &, + std::optional, + bool, + bool + >(&MultiFabRegister::alloc_init) + // ... py::arg("name") + // "..." + ) + .def("alloc_init", + py::overload_cast< + std::string, + ablastr::fields::Direction, + int, + amrex::BoxArray const &, + amrex::DistributionMapping const &, + int, + amrex::IntVect const &, + std::optional, + bool, + bool + >(&MultiFabRegister::alloc_init) + // ... py::arg("name") + // "..." + ) + .def("alias_init", + py::overload_cast< + std::string, + std::string, + int, + std::optional + >(&MultiFabRegister::alias_init) + // ... py::arg("name") + // "..." + ) + .def("alias_init", + py::overload_cast< + std::string, + std::string, + ablastr::fields::Direction, + int, + std::optional + >(&MultiFabRegister::alias_init) // ... py::arg("name") // "..." ) @@ -24,21 +74,49 @@ void init_MultiFabRegister (py::module & m) py::arg("other_level") // "..." ) - .def("has", - &MultiFabRegister::has, - py::arg("name"), - py::arg("level") - // "..." - ) - .def("list", - &MultiFabRegister::list + //.def("has", + // py::overload_cast< + // std::string, + // int + // >(&MultiFabRegister::has) + // //py::arg("name"), + // // "..." + //) + //.def("has", + // py::overload_cast< + // std::string, + // ablastr::fields::Direction, + // int + // >(&MultiFabRegister::has) + // //py::arg("name"), + // // "..." + //) + .def("get", + py::overload_cast< + std::string, + int + >(&MultiFabRegister::get) + //py::arg("name"), // "..." ) - .def("erase", - &MultiFabRegister::erase, - py::arg("name"), - py::arg("level") + .def("get", + py::overload_cast< + std::string, + ablastr::fields::Direction, + int + >(&MultiFabRegister::get) + //py::arg("name"), // "..." ) + //.def("list", + // &MultiFabRegister::list + // // "..." + //) + //.def("erase", + // &MultiFabRegister::erase, + // py::arg("name"), + // py::arg("level") + // // "..." + //) ; } From fe37f928d86b9d8dc0d2b3d8b755bd4e60193fcd Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Thu, 12 Sep 2024 19:19:22 +0200 Subject: [PATCH 172/314] add Venl to field register --- .../FiniteDifferenceSolver/EvolveB.cpp | 4 ++-- .../FiniteDifferenceSolver.H | 4 ++-- Source/FieldSolver/WarpXPushFieldsEM.cpp | 4 ++-- Source/WarpX.H | 5 ----- Source/WarpX.cpp | 19 ++++++++++--------- 5 files changed, 16 insertions(+), 20 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp index aad79ebf2ae..5ed4a1053f2 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp @@ -54,7 +54,7 @@ void FiniteDifferenceSolver::EvolveB ( [[maybe_unused]] ablastr::fields::VectorField const& face_areas, [[maybe_unused]] ablastr::fields::VectorField const& area_mod, [[maybe_unused]] ablastr::fields::VectorField const& ECTRhofield, - [[maybe_unused]] std::array< std::unique_ptr, 3 >& Venl, + [[maybe_unused]] ablastr::fields::VectorField const& Venl, [[maybe_unused]] std::array< std::unique_ptr, 3 >& flag_info_cell, [[maybe_unused]] std::array< std::unique_ptr >, 3 >& borrowing, [[maybe_unused]] int lev, @@ -197,7 +197,7 @@ void FiniteDifferenceSolver::EvolveBCartesianECT ( ablastr::fields::VectorField const& face_areas, ablastr::fields::VectorField const& area_mod, ablastr::fields::VectorField const& ECTRhofield, - std::array< std::unique_ptr, 3 >& Venl, + ablastr::fields::VectorField const& Venl, std::array< std::unique_ptr, 3 >& flag_info_cell, std::array< std::unique_ptr >, 3 >& borrowing, const int lev, amrex::Real const dt ) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 7f4a15ffa0d..9fed0dc2b1e 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -58,7 +58,7 @@ class FiniteDifferenceSolver ablastr::fields::VectorField const& face_areas, ablastr::fields::VectorField const& area_mod, ablastr::fields::VectorField const& ECTRhofield, - std::array< std::unique_ptr, 3 >& Venl, + ablastr::fields::VectorField const& Venl, std::array< std::unique_ptr, 3 >& flag_info_cell, std::array< std::unique_ptr >, 3 >& borrowing, int lev, amrex::Real dt ); @@ -302,7 +302,7 @@ class FiniteDifferenceSolver ablastr::fields::VectorField const& face_areas, ablastr::fields::VectorField const& area_mod, ablastr::fields::VectorField const& ECTRhofield, - std::array< std::unique_ptr, 3 >& Venl, + ablastr::fields::VectorField const& Venl, std::array< std::unique_ptr, 3 >& flag_info_cell, std::array< std::unique_ptr >, 3 >& borrowing, int lev, amrex::Real dt diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 19220e2cdc7..4c72856b9c9 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -861,7 +861,7 @@ WarpX::EvolveB (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_typ m_fields.get_alldirs("face_areas", lev), m_fields.get_alldirs("area_mod", lev), m_fields.get_alldirs("ECTRhofield", lev), - Venl[lev], + m_fields.get_alldirs("Venl", lev), m_flag_info_face[lev], m_borrowing[lev], lev, a_dt ); } else { m_fdtd_solver_cp[lev]->EvolveB( m_fields.get_alldirs("Bfield_cp",lev), @@ -870,7 +870,7 @@ WarpX::EvolveB (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_typ m_fields.get_alldirs("face_areas", lev), m_fields.get_alldirs("area_mod", lev), m_fields.get_alldirs("ECTRhofield", lev), - Venl[lev], + m_fields.get_alldirs("Venl", lev), m_flag_info_face[lev], m_borrowing[lev], lev, a_dt ); } diff --git a/Source/WarpX.H b/Source/WarpX.H index 35d3a0ef758..edc19b19be0 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1494,11 +1494,6 @@ private: * This is only used for the ECT solver.*/ amrex::Vector >, 3 > > m_borrowing; - /** Venl contains the electromotive force for every mesh face, i.e. every entry is - * the corresponding entry in ECTRhofield multiplied by the total area (possibly with enlargement) - * This is only used for the ECT solver.*/ - amrex::Vector, 3 > > Venl; - //EB level set amrex::Vector > m_distance_to_eb; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index c112b45487c..0f4a32b2f44 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -327,9 +327,6 @@ WarpX::WarpX () m_flag_ext_face.resize(nlevs_max); m_borrowing.resize(nlevs_max); - Venl.resize(nlevs_max); - - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { // Create hybrid-PIC model object if needed @@ -2396,12 +2393,16 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm amrex::convert(ba, By_nodal_flag), dm); m_borrowing[lev][2] = std::make_unique>( amrex::convert(ba, Bz_nodal_flag), dm); - AllocInitMultiFab(Venl[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, - guard_cells.ng_FieldSolver, lev, "Venl[x]"); - AllocInitMultiFab(Venl[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, - guard_cells.ng_FieldSolver, lev, "Venl[y]"); - AllocInitMultiFab(Venl[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, - guard_cells.ng_FieldSolver, lev, "Venl[z]"); + + /** Venl contains the electromotive force for every mesh face, i.e. every entry is + * the corresponding entry in ECTRhofield multiplied by the total area (possibly with enlargement) + * This is only used for the ECT solver.*/ + m_fields.alloc_init( "Venl", Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + m_fields.alloc_init( "Venl", Direction{1}, lev, amrex::convert(ba, By_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); + m_fields.alloc_init( "Venl", Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), + dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); /** ECTRhofield is needed only by the ect * solver and it contains the electromotive force density for every mesh face. From b5cfcc09a8481c796673336eb4a1448852378848 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 12 Sep 2024 10:45:51 -0700 Subject: [PATCH 173/314] Fix J functor --- .../Diagnostics/ComputeDiagFunctors/JFunctor.cpp | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp index 10d575a63af..3011d81f825 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp @@ -41,12 +41,9 @@ JFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/ if (m_deposit_current) { // allocate temporary multifab to deposit current density into - using ablastr::fields::Direction; - ablastr::fields::MultiLevelVectorField current_fp_temp; - - warpx.m_fields.alias_init("current_fp_temp", "current_fp", Direction{0}, 0); - warpx.m_fields.alias_init("current_fp_temp", "current_fp", Direction{1}, 0); - warpx.m_fields.alias_init("current_fp_temp", "current_fp", Direction{2}, 0); + ablastr::fields::MultiLevelVectorField current_fp_temp { + warpx.m_fields.get_alldirs("current_fp_temp", m_lev) + }; auto& mypc = warpx.GetPartContainer(); mypc.DepositCurrent(current_fp_temp, warpx.getdt(m_lev), 0.0); @@ -56,11 +53,6 @@ JFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/ for (int idim = 0; idim < 3; ++idim) { current_fp_temp[0][idim]->FillBoundary(warpx.Geom(m_lev).periodicity()); } - - // remove aliases again - warpx.m_fields.erase("current_fp_temp", Direction{0}, 0); - warpx.m_fields.erase("current_fp_temp", Direction{1}, 0); - warpx.m_fields.erase("current_fp_temp", Direction{2}, 0); } WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_mf_src != nullptr, "m_mf_src can't be a nullptr."); From 4de536b1ec3e221c4b515e6c8730723b421cdc22 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 12 Sep 2024 11:03:53 -0700 Subject: [PATCH 174/314] Update J functor --- Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp index 3011d81f825..41bea408326 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp @@ -42,7 +42,7 @@ JFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/ { // allocate temporary multifab to deposit current density into ablastr::fields::MultiLevelVectorField current_fp_temp { - warpx.m_fields.get_alldirs("current_fp_temp", m_lev) + warpx.m_fields.get_alldirs("current_fp", m_lev) }; auto& mypc = warpx.GetPartContainer(); From 48ab69cd537699777b5f0b8f00f5e8783089b0ed Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Thu, 12 Sep 2024 11:08:56 -0700 Subject: [PATCH 175/314] Add Python bindings --- Source/Python/MultiFabRegister.cpp | 96 +++++++++++++++++++++--------- 1 file changed, 67 insertions(+), 29 deletions(-) diff --git a/Source/Python/MultiFabRegister.cpp b/Source/Python/MultiFabRegister.cpp index b7c531c905f..b3432eb7bf0 100644 --- a/Source/Python/MultiFabRegister.cpp +++ b/Source/Python/MultiFabRegister.cpp @@ -16,6 +16,7 @@ void init_MultiFabRegister (py::module & m) using ablastr::fields::MultiFabRegister; py::class_(m, "MultiFabRegister") + .def("alloc_init", py::overload_cast< std::string, @@ -27,10 +28,18 @@ void init_MultiFabRegister (py::module & m) std::optional, bool, bool - >(&MultiFabRegister::alloc_init) - // ... py::arg("name") - // "..." + >(&MultiFabRegister::alloc_init), + py::arg("name"), + py::arg("level"), + py::arg("ba"), + py::arg("dm"), + py::arg("ncomp"), + py::arg("ngrow"), + py::arg("initial_value"), + py::arg("redistribute"), + py::arg("redistribute_on_remake") ) + .def("alloc_init", py::overload_cast< std::string, @@ -43,10 +52,19 @@ void init_MultiFabRegister (py::module & m) std::optional, bool, bool - >(&MultiFabRegister::alloc_init) - // ... py::arg("name") - // "..." + >(&MultiFabRegister::alloc_init), + py::arg("name"), + py::arg("dir"), + py::arg("level"), + py::arg("ba"), + py::arg("dm"), + py::arg("ncomp"), + py::arg("ngrow"), + py::arg("initial_value"), + py::arg("redistribute"), + py::arg("redistribute_on_remake") ) + .def("alias_init", py::overload_cast< std::string, @@ -57,6 +75,7 @@ void init_MultiFabRegister (py::module & m) // ... py::arg("name") // "..." ) + .def("alias_init", py::overload_cast< std::string, @@ -68,29 +87,33 @@ void init_MultiFabRegister (py::module & m) // ... py::arg("name") // "..." ) + .def("alloc_like", &MultiFabRegister::alloc_like, py::arg("other_name"), py::arg("other_level") // "..." ) - //.def("has", - // py::overload_cast< - // std::string, - // int - // >(&MultiFabRegister::has) - // //py::arg("name"), - // // "..." - //) - //.def("has", - // py::overload_cast< - // std::string, - // ablastr::fields::Direction, - // int - // >(&MultiFabRegister::has) - // //py::arg("name"), - // // "..." - //) + + .def("has", + py::overload_cast< + std::string, + int + >(&MultiFabRegister::has) + //py::arg("name"), + // "..." + ) + + .def("has", + py::overload_cast< + std::string, + ablastr::fields::Direction, + int + >(&MultiFabRegister::has) + //py::arg("name"), + // "..." + ) + .def("get", py::overload_cast< std::string, @@ -99,6 +122,7 @@ void init_MultiFabRegister (py::module & m) //py::arg("name"), // "..." ) + .def("get", py::overload_cast< std::string, @@ -108,15 +132,29 @@ void init_MultiFabRegister (py::module & m) //py::arg("name"), // "..." ) + //.def("list", // &MultiFabRegister::list // // "..." //) - //.def("erase", - // &MultiFabRegister::erase, - // py::arg("name"), - // py::arg("level") - // // "..." - //) + + .def("erase", + py::overload_cast< + std::string, + int + >(&MultiFabRegister::erase) + //py::arg("name"), + // "..." + ) + + .def("erase", + py::overload_cast< + std::string, + ablastr::fields::Direction, + int + >(&MultiFabRegister::erase) + //py::arg("name"), + // "..." + ) ; } From a2ad435390d342df0c991c4979b3dbffdff25cbc Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 12 Sep 2024 12:50:52 -0700 Subject: [PATCH 176/314] Fix PML initialization --- Source/BoundaryConditions/PML.cpp | 6 +++--- Source/BoundaryConditions/PML_RZ.cpp | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index b9a9d0edaa0..2307547dac9 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -705,9 +705,9 @@ PML::PML (const int lev, const BoxArray& grid_ba, const amrex::BoxArray ba_Ex = amrex::convert(ba, warpx.m_fields.get("Efield_fp",Direction{0},0)->ixType().toIntVect()); const amrex::BoxArray ba_Ey = amrex::convert(ba, warpx.m_fields.get("Efield_fp",Direction{1},0)->ixType().toIntVect()); const amrex::BoxArray ba_Ez = amrex::convert(ba, warpx.m_fields.get("Efield_fp",Direction{2},0)->ixType().toIntVect()); - warpx.m_fields.alloc_init("pml_E_fp", Direction{0}, lev, ba_Ex, dm, ncompe, nge, 0.0_rt); - warpx.m_fields.alloc_init("pml_E_fp", Direction{1}, lev, ba_Ey, dm, ncompe, nge, 0.0_rt); - warpx.m_fields.alloc_init("pml_E_fp", Direction{2}, lev, ba_Ez, dm, ncompe, nge, 0.0_rt); + WarpX::AllocInitMultiFab(pml_E_fp[0], ba_Ex, dm, ncompe, nge, lev, "pml_E_fp[x]", 0.0_rt); + WarpX::AllocInitMultiFab(pml_E_fp[1], ba_Ey, dm, ncompe, nge, lev, "pml_E_fp[y]", 0.0_rt); + WarpX::AllocInitMultiFab(pml_E_fp[2], ba_Ez, dm, ncompe, nge, lev, "pml_E_fp[z]", 0.0_rt); const amrex::BoxArray ba_Bx = amrex::convert(ba, warpx.m_fields.get("Bfield_fp",Direction{0},0)->ixType().toIntVect()); const amrex::BoxArray ba_By = amrex::convert(ba, warpx.m_fields.get("Bfield_fp",Direction{1},0)->ixType().toIntVect()); diff --git a/Source/BoundaryConditions/PML_RZ.cpp b/Source/BoundaryConditions/PML_RZ.cpp index 10bcb990586..984df7e528f 100644 --- a/Source/BoundaryConditions/PML_RZ.cpp +++ b/Source/BoundaryConditions/PML_RZ.cpp @@ -49,8 +49,8 @@ PML_RZ::PML_RZ (const int lev, const amrex::BoxArray& grid_ba, const amrex::Dist const amrex::MultiFab & Et_fp = *warpx.m_fields.get("Efield_fp",Direction{1},lev); const amrex::BoxArray ba_Er = amrex::convert(grid_ba, Er_fp.ixType().toIntVect()); const amrex::BoxArray ba_Et = amrex::convert(grid_ba, Et_fp.ixType().toIntVect()); - warpx.m_fields.alloc_init("pml_E_fp", Direction{0}, lev, ba_Er, grid_dm, Er_fp.nComp(), Er_fp.nGrowVect(), 0.0_rt); - warpx.m_fields.alloc_init("pml_E_fp", Direction{1}, lev, ba_Et, grid_dm, Et_fp.nComp(), Et_fp.nGrowVect(), 0.0_rt); + WarpX::AllocInitMultiFab(pml_E_fp[0], ba_Er, grid_dm, Er_fp.nComp(), Er_fp.nGrowVect(), lev, "pml_B_fp[0]", 0.0_rt); + WarpX::AllocInitMultiFab(pml_E_fp[1], ba_Et, grid_dm, Et_fp.nComp(), Et_fp.nGrowVect(), lev, "pml_B_fp[1]", 0.0_rt); const amrex::MultiFab & Br_fp = *warpx.m_fields.get("Bfield_fp",Direction{0},lev); const amrex::MultiFab & Bt_fp = *warpx.m_fields.get("Bfield_fp",Direction{1},lev); From e25d806bd67e5df415f6199534df0c5f977d0901 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Thu, 12 Sep 2024 13:13:13 -0700 Subject: [PATCH 177/314] Add `py::arg` helper names to Python bindings --- Source/Python/MultiFabRegister.cpp | 57 +++++++++++++++++------------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/Source/Python/MultiFabRegister.cpp b/Source/Python/MultiFabRegister.cpp index b3432eb7bf0..ec565e4671e 100644 --- a/Source/Python/MultiFabRegister.cpp +++ b/Source/Python/MultiFabRegister.cpp @@ -71,9 +71,11 @@ void init_MultiFabRegister (py::module & m) std::string, int, std::optional - >(&MultiFabRegister::alias_init) - // ... py::arg("name") - // "..." + >(&MultiFabRegister::alias_init), + py::arg("new_name"), + py::arg("alias_name"), + py::arg("level"), + py::arg("initial_value") ) .def("alias_init", @@ -83,25 +85,27 @@ void init_MultiFabRegister (py::module & m) ablastr::fields::Direction, int, std::optional - >(&MultiFabRegister::alias_init) - // ... py::arg("name") - // "..." + >(&MultiFabRegister::alias_init), + py::arg("new_name"), + py::arg("alias_name"), + py::arg("dir"), + py::arg("level"), + py::arg("initial_value") ) .def("alloc_like", &MultiFabRegister::alloc_like, py::arg("other_name"), py::arg("other_level") - // "..." ) .def("has", py::overload_cast< std::string, int - >(&MultiFabRegister::has) - //py::arg("name"), - // "..." + >(&MultiFabRegister::has), + py::arg("name"), + py::arg("level") ) .def("has", @@ -109,18 +113,19 @@ void init_MultiFabRegister (py::module & m) std::string, ablastr::fields::Direction, int - >(&MultiFabRegister::has) - //py::arg("name"), - // "..." + >(&MultiFabRegister::has), + py::arg("name"), + py::arg("dir"), + py::arg("level") ) .def("get", py::overload_cast< std::string, int - >(&MultiFabRegister::get) - //py::arg("name"), - // "..." + >(&MultiFabRegister::get), + py::arg("name"), + py::arg("level") ) .def("get", @@ -128,9 +133,10 @@ void init_MultiFabRegister (py::module & m) std::string, ablastr::fields::Direction, int - >(&MultiFabRegister::get) - //py::arg("name"), - // "..." + >(&MultiFabRegister::get), + py::arg("name"), + py::arg("dir"), + py::arg("level") ) //.def("list", @@ -142,9 +148,9 @@ void init_MultiFabRegister (py::module & m) py::overload_cast< std::string, int - >(&MultiFabRegister::erase) - //py::arg("name"), - // "..." + >(&MultiFabRegister::erase), + py::arg("name"), + py::arg("level") ) .def("erase", @@ -152,9 +158,10 @@ void init_MultiFabRegister (py::module & m) std::string, ablastr::fields::Direction, int - >(&MultiFabRegister::erase) - //py::arg("name"), - // "..." + >(&MultiFabRegister::erase), + py::arg("name"), + py::arg("dir"), + py::arg("level") ) ; } From 8c2b8e195ec37354036106a025e2a72f6669922a Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 12 Sep 2024 13:17:33 -0700 Subject: [PATCH 178/314] Fix typo --- Source/Initialization/WarpXInitData.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 59957ee312e..f8d86c484b2 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -946,7 +946,7 @@ WarpX::InitLevelData (int lev, Real /*time*/) } if (lev > 0) { Efield_aux[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); - m_fields.get("Efield_avg_cp", Direction{i}, lev)->setVal(m_p_ext_field_params->E_external_grid[i]); + m_fields.get("Efield_cp", Direction{i}, lev)->setVal(m_p_ext_field_params->E_external_grid[i]); if (fft_do_time_averaging) { m_fields.get("Efield_avg_cp", Direction{i}, lev)->setVal(m_p_ext_field_params->E_external_grid[i]); } From 0a474f82285628d304c6e00dc4e846ddb821abe4 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 12 Sep 2024 13:45:32 -0700 Subject: [PATCH 179/314] Use new accessor for getField --- Source/BoundaryConditions/PML.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index 2307547dac9..47e7250ec49 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -840,16 +840,16 @@ PML::PML (const int lev, const BoxArray& grid_ba, cdm.define(cba); } - const amrex::BoxArray cba_Ex = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Efield_cp, 1,0).ixType().toIntVect()); - const amrex::BoxArray cba_Ey = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Efield_cp, 1,1).ixType().toIntVect()); - const amrex::BoxArray cba_Ez = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Efield_cp, 1,2).ixType().toIntVect()); + const amrex::BoxArray cba_Ex = amrex::convert(cba, WarpX::GetInstance().m_fields.get("Efield_cp", Direction{0}, 1)->ixType().toIntVect()); + const amrex::BoxArray cba_Ey = amrex::convert(cba, WarpX::GetInstance().m_fields.get("Efield_cp", Direction{1}, 1)->ixType().toIntVect()); + const amrex::BoxArray cba_Ez = amrex::convert(cba, WarpX::GetInstance().m_fields.get("Efield_cp", Direction{2}, 1)->ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_E_cp[0], cba_Ex, cdm, ncompe, nge, lev, "pml_E_cp[x]", 0.0_rt); WarpX::AllocInitMultiFab(pml_E_cp[1], cba_Ey, cdm, ncompe, nge, lev, "pml_E_cp[y]", 0.0_rt); WarpX::AllocInitMultiFab(pml_E_cp[2], cba_Ez, cdm, ncompe, nge, lev, "pml_E_cp[z]", 0.0_rt); - const amrex::BoxArray cba_Bx = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Bfield_cp, 1,0).ixType().toIntVect()); - const amrex::BoxArray cba_By = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Bfield_cp, 1,1).ixType().toIntVect()); - const amrex::BoxArray cba_Bz = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Bfield_cp, 1,2).ixType().toIntVect()); + const amrex::BoxArray cba_Bx = amrex::convert(cba, WarpX::GetInstance().m_fields.get("Bfield_cp", Direction{0}, 1)->ixType().toIntVect()); + const amrex::BoxArray cba_By = amrex::convert(cba, WarpX::GetInstance().m_fields.get("Bfield_cp", Direction{1}, 1)->ixType().toIntVect()); + const amrex::BoxArray cba_Bz = amrex::convert(cba, WarpX::GetInstance().m_fields.get("Bfield_cp", Direction{2}, 1)->ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_B_cp[0], cba_Bx, cdm, ncompb, ngb, lev, "pml_B_cp[x]", 0.0_rt); WarpX::AllocInitMultiFab(pml_B_cp[1], cba_By, cdm, ncompb, ngb, lev, "pml_B_cp[y]", 0.0_rt); WarpX::AllocInitMultiFab(pml_B_cp[2], cba_Bz, cdm, ncompb, ngb, lev, "pml_B_cp[z]", 0.0_rt); From 84b8ec6d518df83d9dc6248583b39007aa12380e Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 12 Sep 2024 13:50:49 -0700 Subject: [PATCH 180/314] Correct MR typo --- Source/BoundaryConditions/PML.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index 47e7250ec49..7f0cfcbc8ef 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -870,9 +870,9 @@ PML::PML (const int lev, const BoxArray& grid_ba, WarpX::AllocInitMultiFab( pml_G_cp, cba_G_nodal, cdm, 3, ngf, lev, "pml_G_cp", 0.0_rt); } - const amrex::BoxArray cba_jx = amrex::convert(cba, WarpX::GetInstance().m_fields.get("current_cp", Direction{0}, 0)->ixType().toIntVect()); - const amrex::BoxArray cba_jy = amrex::convert(cba, WarpX::GetInstance().m_fields.get("current_cp", Direction{1}, 0)->ixType().toIntVect()); - const amrex::BoxArray cba_jz = amrex::convert(cba, WarpX::GetInstance().m_fields.get("current_cp", Direction{2}, 0)->ixType().toIntVect()); + const amrex::BoxArray cba_jx = amrex::convert(cba, WarpX::GetInstance().m_fields.get("current_cp", Direction{0}, 1)->ixType().toIntVect()); + const amrex::BoxArray cba_jy = amrex::convert(cba, WarpX::GetInstance().m_fields.get("current_cp", Direction{1}, 1)->ixType().toIntVect()); + const amrex::BoxArray cba_jz = amrex::convert(cba, WarpX::GetInstance().m_fields.get("current_cp", Direction{2}, 1)->ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_j_cp[0], cba_jx, cdm, 1, ngb, lev, "pml_j_cp[x]", 0.0_rt); WarpX::AllocInitMultiFab(pml_j_cp[1], cba_jy, cdm, 1, ngb, lev, "pml_j_cp[y]", 0.0_rt); WarpX::AllocInitMultiFab(pml_j_cp[2], cba_jz, cdm, 1, ngb, lev, "pml_j_cp[z]", 0.0_rt); From 0b1b2a33a72e82aae1e386664c20e05c8ede0269 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 12 Sep 2024 13:58:30 -0700 Subject: [PATCH 181/314] Fix another typo in MR --- Source/WarpX.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 0f4a32b2f44..07cfcef381e 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2647,9 +2647,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm const std::array cdx = CellSize(lev-1); // Create the MultiFabs for B - m_fields.alloc_init( "Bfield_cp", Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Bfield_cp", Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Bfield_cp", Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Bfield_cp", Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Bfield_cp", Direction{1}, lev, amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Bfield_cp", Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); // Create the MultiFabs for E m_fields.alloc_init( "Efield_cp", Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); From f1e156d9144c1424ed4af148dd461e6bb397bb4b Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Thu, 12 Sep 2024 23:28:36 +0200 Subject: [PATCH 182/314] added distance_to_eb --- Source/EmbeddedBoundary/ParticleScraper.H | 6 +++--- Source/EmbeddedBoundary/WarpXInitEB.cpp | 2 +- Source/Evolve/WarpXEvolve.cpp | 4 ++-- Source/Particles/MultiParticleContainer.H | 2 +- Source/Particles/MultiParticleContainer.cpp | 3 ++- Source/Particles/ParticleBoundaryBuffer.H | 4 +++- Source/Particles/ParticleBoundaryBuffer.cpp | 2 +- Source/Particles/PhysicalParticleContainer.cpp | 14 ++++++++++---- Source/Particles/WarpXParticleContainer.cpp | 7 +++++-- Source/WarpX.H | 7 +++---- Source/WarpX.cpp | 5 ++--- 11 files changed, 33 insertions(+), 23 deletions(-) diff --git a/Source/EmbeddedBoundary/ParticleScraper.H b/Source/EmbeddedBoundary/ParticleScraper.H index c5d9cc68c60..860541542be 100644 --- a/Source/EmbeddedBoundary/ParticleScraper.H +++ b/Source/EmbeddedBoundary/ParticleScraper.H @@ -65,7 +65,7 @@ */ template ::value, int> foo = 0> void -scrapeParticlesAtEB (PC& pc, const amrex::Vector& distance_to_eb, int lev, F&& f) +scrapeParticlesAtEB (PC& pc, ablastr::fields::MultiLevelScalarField const& distance_to_eb, int lev, F&& f) { scrapeParticlesAtEB(pc, distance_to_eb, lev, lev, std::forward(f)); } @@ -108,7 +108,7 @@ scrapeParticlesAtEB (PC& pc, const amrex::Vector& distan */ template ::value, int> foo = 0> void -scrapeParticlesAtEB (PC& pc, const amrex::Vector& distance_to_eb, F&& f) +scrapeParticlesAtEB (PC& pc, ablastr::fields::MultiLevelScalarField const& distance_to_eb, F&& f) { scrapeParticlesAtEB(pc, distance_to_eb, 0, pc.finestLevel(), std::forward(f)); } @@ -153,7 +153,7 @@ scrapeParticlesAtEB (PC& pc, const amrex::Vector& distan */ template ::value, int> foo = 0> void -scrapeParticlesAtEB (PC& pc, const amrex::Vector& distance_to_eb, +scrapeParticlesAtEB (PC& pc, ablastr::fields::MultiLevelScalarField const& distance_to_eb, int lev_min, int lev_max, F&& f) { BL_PROFILE("scrapeParticlesAtEB"); diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index f3e6c5ba7a0..58445e0fae3 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -391,7 +391,7 @@ WarpX::ComputeDistanceToEB () for (int lev=0; lev<=maxLevel(); lev++) { const amrex::EB2::Level& eb_level = eb_is.getLevel(Geom(lev)); auto const eb_fact = fieldEBFactory(lev); - amrex::FillSignedDistance(*m_distance_to_eb[lev], eb_level, eb_fact, 1); + amrex::FillSignedDistance(*m_fields.get("distance_to_e", lev), eb_level, eb_fact, 1); } #endif } diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index ba55eddc588..95790342049 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -543,9 +543,9 @@ void WarpX::HandleParticlesAtBoundaries (int step, amrex::Real cur_time, int num // interact the particles with EB walls (if present) if (EB::enabled()) { - mypc->ScrapeParticlesAtEB(amrex::GetVecOfConstPtrs(m_distance_to_eb)); + mypc->ScrapeParticlesAtEB(m_fields.get_mr_levels("distance_to_e", finest_level)); m_particle_boundary_buffer->gatherParticlesFromEmbeddedBoundaries( - *mypc, amrex::GetVecOfConstPtrs(m_distance_to_eb)); + *mypc, m_fields.get_mr_levels("distance_to_e", finest_level)); mypc->deleteInvalidParticles(); } diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index f15a78df6cd..05850daf07b 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -300,7 +300,7 @@ public: PhysicalParticleContainer& GetPCtmp () { return *pc_tmp; } - void ScrapeParticlesAtEB (const amrex::Vector& distance_to_eb); + void ScrapeParticlesAtEB (ablastr::fields::MultiLevelScalarField const& distance_to_eb); std::string m_B_ext_particle_s = "none"; std::string m_E_ext_particle_s = "none"; diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 560220e9eb9..5ae18b36280 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -964,7 +964,8 @@ void MultiParticleContainer::CheckIonizationProductSpecies() } } -void MultiParticleContainer::ScrapeParticlesAtEB (const amrex::Vector& distance_to_eb) +void MultiParticleContainer::ScrapeParticlesAtEB ( + ablastr::fields::MultiLevelScalarField const& distance_to_eb) { for (auto& pc : allcontainers) { scrapeParticlesAtEB(*pc, distance_to_eb, ParticleBoundaryProcess::Absorb()); diff --git a/Source/Particles/ParticleBoundaryBuffer.H b/Source/Particles/ParticleBoundaryBuffer.H index d33834309ab..24b388be00e 100644 --- a/Source/Particles/ParticleBoundaryBuffer.H +++ b/Source/Particles/ParticleBoundaryBuffer.H @@ -12,6 +12,8 @@ #include "Particles/PinnedMemoryParticleContainer.H" #include "Utils/export.H" +#include + #include @@ -41,7 +43,7 @@ public: void gatherParticlesFromDomainBoundaries (MultiParticleContainer& mypc); void gatherParticlesFromEmbeddedBoundaries ( - MultiParticleContainer& mypc, const amrex::Vector& distance_to_eb + MultiParticleContainer& mypc, ablastr::fields::MultiLevelScalarField const& distance_to_eb ); void redistribute (); diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index bc113e8e3a3..0391dcc6178 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -462,7 +462,7 @@ void ParticleBoundaryBuffer::gatherParticlesFromDomainBoundaries (MultiParticleC } void ParticleBoundaryBuffer::gatherParticlesFromEmbeddedBoundaries ( - MultiParticleContainer& mypc, const amrex::Vector& distance_to_eb) + MultiParticleContainer& mypc, ablastr::fields::MultiLevelScalarField const& distance_to_eb) { if (EB::enabled()) { WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::EB"); diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 1ad43755464..513d3da979e 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1486,8 +1486,11 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int #ifdef AMREX_USE_EB if (EB::enabled()) { - auto &distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); - scrapeParticlesAtEB(*this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); + auto & warpx = WarpX::GetInstance(); + scrapeParticlesAtEB( + *this, + warpx.m_fields.get_mr_levels("distance_to_eb", warpx.finestLevel()), + ParticleBoundaryProcess::Absorb()); } #endif @@ -1986,8 +1989,11 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, #ifdef AMREX_USE_EB if (EB::enabled()) { - auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); - scrapeParticlesAtEB(tmp_pc, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); + auto & warpx = WarpX::GetInstance(); + scrapeParticlesAtEB( + tmp_pc, + warpx.m_fields.get_mr_levels("distance_to_eb", warpx.finestLevel()), + ParticleBoundaryProcess::Absorb()); } #endif diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 42d58b8255a..b3e41f0d04e 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -302,8 +302,11 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, long n, // Remove particles that are inside the embedded boundaries #ifdef AMREX_USE_EB if (EB::enabled()) { - auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); - scrapeParticlesAtEB( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); + auto & warpx = WarpX::GetInstance(); + scrapeParticlesAtEB( + *this, + warpx.m_fields.get_mr_levels("distance_to_eb", warpx.finestLevel()), + ParticleBoundaryProcess::Absorb()); deleteInvalidParticles(); } #endif diff --git a/Source/WarpX.H b/Source/WarpX.H index edc19b19be0..03947e637a5 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -146,7 +146,9 @@ public: [[nodiscard]] HybridPICModel * get_pointer_HybridPICModel () const { return m_hybrid_pic_model.get(); } MultiDiagnostics& GetMultiDiags () {return *multi_diags;} #ifdef AMREX_USE_EB - amrex::Vector >& GetDistanceToEB () {return m_distance_to_eb;} + ablastr::fields::MultiLevelScalarField GetDistanceToEB () { + return m_fields.get_mr_levels("distance_to_eb", finestLevel()); + } #endif ParticleBoundaryBuffer& GetParticleBoundaryBuffer () { return *m_particle_boundary_buffer; } @@ -1494,9 +1496,6 @@ private: * This is only used for the ECT solver.*/ amrex::Vector >, 3 > > m_borrowing; - //EB level set - amrex::Vector > m_distance_to_eb; - // Copy of the coarse aux amrex::Vector > current_buffer_masks; amrex::Vector > gather_buffer_masks; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 07cfcef381e..c92a2d8ce22 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -322,7 +322,6 @@ WarpX::WarpX () Afield_dotMask.resize(nlevs_max); phi_dotMask.resize(nlevs_max); - m_distance_to_eb.resize(nlevs_max); m_flag_info_face.resize(nlevs_max); m_flag_ext_face.resize(nlevs_max); m_borrowing.resize(nlevs_max); @@ -2325,8 +2324,8 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (EB::enabled()) { constexpr int nc_ls = 1; amrex::IntVect const ng_ls(2); - AllocInitMultiFab(m_distance_to_eb[lev], amrex::convert(ba, IntVect::TheNodeVector()), dm, nc_ls, ng_ls, lev, - "m_distance_to_eb"); + //EB level set + m_fields.alloc_init("distance_to_eb", lev, amrex::convert(ba, IntVect::TheNodeVector()), dm, nc_ls, ng_ls, 0.0_rt); // EB info are needed only at the finest level if (lev == maxLevel()) { From cbfa4153e8efe4328feda75467b0119010523e06 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Thu, 12 Sep 2024 15:00:37 -0700 Subject: [PATCH 183/314] Fix bug in allocation of `Efield_cax` --- Source/WarpX.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index c92a2d8ce22..d91d4da24a9 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2771,8 +2771,8 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm // Create the MultiFabs for E m_fields.alloc_init("Efield_cax", Direction{0}, lev, amrex::convert(cba,Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Efield_cax", Direction{1}, lev, amrex::convert(cba,Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Efield_cax", Direction{2}, lev, amrex::convert(cba,Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Efield_cax", Direction{1}, lev, amrex::convert(cba,Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Efield_cax", Direction{2}, lev, amrex::convert(cba,Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); } AllocInitMultiFab(gather_buffer_masks[lev], ba, dm, ncomps, amrex::IntVect(1), lev, "gather_buffer_masks"); From 84db05443abb11ed2889c4ff27d6e1f28b845556 Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Thu, 12 Sep 2024 14:39:14 -0700 Subject: [PATCH 184/314] refactoring Bold in implicitSolvers to work with new format. --- .../ImplicitSolvers/ThetaImplicitEM.H | 8 ----- .../ImplicitSolvers/ThetaImplicitEM.cpp | 36 ++++++++++--------- .../ImplicitSolvers/WarpXImplicitOps.cpp | 23 ++++++------ .../ImplicitSolvers/WarpXSolverVec.cpp | 4 +-- Source/WarpX.H | 14 ++++---- 5 files changed, 42 insertions(+), 43 deletions(-) diff --git a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H index 009c2c7e546..aba66782154 100644 --- a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H +++ b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H @@ -98,14 +98,6 @@ private: */ WarpXSolverVec m_E, m_Eold; - /** - * \brief B is a derived variable from E. Need to save Bold to update B during - * the iterative nonlinear solve for E. Bold is owned here, but only used by WarpX. - * It is not used directly by the nonlinear solver, nor is it the same size as the - * solver vector (size E), and so it should not be WarpXSolverVec type. - */ - amrex::Vector, 3 > > m_Bold; - /** * \brief Update the E and B fields owned by WarpX */ diff --git a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp index 5169b57b56d..daf555226d3 100644 --- a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp +++ b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp @@ -24,17 +24,18 @@ void ThetaImplicitEM::Define ( WarpX* const a_WarpX ) m_E.Define( m_WarpX, "Efield_fp" ); m_Eold.Define( m_E ); - // Define Bold MultiFab + // Define Bold MultiFabs + using ablastr::fields::Direction; const int num_levels = 1; - m_Bold.resize(num_levels); // size is number of levels for (int lev = 0; lev < num_levels; ++lev) { - for (int n=0; n<3; n++) { - const amrex::MultiFab& Bfp = m_WarpX->getField( FieldType::Bfield_fp,lev,n); - m_Bold[lev][n] = std::make_unique( Bfp.boxArray(), - Bfp.DistributionMap(), - Bfp.nComp(), - Bfp.nGrowVect() ); - } + const auto& ba_Bx = m_WarpX->m_fields.get("Bfield_fp",Direction{0},lev)->boxArray(); + const auto& ba_By = m_WarpX->m_fields.get("Bfield_fp",Direction{1},lev)->boxArray(); + const auto& ba_Bz = m_WarpX->m_fields.get("Bfield_fp",Direction{2},lev)->boxArray(); + const auto& dm = m_WarpX->m_fields.get("Bfield_fp",Direction{0},lev)->DistributionMap(); + const amrex::IntVect ngb = m_WarpX->m_fields.get("Bfield_fp",Direction{0},lev)->nGrowVect(); + m_WarpX->m_fields.alloc_init("Bold", Direction{0}, lev, ba_Bx, dm, 1, ngb, 0.0_rt); + m_WarpX->m_fields.alloc_init("Bold", Direction{1}, lev, ba_By, dm, 1, ngb, 0.0_rt); + m_WarpX->m_fields.alloc_init("Bold", Direction{2}, lev, ba_Bz, dm, 1, ngb, 0.0_rt); } // Parse theta-implicit solver specific parameters @@ -89,12 +90,13 @@ void ThetaImplicitEM::OneStep ( const amrex::Real a_time, // Save Eg at the start of the time step m_Eold.Copy( FieldType::Efield_fp ); - const int num_levels = static_cast(m_Bold.size()); + const int num_levels = 1; for (int lev = 0; lev < num_levels; ++lev) { - for (int n=0; n<3; n++) { - const amrex::MultiFab& Bfp = m_WarpX->getField(FieldType::Bfield_fp,lev,n); - amrex::MultiFab& Bold = *m_Bold[lev][n]; - amrex::MultiFab::Copy(Bold, Bfp, 0, 0, 1, Bold.nGrowVect()); + const ablastr::fields::VectorField Bfp = m_WarpX->m_fields.get_alldirs("Bfield_fp",lev); + ablastr::fields::VectorField Bold = m_WarpX->m_fields.get_alldirs("Bold",lev); + for (int n = 0; n < 3; ++n) { + amrex::MultiFab::Copy( *Bold[n], *Bfp[n], 0, 0, Bold[n]->nComp(), + Bold[n]->nGrowVect() ); } } @@ -146,7 +148,8 @@ void ThetaImplicitEM::UpdateWarpXFields ( const WarpXSolverVec& a_E, m_WarpX->SetElectricFieldAndApplyBCs( a_E ); // Update Bfield_fp owned by WarpX - m_WarpX->UpdateMagneticFieldAndApplyBCs( m_Bold, m_theta*a_dt ); + ablastr::fields::MultiLevelVectorField const& Bold = m_WarpX->m_fields.get_mr_levels_alldirs("Bold",0); + m_WarpX->UpdateMagneticFieldAndApplyBCs( Bold, m_theta*a_dt ); } @@ -161,6 +164,7 @@ void ThetaImplicitEM::FinishFieldUpdate ( amrex::Real a_new_time ) const amrex::Real c1 = 1._rt - c0; m_E.linComb( c0, m_E, c1, m_Eold ); m_WarpX->SetElectricFieldAndApplyBCs( m_E ); - m_WarpX->FinishMagneticFieldAndApplyBCs( m_Bold, m_theta ); + ablastr::fields::MultiLevelVectorField const & Bold = m_WarpX->m_fields.get_mr_levels_alldirs("Bold",0); + m_WarpX->FinishMagneticFieldAndApplyBCs( Bold, m_theta ); } diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp index 2e859c01877..bde7368e195 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp @@ -83,20 +83,23 @@ WarpX::SetElectricFieldAndApplyBCs ( const WarpXSolverVec& a_E ) } void -WarpX::UpdateMagneticFieldAndApplyBCs( const amrex::Vector, 3 > >& a_Bn, - amrex::Real a_thetadt ) +WarpX::UpdateMagneticFieldAndApplyBCs( ablastr::fields::MultiLevelVectorField const& a_Bn, + amrex::Real a_thetadt ) { using ablastr::fields::Direction; - amrex::MultiFab::Copy(*m_fields.get("Bfield_fp", Direction{0}, 0), *a_Bn[0][0], 0, 0, ncomps, a_Bn[0][0]->nGrowVect()); - amrex::MultiFab::Copy(*m_fields.get("Bfield_fp", Direction{1}, 0), *a_Bn[0][1], 0, 0, ncomps, a_Bn[0][1]->nGrowVect()); - amrex::MultiFab::Copy(*m_fields.get("Bfield_fp", Direction{2}, 0), *a_Bn[0][2], 0, 0, ncomps, a_Bn[0][2]->nGrowVect()); + for (int lev = 0; lev <= finest_level; ++lev) { + ablastr::fields::VectorField Bfp = m_fields.get_alldirs("Bfield_fp",lev); + amrex::MultiFab::Copy(*Bfp[0], *a_Bn[lev][0], 0, 0, ncomps, a_Bn[lev][0]->nGrowVect()); + amrex::MultiFab::Copy(*Bfp[1], *a_Bn[lev][1], 0, 0, ncomps, a_Bn[lev][1]->nGrowVect()); + amrex::MultiFab::Copy(*Bfp[2], *a_Bn[lev][2], 0, 0, ncomps, a_Bn[lev][2]->nGrowVect()); + } EvolveB(a_thetadt, DtType::Full); ApplyMagneticFieldBCs(); } void -WarpX::FinishMagneticFieldAndApplyBCs( const amrex::Vector, 3 > >& a_Bn, - amrex::Real a_theta ) +WarpX::FinishMagneticFieldAndApplyBCs( ablastr::fields::MultiLevelVectorField const& a_Bn, + amrex::Real a_theta ) { FinishImplicitField(m_fields.get_mr_levels_alldirs("Bfield_fp", 0), a_Bn, a_theta); ApplyMagneticFieldBCs(); @@ -250,9 +253,9 @@ WarpX::FinishImplicitParticleUpdate () } void -WarpX::FinishImplicitField( const ablastr::fields::MultiLevelVectorField& Field_fp, - const amrex::Vector, 3 > >& Field_n, - amrex::Real theta ) +WarpX::FinishImplicitField( ablastr::fields::MultiLevelVectorField const& Field_fp, + ablastr::fields::MultiLevelVectorField const& Field_n, + amrex::Real theta ) { using namespace amrex::literals; diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp index ee146d0ae29..0c502bb15a3 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp @@ -129,8 +129,8 @@ void WarpXSolverVec::Copy ( FieldType a_array_type, } } if (m_scalar_type != FieldType::None) { - const amrex::MultiFab* this_scalar = m_WarpX->getFieldPointer(m_scalar_type,lev,0); - amrex::MultiFab::Copy( *m_scalar_vec[lev], *this_scalar, 0, 0, m_ncomp, + const amrex::MultiFab* this_mf = m_WarpX->m_fields.get(m_scalar_type_name,lev); + amrex::MultiFab::Copy( *m_scalar_vec[lev], *this_mf, 0, 0, m_ncomp, amrex::IntVect::TheZeroVector() ); } } diff --git a/Source/WarpX.H b/Source/WarpX.H index 03947e637a5..c2775c1c833 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -127,14 +127,14 @@ public: void SaveParticlesAtImplicitStepStart (); void FinishImplicitParticleUpdate (); void SetElectricFieldAndApplyBCs ( const WarpXSolverVec& a_E ); - void UpdateMagneticFieldAndApplyBCs ( const amrex::Vector, 3 > >& a_Bn, - amrex::Real a_thetadt ); + void UpdateMagneticFieldAndApplyBCs ( ablastr::fields::MultiLevelVectorField const& a_Bn, + amrex::Real a_thetadt ); void ApplyMagneticFieldBCs (); - void FinishMagneticFieldAndApplyBCs ( const amrex::Vector, 3 > >& a_Bn, - amrex::Real a_theta ); - void FinishImplicitField ( const ablastr::fields::MultiLevelVectorField& Field_fp, - const amrex::Vector, 3 > >& Field_n, - amrex::Real theta ); + void FinishMagneticFieldAndApplyBCs ( ablastr::fields::MultiLevelVectorField const& a_Bn, + amrex::Real a_theta ); + void FinishImplicitField ( const ablastr::fields::MultiLevelVectorField& Field_fp, + const ablastr::fields::MultiLevelVectorField& Field_n, + amrex::Real theta ); void ImplicitComputeRHSE ( amrex::Real dt, WarpXSolverVec& a_Erhs_vec); void ImplicitComputeRHSE (int lev, amrex::Real dt, WarpXSolverVec& a_Erhs_vec); void ImplicitComputeRHSE (int lev, PatchType patch_type, amrex::Real dt, WarpXSolverVec& a_Erhs_vec); From ca52cf3aa14716002cde6693db0ce477fa3396ee Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Thu, 12 Sep 2024 15:07:00 -0700 Subject: [PATCH 185/314] setDotMask working with new format. --- Source/WarpX.H | 4 ++-- Source/WarpX.cpp | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Source/WarpX.H b/Source/WarpX.H index c2775c1c833..f492f711027 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -502,14 +502,14 @@ public: * Get pointer to the amrex::MultiFab containing the dotMask for the specified field */ [[nodiscard]] const amrex::iMultiFab* - getFieldDotMaskPointer (warpx::fields::FieldType field_type, int lev, int dir) const; + getFieldDotMaskPointer (warpx::fields::FieldType field_type, int lev, int dir); /** * \brief * Set the dotMask container */ void SetDotMask( std::unique_ptr& field_dotMask, - warpx::fields::FieldType field_type, int lev, int dir ) const; + std::string field_name, int lev, int dir ); [[nodiscard]] bool DoPML () const {return do_pml;} [[nodiscard]] bool DoFluidSpecies () const {return do_fluid_species;} diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index d91d4da24a9..134ce4d98f2 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -3491,21 +3491,21 @@ WarpX::MakeDistributionMap (int lev, amrex::BoxArray const& ba) } const amrex::iMultiFab* -WarpX::getFieldDotMaskPointer ( FieldType field_type, int lev, int dir ) const +WarpX::getFieldDotMaskPointer ( FieldType field_type, int lev, int dir ) { switch(field_type) { case FieldType::Efield_fp : - SetDotMask( Efield_dotMask[lev][dir], field_type, lev, dir ); + SetDotMask( Efield_dotMask[lev][dir], "Efield_fp", lev, dir ); return Efield_dotMask[lev][dir].get(); case FieldType::Bfield_fp : - SetDotMask( Bfield_dotMask[lev][dir], field_type, lev, dir ); + SetDotMask( Bfield_dotMask[lev][dir], "Bfield_fp", lev, dir ); return Bfield_dotMask[lev][dir].get(); case FieldType::vector_potential_fp : - SetDotMask( Afield_dotMask[lev][dir], field_type, lev, dir ); + SetDotMask( Afield_dotMask[lev][dir], "vector_potential_fp", lev, dir ); return Afield_dotMask[lev][dir].get(); case FieldType::phi_fp : - SetDotMask( phi_dotMask[lev], field_type, lev, 0 ); + SetDotMask( phi_dotMask[lev], "phi_fp", lev, 0 ); return phi_dotMask[lev].get(); default: WARPX_ABORT_WITH_MESSAGE("Invalid field type for dotMask"); @@ -3514,15 +3514,15 @@ WarpX::getFieldDotMaskPointer ( FieldType field_type, int lev, int dir ) const } void WarpX::SetDotMask( std::unique_ptr& field_dotMask, - FieldType field_type, int lev, int dir ) const + std::string field_name, int lev, int dir ) { // Define the dot mask for this field_type needed to properly compute dotProduct() // for field values that have shared locations on different MPI ranks if (field_dotMask != nullptr) { return; } - const amrex::MultiFab* this_field = getFieldPointer(field_type,lev,dir); - const amrex::BoxArray& this_ba = this_field->boxArray(); - const amrex::MultiFab tmp( this_ba, this_field->DistributionMap(), + ablastr::fields::VectorField const& this_field = m_fields.get_alldirs(field_name,lev); + const amrex::BoxArray& this_ba = this_field[dir]->boxArray(); + const amrex::MultiFab tmp( this_ba, this_field[dir]->DistributionMap(), 1, 0, amrex::MFInfo().SetAlloc(false) ); const amrex::Periodicity& period = Geom(lev).periodicity(); field_dotMask = tmp.OwnerMask(period); From d184a42bf703167fc17b3c38e56f52aa5f673dc9 Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Thu, 12 Sep 2024 15:18:36 -0700 Subject: [PATCH 186/314] removed getMultiLevelField from WarpX.H --- Source/WarpX.H | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/Source/WarpX.H b/Source/WarpX.H index f492f711027..f53ccc83efa 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -486,17 +486,6 @@ public: [[nodiscard]] const amrex::MultiFab& getField(warpx::fields::FieldType field_type, int lev, int direction = 0) const; - /** - * \brief - * Get a constant reference to the specified vector field on the different MR levels - * - * \param field_type[in] the field type - * - * \return a vector (which one element per MR level) of arrays of three pointers (for 3 vector components) amrex::MultiFab* containing the field data - */ - [[nodiscard]] const amrex::Vector,3>>& - getMultiLevelField(warpx::fields::FieldType field_type) const; - /** * \brief * Get pointer to the amrex::MultiFab containing the dotMask for the specified field From 00b6c632304cc0ab007c9820adeb32d2a697a0c7 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 12 Sep 2024 15:31:45 -0700 Subject: [PATCH 187/314] `MultiFabRegister`: Start `const`-Correctness --- Source/ablastr/fields/MultiFabRegister.H | 88 +++++++++++++++++++--- Source/ablastr/fields/MultiFabRegister.cpp | 69 ++++++++++++----- 2 files changed, 128 insertions(+), 29 deletions(-) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 5964b6ca732..7725fa49781 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -53,24 +54,43 @@ namespace ablastr::fields */ using ScalarField = amrex::MultiFab*; + /** A read-only scalar field (a MultiFab) + * + * Note: might still have components, e.g., for copies at different times. + */ + using ConstScalarField = amrex::MultiFab const *; + /** A vector field of three MultiFab */ - //using VectorField = ablastr::utils::ConstMap; - using VectorField = std::array; + //using VectorField = ablastr::utils::ConstMap; + using VectorField = std::array; + + /** A read-only vector field of three MultiFab + */ + //using VectorField = ablastr::utils::ConstMap; + using ConstVectorField = std::array; /** A multi-level scalar field */ using MultiLevelScalarField = amrex::Vector; + /** A read-only multi-level scalar field + */ + using ConstMultiLevelScalarField = amrex::Vector; + /** A multi-level vector field */ using MultiLevelVectorField = amrex::Vector; - /** title + /** A read-only multi-level vector field + */ + using ConstMultiLevelVectorField = amrex::Vector; + + /** A class to control the lifetime and properties of a MultiFab (field). * - * body body - * body - * body + * This class is used to own the lifetime of an amrex::MultiFab and to store + * associated information around it regarding unique naming, scalar/vector/tensor + * properties, aliasing, load balancing, etc. */ struct MultiFabOwner { @@ -95,20 +115,23 @@ namespace ablastr::fields std::string m_owner; /** Is this part of a vector/tensor? */ + AMREX_INLINE bool - is_vector () { return m_dir.has_value(); } + is_vector () const { return m_dir.has_value(); } /** Is this an alias MultiFab? * * If yes, that means we do not own the memory. */ + AMREX_INLINE bool - is_alias () { return !m_owner.empty(); } + is_alias () const { return !m_owner.empty(); } }; /** This is a register of fields aka amrex::(i)MultiFabs. * * This is owned by a simulation instance. All used fields should be registered here. + * Internally, this contains @see MultiFabOwner values. */ struct MultiFabRegister { @@ -224,7 +247,7 @@ namespace ablastr::fields has ( std::string name, int level - ); + ) const; /** title * @@ -241,7 +264,7 @@ namespace ablastr::fields std::string name, Direction dir, int level - ); + ) const; /** title * @@ -275,6 +298,38 @@ namespace ablastr::fields int level ); + /** title + * + * body body + * body + * + * @param name ... + * @param level ... + * @return ... + */ + amrex::MultiFab const * + get ( + std::string name, + int level + ) const; + + /** title + * + * body body + * body + * + * @param name ... + * @param dir ... + * @param level ... + * @return ... + */ + amrex::MultiFab const * + get ( + std::string name, + Direction dir, + int level + ) const; + /** title * * Same as get above, but returns all levels for a name. @@ -395,7 +450,7 @@ namespace ablastr::fields mf_name ( std::string name, int level - ); + ) const; /** title * @@ -412,9 +467,18 @@ namespace ablastr::fields std::string name, Direction dir, int level - ); + ) const; private: + amrex::MultiFab* + internal_get ( + std::string key + ); + amrex::MultiFab const * + internal_get ( + std::string key + ) const; + /** data storage: ownership and lifetime control */ std::map< std::string, diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 961711c23dc..fc1876832a9 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -303,7 +303,7 @@ namespace ablastr::fields MultiFabRegister::has ( std::string name, int level - ) + ) const { name = mf_name(name, level); @@ -315,7 +315,7 @@ namespace ablastr::fields std::string name, Direction dir, int level - ) + ) const { name = mf_name(name, dir, level); @@ -323,22 +323,43 @@ namespace ablastr::fields } amrex::MultiFab* - MultiFabRegister::get ( - std::string name, - int level + MultiFabRegister::internal_get ( + std::string key ) { - name = mf_name(name, level); + if (m_mf_register.count(key) == 0) { + // FIXME: temporary, throw a std::runtime_error + return nullptr; + } + amrex::MultiFab & mf = m_mf_register.at(key).m_mf; - if (m_mf_register.count(name) == 0) { + return &mf; + } + + amrex::MultiFab const * + MultiFabRegister::internal_get ( + std::string key + ) const + { + if (m_mf_register.count(key) == 0) { // FIXME: temporary, throw a std::runtime_error return nullptr; } - amrex::MultiFab & mf = m_mf_register[name].m_mf; + amrex::MultiFab const & mf = m_mf_register.at(key).m_mf; return &mf; } + amrex::MultiFab* + MultiFabRegister::get ( + std::string name, + int level + ) + { + name = mf_name(name, level); + return internal_get(name); + } + amrex::MultiFab* MultiFabRegister::get ( std::string name, @@ -347,14 +368,28 @@ namespace ablastr::fields ) { name = mf_name(name, dir, level); + return internal_get(name); + } - if (m_mf_register.count(name) == 0) { - // FIXME: temporary, throw a std::runtime_error - return nullptr; - } - amrex::MultiFab & mf = m_mf_register[name].m_mf; + amrex::MultiFab const * + MultiFabRegister::get ( + std::string name, + int level + ) const + { + name = mf_name(name, level); + return internal_get(name); + } - return &mf; + amrex::MultiFab const * + MultiFabRegister::get ( + std::string name, + Direction dir, + int level + ) const + { + name = mf_name(name, dir, level); + return internal_get(name); } MultiLevelScalarField @@ -476,7 +511,7 @@ namespace ablastr::fields MultiFabRegister::mf_name ( std::string name, int level - ) + ) const { // Add the suffix "[level=level]" return name.append("[level=") @@ -489,9 +524,9 @@ namespace ablastr::fields std::string name, Direction dir, int level - ) + ) const { - // Add the suffix "[level=level]" + // Add the suffix "[dir=dir]" return mf_name( name .append("[dir=") From c22e37ea77221dab9f3d8c0f9df76d0a15fba0a3 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 12 Sep 2024 15:39:35 -0700 Subject: [PATCH 188/314] `MultiFabRegister`: Complete `const`-Correctness --- Source/ablastr/fields/MultiFabRegister.H | 19 ++++++- Source/ablastr/fields/MultiFabRegister.cpp | 63 +++++++++++++++++++++- 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 7725fa49781..69dd18dc457 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -343,6 +343,11 @@ namespace ablastr::fields std::string name, int finest_level ); + ConstMultiLevelScalarField + get_mr_levels ( + std::string name, + int finest_level + ) const; /** title * @@ -357,6 +362,11 @@ namespace ablastr::fields std::string name, int level ); + ConstVectorField + get_alldirs ( + std::string name, + int level + ) const; /** Return a vector field on all MR levels. * @@ -372,6 +382,11 @@ namespace ablastr::fields std::string name, int finest_level ); + ConstMultiLevelVectorField + get_mr_levels_alldirs ( + std::string name, + int finest_level + ) const; /** title * @@ -381,7 +396,7 @@ namespace ablastr::fields * @return ... */ std::vector - list (); + list () const; /** title * @@ -470,7 +485,7 @@ namespace ablastr::fields ) const; private: - amrex::MultiFab* + amrex::MultiFab * internal_get ( std::string key ); diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index fc1876832a9..62bdb95c3c8 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -407,6 +407,21 @@ namespace ablastr::fields return field_on_level; } + ConstMultiLevelScalarField + MultiFabRegister::get_mr_levels ( + std::string name, + int finest_level + ) const + { + ConstMultiLevelScalarField field_on_level; + field_on_level.reserve(finest_level+1); + for (int lvl = 0; lvl <= finest_level; lvl++) + { + field_on_level.push_back(get(name, lvl)); + } + return field_on_level; + } + VectorField MultiFabRegister::get_alldirs ( std::string name, @@ -427,6 +442,26 @@ namespace ablastr::fields return vectorField; } + ConstVectorField + MultiFabRegister::get_alldirs ( + std::string name, + int level + ) const + { + // TODO: Technically, we should search field_on_level via std::unique_copy + std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; + + // insert a new level + ConstVectorField vectorField; + + // insert components + for (Direction dir : all_dirs) + { + vectorField[dir] = get(name, dir, level); + } + return vectorField; + } + MultiLevelVectorField MultiFabRegister::get_mr_levels_alldirs ( std::string name, @@ -453,8 +488,34 @@ namespace ablastr::fields return field_on_level; } + ConstMultiLevelVectorField + MultiFabRegister::get_mr_levels_alldirs ( + std::string name, + int finest_level + ) const + { + ConstMultiLevelVectorField field_on_level; + field_on_level.reserve(finest_level+1); + + // TODO: Technically, we should search field_on_level via std::unique_copy + std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; + + for (int lvl = 0; lvl <= finest_level; lvl++) + { + // insert a new level + field_on_level.push_back(ConstVectorField{}); + + // insert components + for (Direction dir : all_dirs) + { + field_on_level[lvl][dir] = get(name, dir, lvl); + } + } + return field_on_level; + } + std::vector - MultiFabRegister::list () + MultiFabRegister::list () const { std::vector names; names.reserve(m_mf_register.size()); From 58d1e2257425b0e4de1e973214c67e5393a7021f Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 12 Sep 2024 15:43:17 -0700 Subject: [PATCH 189/314] `const`-Correctness: Unbreak Python :) --- Source/Python/MultiFabRegister.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/Python/MultiFabRegister.cpp b/Source/Python/MultiFabRegister.cpp index ec565e4671e..c2c52936793 100644 --- a/Source/Python/MultiFabRegister.cpp +++ b/Source/Python/MultiFabRegister.cpp @@ -103,7 +103,7 @@ void init_MultiFabRegister (py::module & m) py::overload_cast< std::string, int - >(&MultiFabRegister::has), + >(&MultiFabRegister::has, py::const_), py::arg("name"), py::arg("level") ) @@ -113,7 +113,7 @@ void init_MultiFabRegister (py::module & m) std::string, ablastr::fields::Direction, int - >(&MultiFabRegister::has), + >(&MultiFabRegister::has, py::const_), py::arg("name"), py::arg("dir"), py::arg("level") From f6c30b6f45e8a4cd221c4b601b2f155ef2947fba Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 12 Sep 2024 15:45:58 -0700 Subject: [PATCH 190/314] Fix MR bug --- Source/WarpX.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 134ce4d98f2..10706139111 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2646,14 +2646,14 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm const std::array cdx = CellSize(lev-1); // Create the MultiFabs for B - m_fields.alloc_init( "Bfield_cp", Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Bfield_cp", Direction{1}, lev, amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Bfield_cp", Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Bfield_cp", Direction{0}, lev, amrex::convert(cba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Bfield_cp", Direction{1}, lev, amrex::convert(cba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Bfield_cp", Direction{2}, lev, amrex::convert(cba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); // Create the MultiFabs for E - m_fields.alloc_init( "Efield_cp", Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Efield_cp", Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Efield_cp", Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Efield_cp", Direction{0}, lev, amrex::convert(cba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Efield_cp", Direction{1}, lev, amrex::convert(cba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Efield_cp", Direction{2}, lev, amrex::convert(cba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); if (fft_do_time_averaging) { From 87392b0686b917cc93873dbcb89e528731ab7ae4 Mon Sep 17 00:00:00 2001 From: David Grote Date: Thu, 12 Sep 2024 15:37:49 -0700 Subject: [PATCH 191/314] Updates for Python fields wrapper class --- Python/pywarpx/fields.py | 108 ++++++++++++++++------------- Source/Python/MultiFabRegister.cpp | 5 +- Source/Python/WarpX.cpp | 33 ++++++++- 3 files changed, 94 insertions(+), 52 deletions(-) diff --git a/Python/pywarpx/fields.py b/Python/pywarpx/fields.py index 5a194b25758..90b1c6e4fef 100644 --- a/Python/pywarpx/fields.py +++ b/Python/pywarpx/fields.py @@ -77,6 +77,9 @@ class _MultiFABWrapper(object): everytime it is called if this argument is given instead of directly providing the Multifab. + idir: int, optional + For MultiFab that is an element of a vector, the direction number, 0, 1, or 2. + level: int The refinement level @@ -86,9 +89,10 @@ class _MultiFABWrapper(object): ghost cells. """ - def __init__(self, mf=None, mf_name=None, level=0, include_ghosts=False): + def __init__(self, mf=None, mf_name=None, idir=None, level=0, include_ghosts=False): self._mf = mf self.mf_name = mf_name + self.idir = idir self.level = level self.include_ghosts = include_ghosts @@ -116,8 +120,16 @@ def mf(self): else: # Always fetch this anew in case the C++ MultiFab is recreated warpx = libwarpx.libwarpx_so.get_instance() - # All MultiFab names have the level suffix - return warpx.multifab(f"{self.mf_name}[level={self.level}]") + if self.mf_name.startswith('pml'): + # Temporary until pml are updated to new method + name = f"{self.mf_name}[level={self.level}]" + else: + name = self.mf_name + if self.idir is not None: + direction = libwarpx.libwarpx_so.Direction(self.idir) + return warpx.multifab(name, direction, self.level) + else: + return warpx.multifab(name, self.level) @property def shape(self): @@ -573,145 +585,145 @@ def norm0(self, *args): def ExWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="Efield_aux[x]", level=level, include_ghosts=include_ghosts + mf_name="Efield_aux", idir=0, level=level, include_ghosts=include_ghosts ) def EyWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="Efield_aux[y]", level=level, include_ghosts=include_ghosts + mf_name="Efield_aux", idir=1, level=level, include_ghosts=include_ghosts ) def EzWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="Efield_aux[z]", level=level, include_ghosts=include_ghosts + mf_name="Efield_aux", idir=2, level=level, include_ghosts=include_ghosts ) def BxWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="Bfield_aux[x]", level=level, include_ghosts=include_ghosts + mf_name="Bfield_aux", idir=0, level=level, include_ghosts=include_ghosts ) def ByWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="Bfield_aux[y]", level=level, include_ghosts=include_ghosts + mf_name="Bfield_aux", idir=1, level=level, include_ghosts=include_ghosts ) def BzWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="Bfield_aux[z]", level=level, include_ghosts=include_ghosts + mf_name="Bfield_aux", idir=2, level=level, include_ghosts=include_ghosts ) def JxWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="current_fp[x]", level=level, include_ghosts=include_ghosts + mf_name="current_fp", idir=0, level=level, include_ghosts=include_ghosts ) def JyWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="current_fp[y]", level=level, include_ghosts=include_ghosts + mf_name="current_fp", idir=1, level=level, include_ghosts=include_ghosts ) def JzWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="current_fp[z]", level=level, include_ghosts=include_ghosts + mf_name="current_fp", idir=2, level=level, include_ghosts=include_ghosts ) def ExFPWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="Efield_fp[x]", level=level, include_ghosts=include_ghosts + mf_name="Efield_fp", idir=0, level=level, include_ghosts=include_ghosts ) def EyFPWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="Efield_fp[y]", level=level, include_ghosts=include_ghosts + mf_name="Efield_fp", idir=1, level=level, include_ghosts=include_ghosts ) def EzFPWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="Efield_fp[z]", level=level, include_ghosts=include_ghosts + mf_name="Efield_fp", idir=2, level=level, include_ghosts=include_ghosts ) def BxFPWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="Bfield_fp[x]", level=level, include_ghosts=include_ghosts + mf_name="Bfield_fp", idir=0, level=level, include_ghosts=include_ghosts ) def ByFPWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="Bfield_fp[y]", level=level, include_ghosts=include_ghosts + mf_name="Bfield_fp", idir=1, level=level, include_ghosts=include_ghosts ) def BzFPWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="Bfield_fp[z]", level=level, include_ghosts=include_ghosts + mf_name="Bfield_fp", idir=2, level=level, include_ghosts=include_ghosts ) def ExFPExternalWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="Efield_fp_external[x]", level=level, include_ghosts=include_ghosts + mf_name="Efield_fp_external", idir=0, level=level, include_ghosts=include_ghosts ) def EyFPExternalWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="Efield_fp_external[y]", level=level, include_ghosts=include_ghosts + mf_name="Efield_fp_external", idir=1, level=level, include_ghosts=include_ghosts ) def EzFPExternalWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="Efield_fp_external[z]", level=level, include_ghosts=include_ghosts + mf_name="Efield_fp_external", idir=2, level=level, include_ghosts=include_ghosts ) def BxFPExternalWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="Bfield_fp_external[x]", level=level, include_ghosts=include_ghosts + mf_name="Bfield_fp_external", idir=0, level=level, include_ghosts=include_ghosts ) def ByFPExternalWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="Bfield_fp_external[y]", level=level, include_ghosts=include_ghosts + mf_name="Bfield_fp_external", idir=1, level=level, include_ghosts=include_ghosts ) def BzFPExternalWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="Bfield_fp_external[z]", level=level, include_ghosts=include_ghosts + mf_name="Bfield_fp_external", idir=2, level=level, include_ghosts=include_ghosts ) def JxFPWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="current_fp[x]", level=level, include_ghosts=include_ghosts + mf_name="current_fp", idir=0, level=level, include_ghosts=include_ghosts ) def JyFPWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="current_fp[y]", level=level, include_ghosts=include_ghosts + mf_name="current_fp", idir=1, level=level, include_ghosts=include_ghosts ) def JzFPWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="current_fp[z]", level=level, include_ghosts=include_ghosts + mf_name="current_fp", idir=2, level=level, include_ghosts=include_ghosts ) @@ -737,7 +749,7 @@ def GFPWrapper(level=0, include_ghosts=False): def AxFPWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="vector_potential_fp_nodal[x]", + mf_name="vector_potential_fp_nodal", idir=0, level=level, include_ghosts=include_ghosts, ) @@ -745,7 +757,7 @@ def AxFPWrapper(level=0, include_ghosts=False): def AyFPWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="vector_potential_fp_nodal[y]", + mf_name="vector_potential_fp_nodal", idir=1, level=level, include_ghosts=include_ghosts, ) @@ -753,7 +765,7 @@ def AyFPWrapper(level=0, include_ghosts=False): def AzFPWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="vector_potential_fp_nodal[z]", + mf_name="vector_potential_fp_nodal", idir=2, level=level, include_ghosts=include_ghosts, ) @@ -761,55 +773,55 @@ def AzFPWrapper(level=0, include_ghosts=False): def ExCPWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="Efield_cp[x]", level=level, include_ghosts=include_ghosts + mf_name="Efield_cp", idir=0, level=level, include_ghosts=include_ghosts ) def EyCPWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="Efield_cp[y]", level=level, include_ghosts=include_ghosts + mf_name="Efield_cp", idir=1, level=level, include_ghosts=include_ghosts ) def EzCPWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="Efield_cp[z]", level=level, include_ghosts=include_ghosts + mf_name="Efield_cp", idir=2, level=level, include_ghosts=include_ghosts ) def BxCPWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="Bfield_cp[x]", level=level, include_ghosts=include_ghosts + mf_name="Bfield_cp", idir=0, level=level, include_ghosts=include_ghosts ) def ByCPWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="Bfield_cp[y]", level=level, include_ghosts=include_ghosts + mf_name="Bfield_cp", idir=1, level=level, include_ghosts=include_ghosts ) def BzCPWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="Bfield_cp[z]", level=level, include_ghosts=include_ghosts + mf_name="Bfield_cp", idir=2, level=level, include_ghosts=include_ghosts ) def JxCPWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="current_cp[x]", level=level, include_ghosts=include_ghosts + mf_name="current_cp", idir=0, level=level, include_ghosts=include_ghosts ) def JyCPWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="current_cp[y]", level=level, include_ghosts=include_ghosts + mf_name="current_cp", idir=1, level=level, include_ghosts=include_ghosts ) def JzCPWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="current_cp[z]", level=level, include_ghosts=include_ghosts + mf_name="current_cp", idir=2, level=level, include_ghosts=include_ghosts ) @@ -829,55 +841,55 @@ def GCPWrapper(level=0, include_ghosts=False): def EdgeLengthsxWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="edge_lengths[x]", level=level, include_ghosts=include_ghosts + mf_name="edge_lengths", idir=0, level=level, include_ghosts=include_ghosts ) def EdgeLengthsyWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="edge_lengths[y]", level=level, include_ghosts=include_ghosts + mf_name="edge_lengths", idir=1, level=level, include_ghosts=include_ghosts ) def EdgeLengthszWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="edge_lengths[z]", level=level, include_ghosts=include_ghosts + mf_name="edge_lengths", idir=2, level=level, include_ghosts=include_ghosts ) def FaceAreasxWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="face_areas[x]", level=level, include_ghosts=include_ghosts + mf_name="face_areas", idir=0, level=level, include_ghosts=include_ghosts ) def FaceAreasyWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="face_areas[y]", level=level, include_ghosts=include_ghosts + mf_name="face_areas", idir=1, level=level, include_ghosts=include_ghosts ) def FaceAreaszWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="face_areas[z]", level=level, include_ghosts=include_ghosts + mf_name="face_areas", idir=2, level=level, include_ghosts=include_ghosts ) def JxFPAmpereWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="current_fp_ampere[x]", level=level, include_ghosts=include_ghosts + mf_name="current_fp_ampere", idir=0, level=level, include_ghosts=include_ghosts ) def JyFPAmpereWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="current_fp_ampere[y]", level=level, include_ghosts=include_ghosts + mf_name="current_fp_ampere", idir=1, level=level, include_ghosts=include_ghosts ) def JzFPAmpereWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="current_fp_ampere[z]", level=level, include_ghosts=include_ghosts + mf_name="current_fp_ampere", idir=2, level=level, include_ghosts=include_ghosts ) diff --git a/Source/Python/MultiFabRegister.cpp b/Source/Python/MultiFabRegister.cpp index c2c52936793..da0712a8909 100644 --- a/Source/Python/MultiFabRegister.cpp +++ b/Source/Python/MultiFabRegister.cpp @@ -13,7 +13,10 @@ void init_MultiFabRegister (py::module & m) { - using ablastr::fields::MultiFabRegister; + using namespace ablastr::fields; + + py::class_(m, "Direction") + .def(py::init()); py::class_(m, "MultiFabRegister") diff --git a/Source/Python/WarpX.cpp b/Source/Python/WarpX.cpp index 8e7e8f189e0..311d3deee1e 100644 --- a/Source/Python/WarpX.cpp +++ b/Source/Python/WarpX.cpp @@ -55,6 +55,8 @@ namespace warpx { void init_WarpX (py::module& m) { + using ablastr::fields::Direction; + // Expose the WarpX instance m.def("get_instance", [] () { return &WarpX::GetInstance(); }, @@ -118,16 +120,41 @@ void init_WarpX (py::module& m) R"doc(Registry to all WarpX MultiFab (fields).)doc" ) .def("multifab", - [](WarpX const & wx, std::string const multifab_name) { - if (wx.multifab_map.count(multifab_name) > 0) { + [](WarpX & wx, std::string multifab_name, int level) { + if (wx.m_fields.has(multifab_name, level)) { + return wx.m_fields.get(multifab_name, level); + } else if (wx.multifab_map.count(multifab_name) > 0) { return wx.multifab_map.at(multifab_name); } else { throw std::runtime_error("The MultiFab '" + multifab_name + "' is unknown or is not allocated!"); } }, py::arg("multifab_name"), + py::arg("level"), + py::return_value_policy::reference_internal, + R"doc(Return MultiFabs by name and level, e.g., ``\"Efield_aux\"``, ``\"Efield_fp"``, ... + +The physical fields in WarpX have the following naming: + +- ``_fp`` are the "fine" patches, the regular resolution of a current mesh-refinement level +- ``_aux`` are temporary (auxiliar) patches at the same resolution as ``_fp``. + They usually include contributions from other levels and can be interpolated for gather routines of particles. +- ``_cp`` are "coarse" patches, at the same resolution (but not necessary values) as the ``_fp`` of ``level - 1`` + (only for level 1 and higher).)doc" + ) + .def("multifab", + [](WarpX & wx, std::string multifab_name, Direction dir, int level) { + if (wx.m_fields.has(multifab_name, dir, level)) { + return wx.m_fields.get(multifab_name, dir, level); + } else { + throw std::runtime_error("The MultiFab '" + multifab_name + "' is unknown or is not allocated!"); + } + }, + py::arg("multifab_name"), + py::arg("dir"), + py::arg("level"), py::return_value_policy::reference_internal, - R"doc(Return MultiFabs by name, e.g., ``\"Efield_aux[x][level=0]\"``, ``\"Efield_cp[x][level=0]\"``, ... + R"doc(Return MultiFabs by name, direction, and level, e.g., ``\"Efield_aux\"``, ``\"Efield_fp"``, ... The physical fields in WarpX have the following naming: From abcb0877308c8e1cc06779c9936f2b217b64d663 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Sep 2024 22:48:00 +0000 Subject: [PATCH 192/314] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- Python/pywarpx/fields.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Python/pywarpx/fields.py b/Python/pywarpx/fields.py index 90b1c6e4fef..33cb2c7402a 100644 --- a/Python/pywarpx/fields.py +++ b/Python/pywarpx/fields.py @@ -120,7 +120,7 @@ def mf(self): else: # Always fetch this anew in case the C++ MultiFab is recreated warpx = libwarpx.libwarpx_so.get_instance() - if self.mf_name.startswith('pml'): + if self.mf_name.startswith("pml"): # Temporary until pml are updated to new method name = f"{self.mf_name}[level={self.level}]" else: @@ -749,7 +749,8 @@ def GFPWrapper(level=0, include_ghosts=False): def AxFPWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="vector_potential_fp_nodal", idir=0, + mf_name="vector_potential_fp_nodal", + idir=0, level=level, include_ghosts=include_ghosts, ) @@ -757,7 +758,8 @@ def AxFPWrapper(level=0, include_ghosts=False): def AyFPWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="vector_potential_fp_nodal", idir=1, + mf_name="vector_potential_fp_nodal", + idir=1, level=level, include_ghosts=include_ghosts, ) @@ -765,7 +767,8 @@ def AyFPWrapper(level=0, include_ghosts=False): def AzFPWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="vector_potential_fp_nodal", idir=2, + mf_name="vector_potential_fp_nodal", + idir=2, level=level, include_ghosts=include_ghosts, ) From 16cff546230335874785562264a240da351e3b2c Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 12 Sep 2024 16:23:13 -0700 Subject: [PATCH 193/314] Fix typo --- Source/EmbeddedBoundary/WarpXInitEB.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index 58445e0fae3..0b98e13b5ab 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -391,7 +391,7 @@ WarpX::ComputeDistanceToEB () for (int lev=0; lev<=maxLevel(); lev++) { const amrex::EB2::Level& eb_level = eb_is.getLevel(Geom(lev)); auto const eb_fact = fieldEBFactory(lev); - amrex::FillSignedDistance(*m_fields.get("distance_to_e", lev), eb_level, eb_fact, 1); + amrex::FillSignedDistance(*m_fields.get("distance_to_eb", lev), eb_level, eb_fact, 1); } #endif } From 2397d98af9e780c9e0b59d7eaa9d3fd28d9b3068 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 12 Sep 2024 16:46:18 -0700 Subject: [PATCH 194/314] Fix another typo --- Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index 9dc8d85d9d6..8c8cefb95c6 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -648,7 +648,7 @@ FlushFormatPlotfile::WriteAllRawFields( warpx.m_fields.get("Bfield_cp", Direction{0}, lev), warpx.m_fields.get("Bfield_cp", Direction{1}, lev), warpx.m_fields.get("Bfield_cp", Direction{2}, lev), - warpx.m_fields.get("Bfield_cp", Direction{0}, lev), + warpx.m_fields.get("Bfield_fp", Direction{0}, lev), warpx.m_fields.get("Bfield_fp", Direction{1}, lev), warpx.m_fields.get("Bfield_fp", Direction{2}, lev), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards); From 0202ebcaf020e38c7b62da8b40263edaf2da85fe Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Thu, 12 Sep 2024 15:32:07 -0700 Subject: [PATCH 195/314] removing getField FlushFormatPlotFile. --- .../Diagnostics/FlushFormats/FlushFormatPlotfile.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index 8c8cefb95c6..d7cbbd98574 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -615,22 +615,22 @@ FlushFormatPlotfile::WriteAllRawFields( // Averaged fields on fine patch if (WarpX::fft_do_time_averaging) { - WriteRawMF(warpx.getField(FieldType::Efield_avg_fp, lev, 0) , dm, raw_pltname, default_level_prefix, + WriteRawMF(*warpx.m_fields.get("Efield_avg_fp", Direction{0}, lev) , dm, raw_pltname, default_level_prefix, "Ex_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(warpx.getField(FieldType::Efield_avg_fp, lev, 1) , dm, raw_pltname, default_level_prefix, + WriteRawMF(*warpx.m_fields.get("Efield_avg_fp", Direction{1}, lev) , dm, raw_pltname, default_level_prefix, "Ey_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(warpx.getField(FieldType::Efield_avg_fp, lev, 2) , dm, raw_pltname, default_level_prefix, + WriteRawMF(*warpx.m_fields.get("Efield_avg_fp", Direction{2}, lev) , dm, raw_pltname, default_level_prefix, "Ez_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(warpx.getField(FieldType::Bfield_avg_fp, lev, 0) , dm, raw_pltname, default_level_prefix, + WriteRawMF(*warpx.m_fields.get("Bfield_avg_fp", Direction{0}, lev) , dm, raw_pltname, default_level_prefix, "Bx_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(warpx.getField(FieldType::Bfield_avg_fp, lev, 1) , dm, raw_pltname, default_level_prefix, + WriteRawMF(*warpx.m_fields.get("Bfield_avg_fp", Direction{1}, lev) , dm, raw_pltname, default_level_prefix, "By_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(warpx.getField(FieldType::Bfield_avg_fp, lev, 2) , dm, raw_pltname, default_level_prefix, + WriteRawMF(*warpx.m_fields.get("Bfield_avg_fp", Direction{2}, lev) , dm, raw_pltname, default_level_prefix, "Bz_avg_fp", lev, plot_raw_fields_guards); } From 47f51b7052da767bc722c3c25486ecc731c526ab Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Thu, 12 Sep 2024 16:08:23 -0700 Subject: [PATCH 196/314] removed getField and getFieldPointer funs from WarpX. --- .../FlushFormats/FlushFormatCheckpoint.cpp | 36 ++++++++--------- .../HybridPICModel/HybridPICModel.cpp | 7 ++-- Source/WarpX.H | 40 ------------------- Source/WarpX.cpp | 34 ---------------- 4 files changed, 22 insertions(+), 95 deletions(-) diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index d9d3551dfd8..ebdddab830f 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -83,18 +83,18 @@ FlushFormatCheckpoint::WriteToFile ( if (WarpX::fft_do_time_averaging) { - VisMF::Write(warpx.getField(FieldType::Efield_avg_fp, lev, 0), + VisMF::Write(*warpx.m_fields.get("Efield_avg_fp", Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ex_avg_fp")); - VisMF::Write(warpx.getField(FieldType::Efield_avg_fp, lev, 1), + VisMF::Write(*warpx.m_fields.get("Efield_avg_fp", Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ey_avg_fp")); - VisMF::Write(warpx.getField(FieldType::Efield_avg_fp, lev, 2), + VisMF::Write(*warpx.m_fields.get("Efield_avg_fp", Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ez_avg_fp")); - VisMF::Write(warpx.getField(FieldType::Bfield_avg_fp, lev, 0), + VisMF::Write(*warpx.m_fields.get("Bfield_avg_fp", Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bx_avg_fp")); - VisMF::Write(warpx.getField(FieldType::Bfield_avg_fp, lev, 1), + VisMF::Write(*warpx.m_fields.get("Bfield_avg_fp", Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "By_avg_fp")); - VisMF::Write(warpx.getField(FieldType::Bfield_avg_fp, lev, 2), + VisMF::Write(*warpx.m_fields.get("Bfield_avg_fp", Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bz_avg_fp")); } @@ -110,33 +110,33 @@ FlushFormatCheckpoint::WriteToFile ( if (lev > 0) { - VisMF::Write(warpx.getField(FieldType::Efield_cp, lev, 0), + VisMF::Write(*warpx.m_fields.get("Efield_cp", Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ex_cp")); - VisMF::Write(warpx.getField(FieldType::Efield_cp, lev, 1), + VisMF::Write(*warpx.m_fields.get("Efield_cp", Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ey_cp")); - VisMF::Write(warpx.getField(FieldType::Efield_cp, lev, 2), + VisMF::Write(*warpx.m_fields.get("Efield_cp", Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ez_cp")); - VisMF::Write(warpx.getField(FieldType::Bfield_cp, lev, 0), + VisMF::Write(*warpx.m_fields.get("Bfield_cp", Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bx_cp")); - VisMF::Write(warpx.getField(FieldType::Bfield_cp, lev, 1), + VisMF::Write(*warpx.m_fields.get("Bfield_cp", Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "By_cp")); - VisMF::Write(warpx.getField(FieldType::Bfield_cp, lev, 2), + VisMF::Write(*warpx.m_fields.get("Bfield_cp", Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bz_cp")); if (WarpX::fft_do_time_averaging) { - VisMF::Write(warpx.getField(FieldType::Efield_avg_cp, lev, 0), + VisMF::Write(*warpx.m_fields.get("Efield_avg_cp", Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ex_avg_cp")); - VisMF::Write(warpx.getField(FieldType::Efield_avg_cp, lev, 1), + VisMF::Write(*warpx.m_fields.get("Efield_avg_cp", Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ey_avg_cp")); - VisMF::Write(warpx.getField(FieldType::Efield_avg_cp, lev, 2), + VisMF::Write(*warpx.m_fields.get("Efield_avg_cp", Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ez_avg_cp")); - VisMF::Write(warpx.getField(FieldType::Bfield_avg_cp, lev, 0), + VisMF::Write(*warpx.m_fields.get("Bfield_avg_cp", Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bx_avg_cp")); - VisMF::Write(warpx.getField(FieldType::Bfield_avg_cp, lev, 1), + VisMF::Write(*warpx.m_fields.get("Bfield_avg_cp", Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "By_avg_cp")); - VisMF::Write(warpx.getField(FieldType::Bfield_avg_cp, lev, 2), + VisMF::Write(*warpx.m_fields.get("Bfield_avg_cp", Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bz_avg_cp")); } diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 5d2638f5468..5b1cc787920 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -232,9 +232,10 @@ void HybridPICModel::InitData () auto edge_lengths = std::array, 3>(); #ifdef AMREX_USE_EB if (EB::enabled()) { - auto const & edge_lengths_x = warpx.getField(FieldType::edge_lengths, lev, 0); - auto const & edge_lengths_y = warpx.getField(FieldType::edge_lengths, lev, 1); - auto const & edge_lengths_z = warpx.getField(FieldType::edge_lengths, lev, 2); + using ablastr::fields::Direction; + auto const & edge_lengths_x = *warpx.m_fields.get("edge_lengths", Direction{0}, lev); + auto const & edge_lengths_y = *warpx.m_fields.get("edge_lengths", Direction{1}, lev); + auto const & edge_lengths_z = *warpx.m_fields.get("edge_lengths", Direction{2}, lev); edge_lengths = std::array< std::unique_ptr, 3 >{ std::make_unique( diff --git a/Source/WarpX.H b/Source/WarpX.H index f53ccc83efa..52ea7f7fa5c 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -460,32 +460,6 @@ public: static std::map multifab_map; static std::map imultifab_map; - /** - * \brief - * Get a pointer to the field data. - * - * \param field_type[in] the field type - * \param lev[in] the mesh refinement level - * \param direction[in] the field component (0 by default) - * - * \return the pointer to an amrex::MultiFab containing the field data - */ - [[nodiscard]] amrex::MultiFab* - getFieldPointer (warpx::fields::FieldType field_type, int lev, int direction = 0) const; - - /** - * \brief - * Get a constant reference to the field data. - * - * \param field_type[in] the field type - * \param lev[in] the mesh refinement level - * \param direction[in] the field component (0 by default) - * - * \return a constant refernce to an amrex::MultiFab containing the field data - */ - [[nodiscard]] const amrex::MultiFab& - getField(warpx::fields::FieldType field_type, int lev, int direction = 0) const; - /** * \brief * Get pointer to the amrex::MultiFab containing the dotMask for the specified field @@ -1489,20 +1463,6 @@ private: amrex::Vector > current_buffer_masks; amrex::Vector > gather_buffer_masks; - /** - * \brief - * Get a pointer to the field data. Does not check if the pointer - * is not nullptr. - * - * \param field_type[in] the field type - * \param lev[in] the mesh refinement level - * \param direction[in] the field component (0 by default) - * - * \return the pointer to an amrex::MultiFab containing the field data - */ - [[nodiscard]] amrex::MultiFab* - getFieldPointerUnchecked (warpx::fields::FieldType field_type, int lev, int direction = 0) const; - // PML int do_pml = 0; int do_silver_mueller = 0; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 10706139111..95361063300 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -3430,40 +3430,6 @@ WarpX::AliasInitMultiFab ( multifab_map[name_with_suffix] = mf.get(); } -amrex::MultiFab* -WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, const int direction) const -{ - // This function does *not* check if the returned field pointer is != nullptr - - amrex::MultiFab* field_pointer = nullptr; - - amrex::ignore_unused(lev, direction); - - switch(field_type) - { - default: - WARPX_ABORT_WITH_MESSAGE("Invalid field type"); - break; - } - - return field_pointer; -} - -amrex::MultiFab* -WarpX::getFieldPointer (const FieldType field_type, const int lev, const int direction) const -{ - auto* const field_pointer = getFieldPointerUnchecked(field_type, lev, direction); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - field_pointer != nullptr, "Requested field is not initialized!"); - return field_pointer; -} - -const amrex::MultiFab& -WarpX::getField(FieldType field_type, const int lev, const int direction) const -{ - return *getFieldPointer(field_type, lev, direction); -} - amrex::DistributionMapping WarpX::MakeDistributionMap (int lev, amrex::BoxArray const& ba) { From 153aaa2ace1fbb02ac6768b49724467d24ab172e Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Thu, 12 Sep 2024 16:32:37 -0700 Subject: [PATCH 197/314] removing unused names form Fields namespace. --- Source/FieldSolver/Fields.H | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/Source/FieldSolver/Fields.H b/Source/FieldSolver/Fields.H index 6d287646ae6..8cc43306e6e 100644 --- a/Source/FieldSolver/Fields.H +++ b/Source/FieldSolver/Fields.H @@ -17,26 +17,14 @@ namespace warpx::fields None, Efield_fp, Bfield_fp, - current_fp, - current_fp_nodal, - phi_fp, vector_potential_fp, - Efield_cp, - Bfield_cp, - current_cp, - edge_lengths, - face_areas, - Efield_avg_fp, - Bfield_avg_fp, - Efield_avg_cp, - Bfield_avg_cp + phi_fp }; constexpr FieldType ArrayFieldTypes[] = { - FieldType::Efield_fp, FieldType::Bfield_fp, - FieldType::current_fp, FieldType::current_fp_nodal, FieldType::vector_potential_fp, - FieldType::Efield_cp, FieldType::Bfield_cp, FieldType::current_cp, - FieldType::Efield_avg_fp, FieldType::Bfield_avg_fp, FieldType::Efield_avg_cp, FieldType::Bfield_avg_cp}; + FieldType::Efield_fp, + FieldType::Bfield_fp, + FieldType::vector_potential_fp }; inline bool isFieldArray (const FieldType field_type) From 92829f6b60d2df308963fa74c730021af8f6b484 Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Thu, 12 Sep 2024 16:39:41 -0700 Subject: [PATCH 198/314] restoring const declaration to SetDotMask --- Source/WarpX.H | 4 ++-- Source/WarpX.cpp | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Source/WarpX.H b/Source/WarpX.H index 52ea7f7fa5c..d2f2df30633 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -465,14 +465,14 @@ public: * Get pointer to the amrex::MultiFab containing the dotMask for the specified field */ [[nodiscard]] const amrex::iMultiFab* - getFieldDotMaskPointer (warpx::fields::FieldType field_type, int lev, int dir); + getFieldDotMaskPointer (warpx::fields::FieldType field_type, int lev, int dir) const; /** * \brief * Set the dotMask container */ void SetDotMask( std::unique_ptr& field_dotMask, - std::string field_name, int lev, int dir ); + std::string field_name, int lev, int dir ) const; [[nodiscard]] bool DoPML () const {return do_pml;} [[nodiscard]] bool DoFluidSpecies () const {return do_fluid_species;} diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 95361063300..0095b99c57a 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -3457,7 +3457,7 @@ WarpX::MakeDistributionMap (int lev, amrex::BoxArray const& ba) } const amrex::iMultiFab* -WarpX::getFieldDotMaskPointer ( FieldType field_type, int lev, int dir ) +WarpX::getFieldDotMaskPointer ( FieldType field_type, int lev, int dir ) const { switch(field_type) { @@ -3480,13 +3480,13 @@ WarpX::getFieldDotMaskPointer ( FieldType field_type, int lev, int dir ) } void WarpX::SetDotMask( std::unique_ptr& field_dotMask, - std::string field_name, int lev, int dir ) + std::string field_name, int lev, int dir ) const { // Define the dot mask for this field_type needed to properly compute dotProduct() // for field values that have shared locations on different MPI ranks if (field_dotMask != nullptr) { return; } - ablastr::fields::VectorField const& this_field = m_fields.get_alldirs(field_name,lev); + ablastr::fields::ConstVectorField const& this_field = m_fields.get_alldirs(field_name,lev); const amrex::BoxArray& this_ba = this_field[dir]->boxArray(); const amrex::MultiFab tmp( this_ba, this_field[dir]->DistributionMap(), 1, 0, amrex::MFInfo().SetAlloc(false) ); From 088f413075d04872dd8c15a0671b36a7a8faf015 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 12 Sep 2024 16:48:39 -0700 Subject: [PATCH 199/314] Fix another typo --- Source/Evolve/WarpXEvolve.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 95790342049..ba945d6049f 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -543,9 +543,9 @@ void WarpX::HandleParticlesAtBoundaries (int step, amrex::Real cur_time, int num // interact the particles with EB walls (if present) if (EB::enabled()) { - mypc->ScrapeParticlesAtEB(m_fields.get_mr_levels("distance_to_e", finest_level)); + mypc->ScrapeParticlesAtEB(m_fields.get_mr_levels("distance_to_eb", finest_level)); m_particle_boundary_buffer->gatherParticlesFromEmbeddedBoundaries( - *mypc, m_fields.get_mr_levels("distance_to_e", finest_level)); + *mypc, m_fields.get_mr_levels("distance_to_eb", finest_level)); mypc->deleteInvalidParticles(); } From 4dc9bb039b7813b877c50a2068ea58cc496e4d13 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 12 Sep 2024 17:06:39 -0700 Subject: [PATCH 200/314] Edoardo: Fix PML copy `Efield_fp` Co-authored-by: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> --- Source/BoundaryConditions/WarpXFieldBoundaries.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index e56af6492a8..15167af24fe 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -135,10 +135,10 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d if (lev == 0) { if (a_dt_type == DtType::FirstHalf) { if(::isAnyBoundary(field_boundary_lo, field_boundary_hi)){ - auto Efield_fp_new = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); // JRA, new to prevent shadow + auto Efield_fp = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); // JRA, new to prevent shadow auto Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp",max_level); m_fdtd_solver_fp[0]->ApplySilverMuellerBoundary( - Efield_fp_new[lev], Bfield_fp[lev], + Efield_fp[lev], Bfield_fp[lev], Geom(lev).Domain(), dt[lev], field_boundary_lo, field_boundary_hi); } From cdbb3ada1975c3a6bea23fa22bf41c767abf2ce1 Mon Sep 17 00:00:00 2001 From: David Grote Date: Thu, 12 Sep 2024 17:06:39 -0700 Subject: [PATCH 201/314] Fix NullifyMF --- Source/Evolve/WarpXEvolve.cpp | 26 ++++++++++++---------- Source/Utils/WarpXUtil.H | 40 +++++++++++++++++++++++++++++++-- Source/Utils/WarpXUtil.cpp | 42 +++++++++++++++++++++++++++++------ 3 files changed, 87 insertions(+), 21 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index ba945d6049f..0ef6f03608d 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -1195,6 +1195,8 @@ WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, void WarpX::applyMirrors (Real time) { + using ablastr::fields::Direction; + // something to do? if (num_mirrors == 0) { return; @@ -1222,12 +1224,12 @@ WarpX::applyMirrors (Real time) const amrex::Real z_max = std::max(z_max_tmp, z_min+mirror_z_npoints[i_mirror]*dz); // Set each field on the fine patch to zero between z_min and z_max - NullifyMF(m_fields, "Efield_fp[0]", lev, z_min, z_max); - NullifyMF(m_fields, "Efield_fp[1]", lev, z_min, z_max); - NullifyMF(m_fields, "Efield_fp[2]", lev, z_min, z_max); - NullifyMF(m_fields, "Bfield_fp[0]", lev, z_min, z_max); - NullifyMF(m_fields, "Bfield_fp[1]", lev, z_min, z_max); - NullifyMF(m_fields, "Bfield_fp[2]", lev, z_min, z_max); + NullifyMF(m_fields, "Efield_fp", Direction{0}, lev, z_min, z_max); + NullifyMF(m_fields, "Efield_fp", Direction{1}, lev, z_min, z_max); + NullifyMF(m_fields, "Efield_fp", Direction{2}, lev, z_min, z_max); + NullifyMF(m_fields, "Bfield_fp", Direction{0}, lev, z_min, z_max); + NullifyMF(m_fields, "Bfield_fp", Direction{1}, lev, z_min, z_max); + NullifyMF(m_fields, "Bfield_fp", Direction{2}, lev, z_min, z_max); // If div(E)/div(B) cleaning are used, set F/G field to zero NullifyMF(m_fields, "F_fp", lev, z_min, z_max); @@ -1236,12 +1238,12 @@ WarpX::applyMirrors (Real time) if (lev>0) { // Set each field on the coarse patch to zero between z_min and z_max - NullifyMF(m_fields, "Efield_cp[0]", lev, z_min, z_max); - NullifyMF(m_fields, "Efield_cp[1]", lev, z_min, z_max); - NullifyMF(m_fields, "Efield_cp[2]", lev, z_min, z_max); - NullifyMF(m_fields, "Bfield_cp[0]", lev, z_min, z_max); - NullifyMF(m_fields, "Bfield_cp[1]", lev, z_min, z_max); - NullifyMF(m_fields, "Bfield_cp[2]", lev, z_min, z_max); + NullifyMF(m_fields, "Efield_cp", Direction{0}, lev, z_min, z_max); + NullifyMF(m_fields, "Efield_cp", Direction{1}, lev, z_min, z_max); + NullifyMF(m_fields, "Efield_cp", Direction{2}, lev, z_min, z_max); + NullifyMF(m_fields, "Bfield_cp", Direction{0}, lev, z_min, z_max); + NullifyMF(m_fields, "Bfield_cp", Direction{1}, lev, z_min, z_max); + NullifyMF(m_fields, "Bfield_cp", Direction{2}, lev, z_min, z_max); // If div(E)/div(B) cleaning are used, set F/G field to zero NullifyMF(m_fields, "F_cp", lev, z_min, z_max); diff --git a/Source/Utils/WarpXUtil.H b/Source/Utils/WarpXUtil.H index fca0b95ba7f..e35b0cdb313 100644 --- a/Source/Utils/WarpXUtil.H +++ b/Source/Utils/WarpXUtil.H @@ -57,11 +57,47 @@ void CheckGriddingForRZSpectral (); /** Function that sets the value of MultiFab MF to zero. * - * For z between zmin and zmax. + * \param[in] mf Pointer to the MultiFab + * \param[in] lev The mesh refinement level + * \param[in] zmin The minimum z of the range to be mullified + * \param[in] zmin The maximum z of the range to be mullified + */ +void NullifyMFinstance ( + amrex::MultiFab *mf, + int lev, + amrex::Real zmin, + amrex::Real zmax +); + +/** Function that sets the value of MultiFab MF to zero. + * + * \param[in] multifab_map Multifab registry + * \param[in] nf_name Name of Multifab to modify + * \param[in] lev The mesh refinement level + * \param[in] zmin The minimum z of the range to be mullified + * \param[in] zmin The maximum z of the range to be mullified + */ +void NullifyMF ( + ablastr::fields::MultiFabRegister& multifab_map, + std::string const& mf_name, + int lev, + amrex::Real zmin, + amrex::Real zmax +); + +/** Function that sets the value of MultiFab MF to zero. + * + * \param[in] multifab_map Multifab registry + * \param[in] nf_name Name of Multifab to modify + * \param[in] dir Direction, for Multifabs that are components of vectors + * \param[in] lev The mesh refinement level + * \param[in] zmin The minimum z of the range to be mullified + * \param[in] zmin The maximum z of the range to be mullified */ void NullifyMF ( ablastr::fields::MultiFabRegister& multifab_map, - std::string mf_name, + std::string const& mf_name, + ablastr::fields::Direction dir, int lev, amrex::Real zmin, amrex::Real zmax diff --git a/Source/Utils/WarpXUtil.cpp b/Source/Utils/WarpXUtil.cpp index 35d0f08be54..44952091257 100644 --- a/Source/Utils/WarpXUtil.cpp +++ b/Source/Utils/WarpXUtil.cpp @@ -221,18 +221,13 @@ void ConvertLabParamsToBoost() } -void NullifyMF ( - ablastr::fields::MultiFabRegister& multifab_map, - std::string mf_name, +void NullifyMFinstance ( + amrex::MultiFab *mf, int lev, amrex::Real zmin, amrex::Real zmax ) { - WARPX_PROFILE("WarpXUtil::NullifyMF()"); - if (!multifab_map.has(mf_name, lev)) { return; } - - auto * mf = multifab_map.get(mf_name, lev); int const ncomp = mf->nComp(); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) @@ -273,6 +268,39 @@ void NullifyMF ( } } +void NullifyMF ( + ablastr::fields::MultiFabRegister& multifab_map, + std::string const& mf_name, + int lev, + amrex::Real zmin, + amrex::Real zmax +) +{ + WARPX_PROFILE("WarpXUtil::NullifyMF()"); + if (!multifab_map.has(mf_name, lev)) { return; } + + auto * mf = multifab_map.get(mf_name, lev); + + NullifyMFinstance ( mf, lev, zmin, zmax); +} + +void NullifyMF ( + ablastr::fields::MultiFabRegister& multifab_map, + std::string const& mf_name, + ablastr::fields::Direction dir, + int lev, + amrex::Real zmin, + amrex::Real zmax +) +{ + WARPX_PROFILE("WarpXUtil::NullifyMF()"); + if (!multifab_map.has(mf_name, dir, lev)) { return; } + + auto * mf = multifab_map.get(mf_name, dir, lev); + + NullifyMFinstance ( mf, lev, zmin, zmax); +} + namespace WarpXUtilIO{ bool WriteBinaryDataOnFile(const std::string& filename, const amrex::Vector& data) { From 830a2beeadd106f60de51a810b971e47c2667179 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 12 Sep 2024 17:08:30 -0700 Subject: [PATCH 202/314] Fix: Edoardo B->E Copy Paste Co-authored-by: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> --- Source/BoundaryConditions/PML_RZ.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/BoundaryConditions/PML_RZ.cpp b/Source/BoundaryConditions/PML_RZ.cpp index 984df7e528f..d9982a4bb85 100644 --- a/Source/BoundaryConditions/PML_RZ.cpp +++ b/Source/BoundaryConditions/PML_RZ.cpp @@ -49,8 +49,8 @@ PML_RZ::PML_RZ (const int lev, const amrex::BoxArray& grid_ba, const amrex::Dist const amrex::MultiFab & Et_fp = *warpx.m_fields.get("Efield_fp",Direction{1},lev); const amrex::BoxArray ba_Er = amrex::convert(grid_ba, Er_fp.ixType().toIntVect()); const amrex::BoxArray ba_Et = amrex::convert(grid_ba, Et_fp.ixType().toIntVect()); - WarpX::AllocInitMultiFab(pml_E_fp[0], ba_Er, grid_dm, Er_fp.nComp(), Er_fp.nGrowVect(), lev, "pml_B_fp[0]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_E_fp[1], ba_Et, grid_dm, Et_fp.nComp(), Et_fp.nGrowVect(), lev, "pml_B_fp[1]", 0.0_rt); + WarpX::AllocInitMultiFab(pml_E_fp[0], ba_Er, grid_dm, Er_fp.nComp(), Er_fp.nGrowVect(), lev, "pml_E_fp[0]", 0.0_rt); + WarpX::AllocInitMultiFab(pml_E_fp[1], ba_Et, grid_dm, Et_fp.nComp(), Et_fp.nGrowVect(), lev, "pml_E_fp[1]", 0.0_rt); const amrex::MultiFab & Br_fp = *warpx.m_fields.get("Bfield_fp",Direction{0},lev); const amrex::MultiFab & Bt_fp = *warpx.m_fields.get("Bfield_fp",Direction{1},lev); From 0e831f5f37b7f6fe7837b5c03a6a28573d405654 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 12 Sep 2024 17:23:50 -0700 Subject: [PATCH 203/314] MultiFabRegister: Handle !Remake --- Source/ablastr/fields/MultiFabRegister.H | 10 ++++----- Source/ablastr/fields/MultiFabRegister.cpp | 25 ++++++++++++++++------ 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 69dd18dc457..d44806cd315 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -105,8 +105,8 @@ namespace ablastr::fields /** the MR level of this (i)MultiFab */ int m_level = 0; - /** redistribute */ - bool m_redistribute = true; + /** remake distribution map on load balance, @see amrex::AmrCore::RemakeLevel */ + bool m_remake = true; /** redistribute on @see amrex::AmrCore::RemakeLevel */ bool m_redistribute_on_remake = true; @@ -156,7 +156,7 @@ namespace ablastr::fields * @param ncomp ... * @param ngrow ... * @param initial_value ... - * @param redistribute follow the default domain decomposition of the simulation + * @param remake follow the default domain decomposition of the simulation * @param redistribute_on_remake redistribute on @see amrex::AmrCore::RemakeLevel * @return pointer to newly allocated MultiFab */ @@ -169,7 +169,7 @@ namespace ablastr::fields int ncomp, amrex::IntVect const & ngrow, std::optional initial_value = std::nullopt, - bool redistribute = true, + bool remake = true, bool redistribute_on_remake = true ); amrex::MultiFab* @@ -182,7 +182,7 @@ namespace ablastr::fields int ncomp, amrex::IntVect const & ngrow, std::optional initial_value = std::nullopt, - bool redistribute = true, + bool remake = true, bool redistribute_on_remake = true ); diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 62bdb95c3c8..c4eaeeba8ef 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -23,7 +23,7 @@ namespace ablastr::fields int ncomp, amrex::IntVect const & ngrow, std::optional initial_value, - bool redistribute, + bool remake, bool redistribute_on_remake ) { @@ -44,7 +44,7 @@ namespace ablastr::fields {ba, dm, ncomp, ngrow, tag}, std::nullopt, // scalar: no direction level, - redistribute, + remake, redistribute_on_remake, "" // we own the memory } @@ -75,7 +75,7 @@ namespace ablastr::fields int ncomp, amrex::IntVect const & ngrow, std::optional initial_value, - bool redistribute, + bool remake, bool redistribute_on_remake ) { @@ -100,7 +100,7 @@ namespace ablastr::fields {ba, dm, ncomp, ngrow, tag}, dir, level, - redistribute, + remake, redistribute_on_remake, "" // we own the memory } @@ -174,7 +174,7 @@ namespace ablastr::fields {mf_alias, amrex::make_alias, 0, mf_alias.nComp()}, std::nullopt, // scalar: no direction level, - alias.m_redistribute, + alias.m_remake, alias.m_redistribute_on_remake, alias_name } @@ -235,7 +235,7 @@ namespace ablastr::fields {mf_alias, amrex::make_alias, 0, mf_alias.nComp()}, dir, level, - alias.m_redistribute, + alias.m_remake, alias.m_redistribute_on_remake, alias_name } @@ -266,6 +266,13 @@ namespace ablastr::fields for (auto & element : m_mf_register ) { MultiFabOwner & mf_owner = element.second; + + // keep distribution map as it is? + if (!mf_owner.m_remake) { + continue; + } + + // remake MultiFab with new distribution map if (mf_owner.m_level == level && !mf_owner.is_alias()) { amrex::MultiFab & mf = mf_owner.m_mf; amrex::IntVect const & ng = mf.nGrowVect(); @@ -287,6 +294,12 @@ namespace ablastr::fields for (auto & element : m_mf_register ) { MultiFabOwner & mf_owner = element.second; + + // keep distribution map as it is? + if (!mf_owner.m_remake) { + continue; + } + if (mf_owner.m_level == level && mf_owner.is_alias()) { amrex::MultiFab & mf = mf_owner.m_mf; amrex::MultiFab new_mf(mf, amrex::make_alias, 0, mf.nComp()); From a0a609aab9aa0cb7c2efb3295e9e26f2321feecf Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 12 Sep 2024 20:01:53 -0700 Subject: [PATCH 204/314] Fix allocation for ECT solver --- Source/WarpX.cpp | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 0095b99c57a..7ac8330e5c4 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2347,22 +2347,6 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); } if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { - //! EB: Lengths of the mesh edges - m_fields.alloc_init( "edge_lengths", Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), - dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); - m_fields.alloc_init( "edge_lengths", Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), - dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); - m_fields.alloc_init( "edge_lengths", Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), - dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); - - //! EB: Areas of the mesh faces - m_fields.alloc_init( "face_areas", Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), - dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); - m_fields.alloc_init( "face_areas", Direction{1}, lev, amrex::convert(ba, By_nodal_flag), - dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); - m_fields.alloc_init( "face_areas", Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), - dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); - AllocInitMultiFab(m_flag_info_face[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_flag_info_face[x]"); AllocInitMultiFab(m_flag_info_face[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, From f6c314aa3d4f6c0088e5485568c2be13f125d16c Mon Sep 17 00:00:00 2001 From: Justin Angus Date: Thu, 12 Sep 2024 20:28:14 -0700 Subject: [PATCH 205/314] Efield_fp_new moved back to Efield_fp --- Source/BoundaryConditions/WarpXFieldBoundaries.cpp | 2 +- Source/FieldSolver/ElectrostaticSolver.cpp | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index 15167af24fe..a9f2be4c687 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -135,7 +135,7 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d if (lev == 0) { if (a_dt_type == DtType::FirstHalf) { if(::isAnyBoundary(field_boundary_lo, field_boundary_hi)){ - auto Efield_fp = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); // JRA, new to prevent shadow + auto Efield_fp = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); auto Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp",max_level); m_fdtd_solver_fp[0]->ApplySilverMuellerBoundary( Efield_fp[lev], Bfield_fp[lev], diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index ae9166d2ac8..e5ee4da6a6d 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -141,9 +141,9 @@ WarpX::AddBoundaryField () self_fields_verbosity ); // Compute the corresponding electric and magnetic field, from the potential phi. - auto Efield_fp_new = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); // JRA, new to prevent shadow + auto Efield_fp = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); auto Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp",max_level); - computeE( Efield_fp_new, phi, beta ); + computeE( Efield_fp, phi, beta ); computeB( Bfield_fp, phi, beta ); // de-allocate temporary @@ -230,9 +230,9 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) pc.self_fields_verbosity ); // Compute the corresponding electric and magnetic field, from the potential phi - auto Efield_fp_new = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); // JRA, new to prevent shadow + auto Efield_fp = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); auto Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp",max_level); - computeE( Efield_fp_new, phi, beta ); + computeE( Efield_fp, phi, beta ); computeB( Bfield_fp, phi, beta ); // de-allocate temporary @@ -309,11 +309,11 @@ WarpX::AddSpaceChargeFieldLabFrame () // Compute the electric field. Note that if an EB is used the electric // field will be calculated in the computePhi call. - auto Efield_fp_new = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); // JRA, new to prevent shadow + auto Efield_fp = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); auto Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp",max_level); - if (!EB::enabled()) { computeE( Efield_fp_new, phi_fp, beta ); } + if (!EB::enabled()) { computeE( Efield_fp, phi_fp, beta ); } else { - if (IsPythonCallbackInstalled("poissonsolver")) { computeE(Efield_fp_new, phi_fp, beta); } + if (IsPythonCallbackInstalled("poissonsolver")) { computeE(Efield_fp, phi_fp, beta); } } // Compute the magnetic field From 396a035c4590afd7d67b8b1e7ba8da789ab73eb1 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 12 Sep 2024 21:23:07 -0700 Subject: [PATCH 206/314] Fix bug when alias is being redistributed --- Source/ablastr/fields/MultiFabRegister.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index c4eaeeba8ef..3c388d46474 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -301,7 +301,7 @@ namespace ablastr::fields } if (mf_owner.m_level == level && mf_owner.is_alias()) { - amrex::MultiFab & mf = mf_owner.m_mf; + amrex::MultiFab & mf = m_mf_register[mf_owner.m_owner].m_mf; amrex::MultiFab new_mf(mf, amrex::make_alias, 0, mf.nComp()); // no copy via Redistribute: the owner was already redistributed From 29bdf42c661d1275f1cdcabfe9881453fd86648d Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 12 Sep 2024 22:19:05 -0700 Subject: [PATCH 207/314] Fix RZ field functor --- Source/Diagnostics/BTDiagnostics.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 01e63d6a3b0..4689cc1b563 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -701,11 +701,11 @@ BTDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) } else if ( m_cellcenter_varnames_fields[comp] == "Bz" ){ m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{2}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "jr" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", lev), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", Direction{0}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "jt" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", lev), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", Direction{1}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "jz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", lev), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", Direction{2}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "rho" ){ m_cell_center_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, false, -1, false, ncomp); } From a5592be26b450cf360859b018df23cb2c5b0a180 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 12 Sep 2024 23:25:15 -0700 Subject: [PATCH 208/314] Update PMLs --- Source/BoundaryConditions/PML.H | 87 ++--- Source/BoundaryConditions/PML.cpp | 319 +++++++----------- Source/BoundaryConditions/WarpXEvolvePML.cpp | 23 +- .../WarpXFieldBoundaries.cpp | 4 +- .../FlushFormats/FlushFormatCheckpoint.cpp | 1 + Source/Diagnostics/WarpXIO.cpp | 2 +- Source/Evolve/WarpXEvolve.cpp | 2 +- .../FiniteDifferenceSolver/EvolveBPML.cpp | 16 +- .../FiniteDifferenceSolver/EvolveEPML.cpp | 20 +- .../FiniteDifferenceSolver.H | 28 +- Source/FieldSolver/WarpXPushFieldsEM.cpp | 38 ++- Source/Parallelization/WarpXComm.cpp | 32 +- Source/Utils/WarpXMovingWindow.cpp | 16 +- 13 files changed, 242 insertions(+), 346 deletions(-) diff --git a/Source/BoundaryConditions/PML.H b/Source/BoundaryConditions/PML.H index ba9e2c3ab5d..f6576500c3e 100644 --- a/Source/BoundaryConditions/PML.H +++ b/Source/BoundaryConditions/PML.H @@ -17,6 +17,8 @@ # include "FieldSolver/SpectralSolver/SpectralSolver.H" #endif +#include + #include #include #include @@ -154,23 +156,6 @@ public: void ComputePMLFactors (amrex::Real dt); - std::array GetE_fp (); - std::array GetB_fp (); - std::array Getj_fp (); - std::array GetE_cp (); - std::array GetB_cp (); - std::array Getj_cp (); - std::array Get_edge_lengths (); - std::array Get_face_areas (); - - // Used when WarpX::do_pml_dive_cleaning = true - amrex::MultiFab* GetF_fp (); - amrex::MultiFab* GetF_cp (); - - // Used when WarpX::do_pml_divb_cleaning = true - amrex::MultiFab* GetG_fp (); - amrex::MultiFab* GetG_cp (); - [[nodiscard]] const MultiSigmaBox& GetMultiSigmaBox_fp () const { return *sigba_fp; @@ -182,35 +167,33 @@ public: } #ifdef WARPX_USE_FFT - void PushPSATD (int lev); + void PushPSATD (ablastr::fields::MultiFabRegister& fields, int lev); #endif - void CopyJtoPMLs (const std::array& j_fp, - const std::array& j_cp); + void CopyJtoPMLs (ablastr::fields::MultiFabRegister& fields, int lev); - void Exchange (const std::array& mf_pml, - const std::array& mf, + void Exchange (ablastr::fields::VectorField mf_pml, + ablastr::fields::VectorField mf, + const PatchType& patch_type, + int do_pml_in_domain); + void Exchange (amrex::MultiFab* mf_pml, + amrex::MultiFab* mf, const PatchType& patch_type, int do_pml_in_domain); - void CopyJtoPMLs (PatchType patch_type, - const std::array& jp); - - void ExchangeF (amrex::MultiFab* F_fp, amrex::MultiFab* F_cp, int do_pml_in_domain); - void ExchangeF (PatchType patch_type, amrex::MultiFab* Fp, int do_pml_in_domain); - - void ExchangeG (amrex::MultiFab* G_fp, amrex::MultiFab* G_cp, int do_pml_in_domain); - void ExchangeG (PatchType patch_type, amrex::MultiFab* Gp, int do_pml_in_domain); + void CopyJtoPMLs ( + ablastr::fields::MultiFabRegister& fields, + PatchType patch_type, + int lev + ); - void FillBoundaryE (PatchType patch_type, std::optional nodal_sync=std::nullopt); - void FillBoundaryB (PatchType patch_type, std::optional nodal_sync=std::nullopt); - void FillBoundaryF (PatchType patch_type, std::optional nodal_sync=std::nullopt); - void FillBoundaryG (PatchType patch_type, std::optional nodal_sync=std::nullopt); + void FillBoundary (ablastr::fields::VectorField mf_pml, PatchType patch_type, std::optional nodal_sync=std::nullopt); + void FillBoundary (amrex::MultiFab & mf_pml, PatchType patch_type, std::optional nodal_sync=std::nullopt); [[nodiscard]] bool ok () const { return m_ok; } - void CheckPoint (const std::string& dir) const; - void Restart (const std::string& dir); + void CheckPoint (ablastr::fields::MultiFabRegister& fields, const std::string& dir) const; + void Restart (ablastr::fields::MultiFabRegister& fields, const std::string& dir); static void Exchange (amrex::MultiFab& pml, amrex::MultiFab& reg, const amrex::Geometry& geom, int do_pml_in_domain); @@ -226,24 +209,6 @@ private: const amrex::Geometry* m_geom; const amrex::Geometry* m_cgeom; - std::array,3> pml_E_fp; - std::array,3> pml_B_fp; - std::array,3> pml_j_fp; - - std::array,3> pml_edge_lengths; - - std::array,3> pml_E_cp; - std::array,3> pml_B_cp; - std::array,3> pml_j_cp; - - // Used when WarpX::do_pml_dive_cleaning = true - std::unique_ptr pml_F_fp; - std::unique_ptr pml_F_cp; - - // Used when WarpX::do_pml_divb_cleaning = true - std::unique_ptr pml_G_fp; - std::unique_ptr pml_G_cp; - std::unique_ptr sigba_fp; std::unique_ptr sigba_cp; @@ -292,13 +257,15 @@ private: }; #ifdef WARPX_USE_FFT -void PushPMLPSATDSinglePatch( int lev, +void PushPMLPSATDSinglePatch ( + int lev, SpectralSolver& solver, - std::array,3>& pml_E, - std::array,3>& pml_B, - std::unique_ptr& pml_F, - std::unique_ptr& pml_G, - const amrex::IntVect& fill_guards); + ablastr::fields::VectorField& pml_E, + ablastr::fields::VectorField& pml_B, + ablastr::fields::ScalarField pml_F, + ablastr::fields::ScalarField pml_G, + const amrex::IntVect& fill_guards +); #endif #endif diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index 7f0cfcbc8ef..1aed10ce654 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -705,30 +705,30 @@ PML::PML (const int lev, const BoxArray& grid_ba, const amrex::BoxArray ba_Ex = amrex::convert(ba, warpx.m_fields.get("Efield_fp",Direction{0},0)->ixType().toIntVect()); const amrex::BoxArray ba_Ey = amrex::convert(ba, warpx.m_fields.get("Efield_fp",Direction{1},0)->ixType().toIntVect()); const amrex::BoxArray ba_Ez = amrex::convert(ba, warpx.m_fields.get("Efield_fp",Direction{2},0)->ixType().toIntVect()); - WarpX::AllocInitMultiFab(pml_E_fp[0], ba_Ex, dm, ncompe, nge, lev, "pml_E_fp[x]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_E_fp[1], ba_Ey, dm, ncompe, nge, lev, "pml_E_fp[y]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_E_fp[2], ba_Ez, dm, ncompe, nge, lev, "pml_E_fp[z]", 0.0_rt); + warpx.m_fields.alloc_init("pml_E_fp", Direction{0}, lev, ba_Ex, dm, ncompe, nge, 0.0_rt, false, false); + warpx.m_fields.alloc_init("pml_E_fp", Direction{1}, lev, ba_Ey, dm, ncompe, nge, 0.0_rt, false, false); + warpx.m_fields.alloc_init("pml_E_fp", Direction{2}, lev, ba_Ez, dm, ncompe, nge, 0.0_rt, false, false); const amrex::BoxArray ba_Bx = amrex::convert(ba, warpx.m_fields.get("Bfield_fp",Direction{0},0)->ixType().toIntVect()); const amrex::BoxArray ba_By = amrex::convert(ba, warpx.m_fields.get("Bfield_fp",Direction{1},0)->ixType().toIntVect()); const amrex::BoxArray ba_Bz = amrex::convert(ba, warpx.m_fields.get("Bfield_fp",Direction{2},0)->ixType().toIntVect()); - WarpX::AllocInitMultiFab(pml_B_fp[0], ba_Bx, dm, ncompb, ngb, lev, "pml_B_fp[x]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_B_fp[1], ba_By, dm, ncompb, ngb, lev, "pml_B_fp[y]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_B_fp[2], ba_Bz, dm, ncompb, ngb, lev, "pml_B_fp[z]", 0.0_rt); + warpx.m_fields.alloc_init("pml_B_fp", Direction{0}, lev, ba_Bx, dm, ncompb, ngb, 0.0_rt, false, false); + warpx.m_fields.alloc_init("pml_B_fp", Direction{1}, lev, ba_By, dm, ncompb, ngb, 0.0_rt, false, false); + warpx.m_fields.alloc_init("pml_B_fp", Direction{2}, lev, ba_Bz, dm, ncompb, ngb, 0.0_rt, false, false); const amrex::BoxArray ba_jx = amrex::convert(ba, WarpX::GetInstance().m_fields.get("current_fp", Direction{0}, 0)->ixType().toIntVect()); const amrex::BoxArray ba_jy = amrex::convert(ba, WarpX::GetInstance().m_fields.get("current_fp", Direction{1}, 0)->ixType().toIntVect()); const amrex::BoxArray ba_jz = amrex::convert(ba, WarpX::GetInstance().m_fields.get("current_fp", Direction{2}, 0)->ixType().toIntVect()); - WarpX::AllocInitMultiFab(pml_j_fp[0], ba_jx, dm, 1, ngb, lev, "pml_j_fp[x]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_j_fp[1], ba_jy, dm, 1, ngb, lev, "pml_j_fp[y]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_j_fp[2], ba_jz, dm, 1, ngb, lev, "pml_j_fp[z]", 0.0_rt); + warpx.m_fields.alloc_init("pml_j_fp", Direction{0}, lev, ba_jx, dm, 1, ngb, 0.0_rt, false, false); + warpx.m_fields.alloc_init("pml_j_fp", Direction{1}, lev, ba_jy, dm, 1, ngb, 0.0_rt, false, false); + warpx.m_fields.alloc_init("pml_j_fp", Direction{2}, lev, ba_jz, dm, 1, ngb, 0.0_rt, false, false); #ifdef AMREX_USE_EB if (eb_enabled) { const amrex::IntVect max_guard_EB_vect = amrex::IntVect(max_guard_EB); - WarpX::AllocInitMultiFab(pml_edge_lengths[0], ba_Ex, dm, WarpX::ncomps, max_guard_EB_vect, lev, "pml_edge_lengths[x]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_edge_lengths[1], ba_Ey, dm, WarpX::ncomps, max_guard_EB_vect, lev, "pml_edge_lengths[y]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_edge_lengths[2], ba_Ez, dm, WarpX::ncomps, max_guard_EB_vect, lev, "pml_edge_lengths[z]", 0.0_rt); + warpx.m_fields.alloc_init("pml_edge_lengths", Direction{0}, lev, ba_Ex, dm, WarpX::ncomps, max_guard_EB_vect, 0.0_rt, false, false); + warpx.m_fields.alloc_init("pml_edge_lengths", Direction{1}, lev, ba_Ey, dm, WarpX::ncomps, max_guard_EB_vect, 0.0_rt, false, false); + warpx.m_fields.alloc_init("pml_edge_lengths", Direction{2}, lev, ba_Ez, dm, WarpX::ncomps, max_guard_EB_vect, 0.0_rt, false, false); if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::Yee || WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC || @@ -736,7 +736,7 @@ PML::PML (const int lev, const BoxArray& grid_ba, auto const eb_fact = fieldEBFactory(); - auto t_pml_edge_lengths = ablastr::fields::a2m(pml_edge_lengths); + ablastr::fields::VectorField t_pml_edge_lengths = warpx.m_fields.get_alldirs("pml_edge_lengths", lev); WarpX::ComputeEdgeLengths(t_pml_edge_lengths, eb_fact); WarpX::ScaleEdges(t_pml_edge_lengths, WarpX::CellSize(lev)); @@ -748,7 +748,7 @@ PML::PML (const int lev, const BoxArray& grid_ba, if (m_dive_cleaning) { const amrex::BoxArray ba_F_nodal = amrex::convert(ba, amrex::IntVect::TheNodeVector()); - WarpX::AllocInitMultiFab(pml_F_fp, ba_F_nodal, dm, 3, ngf, lev, "pml_F_fp", 0.0_rt); + warpx.m_fields.alloc_init("pml_F_fp", lev, ba_F_nodal, dm, 3, ngf, 0.0_rt, false, false); } if (m_divb_cleaning) @@ -758,7 +758,7 @@ PML::PML (const int lev, const BoxArray& grid_ba, (grid_type == GridType::Collocated) ? amrex::IntVect::TheNodeVector() : amrex::IntVect::TheCellVector(); const amrex::BoxArray ba_G_nodal = amrex::convert(ba, G_nodal_flag); - WarpX::AllocInitMultiFab(pml_G_fp, ba_G_nodal, dm, 3, ngf, lev, "pml_G_fp", 0.0_rt); + warpx.m_fields.alloc_init("pml_G_fp", lev, ba_G_nodal, dm, 3, ngf, 0.0_rt, false, false); } Box single_domain_box = is_single_box_domain ? domain0 : Box(); @@ -843,21 +843,21 @@ PML::PML (const int lev, const BoxArray& grid_ba, const amrex::BoxArray cba_Ex = amrex::convert(cba, WarpX::GetInstance().m_fields.get("Efield_cp", Direction{0}, 1)->ixType().toIntVect()); const amrex::BoxArray cba_Ey = amrex::convert(cba, WarpX::GetInstance().m_fields.get("Efield_cp", Direction{1}, 1)->ixType().toIntVect()); const amrex::BoxArray cba_Ez = amrex::convert(cba, WarpX::GetInstance().m_fields.get("Efield_cp", Direction{2}, 1)->ixType().toIntVect()); - WarpX::AllocInitMultiFab(pml_E_cp[0], cba_Ex, cdm, ncompe, nge, lev, "pml_E_cp[x]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_E_cp[1], cba_Ey, cdm, ncompe, nge, lev, "pml_E_cp[y]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_E_cp[2], cba_Ez, cdm, ncompe, nge, lev, "pml_E_cp[z]", 0.0_rt); + warpx.m_fields.alloc_init("pml_E_cp", Direction{0}, lev, cba_Ex, cdm, ncompe, nge, 0.0_rt, false, false); + warpx.m_fields.alloc_init("pml_E_cp", Direction{1}, lev, cba_Ey, cdm, ncompe, nge, 0.0_rt, false, false); + warpx.m_fields.alloc_init("pml_E_cp", Direction{2}, lev, cba_Ez, cdm, ncompe, nge, 0.0_rt, false, false); const amrex::BoxArray cba_Bx = amrex::convert(cba, WarpX::GetInstance().m_fields.get("Bfield_cp", Direction{0}, 1)->ixType().toIntVect()); const amrex::BoxArray cba_By = amrex::convert(cba, WarpX::GetInstance().m_fields.get("Bfield_cp", Direction{1}, 1)->ixType().toIntVect()); const amrex::BoxArray cba_Bz = amrex::convert(cba, WarpX::GetInstance().m_fields.get("Bfield_cp", Direction{2}, 1)->ixType().toIntVect()); - WarpX::AllocInitMultiFab(pml_B_cp[0], cba_Bx, cdm, ncompb, ngb, lev, "pml_B_cp[x]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_B_cp[1], cba_By, cdm, ncompb, ngb, lev, "pml_B_cp[y]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_B_cp[2], cba_Bz, cdm, ncompb, ngb, lev, "pml_B_cp[z]", 0.0_rt); + warpx.m_fields.alloc_init("pml_B_cp", Direction{0}, lev, cba_Bx, cdm, ncompb, ngb, 0.0_rt, false, false); + warpx.m_fields.alloc_init("pml_B_cp", Direction{1}, lev, cba_By, cdm, ncompb, ngb, 0.0_rt, false, false); + warpx.m_fields.alloc_init("pml_B_cp", Direction{2}, lev, cba_Bz, cdm, ncompb, ngb, 0.0_rt, false, false); if (m_dive_cleaning) { const amrex::BoxArray cba_F_nodal = amrex::convert(cba, amrex::IntVect::TheNodeVector()); - WarpX::AllocInitMultiFab(pml_F_cp, cba_F_nodal, cdm, 3, ngf, lev, "pml_F_cp", 0.0_rt); + warpx.m_fields.alloc_init("pml_F_cp", lev, cba_F_nodal, cdm, 3, ngf, 0.0_rt, false, false); } if (m_divb_cleaning) @@ -867,15 +867,15 @@ PML::PML (const int lev, const BoxArray& grid_ba, (grid_type == GridType::Collocated) ? amrex::IntVect::TheNodeVector() : amrex::IntVect::TheCellVector(); const amrex::BoxArray cba_G_nodal = amrex::convert(cba, G_nodal_flag); - WarpX::AllocInitMultiFab( pml_G_cp, cba_G_nodal, cdm, 3, ngf, lev, "pml_G_cp", 0.0_rt); + warpx.m_fields.alloc_init("pml_G_cp", lev, cba_G_nodal, cdm, 3, ngf, 0.0_rt, false, false); } const amrex::BoxArray cba_jx = amrex::convert(cba, WarpX::GetInstance().m_fields.get("current_cp", Direction{0}, 1)->ixType().toIntVect()); const amrex::BoxArray cba_jy = amrex::convert(cba, WarpX::GetInstance().m_fields.get("current_cp", Direction{1}, 1)->ixType().toIntVect()); const amrex::BoxArray cba_jz = amrex::convert(cba, WarpX::GetInstance().m_fields.get("current_cp", Direction{2}, 1)->ixType().toIntVect()); - WarpX::AllocInitMultiFab(pml_j_cp[0], cba_jx, cdm, 1, ngb, lev, "pml_j_cp[x]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_j_cp[1], cba_jy, cdm, 1, ngb, lev, "pml_j_cp[y]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_j_cp[2], cba_jz, cdm, 1, ngb, lev, "pml_j_cp[z]", 0.0_rt); + warpx.m_fields.alloc_init("pml_j_cp", Direction{0}, lev, cba_jx, cdm, 1, ngb, 0.0_rt, false, false); + warpx.m_fields.alloc_init("pml_j_cp", Direction{1}, lev, cba_jy, cdm, 1, ngb, 0.0_rt, false, false); + warpx.m_fields.alloc_init("pml_j_cp", Direction{2}, lev, cba_jz, cdm, 1, ngb, 0.0_rt, false, false); single_domain_box = is_single_box_domain ? cdomain : Box(); sigba_cp = std::make_unique(cba, cdm, grid_cba_reduced, cgeom->CellSize(), @@ -1050,96 +1050,32 @@ PML::ComputePMLFactors (amrex::Real dt) } } -std::array -PML::GetE_fp () -{ - return {pml_E_fp[0].get(), pml_E_fp[1].get(), pml_E_fp[2].get()}; -} - -std::array -PML::GetB_fp () -{ - return {pml_B_fp[0].get(), pml_B_fp[1].get(), pml_B_fp[2].get()}; -} - -std::array -PML::Getj_fp () -{ - return {pml_j_fp[0].get(), pml_j_fp[1].get(), pml_j_fp[2].get()}; -} - -std::array -PML::GetE_cp () -{ - return {pml_E_cp[0].get(), pml_E_cp[1].get(), pml_E_cp[2].get()}; -} - -std::array -PML::GetB_cp () -{ - return {pml_B_cp[0].get(), pml_B_cp[1].get(), pml_B_cp[2].get()}; -} - -std::array -PML::Getj_cp () -{ - return {pml_j_cp[0].get(), pml_j_cp[1].get(), pml_j_cp[2].get()}; -} - -std::array -PML::Get_edge_lengths() -{ - return {pml_edge_lengths[0].get(), pml_edge_lengths[1].get(), pml_edge_lengths[2].get()}; -} - - -MultiFab* -PML::GetF_fp () -{ - return pml_F_fp.get(); -} - -MultiFab* -PML::GetF_cp () -{ - return pml_F_cp.get(); -} - -MultiFab* -PML::GetG_fp () +void +PML::CopyJtoPMLs ( + ablastr::fields::MultiFabRegister& fields, + PatchType patch_type, + int lev +) { - return pml_G_fp.get(); -} + using ablastr::fields::Direction; -MultiFab* -PML::GetG_cp () -{ - return pml_G_cp.get(); -} + bool const has_j_fp = fields.has("j_fp", Direction{0}, lev); + bool const has_pml_j_fp = fields.has("pml_j_fp", Direction{0}, lev); + bool const has_j_cp = fields.has("j_cp", Direction{0}, lev); + bool const has_pml_j_cp = fields.has("pml_j_cp", Direction{0}, lev); -void PML::Exchange (const std::array& mf_pml, - const std::array& mf, - const PatchType& patch_type, - const int do_pml_in_domain) -{ - const amrex::Geometry& geom = (patch_type == PatchType::fine) ? *m_geom : *m_cgeom; - if (mf_pml[0] && mf[0]) { Exchange(*mf_pml[0], *mf[0], geom, do_pml_in_domain); } - if (mf_pml[1] && mf[1]) { Exchange(*mf_pml[1], *mf[1], geom, do_pml_in_domain); } - if (mf_pml[2] && mf[2]) { Exchange(*mf_pml[2], *mf[2], geom, do_pml_in_domain); } -} - -void -PML::CopyJtoPMLs (PatchType patch_type, - const std::array& jp) -{ - if (patch_type == PatchType::fine && pml_j_fp[0] && jp[0]) + if (patch_type == PatchType::fine && has_pml_j_fp && has_j_fp) { + ablastr::fields::VectorField pml_j_fp = fields.get_alldirs("pml_j_fp", lev); + ablastr::fields::VectorField jp = fields.get_alldirs("j_fp", lev); CopyToPML(*pml_j_fp[0], *jp[0], *m_geom); CopyToPML(*pml_j_fp[1], *jp[1], *m_geom); CopyToPML(*pml_j_fp[2], *jp[2], *m_geom); } - else if (patch_type == PatchType::coarse && pml_j_cp[0] && jp[0]) + else if (patch_type == PatchType::coarse && has_j_cp && has_pml_j_cp) { + ablastr::fields::VectorField pml_j_cp = fields.get_alldirs("pml_j_cp", lev); + ablastr::fields::VectorField jp = fields.get_alldirs("j_cp", lev); CopyToPML(*pml_j_cp[0], *jp[0], *m_cgeom); CopyToPML(*pml_j_cp[1], *jp[1], *m_cgeom); CopyToPML(*pml_j_cp[2], *jp[2], *m_cgeom); @@ -1147,46 +1083,33 @@ PML::CopyJtoPMLs (PatchType patch_type, } void -PML::CopyJtoPMLs (const std::array& j_fp, - const std::array& j_cp) +PML::CopyJtoPMLs ( + ablastr::fields::MultiFabRegister& fields, + int lev +) { - CopyJtoPMLs(PatchType::fine, j_fp); - CopyJtoPMLs(PatchType::coarse, j_cp); + CopyJtoPMLs(fields, PatchType::fine, lev); + CopyJtoPMLs(fields, PatchType::coarse, lev); } -void -PML::ExchangeF (amrex::MultiFab* F_fp, amrex::MultiFab* F_cp, int do_pml_in_domain) -{ - ExchangeF(PatchType::fine, F_fp, do_pml_in_domain); - ExchangeF(PatchType::coarse, F_cp, do_pml_in_domain); -} - -void -PML::ExchangeF (PatchType patch_type, amrex::MultiFab* Fp, int do_pml_in_domain) -{ - if (patch_type == PatchType::fine && pml_F_fp && Fp) { - Exchange(*pml_F_fp, *Fp, *m_geom, do_pml_in_domain); - } else if (patch_type == PatchType::coarse && pml_F_cp && Fp) { - Exchange(*pml_F_cp, *Fp, *m_cgeom, do_pml_in_domain); - } -} - -void PML::ExchangeG (amrex::MultiFab* G_fp, amrex::MultiFab* G_cp, int do_pml_in_domain) +void PML::Exchange (ablastr::fields::VectorField mf_pml, + ablastr::fields::VectorField mf, + const PatchType& patch_type, + const int do_pml_in_domain) { - ExchangeG(PatchType::fine, G_fp, do_pml_in_domain); - ExchangeG(PatchType::coarse, G_cp, do_pml_in_domain); + const amrex::Geometry& geom = (patch_type == PatchType::fine) ? *m_geom : *m_cgeom; + if (mf_pml[0] && mf[0]) { Exchange(*mf_pml[0], *mf[0], geom, do_pml_in_domain); } + if (mf_pml[1] && mf[1]) { Exchange(*mf_pml[1], *mf[1], geom, do_pml_in_domain); } + if (mf_pml[2] && mf[2]) { Exchange(*mf_pml[2], *mf[2], geom, do_pml_in_domain); } } -void PML::ExchangeG (PatchType patch_type, amrex::MultiFab* Gp, int do_pml_in_domain) +void PML::Exchange (amrex::MultiFab* mf_pml, + amrex::MultiFab* mf, + const PatchType& patch_type, + const int do_pml_in_domain) { - if (patch_type == PatchType::fine && pml_G_fp && Gp) - { - Exchange(*pml_G_fp, *Gp, *m_geom, do_pml_in_domain); - } - else if (patch_type == PatchType::coarse && pml_G_cp && Gp) - { - Exchange(*pml_G_cp, *Gp, *m_cgeom, do_pml_in_domain); - } + const amrex::Geometry& geom = (patch_type == PatchType::fine) ? *m_geom : *m_cgeom; + if (mf_pml && mf) { Exchange(*mf_pml, *mf, geom, do_pml_in_domain); } } void @@ -1280,74 +1203,40 @@ PML::CopyToPML (MultiFab& pml, MultiFab& reg, const Geometry& geom) } void -PML::FillBoundaryE (PatchType patch_type, std::optional nodal_sync) +PML::FillBoundary (ablastr::fields::VectorField mf_pml, PatchType patch_type, std::optional nodal_sync) { - if (patch_type == PatchType::fine && pml_E_fp[0] && pml_E_fp[0]->nGrowVect().max() > 0) - { - const auto& period = m_geom->periodicity(); - const Vector mf{pml_E_fp[0].get(),pml_E_fp[1].get(),pml_E_fp[2].get()}; - ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period, nodal_sync); - } - else if (patch_type == PatchType::coarse && pml_E_cp[0] && pml_E_cp[0]->nGrowVect().max() > 0) - { - const auto& period = m_cgeom->periodicity(); - const Vector mf{pml_E_cp[0].get(),pml_E_cp[1].get(),pml_E_cp[2].get()}; - ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period, nodal_sync); - } -} + const auto& period = + (patch_type == PatchType::fine) ? + m_geom->periodicity() : + m_cgeom->periodicity(); -void -PML::FillBoundaryB (PatchType patch_type, std::optional nodal_sync) -{ - if (patch_type == PatchType::fine && pml_B_fp[0]) - { - const auto& period = m_geom->periodicity(); - const Vector mf{pml_B_fp[0].get(),pml_B_fp[1].get(),pml_B_fp[2].get()}; - ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period, nodal_sync); - } - else if (patch_type == PatchType::coarse && pml_B_cp[0]) - { - const auto& period = m_cgeom->periodicity(); - const Vector mf{pml_B_cp[0].get(),pml_B_cp[1].get(),pml_B_cp[2].get()}; - ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period, nodal_sync); - } + const Vector mf{mf_pml[0], mf_pml[1], mf_pml[2]}; + ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period, nodal_sync); } void -PML::FillBoundaryF (PatchType patch_type, std::optional nodal_sync) +PML::FillBoundary (amrex::MultiFab & mf_pml, PatchType patch_type, std::optional nodal_sync) { - if (patch_type == PatchType::fine && pml_F_fp && pml_F_fp->nGrowVect().max() > 0) - { - const auto& period = m_geom->periodicity(); - ablastr::utils::communication::FillBoundary(*pml_F_fp, WarpX::do_single_precision_comms, period, nodal_sync); - } - else if (patch_type == PatchType::coarse && pml_F_cp && pml_F_cp->nGrowVect().max() > 0) - { - const auto& period = m_cgeom->periodicity(); - ablastr::utils::communication::FillBoundary(*pml_F_cp, WarpX::do_single_precision_comms, period, nodal_sync); - } -} + const auto& period = + (patch_type == PatchType::fine) ? + m_geom->periodicity() : + m_cgeom->periodicity(); -void -PML::FillBoundaryG (PatchType patch_type, std::optional nodal_sync) -{ - if (patch_type == PatchType::fine && pml_G_fp && pml_G_fp->nGrowVect().max() > 0) - { - const auto& period = m_geom->periodicity(); - ablastr::utils::communication::FillBoundary(*pml_G_fp, WarpX::do_single_precision_comms, period, nodal_sync); - } - else if (patch_type == PatchType::coarse && pml_G_cp && pml_G_cp->nGrowVect().max() > 0) - { - const auto& period = m_cgeom->periodicity(); - ablastr::utils::communication::FillBoundary(*pml_G_cp, WarpX::do_single_precision_comms, period, nodal_sync); - } + ablastr::utils::communication::FillBoundary(mf_pml, WarpX::do_single_precision_comms, period, nodal_sync); } void -PML::CheckPoint (const std::string& dir) const +PML::CheckPoint ( + ablastr::fields::MultiFabRegister& fields, + const std::string& dir +) const { - if (pml_E_fp[0]) + using ablastr::fields::Direction; + + if (fields.has("pml_E_fp", Direction{0}, 0)) { + ablastr::fields::VectorField pml_E_fp = fields.get_alldirs("pml_E_fp", 0); + ablastr::fields::VectorField pml_B_fp = fields.get_alldirs("pml_B_fp", 0); VisMF::AsyncWrite(*pml_E_fp[0], dir+"_Ex_fp"); VisMF::AsyncWrite(*pml_E_fp[1], dir+"_Ey_fp"); VisMF::AsyncWrite(*pml_E_fp[2], dir+"_Ez_fp"); @@ -1356,8 +1245,10 @@ PML::CheckPoint (const std::string& dir) const VisMF::AsyncWrite(*pml_B_fp[2], dir+"_Bz_fp"); } - if (pml_E_cp[0]) + if (fields.has("pml_E_cp", Direction{0}, 0)) { + ablastr::fields::VectorField pml_E_cp = fields.get_alldirs("pml_E_cp", 0); + ablastr::fields::VectorField pml_B_cp = fields.get_alldirs("pml_B_cp", 0); VisMF::AsyncWrite(*pml_E_cp[0], dir+"_Ex_cp"); VisMF::AsyncWrite(*pml_E_cp[1], dir+"_Ey_cp"); VisMF::AsyncWrite(*pml_E_cp[2], dir+"_Ez_cp"); @@ -1368,10 +1259,17 @@ PML::CheckPoint (const std::string& dir) const } void -PML::Restart (const std::string& dir) +PML::Restart ( + ablastr::fields::MultiFabRegister& fields, + const std::string& dir +) { - if (pml_E_fp[0]) + using ablastr::fields::Direction; + + if (fields.has("pml_E_fp", Direction{0}, 0)) { + ablastr::fields::VectorField pml_E_fp = fields.get_alldirs("pml_E_fp", 0); + ablastr::fields::VectorField pml_B_fp = fields.get_alldirs("pml_B_fp", 0); VisMF::Read(*pml_E_fp[0], dir+"_Ex_fp"); VisMF::Read(*pml_E_fp[1], dir+"_Ey_fp"); VisMF::Read(*pml_E_fp[2], dir+"_Ez_fp"); @@ -1380,8 +1278,10 @@ PML::Restart (const std::string& dir) VisMF::Read(*pml_B_fp[2], dir+"_Bz_fp"); } - if (pml_E_cp[0]) + if (fields.has("pml_E_cp", Direction{0}, 0)) { + ablastr::fields::VectorField pml_E_cp = fields.get_alldirs("pml_E_cp", 0); + ablastr::fields::VectorField pml_B_cp = fields.get_alldirs("pml_B_cp", 0); VisMF::Read(*pml_E_cp[0], dir+"_Ex_cp"); VisMF::Read(*pml_E_cp[1], dir+"_Ey_cp"); VisMF::Read(*pml_E_cp[2], dir+"_Ez_cp"); @@ -1393,11 +1293,20 @@ PML::Restart (const std::string& dir) #ifdef WARPX_USE_FFT void -PML::PushPSATD (const int lev) { +PML::PushPSATD (ablastr::fields::MultiFabRegister& fields, const int lev) +{ + ablastr::fields::VectorField pml_E_fp = fields.get_alldirs("pml_E_fp", lev); + ablastr::fields::VectorField pml_B_fp = fields.get_alldirs("pml_B_fp", lev); + ablastr::fields::ScalarField pml_F_fp = fields.get("pml_F_fp", lev); + ablastr::fields::ScalarField pml_G_fp = fields.get("pml_G_fp", lev); // Update the fields on the fine and coarse patch PushPMLPSATDSinglePatch(lev, *spectral_solver_fp, pml_E_fp, pml_B_fp, pml_F_fp, pml_G_fp, m_fill_guards_fields); if (spectral_solver_cp) { + ablastr::fields::VectorField pml_E_cp = fields.get_alldirs("pml_E_cp", lev); + ablastr::fields::VectorField pml_B_cp = fields.get_alldirs("pml_B_cp", lev); + ablastr::fields::ScalarField pml_F_cp = fields.get("pml_F_cp", lev); + ablastr::fields::ScalarField pml_G_cp = fields.get("pml_G_cp", lev); PushPMLPSATDSinglePatch(lev, *spectral_solver_cp, pml_E_cp, pml_B_cp, pml_F_cp, pml_G_cp, m_fill_guards_fields); } } @@ -1406,10 +1315,10 @@ void PushPMLPSATDSinglePatch ( const int lev, SpectralSolver& solver, - std::array,3>& pml_E, - std::array,3>& pml_B, - std::unique_ptr& pml_F, - std::unique_ptr& pml_G, + ablastr::fields::VectorField& pml_E, + ablastr::fields::VectorField& pml_B, + ablastr::fields::ScalarField pml_F, + ablastr::fields::ScalarField pml_G, const amrex::IntVect& fill_guards) { const SpectralFieldIndex& Idx = solver.m_spectral_index; diff --git a/Source/BoundaryConditions/WarpXEvolvePML.cpp b/Source/BoundaryConditions/WarpXEvolvePML.cpp index 0010b3e4544..3b667bb48b3 100644 --- a/Source/BoundaryConditions/WarpXEvolvePML.cpp +++ b/Source/BoundaryConditions/WarpXEvolvePML.cpp @@ -86,10 +86,10 @@ WarpX::DampPML_Cartesian (const int lev, PatchType patch_type) if (pml[lev]->ok()) { - const auto& pml_E = (patch_type == PatchType::fine) ? pml[lev]->GetE_fp() : pml[lev]->GetE_cp(); - const auto& pml_B = (patch_type == PatchType::fine) ? pml[lev]->GetB_fp() : pml[lev]->GetB_cp(); - const auto& pml_F = (patch_type == PatchType::fine) ? pml[lev]->GetF_fp() : pml[lev]->GetF_cp(); - const auto& pml_G = (patch_type == PatchType::fine) ? pml[lev]->GetG_fp() : pml[lev]->GetG_cp(); + const auto& pml_E = (patch_type == PatchType::fine) ? m_fields.get_alldirs("pml_E_fp", lev) : m_fields.get_alldirs("pml_E_cp", lev); + const auto& pml_B = (patch_type == PatchType::fine) ? m_fields.get_alldirs("pml_B_fp", lev) : m_fields.get_alldirs("pml_B_cp", lev); + const auto& pml_F = (patch_type == PatchType::fine) ? m_fields.get("pml_F_fp", lev) : m_fields.get("pml_F_cp", lev); + const auto& pml_G = (patch_type == PatchType::fine) ? m_fields.get("pml_G_fp", lev) : m_fields.get("pml_G_cp", lev); const auto& sigba = (patch_type == PatchType::fine) ? pml[lev]->GetMultiSigmaBox_fp() : pml[lev]->GetMultiSigmaBox_cp(); @@ -249,7 +249,7 @@ WarpX::DampJPML (int lev, PatchType patch_type) if (pml[lev]->ok()) { - const auto& pml_j = (patch_type == PatchType::fine) ? pml[lev]->Getj_fp() : pml[lev]->Getj_cp(); + const auto& pml_j = (patch_type == PatchType::fine) ? m_fields.get_alldirs("pml_j_fp", lev) : m_fields.get_alldirs("pml_j_cp", lev); const auto& sigba = (patch_type == PatchType::fine) ? pml[lev]->GetMultiSigmaBox_fp() : pml[lev]->GetMultiSigmaBox_cp(); @@ -278,7 +278,7 @@ WarpX::DampJPML (int lev, PatchType patch_type) // Skip the field update if this gridpoint is inside the embedded boundary amrex::Array4 eb_lxfab, eb_lyfab, eb_lzfab; if (EB::enabled()) { - const auto &pml_edge_lenghts = pml[lev]->Get_edge_lengths(); + const auto &pml_edge_lenghts = m_fields.get_alldirs("pmg_edge_lengths", lev); eb_lxfab = pml_edge_lenghts[0]->array(mfi); eb_lyfab = pml_edge_lenghts[1]->array(mfi); @@ -348,16 +348,7 @@ WarpX::CopyJPML () for (int lev = 0; lev <= finest_level; ++lev) { if (pml[lev] && pml[lev]->ok()){ - pml[lev]->CopyJtoPMLs({ - m_fields.get("current_fp", Direction{0}, lev), - m_fields.get("current_fp", Direction{1}, lev), - m_fields.get("current_fp", Direction{2}, lev) - }, { - m_fields.get("current_cp", Direction{0}, lev), - m_fields.get("current_cp", Direction{1}, lev), - m_fields.get("current_cp", Direction{2}, lev) - } - ); + pml[lev]->CopyJtoPMLs(m_fields, lev); } } } diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index a9f2be4c687..2121fce1d8c 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -65,7 +65,7 @@ void WarpX::ApplyEfieldBoundary(const int lev, PatchType patch_type) // apply pec on split E-fields in PML region const bool split_pml_field = true; PEC::ApplyPECtoEfield( - pml[lev]->GetE_fp(), + m_fields.get_alldirs("pml_E_fp",lev), field_boundary_lo, field_boundary_hi, get_ng_fieldgather(), Geom(lev), lev, patch_type, ref_ratio, @@ -83,7 +83,7 @@ void WarpX::ApplyEfieldBoundary(const int lev, PatchType patch_type) // apply pec on split E-fields in PML region const bool split_pml_field = true; PEC::ApplyPECtoEfield( - pml[lev]->GetE_cp(), + m_fields.get_alldirs("pml_E_cp",lev), field_boundary_lo, field_boundary_hi, get_ng_fieldgather(), Geom(lev), lev, patch_type, ref_ratio, diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index ebdddab830f..8a9c3c13ccc 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -154,6 +154,7 @@ FlushFormatCheckpoint::WriteToFile ( if (warpx.DoPML()) { if (warpx.GetPML(lev)) { warpx.GetPML(lev)->CheckPoint( + warpx.m_fields, amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "pml")); } #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp index b932620c7b8..47583ec99de 100644 --- a/Source/Diagnostics/WarpXIO.cpp +++ b/Source/Diagnostics/WarpXIO.cpp @@ -386,7 +386,7 @@ WarpX::InitFromCheckpoint () { for (int lev = 0; lev < nlevs; ++lev) { if (pml[lev]) { - pml[lev]->Restart(amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "pml")); + pml[lev]->Restart(m_fields, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "pml")); } #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev]) { diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 0ef6f03608d..dd98a9d8e77 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -799,7 +799,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) { if (do_pml && pml[lev]->ok()) { - pml[lev]->PushPSATD(lev); + pml[lev]->PushPSATD(m_fields, lev); } ApplyEfieldBoundary(lev, PatchType::fine); if (lev > 0) { ApplyEfieldBoundary(lev, PatchType::coarse); } diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp index 083ed5cde41..8dc1f29165a 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp @@ -41,18 +41,26 @@ using namespace amrex; * \brief Update the B field, over one timestep */ void FiniteDifferenceSolver::EvolveBPML ( - std::array< amrex::MultiFab*, 3 > Bfield, - ablastr::fields::VectorField const Efield, + ablastr::fields::MultiFabRegister& fields, + PatchType patch_type, + int level, amrex::Real const dt, - const bool dive_cleaning) { + const bool dive_cleaning +) +{ // Select algorithm (The choice of algorithm is a runtime option, // but we compile code for each algorithm, using templates) #ifdef WARPX_DIM_RZ - amrex::ignore_unused(Bfield, Efield, dt, dive_cleaning); + amrex::ignore_unused(fields, dt, dive_cleaning); WARPX_ABORT_WITH_MESSAGE( "PML are not implemented in cylindrical geometry."); #else + ablastr::fields::VectorField Bfield = (patch_type == PatchType::fine) ? + fields.get_alldirs("pml_B_fp", level) : fields.get_alldirs("pml_B_cp", level); + ablastr::fields::VectorField Efield = (patch_type == PatchType::fine) ? + fields.get_alldirs("pml_E_fp", level) : fields.get_alldirs("pml_E_cp", level); + if (m_grid_type == ablastr::utils::enums::GridType::Collocated) { EvolveBPMLCartesian (Bfield, Efield, dt, dive_cleaning); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp index a1ba6e44a8c..9dc371ed938 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp @@ -45,21 +45,29 @@ using namespace amrex; * \brief Update the E field, over one timestep */ void FiniteDifferenceSolver::EvolveEPML ( - std::array< amrex::MultiFab*, 3 > Efield, - std::array< amrex::MultiFab*, 3 > const Bfield, - std::array< amrex::MultiFab*, 3 > const Jfield, - std::array< amrex::MultiFab*, 3 > const edge_lengths, - amrex::MultiFab* const Ffield, + ablastr::fields::MultiFabRegister& fields, + PatchType patch_type, + int level, MultiSigmaBox const& sigba, amrex::Real const dt, bool pml_has_particles ) { // Select algorithm (The choice of algorithm is a runtime option, // but we compile code for each algorithm, using templates) #ifdef WARPX_DIM_RZ - amrex::ignore_unused(Efield, Bfield, Jfield, Ffield, sigba, dt, pml_has_particles, edge_lengths); + amrex::ignore_unused(fields, patch_type, level, sigba, dt, pml_has_particles); WARPX_ABORT_WITH_MESSAGE( "PML are not implemented in cylindrical geometry."); #else + ablastr::fields::VectorField Efield = (patch_type == PatchType::fine) ? + fields.get_alldirs("pml_E_fp", level) : fields.get_alldirs("pml_E_cp", level); + ablastr::fields::VectorField Bfield = (patch_type == PatchType::fine) ? + fields.get_alldirs("pml_B_fp", level) : fields.get_alldirs("pml_B_cp", level); + ablastr::fields::VectorField Jfield = (patch_type == PatchType::fine) ? + fields.get_alldirs("pml_j_fp", level) : fields.get_alldirs("pml_j_cp", level); + ablastr::fields::VectorField edge_lengths = fields.get_alldirs("pml_edge_lengths", level); + amrex::MultiFab* const Ffield = (patch_type == PatchType::fine) ? + fields.get("pml_F_fp", level) : fields.get("pml_F_cp", level); + if (m_grid_type == GridType::Collocated) { EvolveEPMLCartesian ( diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 9fed0dc2b1e..0ec0bb7b338 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -121,18 +121,22 @@ class FiniteDifferenceSolver amrex::Real dt, std::unique_ptr const& macroscopic_properties); - void EvolveBPML ( std::array< amrex::MultiFab*, 3 > Bfield, - ablastr::fields::VectorField const Efield, - amrex::Real dt, - bool dive_cleaning); - - void EvolveEPML ( ablastr::fields::VectorField Efield, - std::array< amrex::MultiFab*, 3 > Bfield, - std::array< amrex::MultiFab*, 3 > Jfield, - std::array< amrex::MultiFab*, 3 > edge_lengths, - amrex::MultiFab* Ffield, - MultiSigmaBox const& sigba, - amrex::Real dt, bool pml_has_particles ); + void EvolveBPML ( + ablastr::fields::MultiFabRegister& fields, + PatchType patch_type, + int level, + amrex::Real dt, + bool dive_cleaning + ); + + void EvolveEPML ( + ablastr::fields::MultiFabRegister& fields, + PatchType patch_type, + int level, + MultiSigmaBox const& sigba, + amrex::Real dt, + bool pml_has_particles + ); void EvolveFPML ( amrex::MultiFab* Ffield, ablastr::fields::VectorField Efield, diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 4c72856b9c9..3997f243c15 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -818,7 +818,7 @@ WarpX::PushPSATD () { if (pml[lev] && pml[lev]->ok()) { - pml[lev]->PushPSATD(lev); + pml[lev]->PushPSATD(m_fields, lev); } ApplyEfieldBoundary(lev, PatchType::fine); if (lev > 0) { ApplyEfieldBoundary(lev, PatchType::coarse); } @@ -878,10 +878,10 @@ WarpX::EvolveB (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_typ if (do_pml && pml[lev]->ok()) { if (patch_type == PatchType::fine) { m_fdtd_solver_fp[lev]->EvolveBPML( - pml[lev]->GetB_fp(), pml[lev]->GetE_fp(), a_dt, WarpX::do_dive_cleaning); + m_fields, patch_type, lev, a_dt, WarpX::do_dive_cleaning); } else { m_fdtd_solver_cp[lev]->EvolveBPML( - pml[lev]->GetB_cp(), pml[lev]->GetE_cp(), a_dt, WarpX::do_dive_cleaning); + m_fields, patch_type, lev, a_dt, WarpX::do_dive_cleaning); } } @@ -938,16 +938,16 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) if (do_pml && pml[lev]->ok()) { if (patch_type == PatchType::fine) { m_fdtd_solver_fp[lev]->EvolveEPML( - pml[lev]->GetE_fp(), pml[lev]->GetB_fp(), - pml[lev]->Getj_fp(), pml[lev]->Get_edge_lengths(), - pml[lev]->GetF_fp(), + m_fields, + patch_type, + lev, pml[lev]->GetMultiSigmaBox_fp(), a_dt, pml_has_particles ); } else { m_fdtd_solver_cp[lev]->EvolveEPML( - pml[lev]->GetE_cp(), pml[lev]->GetB_cp(), - pml[lev]->Getj_cp(), pml[lev]->Get_edge_lengths(), - pml[lev]->GetF_cp(), + m_fields, + patch_type, + lev, pml[lev]->GetMultiSigmaBox_cp(), a_dt, pml_has_particles ); } @@ -1021,10 +1021,14 @@ WarpX::EvolveF (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_typ if (do_pml && pml[lev]->ok()) { if (patch_type == PatchType::fine) { m_fdtd_solver_fp[lev]->EvolveFPML( - pml[lev]->GetF_fp(), pml[lev]->GetE_fp(), a_dt ); + m_fields.get("F_fp", lev), + m_fields.get_alldirs("Efield_fp", lev), + a_dt ); } else { m_fdtd_solver_cp[lev]->EvolveFPML( - pml[lev]->GetF_cp(), pml[lev]->GetE_cp(), a_dt ); + m_fields.get("F_cp", lev), + m_fields.get_alldirs("Efield_cp", lev), + a_dt ); } } } @@ -1119,16 +1123,16 @@ WarpX::MacroscopicEvolveE (int lev, PatchType patch_type, amrex::Real a_dt) { if (do_pml && pml[lev]->ok()) { if (patch_type == PatchType::fine) { m_fdtd_solver_fp[lev]->EvolveEPML( - pml[lev]->GetE_fp(), pml[lev]->GetB_fp(), - pml[lev]->Getj_fp(), pml[lev]->Get_edge_lengths(), - pml[lev]->GetF_fp(), + m_fields, + patch_type, + lev, pml[lev]->GetMultiSigmaBox_fp(), a_dt, pml_has_particles ); } else { m_fdtd_solver_cp[lev]->EvolveEPML( - pml[lev]->GetE_cp(), pml[lev]->GetB_cp(), - pml[lev]->Getj_cp(), pml[lev]->Get_edge_lengths(), - pml[lev]->GetF_cp(), + m_fields, + patch_type, + lev, pml[lev]->GetMultiSigmaBox_cp(), a_dt, pml_has_particles ); } diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 9d2cac9ccfb..44b96b93119 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -729,10 +729,12 @@ WarpX::FillBoundaryE (const int lev, const PatchType patch_type, const amrex::In if (pml[lev] && pml[lev]->ok()) { const std::array mf_pml = - (patch_type == PatchType::fine) ? pml[lev]->GetE_fp() : pml[lev]->GetE_cp(); + (patch_type == PatchType::fine) ? + m_fields.get_alldirs("pml_E_fp", finest_level) : + m_fields.get_alldirs("pml_E_cp", finest_level); pml[lev]->Exchange(mf_pml, mf, patch_type, do_pml_in_domain); - pml[lev]->FillBoundaryE(patch_type, nodal_sync); + pml[lev]->FillBoundary(mf_pml, patch_type, nodal_sync); } #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) @@ -788,10 +790,12 @@ WarpX::FillBoundaryB (const int lev, const PatchType patch_type, const amrex::In if (pml[lev] && pml[lev]->ok()) { const std::array mf_pml = - (patch_type == PatchType::fine) ? pml[lev]->GetB_fp() : pml[lev]->GetB_cp(); + (patch_type == PatchType::fine) ? + m_fields.get_alldirs("pml_B_fp", finest_level) : + m_fields.get_alldirs("pml_B_cp", finest_level); pml[lev]->Exchange(mf_pml, mf, patch_type, do_pml_in_domain); - pml[lev]->FillBoundaryB(patch_type, nodal_sync); + pml[lev]->FillBoundary(mf_pml, patch_type, nodal_sync); } #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) @@ -944,8 +948,8 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng, std::optionalok()) { - if (m_fields.has("F_fp", lev)) { pml[lev]->ExchangeF(patch_type, m_fields.get("F_fp", lev), do_pml_in_domain); } - pml[lev]->FillBoundaryF(patch_type, nodal_sync); + if (m_fields.has("pml_F_fp", lev) && m_fields.has("F_fp", lev)) { pml[lev]->Exchange(m_fields.get("pml_F_fp", lev), m_fields.get("F_fp", lev), patch_type, do_pml_in_domain); } + pml[lev]->FillBoundary(*m_fields.get("pml_F_fp", lev), patch_type, nodal_sync); } if (m_fields.has("F_fp", lev)) @@ -959,8 +963,8 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng, std::optionalok()) { - if (m_fields.has("F_cp", lev)) { pml[lev]->ExchangeF(patch_type, m_fields.get("F_cp", lev), do_pml_in_domain); } - pml[lev]->FillBoundaryF(patch_type, nodal_sync); + if (m_fields.has("pml_F_cp", lev) && m_fields.has("F_cp", lev)) { pml[lev]->Exchange(m_fields.get("pml_F_cp", lev), m_fields.get("F_cp", lev), patch_type, do_pml_in_domain); } + pml[lev]->FillBoundary(*m_fields.get("pml_F_cp", lev), patch_type, nodal_sync); } if (m_fields.has("F_cp", lev)) @@ -988,10 +992,10 @@ void WarpX::FillBoundaryG (int lev, PatchType patch_type, IntVect ng, std::optio { if (do_pml && pml[lev] && pml[lev]->ok()) { - if (m_fields.has("G_fp",lev)) { - pml[lev]->ExchangeG(patch_type, m_fields.get("G_fp",lev), do_pml_in_domain); + if (m_fields.has("pml_G_fp",lev) && m_fields.has("G_fp",lev)) { + pml[lev]->Exchange(m_fields.get("pml_G_fp", lev), m_fields.get("G_fp", lev), patch_type, do_pml_in_domain); } - pml[lev]->FillBoundaryG(patch_type, nodal_sync); + pml[lev]->FillBoundary(*m_fields.get("pml_G_fp", lev), patch_type, nodal_sync); } if (m_fields.has("G_fp",lev)) @@ -1006,10 +1010,10 @@ void WarpX::FillBoundaryG (int lev, PatchType patch_type, IntVect ng, std::optio { if (do_pml && pml[lev] && pml[lev]->ok()) { - if (m_fields.has("G_cp",lev)) { - pml[lev]->ExchangeG(patch_type, m_fields.get("G_cp",lev), do_pml_in_domain); + if (m_fields.has("pml_G_cp",lev) && m_fields.has("G_cp",lev)) { + pml[lev]->Exchange(m_fields.get("pml_G_cp", lev), m_fields.get("G_cp", lev), patch_type, do_pml_in_domain); } - pml[lev]->FillBoundaryG(patch_type, nodal_sync); + pml[lev]->FillBoundary(*m_fields.get("pml_G_cp", lev), patch_type, nodal_sync); } if (m_fields.has("G_cp",lev)) diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 5488c2238fd..dbd027e90f7 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -259,8 +259,8 @@ WarpX::MoveWindow (const int step, bool move_j) shiftMF(*m_fields.get("current_fp", Direction{dim}, lev), geom[lev], num_shift, dir, lev, do_update_cost); } if (pml[lev] && pml[lev]->ok()) { - const std::array& pml_B = pml[lev]->GetB_fp(); - const std::array& pml_E = pml[lev]->GetE_fp(); + const std::array& pml_B = m_fields.get_alldirs("pml_B_fp", lev); + const std::array& pml_E = m_fields.get_alldirs("pml_E_fp", lev); shiftMF(*pml_B[dim], geom[lev], num_shift, dir, lev, dont_update_cost); shiftMF(*pml_E[dim], geom[lev], num_shift, dir, lev, dont_update_cost); } @@ -290,8 +290,8 @@ WarpX::MoveWindow (const int step, bool move_j) shiftMF(*m_fields.get("current_cp", Direction{dim}, lev), geom[lev-1], num_shift_crse, dir, lev, do_update_cost); } if (do_pml && pml[lev]->ok()) { - const std::array& pml_B = pml[lev]->GetB_cp(); - const std::array& pml_E = pml[lev]->GetE_cp(); + const std::array& pml_B = m_fields.get_alldirs("pml_B_cp", lev); + const std::array& pml_E = m_fields.get_alldirs("pml_E_cp", lev); shiftMF(*pml_B[dim], geom[lev-1], num_shift_crse, dir, lev, dont_update_cost); shiftMF(*pml_E[dim], geom[lev-1], num_shift_crse, dir, lev, dont_update_cost); } @@ -317,7 +317,7 @@ WarpX::MoveWindow (const int step, bool move_j) // Fine grid if (do_pml && pml[lev]->ok()) { - amrex::MultiFab* pml_F = pml[lev]->GetF_fp(); + amrex::MultiFab* pml_F = m_fields.get("pml_F_fp", lev); shiftMF(*pml_F, geom[lev], num_shift, dir, lev, dont_update_cost); } if (lev > 0) @@ -325,7 +325,7 @@ WarpX::MoveWindow (const int step, bool move_j) // Coarse grid if (do_pml && pml[lev]->ok()) { - amrex::MultiFab* pml_F = pml[lev]->GetF_cp(); + amrex::MultiFab* pml_F = m_fields.get("pml_F_cp", lev); shiftMF(*pml_F, geom[lev-1], num_shift_crse, dir, lev, dont_update_cost); } } @@ -350,7 +350,7 @@ WarpX::MoveWindow (const int step, bool move_j) // Fine grid if (do_pml && pml[lev]->ok()) { - amrex::MultiFab* pml_G = pml[lev]->GetG_fp(); + amrex::MultiFab* pml_G = m_fields.get("pml_G_fp", lev); shiftMF(*pml_G, geom[lev], num_shift, dir, lev, dont_update_cost); } if (lev > 0) @@ -358,7 +358,7 @@ WarpX::MoveWindow (const int step, bool move_j) // Coarse grid if (do_pml && pml[lev]->ok()) { - amrex::MultiFab* pml_G = pml[lev]->GetG_cp(); + amrex::MultiFab* pml_G = m_fields.get("pml_G_cp", lev); shiftMF(*pml_G, geom[lev-1], num_shift_crse, dir, lev, dont_update_cost); } } From 8b73153ef31093ed25aa5cf14fd57cc702c16208 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 12 Sep 2024 23:27:18 -0700 Subject: [PATCH 209/314] Fix RZ Warnings --- Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp index 8dc1f29165a..9a83b4bc072 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp @@ -52,7 +52,7 @@ void FiniteDifferenceSolver::EvolveBPML ( // Select algorithm (The choice of algorithm is a runtime option, // but we compile code for each algorithm, using templates) #ifdef WARPX_DIM_RZ - amrex::ignore_unused(fields, dt, dive_cleaning); + amrex::ignore_unused(fields, patch_type, level, dt, dive_cleaning); WARPX_ABORT_WITH_MESSAGE( "PML are not implemented in cylindrical geometry."); #else From e7c3713d117384be1cd911bad06ffef339c326ed Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 13 Sep 2024 01:22:30 -0700 Subject: [PATCH 210/314] Start Reworking Evolve/Sync Calls Towards throw-able `get`. --- Examples/CMakeLists.txt | 1 + Source/Evolve/WarpXEvolve.cpp | 54 ++++---------- .../FiniteDifferenceSolver/EvolveB.cpp | 27 +++++-- .../FiniteDifferenceSolver.H | 12 +-- .../MagnetostaticSolver.cpp | 5 +- Source/FieldSolver/WarpXPushFieldsEM.cpp | 26 +++---- .../FieldSolver/WarpXPushFieldsHybridPIC.cpp | 5 +- Source/Parallelization/WarpXComm.cpp | 29 +++++--- Source/Particles/LaserParticleContainer.H | 15 ++-- Source/Particles/LaserParticleContainer.cpp | 31 +++++--- Source/Particles/MultiParticleContainer.H | 20 ++--- Source/Particles/MultiParticleContainer.cpp | 32 ++++---- Source/Particles/PhotonParticleContainer.H | 36 +++------ Source/Particles/PhotonParticleContainer.cpp | 22 ++---- Source/Particles/PhysicalParticleContainer.H | 56 +++----------- .../Particles/PhysicalParticleContainer.cpp | 73 ++++++++++++------- .../RigidInjectedParticleContainer.H | 34 ++------- .../RigidInjectedParticleContainer.cpp | 22 ++---- Source/Particles/WarpXParticleContainer.H | 11 +-- Source/WarpX.H | 9 +-- Source/ablastr/fields/MultiFabRegister.cpp | 2 + 21 files changed, 211 insertions(+), 311 deletions(-) diff --git a/Examples/CMakeLists.txt b/Examples/CMakeLists.txt index 7ebb1465be4..3df32ed0f95 100644 --- a/Examples/CMakeLists.txt +++ b/Examples/CMakeLists.txt @@ -129,6 +129,7 @@ function(add_warpx_test set(runtime_params "amrex.abort_on_unused_inputs = 1" "amrex.throw_exception = 1" + "amrex.signal_handling = 0" "warpx.always_warn_immediately = 1" "warpx.do_dynamic_scheduling = 0" "warpx.serialize_initial_conditions = 1" diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index dd98a9d8e77..16fe79c4ca6 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -571,9 +571,7 @@ void WarpX::SyncCurrentAndRho () ? "current_fp_vay" : "current_fp"; // TODO Replace current_cp with current_cp_vay once Vay deposition is implemented with MR - SyncCurrent(m_fields.get_mr_levels_alldirs(current_fp_string, finest_level), - m_fields.get_mr_levels_alldirs("current_cp", finest_level), - m_fields.get_mr_levels_alldirs("current_buf", finest_level) ); + SyncCurrent(current_fp_string); SyncRho(); } @@ -585,9 +583,7 @@ void WarpX::SyncCurrentAndRho () if (!current_correction && current_deposition_algo != CurrentDepositionAlgo::Vay) { - SyncCurrent(m_fields.get_mr_levels_alldirs("current_fp", finest_level), - m_fields.get_mr_levels_alldirs("current_cp", finest_level), - m_fields.get_mr_levels_alldirs("current_buf", finest_level) ); + SyncCurrent("current_fp"); SyncRho(); } @@ -603,9 +599,7 @@ void WarpX::SyncCurrentAndRho () } else // FDTD { - SyncCurrent(m_fields.get_mr_levels_alldirs("current_fp", finest_level), - m_fields.get_mr_levels_alldirs("current_cp", finest_level), - m_fields.get_mr_levels_alldirs("current_buf", finest_level) ); + SyncCurrent("current_fp"); SyncRho(); } @@ -692,10 +686,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // namely 'current_fp_nodal': SyncCurrent stores the result of its centering // into 'current_fp' and then performs both filtering, if used, and exchange // of guard cells. - SyncCurrent( - m_fields.get_mr_levels_alldirs( "current_fp", finest_level), - m_fields.get_mr_levels_alldirs( "current_cp", finest_level), - m_fields.get_mr_levels_alldirs( "current_buf", finest_level) ); + SyncCurrent("current_fp"); // Forward FFT of J PSATDForwardTransformJ( m_fields.get_mr_levels_alldirs( "current_fp", finest_level), @@ -731,10 +722,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // namely 'current_fp_nodal': SyncCurrent stores the result of its centering // into 'current_fp' and then performs both filtering, if used, and exchange // of guard cells. - SyncCurrent( - m_fields.get_mr_levels_alldirs( "current_fp", finest_level), - m_fields.get_mr_levels_alldirs( "current_cp", finest_level), - m_fields.get_mr_levels_alldirs( "current_buf", finest_level) ); + SyncCurrent("current_fp"); // Forward FFT of J PSATDForwardTransformJ( m_fields.get_mr_levels_alldirs( "current_fp", finest_level), @@ -1117,28 +1105,16 @@ WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, current_fp_string = "current_fp"; } - mypc->Evolve(lev, - *m_fields.get("Efield_aux", Direction{0}, lev), - *m_fields.get("Efield_aux", Direction{1}, lev), - *m_fields.get("Efield_aux", Direction{2}, lev), - *m_fields.get("Bfield_aux", Direction{0}, lev), - *m_fields.get("Bfield_aux", Direction{1}, lev), - *m_fields.get("Bfield_aux", Direction{2}, lev), - *m_fields.get(current_fp_string, Direction{0}, lev), - *m_fields.get(current_fp_string, Direction{1}, lev), - *m_fields.get(current_fp_string, Direction{2}, lev), - m_fields.get("current_buf", Direction{0}, lev), - m_fields.get("current_buf", Direction{1}, lev), - m_fields.get("current_buf", Direction{2}, lev), - m_fields.get("rho_fp",lev), - m_fields.get("rho_buf", lev), - m_fields.get("Efield_cax", Direction{0}, lev), - m_fields.get("Efield_cax", Direction{1}, lev), - m_fields.get("Efield_cax", Direction{2}, lev), - m_fields.get("Bfield_cax", Direction{0}, lev), - m_fields.get("Bfield_cax", Direction{1}, lev), - m_fields.get("Bfield_cax", Direction{2}, lev), - cur_time, dt[lev], a_dt_type, skip_current, push_type); + mypc->Evolve( + m_fields, + lev, + current_fp_string, + cur_time, + dt[lev], + a_dt_type, + skip_current, + push_type + ); if (! skip_current) { #ifdef WARPX_DIM_RZ // This is called after all particles have deposited their current and charge. diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp index 5ed4a1053f2..44c297034a6 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp @@ -48,18 +48,18 @@ using namespace amrex; * \brief Update the B field, over one timestep */ void FiniteDifferenceSolver::EvolveB ( - [[maybe_unused]] ablastr::fields::VectorField const& Bfield, - [[maybe_unused]] ablastr::fields::VectorField const& Efield, - [[maybe_unused]] amrex::MultiFab const * Gfield, - [[maybe_unused]] ablastr::fields::VectorField const& face_areas, - [[maybe_unused]] ablastr::fields::VectorField const& area_mod, - [[maybe_unused]] ablastr::fields::VectorField const& ECTRhofield, - [[maybe_unused]] ablastr::fields::VectorField const& Venl, + ablastr::fields::MultiFabRegister& fields, + int lev, + PatchType patch_type, [[maybe_unused]] std::array< std::unique_ptr, 3 >& flag_info_cell, [[maybe_unused]] std::array< std::unique_ptr >, 3 >& borrowing, - [[maybe_unused]] int lev, [[maybe_unused]] amrex::Real const dt ) { + ablastr::fields::VectorField Bfield = patch_type == PatchType::fine ? + fields.get_alldirs("Bfield_fp", lev) : fields.get_alldirs("Bfield_cp", lev); + ablastr::fields::VectorField Efield = patch_type == PatchType::fine ? + fields.get_alldirs("Bfield_fp", lev) : fields.get_alldirs("Efield_cp", lev); + // Select algorithm (The choice of algorithm is a runtime option, // but we compile code for each algorithm, using templates) #ifdef WARPX_DIM_RZ @@ -68,6 +68,17 @@ void FiniteDifferenceSolver::EvolveB ( EvolveBCylindrical ( Bfield, Efield, lev, dt ); #else + amrex::MultiFab const * Gfield = patch_type == PatchType::fine ? + fields.get("G_fp", lev) : fields.get("G_fp", lev); + ablastr::fields::VectorField face_areas = patch_type == PatchType::fine ? + fields.get_alldirs("face_areas", lev) : fields.get_alldirs("face_areas", lev); + ablastr::fields::VectorField area_mod = patch_type == PatchType::fine ? + fields.get_alldirs("area_mod", lev) : fields.get_alldirs("area_mod", lev); + ablastr::fields::VectorField ECTRhofield = patch_type == PatchType::fine ? + fields.get_alldirs("ECTRhofield", lev) : fields.get_alldirs("ECTRhofield", lev); + ablastr::fields::VectorField Venl = patch_type == PatchType::fine ? + fields.get_alldirs("Venl", lev) : fields.get_alldirs("Venl", lev); + if (m_grid_type == GridType::Collocated) { EvolveBCartesian ( Bfield, Efield, Gfield, lev, dt ); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 0ec0bb7b338..1d6a77ae068 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -52,16 +52,12 @@ class FiniteDifferenceSolver std::array cell_size, ablastr::utils::enums::GridType grid_type ); - void EvolveB ( ablastr::fields::VectorField const& Bfield, - ablastr::fields::VectorField const& Efield, - amrex::MultiFab const * Gfield, - ablastr::fields::VectorField const& face_areas, - ablastr::fields::VectorField const& area_mod, - ablastr::fields::VectorField const& ECTRhofield, - ablastr::fields::VectorField const& Venl, + void EvolveB ( ablastr::fields::MultiFabRegister& fields, + int lev, + PatchType patch_type, std::array< std::unique_ptr, 3 >& flag_info_cell, std::array< std::unique_ptr >, 3 >& borrowing, - int lev, amrex::Real dt ); + amrex::Real dt ); void EvolveE ( ablastr::fields::VectorField const& Efield, ablastr::fields::VectorField const& Bfield, diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp index c5df9a17e39..3d5fdad264e 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp @@ -114,10 +114,7 @@ WarpX::AddMagnetostaticFieldLabFrame() } #endif - SyncCurrent( - m_fields.get_mr_levels_alldirs("current_fp", finest_level), - m_fields.get_mr_levels_alldirs("current_cp", finest_level), - m_fields.get_mr_levels_alldirs("current_buf", finest_level) ); + SyncCurrent("current_fp"); // set the boundary and current density potentials setVectorPotentialBC(m_fields.get_mr_levels_alldirs("vector_potential_fp_nodal", finest_level)); diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 3997f243c15..3a03015685e 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -743,7 +743,7 @@ WarpX::PushPSATD () PSATDBackwardTransformJ(current_fp, current_cp); // Synchronize J and rho - SyncCurrent(current_fp, current_cp, current_buf); + SyncCurrent("current_fp"); SyncRho(); } else if (current_deposition_algo == CurrentDepositionAlgo::Vay) @@ -855,23 +855,15 @@ WarpX::EvolveB (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_typ { // Evolve B field in regular cells if (patch_type == PatchType::fine) { - m_fdtd_solver_fp[lev]->EvolveB( m_fields.get_alldirs("Bfield_fp",lev), - m_fields.get_alldirs("Efield_fp",lev), - m_fields.get("G_fp", lev), - m_fields.get_alldirs("face_areas", lev), - m_fields.get_alldirs("area_mod", lev), - m_fields.get_alldirs("ECTRhofield", lev), - m_fields.get_alldirs("Venl", lev), - m_flag_info_face[lev], m_borrowing[lev], lev, a_dt ); + m_fdtd_solver_fp[lev]->EvolveB( m_fields, + lev, + patch_type, + m_flag_info_face[lev], m_borrowing[lev], a_dt ); } else { - m_fdtd_solver_cp[lev]->EvolveB( m_fields.get_alldirs("Bfield_cp",lev), - m_fields.get_alldirs("Efield_cp",lev), - m_fields.get("G_fp", lev), - m_fields.get_alldirs("face_areas", lev), - m_fields.get_alldirs("area_mod", lev), - m_fields.get_alldirs("ECTRhofield", lev), - m_fields.get_alldirs("Venl", lev), - m_flag_info_face[lev], m_borrowing[lev], lev, a_dt ); + m_fdtd_solver_cp[lev]->EvolveB( m_fields, + lev, + patch_type, + m_flag_info_face[lev], m_borrowing[lev], a_dt ); } // Evolve B field in PML cells diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index 1728c0c837f..7ba912d96ce 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -200,10 +200,7 @@ void WarpX::HybridPICDepositInitialRhoAndJ () mypc->DepositCharge(amrex::GetVecOfPtrs(rho_fp_temp), 0._rt); mypc->DepositCurrent(va2vm(current_fp_temp), dt[0], 0._rt); SyncRho(amrex::GetVecOfPtrs(rho_fp_temp), m_fields.get_mr_levels("rho_cp", finest_level), m_fields.get_mr_levels("rho_buf", finest_level)); - SyncCurrent( - m_fields.get_mr_levels_alldirs("current_fp", finest_level), - m_fields.get_mr_levels_alldirs("current_cp", finest_level), - m_fields.get_mr_levels_alldirs("current_buf", finest_level) ); + SyncCurrent("current_fp"); for (int lev=0; lev <= finest_level; ++lev) { // SyncCurrent does not include a call to FillBoundary, but it is needed // for the hybrid-PIC solver since current values are interpolated to diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 44b96b93119..c5e50ef4454 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -995,7 +995,9 @@ void WarpX::FillBoundaryG (int lev, PatchType patch_type, IntVect ng, std::optio if (m_fields.has("pml_G_fp",lev) && m_fields.has("G_fp",lev)) { pml[lev]->Exchange(m_fields.get("pml_G_fp", lev), m_fields.get("G_fp", lev), patch_type, do_pml_in_domain); } - pml[lev]->FillBoundary(*m_fields.get("pml_G_fp", lev), patch_type, nodal_sync); + if (m_fields.has("pml_G_fp",lev)) { + pml[lev]->FillBoundary(*m_fields.get("pml_G_fp", lev), patch_type, nodal_sync); + } } if (m_fields.has("G_fp",lev)) @@ -1051,15 +1053,14 @@ WarpX::FillBoundaryAux (int lev, IntVect ng) } void -WarpX::SyncCurrent ( - const ablastr::fields::MultiLevelVectorField& J_fp, - const ablastr::fields::MultiLevelVectorField& J_cp, - const ablastr::fields::MultiLevelVectorField& J_buffer) +WarpX::SyncCurrent (std::string current_fp_string) { using ablastr::fields::Direction; WARPX_PROFILE("WarpX::SyncCurrent()"); + ablastr::fields::MultiLevelVectorField J_fp = m_fields.get_mr_levels_alldirs(current_fp_string, finest_level); + // If warpx.do_current_centering = 1, center currents from nodal grid to staggered grid if (do_current_centering) { @@ -1173,6 +1174,7 @@ WarpX::SyncCurrent ( } }); // Now it's safe to apply filter and sumboundary on J_cp + ablastr::fields::MultiLevelVectorField J_cp = m_fields.get_mr_levels_alldirs("current_cp", finest_level); if (use_filter) { ApplyFilterJ(J_cp, lev+1, idim); @@ -1187,12 +1189,15 @@ WarpX::SyncCurrent ( // filtering depends on the level. This is also done before any // same-level communication because it's easier this way to // avoid double counting. + ablastr::fields::MultiLevelVectorField J_cp = m_fields.get_mr_levels_alldirs("current_cp", finest_level); J_cp[lev][Direction{idim}]->setVal(0.0); ablastr::coarsen::average::Coarsen(*J_cp[lev][Direction{idim}], *J_fp[lev][Direction{idim}], refRatio(lev-1)); - if (J_buffer[lev][Direction{idim}]) + if (m_fields.has("current_buf", Direction{idim}, lev)) { + ablastr::fields::MultiLevelVectorField J_buffer = m_fields.get_mr_levels_alldirs("current_buf", finest_level); + IntVect const& ng = J_cp[lev][Direction{idim}]->nGrowVect(); AMREX_ASSERT(ng.allLE(J_buffer[lev][Direction{idim}]->nGrowVect())); MultiFab::Add(*J_buffer[lev][Direction{idim}], *J_cp[lev][Direction{idim}], @@ -1218,10 +1223,14 @@ WarpX::SyncCurrent ( void WarpX::SyncRho () { - SyncRho( - m_fields.get_mr_levels("rho_fp", finest_level), - m_fields.get_mr_levels("rho_cp", finest_level), - m_fields.get_mr_levels("rho_buf", finest_level)); + ablastr::fields::MultiLevelScalarField rho_fp = m_fields.has("rho_fp", 0) ? + m_fields.get_mr_levels("rho_fp", finest_level) : ablastr::fields::MultiLevelScalarField{1}; + ablastr::fields::MultiLevelScalarField rho_cp = m_fields.has("rho_cp", 0) ? + m_fields.get_mr_levels("rho_cp", finest_level) : ablastr::fields::MultiLevelScalarField{1}; + ablastr::fields::MultiLevelScalarField rho_buf = m_fields.has("rho_buf", 0) ? + m_fields.get_mr_levels("rho_buf", finest_level) : ablastr::fields::MultiLevelScalarField{1}; + + SyncRho(rho_fp, rho_cp, rho_buf); } void diff --git a/Source/Particles/LaserParticleContainer.H b/Source/Particles/LaserParticleContainer.H index 197cb897602..bb969747927 100644 --- a/Source/Particles/LaserParticleContainer.H +++ b/Source/Particles/LaserParticleContainer.H @@ -64,16 +64,11 @@ public: void WriteHeader (std::ostream& os) const final; - void Evolve (int lev, - const amrex::MultiFab&, const amrex::MultiFab&, const amrex::MultiFab&, - const amrex::MultiFab&, const amrex::MultiFab&, const amrex::MultiFab&, - amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, - amrex::MultiFab*, amrex::MultiFab*, amrex::MultiFab*, - amrex::MultiFab* rho, amrex::MultiFab* crho, - const amrex::MultiFab*, const amrex::MultiFab*, const amrex::MultiFab*, - const amrex::MultiFab*, const amrex::MultiFab*, const amrex::MultiFab*, - amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, - bool skip_deposition=false, PushType push_type=PushType::Explicit) final; + void Evolve (ablastr::fields::MultiFabRegister& fields, + int lev, + std::string current_fp_string, + amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, + bool skip_deposition=false, PushType push_type=PushType::Explicit) final; void PushP (int lev, amrex::Real dt, const amrex::MultiFab& , diff --git a/Source/Particles/LaserParticleContainer.cpp b/Source/Particles/LaserParticleContainer.cpp index e9509a1ef40..626238a99cf 100644 --- a/Source/Particles/LaserParticleContainer.cpp +++ b/Source/Particles/LaserParticleContainer.cpp @@ -558,14 +558,9 @@ LaserParticleContainer::InitData (int lev) } void -LaserParticleContainer::Evolve (int lev, - const MultiFab&, const MultiFab&, const MultiFab&, - const MultiFab&, const MultiFab&, const MultiFab&, - MultiFab& jx, MultiFab& jy, MultiFab& jz, - MultiFab* cjx, MultiFab* cjy, MultiFab* cjz, - MultiFab* rho, MultiFab* crho, - const MultiFab*, const MultiFab*, const MultiFab*, - const MultiFab*, const MultiFab*, const MultiFab*, +LaserParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, + int lev, + std::string current_fp_string, Real t, Real dt, DtType /*a_dt_type*/, bool skip_deposition, PushType push_type) { WARPX_PROFILE("LaserParticleContainer::Evolve()"); @@ -588,7 +583,8 @@ LaserParticleContainer::Evolve (int lev, amrex::LayoutData* cost = WarpX::getCosts(lev); - const bool has_buffer = cjx; + const bool has_rho = fields.has("rho_fp", lev); + const bool has_buffer = fields.has("current_buf", lev); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) @@ -628,11 +624,13 @@ LaserParticleContainer::Evolve (int lev, np_current = 0; } - if (rho && ! skip_deposition && ! do_not_deposit) { + if (has_rho && ! skip_deposition && ! do_not_deposit) { int* AMREX_RESTRICT ion_lev = nullptr; + amrex::MultiFab* rho = fields.get("rho_fp", lev); DepositCharge(pti, wp, ion_lev, rho, 0, 0, np_current, thread_num, lev, lev); if (has_buffer) { + amrex::MultiFab* crho = fields.get("rho_buf", lev); DepositCharge(pti, wp, ion_lev, crho, 0, np_current, np-np_current, thread_num, lev, lev-1); } @@ -660,6 +658,7 @@ LaserParticleContainer::Evolve (int lev, WARPX_PROFILE_VAR_STOP(blp_pp); // Current Deposition + using ablastr::fields::Direction; if (!skip_deposition) { // Deposit at t_{n+1/2} @@ -667,13 +666,19 @@ LaserParticleContainer::Evolve (int lev, int* ion_lev = nullptr; // Deposit inside domains - DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, &jx, &jy, &jz, + amrex::MultiFab * jx = fields.get(current_fp_string, Direction{0}, lev); + amrex::MultiFab * jy = fields.get(current_fp_string, Direction{1}, lev); + amrex::MultiFab * jz = fields.get(current_fp_string, Direction{2}, lev); + DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, jx, jy, jz, 0, np_current, thread_num, lev, lev, dt, relative_time, push_type); if (has_buffer) { // Deposit in buffers + amrex::MultiFab * cjx = fields.get("current_buf", Direction{0}, lev); + amrex::MultiFab * cjy = fields.get("current_buf", Direction{1}, lev); + amrex::MultiFab * cjz = fields.get("current_buf", Direction{2}, lev); DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, cjx, cjy, cjz, np_current, np-np_current, thread_num, lev, lev-1, dt, relative_time, push_type); @@ -681,11 +686,13 @@ LaserParticleContainer::Evolve (int lev, } - if (rho && ! skip_deposition && ! do_not_deposit) { + if (has_rho && ! skip_deposition && ! do_not_deposit) { int* AMREX_RESTRICT ion_lev = nullptr; + amrex::MultiFab* rho = fields.get("rho_fp", lev); DepositCharge(pti, wp, ion_lev, rho, 1, 0, np_current, thread_num, lev, lev); if (has_buffer) { + amrex::MultiFab* crho = fields.get("rho_buf", lev); DepositCharge(pti, wp, ion_lev, crho, 1, np_current, np-np_current, thread_num, lev, lev-1); } diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index 05850daf07b..3334a5233cc 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -104,16 +104,16 @@ public: * field solve, and pushing the particles, for all the species in the MultiParticleContainer. * This is the electromagnetic version. */ - void Evolve (int lev, - const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, - const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, - amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, - amrex::MultiFab* cjx, amrex::MultiFab* cjy, amrex::MultiFab* cjz, - amrex::MultiFab* rho, amrex::MultiFab* crho, - const amrex::MultiFab* cEx, const amrex::MultiFab* cEy, const amrex::MultiFab* cEz, - const amrex::MultiFab* cBx, const amrex::MultiFab* cBy, const amrex::MultiFab* cBz, - amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, bool skip_deposition=false, - PushType push_type=PushType::Explicit); + void Evolve ( + ablastr::fields::MultiFabRegister& fields, + int lev, + std::string current_fp_string, + amrex::Real t, + amrex::Real dt, + DtType a_dt_type=DtType::Full, + bool skip_deposition=false, + PushType push_type=PushType::Explicit + ); /** * \brief This pushes the particle positions by one time step for all the species in the diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 5ae18b36280..d367468c36a 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -458,30 +458,26 @@ MultiParticleContainer::InitMultiPhysicsModules () } void -MultiParticleContainer::Evolve (int lev, - const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, - const MultiFab& Bx, const MultiFab& By, const MultiFab& Bz, - MultiFab& jx, MultiFab& jy, MultiFab& jz, - MultiFab* cjx, MultiFab* cjy, MultiFab* cjz, - MultiFab* rho, MultiFab* crho, - const MultiFab* cEx, const MultiFab* cEy, const MultiFab* cEz, - const MultiFab* cBx, const MultiFab* cBy, const MultiFab* cBz, +MultiParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, + int lev, + std::string current_fp_string, Real t, Real dt, DtType a_dt_type, bool skip_deposition, PushType push_type) { if (! skip_deposition) { - jx.setVal(0.0); - jy.setVal(0.0); - jz.setVal(0.0); - if (cjx) { cjx->setVal(0.0); } - if (cjy) { cjy->setVal(0.0); } - if (cjz) { cjz->setVal(0.0); } - if (rho) { rho->setVal(0.0); } - if (crho) { crho->setVal(0.0); } + using ablastr::fields::Direction; + + fields.get(current_fp_string, Direction{0}, lev)->setVal(0.0); + fields.get(current_fp_string, Direction{1}, lev)->setVal(0.0); + fields.get(current_fp_string, Direction{2}, lev)->setVal(0.0); + if (fields.has("current_buf", Direction{0}, lev)) { fields.get("current_buf", Direction{0}, lev)->setVal(0.0); } + if (fields.has("current_buf", Direction{1}, lev)) { fields.get("current_buf", Direction{1}, lev)->setVal(0.0); } + if (fields.has("current_buf", Direction{1}, lev)) { fields.get("current_buf", Direction{2}, lev)->setVal(0.0); } + if (fields.has("rho_fp", lev)) { fields.get("rho_fp", lev)->setVal(0.0); } + if (fields.has("rho_buf", lev)) { fields.get("rho_buf", lev)->setVal(0.0); } } for (auto& pc : allcontainers) { - pc->Evolve(lev, Ex, Ey, Ez, Bx, By, Bz, jx, jy, jz, cjx, cjy, cjz, - rho, crho, cEx, cEy, cEz, cBx, cBy, cBz, t, dt, a_dt_type, skip_deposition, push_type); + pc->Evolve(fields, lev, current_fp_string, t, dt, a_dt_type, skip_deposition, push_type); } } diff --git a/Source/Particles/PhotonParticleContainer.H b/Source/Particles/PhotonParticleContainer.H index 34afac53482..18796519918 100644 --- a/Source/Particles/PhotonParticleContainer.H +++ b/Source/Particles/PhotonParticleContainer.H @@ -46,34 +46,16 @@ public: void InitData() override; - void Evolve (int lev, - const amrex::MultiFab& Ex, - const amrex::MultiFab& Ey, - const amrex::MultiFab& Ez, - const amrex::MultiFab& Bx, - const amrex::MultiFab& By, - const amrex::MultiFab& Bz, - amrex::MultiFab& jx, - amrex::MultiFab& jy, - amrex::MultiFab& jz, - amrex::MultiFab* cjx, - amrex::MultiFab* cjy, - amrex::MultiFab* cjz, - amrex::MultiFab* rho, - amrex::MultiFab* crho, - const amrex::MultiFab* cEx, - const amrex::MultiFab* cEy, - const amrex::MultiFab* cEz, - const amrex::MultiFab* cBx, - const amrex::MultiFab* cBy, - const amrex::MultiFab* cBz, - amrex::Real t, - amrex::Real dt, - DtType a_dt_type=DtType::Full, - bool skip_deposition=false, - PushType push_type=PushType::Explicit) override; + void Evolve (ablastr::fields::MultiFabRegister& fields, + int lev, + std::string current_fp_string, + amrex::Real t, + amrex::Real dt, + DtType a_dt_type=DtType::Full, + bool skip_deposition=false, + PushType push_type=PushType::Explicit) override; - void PushPX(WarpXParIter& pti, + void PushPX (WarpXParIter& pti, amrex::FArrayBox const * exfab, amrex::FArrayBox const * eyfab, amrex::FArrayBox const * ezfab, diff --git a/Source/Particles/PhotonParticleContainer.cpp b/Source/Particles/PhotonParticleContainer.cpp index 1f15d5210f5..ba33836fce1 100644 --- a/Source/Particles/PhotonParticleContainer.cpp +++ b/Source/Particles/PhotonParticleContainer.cpp @@ -229,27 +229,17 @@ PhotonParticleContainer::PushPX (WarpXParIter& pti, } void -PhotonParticleContainer::Evolve (int lev, - const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, - const MultiFab& Bx, const MultiFab& By, const MultiFab& Bz, - MultiFab& jx, MultiFab& jy, MultiFab& jz, - MultiFab* cjx, MultiFab* cjy, MultiFab* cjz, - MultiFab* rho, MultiFab* crho, - const MultiFab* cEx, const MultiFab* cEy, const MultiFab* cEz, - const MultiFab* cBx, const MultiFab* cBy, const MultiFab* cBz, +PhotonParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, + int lev, + std::string current_fp_string, Real t, Real dt, DtType a_dt_type, bool skip_deposition, PushType push_type) { // This does gather, push and deposit. // Push and deposit have been re-written for photons - PhysicalParticleContainer::Evolve (lev, - Ex, Ey, Ez, - Bx, By, Bz, - jx, jy, jz, - cjx, cjy, cjz, - rho, crho, - cEx, cEy, cEz, - cBx, cBy, cBz, + PhysicalParticleContainer::Evolve (fields, + lev, + current_fp_string, t, dt, a_dt_type, skip_deposition, push_type); } diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index 5d9b41b8b75..b3e9f258816 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -81,27 +81,9 @@ public: * \brief Evolve is the central function PhysicalParticleContainer that * advances plasma particles for a time dt (typically one timestep). * + * \param fields the WarpX field register * \param lev level on which particles are living - * \param Ex MultiFab from which field Ex is gathered - * \param Ey MultiFab from which field Ey is gathered - * \param Ez MultiFab from which field Ez is gathered - * \param Bx MultiFab from which field Bx is gathered - * \param By MultiFab from which field By is gathered - * \param Bz MultiFab from which field Bz is gathered - * \param jx MultiFab to which the particles' current jx is deposited - * \param jy MultiFab to which the particles' current jy is deposited - * \param jz MultiFab to which the particles' current jz is deposited - * \param cjx Same as jx (coarser, from lev-1), when using deposition buffers - * \param cjy Same as jy (coarser, from lev-1), when using deposition buffers - * \param cjz Same as jz (coarser, from lev-1), when using deposition buffers - * \param rho MultiFab to which the particles' charge is deposited - * \param crho Same as rho (coarser, from lev-1), when using deposition buffers - * \param cEx Same as Ex (coarser, from lev-1), when using gather buffers - * \param cEy Same as Ey (coarser, from lev-1), when using gather buffers - * \param cEz Same as Ez (coarser, from lev-1), when using gather buffers - * \param cBx Same as Bx (coarser, from lev-1), when using gather buffers - * \param cBy Same as By (coarser, from lev-1), when using gather buffers - * \param cBz Same as Bz (coarser, from lev-1), when using gather buffers + * \param current_fp_string current coarse or fine patch identifier in fields * \param t current physical time * \param dt time step by which particles are advanced * \param a_dt_type type of time step (used for sub-cycling) @@ -112,32 +94,14 @@ public: * field gather, particle push and current deposition for all particles * in the box. */ - void Evolve (int lev, - const amrex::MultiFab& Ex, - const amrex::MultiFab& Ey, - const amrex::MultiFab& Ez, - const amrex::MultiFab& Bx, - const amrex::MultiFab& By, - const amrex::MultiFab& Bz, - amrex::MultiFab& jx, - amrex::MultiFab& jy, - amrex::MultiFab& jz, - amrex::MultiFab* cjx, - amrex::MultiFab* cjy, - amrex::MultiFab* cjz, - amrex::MultiFab* rho, - amrex::MultiFab* crho, - const amrex::MultiFab* cEx, - const amrex::MultiFab* cEy, - const amrex::MultiFab* cEz, - const amrex::MultiFab* cBx, - const amrex::MultiFab* cBy, - const amrex::MultiFab* cBz, - amrex::Real t, - amrex::Real dt, - DtType a_dt_type=DtType::Full, - bool skip_deposition=false, - PushType push_type=PushType::Explicit) override; + void Evolve (ablastr::fields::MultiFabRegister& fields, + int lev, + std::string current_fp_string, + amrex::Real t, + amrex::Real dt, + DtType a_dt_type=DtType::Full, + bool skip_deposition=false, + PushType push_type=PushType::Explicit) override; virtual void PushPX (WarpXParIter& pti, amrex::FArrayBox const * exfab, diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 513d3da979e..c3725cb1239 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -2025,14 +2025,9 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, } void -PhysicalParticleContainer::Evolve (int lev, - const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, - const MultiFab& Bx, const MultiFab& By, const MultiFab& Bz, - MultiFab& jx, MultiFab& jy, MultiFab& jz, - MultiFab* cjx, MultiFab* cjy, MultiFab* cjz, - MultiFab* rho, MultiFab* crho, - const MultiFab* cEx, const MultiFab* cEy, const MultiFab* cEz, - const MultiFab* cBx, const MultiFab* cBy, const MultiFab* cBz, +PhysicalParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, + int lev, + std::string current_fp_string, Real /*t*/, Real dt, DtType a_dt_type, bool skip_deposition, PushType push_type) { @@ -2047,7 +2042,18 @@ PhysicalParticleContainer::Evolve (int lev, const iMultiFab* current_masks = WarpX::CurrentBufferMasks(lev); const iMultiFab* gather_masks = WarpX::GatherBufferMasks(lev); - const bool has_buffer = cEx || cjx; + using ablastr::fields::Direction; + const bool has_rho = fields.has("rho_fp", lev); + const bool has_cjx = fields.has("current_buf", Direction{0}, lev); + const bool has_cEx = fields.has("Efield_cax", Direction{0}, lev); + const bool has_buffer = has_cEx || has_cjx; + + amrex::MultiFab & Ex = *fields.get("Efield_aux", Direction{0}, lev); + amrex::MultiFab & Ey = *fields.get("Efield_aux", Direction{1}, lev); + amrex::MultiFab & Ez = *fields.get("Efield_aux", Direction{2}, lev); + amrex::MultiFab & Bx = *fields.get("Bfield_aux", Direction{0}, lev); + amrex::MultiFab & By = *fields.get("Bfield_aux", Direction{1}, lev); + amrex::MultiFab & Bz = *fields.get("Bfield_aux", Direction{2}, lev); if (m_do_back_transformed_particles) { @@ -2135,17 +2141,19 @@ PhysicalParticleContainer::Evolve (int lev, pti, lev, current_masks, gather_masks ); } - const long np_current = (cjx) ? nfine_current : np; + const long np_current = has_cjx ? nfine_current : np; - if (rho && ! skip_deposition && ! do_not_deposit) { + if (has_rho && ! skip_deposition && ! do_not_deposit) { // Deposit charge before particle push, in component 0 of MultiFab rho. const int* const AMREX_RESTRICT ion_lev = (do_field_ionization)? pti.GetiAttribs(particle_icomps["ionizationLevel"]).dataPtr():nullptr; + amrex::MultiFab* rho = fields.get("rho_fp", lev); DepositCharge(pti, wp, ion_lev, rho, 0, 0, np_current, thread_num, lev, lev); if (has_buffer){ + amrex::MultiFab* crho = fields.get("rho_buf", lev); DepositCharge(pti, wp, ion_lev, crho, 0, np_current, np-np_current, thread_num, lev, lev-1); } @@ -2153,7 +2161,7 @@ PhysicalParticleContainer::Evolve (int lev, if (! do_not_push) { - const long np_gather = (cEx) ? nfine_gather : np; + const long np_gather = has_cEx ? nfine_gather : np; int e_is_nodal = Ex.is_nodal() and Ey.is_nodal() and Ez.is_nodal(); @@ -2180,13 +2188,20 @@ PhysicalParticleContainer::Evolve (int lev, const IntVect& ref_ratio = WarpX::RefRatio(lev-1); const Box& cbox = amrex::coarsen(box,ref_ratio); + amrex::MultiFab & cEx = *fields.get("Efield_cax", Direction{0}, lev); + amrex::MultiFab & cEy = *fields.get("Efield_cax", Direction{1}, lev); + amrex::MultiFab & cEz = *fields.get("Efield_cax", Direction{2}, lev); + amrex::MultiFab & cBx = *fields.get("Bfield_cax", Direction{0}, lev); + amrex::MultiFab & cBy = *fields.get("Bfield_cax", Direction{1}, lev); + amrex::MultiFab & cBz = *fields.get("Bfield_cax", Direction{2}, lev); + // Data on the grid - FArrayBox const* cexfab = &(*cEx)[pti]; - FArrayBox const* ceyfab = &(*cEy)[pti]; - FArrayBox const* cezfab = &(*cEz)[pti]; - FArrayBox const* cbxfab = &(*cBx)[pti]; - FArrayBox const* cbyfab = &(*cBy)[pti]; - FArrayBox const* cbzfab = &(*cBz)[pti]; + FArrayBox const* cexfab = &cEx[pti]; + FArrayBox const* ceyfab = &cEy[pti]; + FArrayBox const* cezfab = &cEz[pti]; + FArrayBox const* cbxfab = &cBx[pti]; + FArrayBox const* cbyfab = &cBy[pti]; + FArrayBox const* cbzfab = &cBz[pti]; if (WarpX::use_fdtd_nci_corr) { @@ -2197,23 +2212,23 @@ PhysicalParticleContainer::Evolve (int lev, applyNCIFilter(lev-1, cbox, exeli, eyeli, ezeli, bxeli, byeli, bzeli, filtered_Ex, filtered_Ey, filtered_Ez, filtered_Bx, filtered_By, filtered_Bz, - (*cEx)[pti], (*cEy)[pti], (*cEz)[pti], - (*cBx)[pti], (*cBy)[pti], (*cBz)[pti], + cEx[pti], cEy[pti], cEz[pti], + cBx[pti], cBy[pti], cBz[pti], cexfab, ceyfab, cezfab, cbxfab, cbyfab, cbzfab); } // Field gather and push for particles in gather buffers - e_is_nodal = cEx->is_nodal() and cEy->is_nodal() and cEz->is_nodal(); + e_is_nodal = cEx.is_nodal() and cEy.is_nodal() and cEz.is_nodal(); if (push_type == PushType::Explicit) { PushPX(pti, cexfab, ceyfab, cezfab, cbxfab, cbyfab, cbzfab, - cEx->nGrowVect(), e_is_nodal, + cEx.nGrowVect(), e_is_nodal, nfine_gather, np-nfine_gather, lev, lev-1, dt, ScaleFields(false), a_dt_type); } else if (push_type == PushType::Implicit) { ImplicitPushXP(pti, cexfab, ceyfab, cezfab, cbxfab, cbyfab, cbzfab, - cEx->nGrowVect(), e_is_nodal, + cEx.nGrowVect(), e_is_nodal, nfine_gather, np-nfine_gather, lev, lev-1, dt, ScaleFields(false), a_dt_type); } @@ -2231,13 +2246,19 @@ PhysicalParticleContainer::Evolve (int lev, pti.GetiAttribs(particle_icomps["ionizationLevel"]).dataPtr():nullptr; // Deposit inside domains - DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, &jx, &jy, &jz, + amrex::MultiFab * jx = fields.get(current_fp_string, Direction{0}, lev); + amrex::MultiFab * jy = fields.get(current_fp_string, Direction{1}, lev); + amrex::MultiFab * jz = fields.get(current_fp_string, Direction{2}, lev); + DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, jx, jy, jz, 0, np_current, thread_num, lev, lev, dt, relative_time, push_type); if (has_buffer) { // Deposit in buffers + amrex::MultiFab * cjx = fields.get("current_buf", Direction{0}, lev); + amrex::MultiFab * cjy = fields.get("current_buf", Direction{1}, lev); + amrex::MultiFab * cjz = fields.get("current_buf", Direction{2}, lev); DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, cjx, cjy, cjz, np_current, np-np_current, thread_num, lev, lev-1, dt, relative_time, push_type); @@ -2245,10 +2266,11 @@ PhysicalParticleContainer::Evolve (int lev, } // end of "if electrostatic_solver_id == ElectrostaticSolverAlgo::None" } // end of "if do_not_push" - if (rho && ! skip_deposition && ! do_not_deposit) { + if (has_rho && ! skip_deposition && ! do_not_deposit) { // Deposit charge after particle push, in component 1 of MultiFab rho. // (Skipped for electrostatic solver, as this may lead to out-of-bounds) if (WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::None) { + amrex::MultiFab* rho = fields.get("rho_fp", lev); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(rho->nComp() >= 2, "Cannot deposit charge in rho component 1: only component 0 is allocated!"); @@ -2258,6 +2280,7 @@ PhysicalParticleContainer::Evolve (int lev, DepositCharge(pti, wp, ion_lev, rho, 1, 0, np_current, thread_num, lev, lev); if (has_buffer){ + amrex::MultiFab* crho = fields.get("rho_buf", lev); DepositCharge(pti, wp, ion_lev, crho, 1, np_current, np-np_current, thread_num, lev, lev-1); } diff --git a/Source/Particles/RigidInjectedParticleContainer.H b/Source/Particles/RigidInjectedParticleContainer.H index bc20420ea6e..0f71161e4b1 100644 --- a/Source/Particles/RigidInjectedParticleContainer.H +++ b/Source/Particles/RigidInjectedParticleContainer.H @@ -61,32 +61,14 @@ public: virtual void RemapParticles(); - void Evolve (int lev, - const amrex::MultiFab& Ex, - const amrex::MultiFab& Ey, - const amrex::MultiFab& Ez, - const amrex::MultiFab& Bx, - const amrex::MultiFab& By, - const amrex::MultiFab& Bz, - amrex::MultiFab& jx, - amrex::MultiFab& jy, - amrex::MultiFab& jz, - amrex::MultiFab* cjx, - amrex::MultiFab* cjy, - amrex::MultiFab* cjz, - amrex::MultiFab* rho, - amrex::MultiFab* crho, - const amrex::MultiFab* cEx, - const amrex::MultiFab* cEy, - const amrex::MultiFab* cEz, - const amrex::MultiFab* cBx, - const amrex::MultiFab* cBy, - const amrex::MultiFab* cBz, - amrex::Real t, - amrex::Real dt, - DtType a_dt_type=DtType::Full, - bool skip_deposition=false, - PushType push_type=PushType::Explicit) override; + void Evolve (ablastr::fields::MultiFabRegister& fields, + int lev, + std::string current_fp_string, + amrex::Real t, + amrex::Real dt, + DtType a_dt_type=DtType::Full, + bool skip_deposition=false, + PushType push_type=PushType::Explicit) override; void PushPX (WarpXParIter& pti, amrex::FArrayBox const * exfab, diff --git a/Source/Particles/RigidInjectedParticleContainer.cpp b/Source/Particles/RigidInjectedParticleContainer.cpp index c3ec4c41131..5228eb3c606 100644 --- a/Source/Particles/RigidInjectedParticleContainer.cpp +++ b/Source/Particles/RigidInjectedParticleContainer.cpp @@ -291,14 +291,9 @@ RigidInjectedParticleContainer::PushPX (WarpXParIter& pti, } void -RigidInjectedParticleContainer::Evolve (int lev, - const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, - const MultiFab& Bx, const MultiFab& By, const MultiFab& Bz, - MultiFab& jx, MultiFab& jy, MultiFab& jz, - MultiFab* cjx, MultiFab* cjy, MultiFab* cjz, - MultiFab* rho, MultiFab* crho, - const MultiFab* cEx, const MultiFab* cEy, const MultiFab* cEz, - const MultiFab* cBx, const MultiFab* cBy, const MultiFab* cBz, +RigidInjectedParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, + int lev, + std::string current_fp_string, Real t, Real dt, DtType a_dt_type, bool skip_deposition, PushType push_type) { @@ -317,14 +312,9 @@ RigidInjectedParticleContainer::Evolve (int lev, done_injecting_lev = ((zinject_plane_levels[lev] < plo[WARPX_ZINDEX] && WarpX::moving_window_v + WarpX::beta_boost*PhysConst::c >= 0.) || (zinject_plane_levels[lev] > phi[WARPX_ZINDEX] && WarpX::moving_window_v + WarpX::beta_boost*PhysConst::c <= 0.)); - PhysicalParticleContainer::Evolve (lev, - Ex, Ey, Ez, - Bx, By, Bz, - jx, jy, jz, - cjx, cjy, cjz, - rho, crho, - cEx, cEy, cEz, - cBx, cBy, cBz, + PhysicalParticleContainer::Evolve (fields, + lev, + current_fp_string, t, dt, a_dt_type, skip_deposition, push_type); } diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index c2f1f9b5265..4af9bee3734 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -147,14 +147,9 @@ public: * particles for a time dt (typically one timestep). It is a pure virtual * function for flexibility. */ - virtual void Evolve (int lev, - const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, - const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, - amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, - amrex::MultiFab* cjx, amrex::MultiFab* cjy, amrex::MultiFab* cjz, - amrex::MultiFab* rho, amrex::MultiFab* crho, - const amrex::MultiFab* cEx, const amrex::MultiFab* cEy, const amrex::MultiFab* cEz, - const amrex::MultiFab* cBx, const amrex::MultiFab* cBy, const amrex::MultiFab* cBz, + virtual void Evolve (ablastr::fields::MultiFabRegister& fields, + int lev, + std::string current_fp_string, amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, bool skip_deposition=false, PushType push_type=PushType::Explicit) = 0; diff --git a/Source/WarpX.H b/Source/WarpX.H index d2f2df30633..1fc7bc2b1fb 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -790,14 +790,9 @@ public: * Then, for each MR level, including level 0, apply filter and sum guard * cells across levels. * - * \param[in,out] J_fp reference to fine-patch current \c MultiFab (all MR levels) - * \param[in,out] J_cp reference to coarse-patch current \c MultiFab (all MR levels) - * \param[in,out] J_buffer reference to buffer current \c MultiFab (all MR levels) + * \param[in] current_fp_string the coarse of fine patch to use for current */ - void SyncCurrent ( - const ablastr::fields::MultiLevelVectorField& J_fp, - const ablastr::fields::MultiLevelVectorField& J_cp, - const ablastr::fields::MultiLevelVectorField& J_buffer); + void SyncCurrent (std::string current_fp_string); void SyncRho (); diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 3c388d46474..b42ead54022 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -342,6 +342,7 @@ namespace ablastr::fields { if (m_mf_register.count(key) == 0) { // FIXME: temporary, throw a std::runtime_error + // throw std::runtime_error("MultiFabRegister::get name does not exist in register: " + key); return nullptr; } amrex::MultiFab & mf = m_mf_register.at(key).m_mf; @@ -356,6 +357,7 @@ namespace ablastr::fields { if (m_mf_register.count(key) == 0) { // FIXME: temporary, throw a std::runtime_error + // throw std::runtime_error("MultiFabRegister::get name does not exist in register: " + key); return nullptr; } amrex::MultiFab const & mf = m_mf_register.at(key).m_mf; From 893c79c8a7438236e6d32ba0fb30c0e9929d371e Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 13 Sep 2024 01:56:23 -0700 Subject: [PATCH 211/314] LWFA 3D: Passes with Register on Throw Modernize FDTD `UpdateE`. --- .../FiniteDifferenceSolver/EvolveB.cpp | 38 +++++++++++----- .../FiniteDifferenceSolver/EvolveE.cpp | 45 +++++++++++++++---- .../FiniteDifferenceSolver.H | 13 +++--- .../ImplicitSolvers/WarpXImplicitOps.cpp | 24 +++++----- Source/FieldSolver/WarpXPushFieldsEM.cpp | 24 +++++----- Source/Utils/WarpXMovingWindow.cpp | 8 ++-- 6 files changed, 92 insertions(+), 60 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp index 44c297034a6..0a69db6076d 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp @@ -55,10 +55,11 @@ void FiniteDifferenceSolver::EvolveB ( [[maybe_unused]] std::array< std::unique_ptr >, 3 >& borrowing, [[maybe_unused]] amrex::Real const dt ) { + using ablastr::fields::Direction; ablastr::fields::VectorField Bfield = patch_type == PatchType::fine ? fields.get_alldirs("Bfield_fp", lev) : fields.get_alldirs("Bfield_cp", lev); ablastr::fields::VectorField Efield = patch_type == PatchType::fine ? - fields.get_alldirs("Bfield_fp", lev) : fields.get_alldirs("Efield_cp", lev); + fields.get_alldirs("Efield_fp", lev) : fields.get_alldirs("Efield_cp", lev); // Select algorithm (The choice of algorithm is a runtime option, // but we compile code for each algorithm, using templates) @@ -68,16 +69,31 @@ void FiniteDifferenceSolver::EvolveB ( EvolveBCylindrical ( Bfield, Efield, lev, dt ); #else - amrex::MultiFab const * Gfield = patch_type == PatchType::fine ? - fields.get("G_fp", lev) : fields.get("G_fp", lev); - ablastr::fields::VectorField face_areas = patch_type == PatchType::fine ? - fields.get_alldirs("face_areas", lev) : fields.get_alldirs("face_areas", lev); - ablastr::fields::VectorField area_mod = patch_type == PatchType::fine ? - fields.get_alldirs("area_mod", lev) : fields.get_alldirs("area_mod", lev); - ablastr::fields::VectorField ECTRhofield = patch_type == PatchType::fine ? - fields.get_alldirs("ECTRhofield", lev) : fields.get_alldirs("ECTRhofield", lev); - ablastr::fields::VectorField Venl = patch_type == PatchType::fine ? - fields.get_alldirs("Venl", lev) : fields.get_alldirs("Venl", lev); + amrex::MultiFab const * Gfield = nullptr; + if (fields.has("G_fp", lev)) { + Gfield = patch_type == PatchType::fine ? + fields.get("G_fp", lev) : fields.get("G_cp", lev); + } + ablastr::fields::VectorField face_areas; + if (fields.has("face_areas", Direction{0}, lev)) { + face_areas = patch_type == PatchType::fine ? + fields.get_alldirs("face_areas", lev) : fields.get_alldirs("face_areas", lev); + } + ablastr::fields::VectorField area_mod; + if (fields.has("face_areas", Direction{0}, lev)) { + area_mod = patch_type == PatchType::fine ? + fields.get_alldirs("area_mod", lev) : fields.get_alldirs("area_mod", lev); + } + ablastr::fields::VectorField ECTRhofield; + if (fields.has("ECTRhofield", Direction{0}, lev)) { + ECTRhofield = patch_type == PatchType::fine ? + fields.get_alldirs("ECTRhofield", lev) : fields.get_alldirs("ECTRhofield", lev); + } + ablastr::fields::VectorField Venl; + if (fields.has("Venl", Direction{0}, lev)) { + Venl = patch_type == PatchType::fine ? + fields.get_alldirs("Venl", lev) : fields.get_alldirs("Venl", lev); + } if (m_grid_type == GridType::Collocated) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp index ff24a289915..ac18b2f7fae 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp @@ -50,17 +50,44 @@ using namespace ablastr::fields; * \brief Update the E field, over one timestep */ void FiniteDifferenceSolver::EvolveE ( + ablastr::fields::MultiFabRegister & fields, + int lev, + PatchType patch_type, ablastr::fields::VectorField const& Efield, - ablastr::fields::VectorField const& Bfield, - ablastr::fields::VectorField const& Jfield, - VectorField const& edge_lengths, - VectorField const& face_areas, - VectorField const& ECTRhofield, - amrex::MultiFab const* Ffield, - int lev, amrex::Real const dt ) { + amrex::Real const dt +) +{ + using ablastr::fields::Direction; + ablastr::fields::VectorField Bfield = patch_type == PatchType::fine ? + fields.get_alldirs("Bfield_fp", lev) : fields.get_alldirs("Bfield_cp", lev); + ablastr::fields::VectorField Jfield = patch_type == PatchType::fine ? + fields.get_alldirs("current_fp", lev) : fields.get_alldirs("current_cp", lev); + + amrex::MultiFab* Ffield = nullptr; + if (fields.has("F_fp", Direction{0}, lev)) { + Ffield = patch_type == PatchType::fine ? + fields.get("F_fp", lev) : fields.get("F_cp", lev); + } - if (m_fdtd_algo != ElectromagneticSolverAlgo::ECT) { - amrex::ignore_unused(face_areas, ECTRhofield); + ablastr::fields::VectorField edge_lengths; + if (fields.has("edge_lengths", Direction{0}, lev)) { + edge_lengths = patch_type == PatchType::fine ? + fields.get_alldirs("edge_lengths", lev) : fields.get_alldirs("edge_lengths", lev); + } + ablastr::fields::VectorField face_areas; + if (fields.has("face_areas", Direction{0}, lev)) { + face_areas = patch_type == PatchType::fine ? + fields.get_alldirs("face_areas", lev) : fields.get_alldirs("face_areas", lev); + } + ablastr::fields::VectorField area_mod; + if (fields.has("face_areas", Direction{0}, lev)) { + area_mod = patch_type == PatchType::fine ? + fields.get_alldirs("area_mod", lev) : fields.get_alldirs("area_mod", lev); + } + ablastr::fields::VectorField ECTRhofield; + if (fields.has("ECTRhofield", Direction{0}, lev)) { + ECTRhofield = patch_type == PatchType::fine ? + fields.get_alldirs("ECTRhofield", lev) : fields.get_alldirs("ECTRhofield", lev); } // Select algorithm (The choice of algorithm is a runtime option, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 1d6a77ae068..c98ea2b7d3f 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -59,14 +59,11 @@ class FiniteDifferenceSolver std::array< std::unique_ptr >, 3 >& borrowing, amrex::Real dt ); - void EvolveE ( ablastr::fields::VectorField const& Efield, - ablastr::fields::VectorField const& Bfield, - ablastr::fields::VectorField const& Jfield, - ablastr::fields::VectorField const& edge_lengths, - ablastr::fields::VectorField const& face_areas, - ablastr::fields::VectorField const& ECTRhofield, - amrex::MultiFab const* Ffield, - int lev, amrex::Real dt ); + void EvolveE ( ablastr::fields::MultiFabRegister & fields, + int lev, + PatchType patch_type, + ablastr::fields::VectorField const& Efield, + amrex::Real dt ); void EvolveF ( amrex::MultiFab* Ffield, ablastr::fields::VectorField const& Efield, diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp index bde7368e195..06ec79779dc 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp @@ -340,21 +340,17 @@ WarpX::ImplicitComputeRHSE (int lev, PatchType patch_type, amrex::Real a_dt, War // a_Erhs_vec storing only the RHS of the update equation. I.e., // c^2*dt*(curl(B^{n+theta} - mu0*J^{n+1/2}) if (patch_type == PatchType::fine) { - m_fdtd_solver_fp[lev]->EvolveE( a_Erhs_vec.getArrayVec()[lev], - m_fields.get_alldirs("Bfield_fp", lev), - m_fields.get_alldirs("current_fp", lev), - m_fields.get_alldirs("edge_lengths", lev), - m_fields.get_alldirs("face_areas", lev), - m_fields.get_alldirs("ECTRhofield", lev), - m_fields.get("F_fp", lev), lev, a_dt ); + m_fdtd_solver_fp[lev]->EvolveE( m_fields, + lev, + patch_type, + a_Erhs_vec.getArrayVec()[lev], + a_dt ); } else { - m_fdtd_solver_cp[lev]->EvolveE( a_Erhs_vec.getArrayVec()[lev], - m_fields.get_alldirs("Bfield_fp", lev), - m_fields.get_alldirs("current_cp", lev), - m_fields.get_alldirs("edge_lengths", lev), - m_fields.get_alldirs("face_areas", lev), - m_fields.get_alldirs("ECTRhofield", lev), - m_fields.get("F_cp", lev), lev, a_dt ); + m_fdtd_solver_cp[lev]->EvolveE( m_fields, + lev, + patch_type, + a_Erhs_vec.getArrayVec()[lev], + a_dt ); } // Compute Efield_rhs in PML cells by calling EvolveEPML diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 3a03015685e..f4ec24a57d4 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -909,21 +909,17 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) { // Evolve E field in regular cells if (patch_type == PatchType::fine) { - m_fdtd_solver_fp[lev]->EvolveE( m_fields.get_alldirs("Efield_fp",lev), - m_fields.get_alldirs("Bfield_fp", lev), - m_fields.get_alldirs("current_fp", lev), - m_fields.get_alldirs("edge_lengths", lev), - m_fields.get_alldirs("face_areas", lev), - m_fields.get_alldirs("ECTRhofield", lev), - m_fields.get("F_fp", lev), lev, a_dt ); + m_fdtd_solver_fp[lev]->EvolveE( m_fields, + lev, + patch_type, + m_fields.get_alldirs("Efield_fp",lev), + a_dt ); } else { - m_fdtd_solver_cp[lev]->EvolveE( m_fields.get_alldirs("Efield_cp",lev), - m_fields.get_alldirs("Bfield_cp", lev), - m_fields.get_alldirs("current_cp", lev), - m_fields.get_alldirs("edge_lengths", lev), - m_fields.get_alldirs("face_areas", lev), - m_fields.get_alldirs("ECTRhofield", lev), - m_fields.get("F_cp", lev), lev, a_dt ); + m_fdtd_solver_cp[lev]->EvolveE( m_fields, + lev, + patch_type, + m_fields.get_alldirs("Efield_cp",lev), + a_dt ); } // Evolve E field in PML cells diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index dbd027e90f7..4e286a8e1e5 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -153,10 +153,6 @@ WarpX::MoveWindow (const int step, bool move_j) ablastr::fields::MultiLevelVectorField Efield_aux = m_fields.get_mr_levels_alldirs("Efield_aux", finest_level); ablastr::fields::MultiLevelVectorField Bfield_aux = m_fields.get_mr_levels_alldirs("Bfield_aux", finest_level); - ablastr::fields::MultiLevelVectorField Efield_avg_fp = m_fields.get_mr_levels_alldirs("Efield_avg_fp", finest_level); - ablastr::fields::MultiLevelVectorField Bfield_avg_fp = m_fields.get_mr_levels_alldirs("Bfield_avg_fp", finest_level); - ablastr::fields::MultiLevelVectorField Efield_avg_cp = m_fields.get_mr_levels_alldirs("Efield_avg_cp", finest_level); - ablastr::fields::MultiLevelVectorField Bfield_avg_cp = m_fields.get_mr_levels_alldirs("Bfield_avg_cp", finest_level); // Update the continuous position of the moving window, // and of the plasma injection @@ -250,6 +246,8 @@ WarpX::MoveWindow (const int step, bool move_j) shiftMF(*m_fields.get("Efield_fp",Direction{0},lev), geom[lev], num_shift, dir, lev, do_update_cost, m_p_ext_field_params->E_external_grid[dim], use_Eparser, Efield_parser); if (fft_do_time_averaging) { + ablastr::fields::MultiLevelVectorField Efield_avg_fp = m_fields.get_mr_levels_alldirs("Efield_avg_fp", finest_level); + ablastr::fields::MultiLevelVectorField Bfield_avg_fp = m_fields.get_mr_levels_alldirs("Bfield_avg_fp", finest_level); shiftMF(*Bfield_avg_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost, m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); shiftMF(*Efield_avg_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost, @@ -281,6 +279,8 @@ WarpX::MoveWindow (const int step, bool move_j) shiftMF(*Bfield_aux[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost); shiftMF(*Efield_aux[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost); if (fft_do_time_averaging) { + ablastr::fields::MultiLevelVectorField Efield_avg_cp = m_fields.get_mr_levels_alldirs("Efield_avg_cp", finest_level); + ablastr::fields::MultiLevelVectorField Bfield_avg_cp = m_fields.get_mr_levels_alldirs("Bfield_avg_cp", finest_level); shiftMF(*Bfield_avg_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost, m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); shiftMF(*Efield_avg_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost, From edb9be60054d6e8fc287497eebc4e8375014b5f3 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 13 Sep 2024 02:13:47 -0700 Subject: [PATCH 212/314] PML: DivE/B Cleaning --- Source/BoundaryConditions/WarpXEvolvePML.cpp | 22 ++++++++++++------- .../FiniteDifferenceSolver/EvolveEPML.cpp | 13 ++++++++--- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/Source/BoundaryConditions/WarpXEvolvePML.cpp b/Source/BoundaryConditions/WarpXEvolvePML.cpp index 3b667bb48b3..087cf36a2a9 100644 --- a/Source/BoundaryConditions/WarpXEvolvePML.cpp +++ b/Source/BoundaryConditions/WarpXEvolvePML.cpp @@ -88,10 +88,7 @@ WarpX::DampPML_Cartesian (const int lev, PatchType patch_type) { const auto& pml_E = (patch_type == PatchType::fine) ? m_fields.get_alldirs("pml_E_fp", lev) : m_fields.get_alldirs("pml_E_cp", lev); const auto& pml_B = (patch_type == PatchType::fine) ? m_fields.get_alldirs("pml_B_fp", lev) : m_fields.get_alldirs("pml_B_cp", lev); - const auto& pml_F = (patch_type == PatchType::fine) ? m_fields.get("pml_F_fp", lev) : m_fields.get("pml_F_cp", lev); - const auto& pml_G = (patch_type == PatchType::fine) ? m_fields.get("pml_G_fp", lev) : m_fields.get("pml_G_cp", lev); - const auto& sigba = (patch_type == PatchType::fine) ? pml[lev]->GetMultiSigmaBox_fp() - : pml[lev]->GetMultiSigmaBox_cp(); + const auto& sigba = (patch_type == PatchType::fine) ? pml[lev]->GetMultiSigmaBox_fp() : pml[lev]->GetMultiSigmaBox_cp(); const amrex::IntVect Ex_stag = pml_E[0]->ixType().toIntVect(); const amrex::IntVect Ey_stag = pml_E[1]->ixType().toIntVect(); @@ -102,12 +99,16 @@ WarpX::DampPML_Cartesian (const int lev, PatchType patch_type) const amrex::IntVect Bz_stag = pml_B[2]->ixType().toIntVect(); amrex::IntVect F_stag; - if (pml_F) { + if (m_fields.has("pml_F_fp", lev)) { + amrex::MultiFab* pml_F = (patch_type == PatchType::fine) ? + m_fields.get("pml_F_fp", lev) : m_fields.get("pml_F_cp", lev); F_stag = pml_F->ixType().toIntVect(); } amrex::IntVect G_stag; - if (pml_G) { + if (m_fields.has("pml_G_fp", lev)) { + amrex::MultiFab* pml_G = (patch_type == PatchType::fine) ? + m_fields.get("pml_G_fp", lev) : m_fields.get("pml_G_cp", lev); G_stag = pml_G->ixType().toIntVect(); } @@ -198,7 +199,9 @@ WarpX::DampPML_Cartesian (const int lev, PatchType patch_type) // For warpx_damp_pml_F(), mfi.nodaltilebox is used in the ParallelFor loop and here we // use mfi.tilebox. However, it does not matter because in damp_pml, where nodaltilebox // is used, only a simple multiplication is performed. - if (pml_F) { + if (m_fields.has("pml_F_fp", lev)) { + amrex::MultiFab* pml_F = (patch_type == PatchType::fine) ? + m_fields.get("pml_F_fp", lev) : m_fields.get("pml_F_cp", lev); const Box& tnd = mfi.nodaltilebox(); auto const& pml_F_fab = pml_F->array(mfi); amrex::ParallelFor(tnd, [=] AMREX_GPU_DEVICE (int i, int j, int k) @@ -209,7 +212,10 @@ WarpX::DampPML_Cartesian (const int lev, PatchType patch_type) } // Damp G when WarpX::do_divb_cleaning = true - if (pml_G) { + if (m_fields.has("pml_G_fp", lev)) { + amrex::MultiFab* pml_G = (patch_type == PatchType::fine) ? + m_fields.get("pml_G_fp", lev) : m_fields.get("pml_G_cp", lev); + const Box& tb = mfi.tilebox(G_stag); auto const& pml_G_fab = pml_G->array(mfi); amrex::ParallelFor(tb, [=] AMREX_GPU_DEVICE (int i, int j, int k) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp index 9dc371ed938..4da403156b0 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp @@ -58,15 +58,22 @@ void FiniteDifferenceSolver::EvolveEPML ( WARPX_ABORT_WITH_MESSAGE( "PML are not implemented in cylindrical geometry."); #else + using ablastr::fields::Direction; ablastr::fields::VectorField Efield = (patch_type == PatchType::fine) ? fields.get_alldirs("pml_E_fp", level) : fields.get_alldirs("pml_E_cp", level); ablastr::fields::VectorField Bfield = (patch_type == PatchType::fine) ? fields.get_alldirs("pml_B_fp", level) : fields.get_alldirs("pml_B_cp", level); ablastr::fields::VectorField Jfield = (patch_type == PatchType::fine) ? fields.get_alldirs("pml_j_fp", level) : fields.get_alldirs("pml_j_cp", level); - ablastr::fields::VectorField edge_lengths = fields.get_alldirs("pml_edge_lengths", level); - amrex::MultiFab* const Ffield = (patch_type == PatchType::fine) ? - fields.get("pml_F_fp", level) : fields.get("pml_F_cp", level); + ablastr::fields::VectorField edge_lengths; + if (fields.has("pml_edge_lengths", Direction{0}, level)) { + edge_lengths = fields.get_alldirs("pml_edge_lengths", level); + } + amrex::MultiFab * Ffield = nullptr; + if (fields.has("pml_F_fp", level)) { + Ffield = (patch_type == PatchType::fine) ? + fields.get("pml_F_fp", level) : fields.get("pml_F_cp", level); + } if (m_grid_type == GridType::Collocated) { From bc6d333ca08fadc183d3421ed1239dc7e00e8a99 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 13 Sep 2024 02:23:42 -0700 Subject: [PATCH 213/314] More DivE/DivB Cleaning --- Source/Parallelization/WarpXComm.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index c5e50ef4454..36f551e59e1 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -948,8 +948,12 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng, std::optionalok()) { - if (m_fields.has("pml_F_fp", lev) && m_fields.has("F_fp", lev)) { pml[lev]->Exchange(m_fields.get("pml_F_fp", lev), m_fields.get("F_fp", lev), patch_type, do_pml_in_domain); } - pml[lev]->FillBoundary(*m_fields.get("pml_F_fp", lev), patch_type, nodal_sync); + if (m_fields.has("pml_F_fp", lev) && m_fields.has("F_fp", lev)) { + pml[lev]->Exchange(m_fields.get("pml_F_fp", lev), m_fields.get("F_fp", lev), patch_type, do_pml_in_domain); + } + if (m_fields.has("pml_F_fp", lev)) { + pml[lev]->FillBoundary(*m_fields.get("pml_F_fp", lev), patch_type, nodal_sync); + } } if (m_fields.has("F_fp", lev)) @@ -963,8 +967,12 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng, std::optionalok()) { - if (m_fields.has("pml_F_cp", lev) && m_fields.has("F_cp", lev)) { pml[lev]->Exchange(m_fields.get("pml_F_cp", lev), m_fields.get("F_cp", lev), patch_type, do_pml_in_domain); } - pml[lev]->FillBoundary(*m_fields.get("pml_F_cp", lev), patch_type, nodal_sync); + if (m_fields.has("pml_F_cp", lev) && m_fields.has("F_cp", lev)) { + pml[lev]->Exchange(m_fields.get("pml_F_cp", lev), m_fields.get("F_cp", lev), patch_type, do_pml_in_domain); + } + if (m_fields.has("pml_F_cp", lev)) { + pml[lev]->FillBoundary(*m_fields.get("pml_F_cp", lev), patch_type, nodal_sync); + } } if (m_fields.has("F_cp", lev)) From dc970b83082c0d519677ab9df19cea6ac8d8cace Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 13 Sep 2024 04:36:16 -0700 Subject: [PATCH 214/314] Fix compilation --- Source/Particles/LaserParticleContainer.cpp | 4 +++- Source/Particles/PhysicalParticleContainer.cpp | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Source/Particles/LaserParticleContainer.cpp b/Source/Particles/LaserParticleContainer.cpp index 626238a99cf..ee3954546c7 100644 --- a/Source/Particles/LaserParticleContainer.cpp +++ b/Source/Particles/LaserParticleContainer.cpp @@ -563,6 +563,8 @@ LaserParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, std::string current_fp_string, Real t, Real dt, DtType /*a_dt_type*/, bool skip_deposition, PushType push_type) { + using ablastr::fields::Direction; + WARPX_PROFILE("LaserParticleContainer::Evolve()"); WARPX_PROFILE_VAR_NS("LaserParticleContainer::Evolve::ParticlePush", blp_pp); @@ -579,7 +581,7 @@ LaserParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, // Update laser profile m_up_laser_profile->update(t_lab); - BL_ASSERT(OnSameGrids(lev,jx)); + BL_ASSERT(OnSameGrids(lev, *fields.get("current_fp", Direction{0}, lev))); amrex::LayoutData* cost = WarpX::getCosts(lev); diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index c3725cb1239..a5deb3640c3 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -2031,11 +2031,12 @@ PhysicalParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, Real /*t*/, Real dt, DtType a_dt_type, bool skip_deposition, PushType push_type) { + using ablastr::fields::Direction; WARPX_PROFILE("PhysicalParticleContainer::Evolve()"); WARPX_PROFILE_VAR_NS("PhysicalParticleContainer::Evolve::GatherAndPush", blp_fg); - BL_ASSERT(OnSameGrids(lev,jx)); + BL_ASSERT(OnSameGrids(lev, *fields.get("current_fp", Direction{0}, lev))); amrex::LayoutData* cost = WarpX::getCosts(lev); From 3e779f3d0b4dc9758bc96dab23848a1541a4144f Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 13 Sep 2024 06:11:43 -0700 Subject: [PATCH 215/314] Allow traceback and CI test --- Examples/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Examples/CMakeLists.txt b/Examples/CMakeLists.txt index 3df32ed0f95..14943c93548 100644 --- a/Examples/CMakeLists.txt +++ b/Examples/CMakeLists.txt @@ -129,7 +129,7 @@ function(add_warpx_test set(runtime_params "amrex.abort_on_unused_inputs = 1" "amrex.throw_exception = 1" - "amrex.signal_handling = 0" + "amrex.signal_handling = 1" "warpx.always_warn_immediately = 1" "warpx.do_dynamic_scheduling = 0" "warpx.serialize_initial_conditions = 1" From c4fff157576c6774946a4ddbc8b31fa43218a3f6 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 13 Sep 2024 06:15:35 -0700 Subject: [PATCH 216/314] Fix MR test --- Source/Parallelization/WarpXComm.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 36f551e59e1..5b79a79db4a 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -1232,11 +1232,14 @@ WarpX::SyncCurrent (std::string current_fp_string) void WarpX::SyncRho () { ablastr::fields::MultiLevelScalarField rho_fp = m_fields.has("rho_fp", 0) ? - m_fields.get_mr_levels("rho_fp", finest_level) : ablastr::fields::MultiLevelScalarField{1}; - ablastr::fields::MultiLevelScalarField rho_cp = m_fields.has("rho_cp", 0) ? - m_fields.get_mr_levels("rho_cp", finest_level) : ablastr::fields::MultiLevelScalarField{1}; - ablastr::fields::MultiLevelScalarField rho_buf = m_fields.has("rho_buf", 0) ? - m_fields.get_mr_levels("rho_buf", finest_level) : ablastr::fields::MultiLevelScalarField{1}; + m_fields.get_mr_levels("rho_fp", finest_level) : + ablastr::fields::MultiLevelScalarField{static_cast(finest_level+1)}; + ablastr::fields::MultiLevelScalarField rho_cp = m_fields.has("rho_cp", 1) ? + m_fields.get_mr_levels("rho_cp", finest_level) : + ablastr::fields::MultiLevelScalarField{static_cast(finest_level+1)}; + ablastr::fields::MultiLevelScalarField rho_buf = m_fields.has("rho_buf", 1) ? + m_fields.get_mr_levels("rho_buf", finest_level) : + ablastr::fields::MultiLevelScalarField{static_cast(finest_level+1)}; SyncRho(rho_fp, rho_cp, rho_buf); } From 95a61a386742dccfcdfea3891a81f4bedbd0f0f9 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 13 Sep 2024 09:30:51 -0700 Subject: [PATCH 217/314] Cleanup: remove ConstMap --- Source/ablastr/utils/ConstMap.H | 48 --------------------------------- 1 file changed, 48 deletions(-) delete mode 100644 Source/ablastr/utils/ConstMap.H diff --git a/Source/ablastr/utils/ConstMap.H b/Source/ablastr/utils/ConstMap.H deleted file mode 100644 index 58d0519fc6d..00000000000 --- a/Source/ablastr/utils/ConstMap.H +++ /dev/null @@ -1,48 +0,0 @@ -/* Copyright 2024 The ABLASTR Community - * - * This file is part of ABLASTR. - * - * License: BSD-3-Clause-LBNL - * Authors: Axel Huebl - */ - -#ifndef ABLASTR_CONST_MAP_H_ -#define ABLASTR_CONST_MAP_H_ - -#include "TextMsg.H" - -#include -#include - - -namespace ablastr::utils -{ - /** This is a std::map where operator[] throws if the key is missing - */ - template< typename Key, typename T > - struct ConstMap : public std::map< Key, T > - { - T& operator[]( const Key& key ) - { - return this->at(key); - } - - T& operator[]( Key&& key ) - { - return this->at(key); - } - - T& operator[]( const Key& key ) const - { - return this->at(key); - } - - T& operator[]( Key&& key ) const - { - return this->at(key); - } - }; - -} // ablastr::utils - -#endif // ABLASTR_CONST_MAP_H_ From 1e426f5cbeae9e2a37396fb46c181e389a5d6e79 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 13 Sep 2024 09:31:10 -0700 Subject: [PATCH 218/314] Cleanup: remove ConstMap --- Source/ablastr/fields/MultiFabRegister.H | 2 -- 1 file changed, 2 deletions(-) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index d44806cd315..e4313959f29 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -8,8 +8,6 @@ #ifndef ABLASTR_FIELDS_MF_REGISTER_H #define ABLASTR_FIELDS_MF_REGISTER_H -#include "ablastr/utils/ConstMap.H" - #include #include #include From 73a0673841c899fd7132105c316cbd6ddcda39fc Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 13 Sep 2024 09:52:40 -0700 Subject: [PATCH 219/314] Fix divE cleaning in PML --- Source/FieldSolver/WarpXPushFieldsEM.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index f4ec24a57d4..ca8288f8712 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -1009,13 +1009,13 @@ WarpX::EvolveF (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_typ if (do_pml && pml[lev]->ok()) { if (patch_type == PatchType::fine) { m_fdtd_solver_fp[lev]->EvolveFPML( - m_fields.get("F_fp", lev), - m_fields.get_alldirs("Efield_fp", lev), + m_fields.get("pml_F_fp", lev), + m_fields.get_alldirs("pml_E_fp", lev), a_dt ); } else { m_fdtd_solver_cp[lev]->EvolveFPML( - m_fields.get("F_cp", lev), - m_fields.get_alldirs("Efield_cp", lev), + m_fields.get("pml_F_cp", lev), + m_fields.get_alldirs("pml_E_cp", lev), a_dt ); } } From 55fb4fdb11fcdd1cad3be96acd6e6fd844e9194e Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 13 Sep 2024 10:19:21 -0700 Subject: [PATCH 220/314] Some code cleanup --- Source/Parallelization/WarpXComm.cpp | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 5b79a79db4a..aee60a10ff7 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -103,14 +103,10 @@ WarpX::UpdateAuxilaryDataStagToNodal () ablastr::fields::MultiLevelVectorField const& Efield_fp = m_fields.get_mr_levels_alldirs("Efield_fp", finest_level); ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); - ablastr::fields::MultiLevelVectorField const& Efield_cp = m_fields.get_mr_levels_alldirs("Efield_cp", finest_level); - ablastr::fields::MultiLevelVectorField const& Bfield_cp = m_fields.get_mr_levels_alldirs("Bfield_cp", finest_level); ablastr::fields::MultiLevelVectorField const& Efield_avg_fp = m_fields.get_mr_levels_alldirs("Efield_avg_fp", finest_level); ablastr::fields::MultiLevelVectorField const& Bfield_avg_fp = m_fields.get_mr_levels_alldirs("Bfield_avg_fp", finest_level); ablastr::fields::MultiLevelVectorField const& Efield_aux = m_fields.get_mr_levels_alldirs("Efield_aux", finest_level); ablastr::fields::MultiLevelVectorField const& Bfield_aux = m_fields.get_mr_levels_alldirs("Bfield_aux", finest_level); - ablastr::fields::MultiLevelVectorField const& Efield_cax = m_fields.get_mr_levels_alldirs("Efield_cax", finest_level); - ablastr::fields::MultiLevelVectorField const& Bfield_cax = m_fields.get_mr_levels_alldirs("Bfield_cax", finest_level); ablastr::fields::MultiLevelVectorField const & Bmf = WarpX::fft_do_time_averaging ? Bfield_avg_fp : Bfield_fp; ablastr::fields::MultiLevelVectorField const & Emf = WarpX::fft_do_time_averaging ? Efield_avg_fp : Efield_fp; @@ -222,13 +218,13 @@ WarpX::UpdateAuxilaryDataStagToNodal () const amrex::IntVect& refinement_ratio = refRatio(lev-1); - const amrex::IntVect& Bx_fp_stag = Bfield_fp[lev][0]->ixType().toIntVect(); - const amrex::IntVect& By_fp_stag = Bfield_fp[lev][1]->ixType().toIntVect(); - const amrex::IntVect& Bz_fp_stag = Bfield_fp[lev][2]->ixType().toIntVect(); + const amrex::IntVect& Bx_fp_stag = m_fields.get("Bfield_fp",Direction{0},lev)->ixType().toIntVect(); + const amrex::IntVect& By_fp_stag = m_fields.get("Bfield_fp",Direction{1},lev)->ixType().toIntVect(); + const amrex::IntVect& Bz_fp_stag = m_fields.get("Bfield_fp",Direction{2},lev)->ixType().toIntVect(); - const amrex::IntVect& Bx_cp_stag = Bfield_cp[lev][0]->ixType().toIntVect(); - const amrex::IntVect& By_cp_stag = Bfield_cp[lev][1]->ixType().toIntVect(); - const amrex::IntVect& Bz_cp_stag = Bfield_cp[lev][2]->ixType().toIntVect(); + const amrex::IntVect& Bx_cp_stag = m_fields.get("Bfield_cp",Direction{0},lev)->ixType().toIntVect(); + const amrex::IntVect& By_cp_stag = m_fields.get("Bfield_cp",Direction{1},lev)->ixType().toIntVect(); + const amrex::IntVect& Bz_cp_stag = m_fields.get("Bfield_cp",Direction{2},lev)->ixType().toIntVect(); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) @@ -238,12 +234,12 @@ WarpX::UpdateAuxilaryDataStagToNodal () Array4 const& bx_aux = Bfield_aux[lev][0]->array(mfi); Array4 const& by_aux = Bfield_aux[lev][1]->array(mfi); Array4 const& bz_aux = Bfield_aux[lev][2]->array(mfi); - Array4 const& bx_fp = Bfield_fp[lev][0]->const_array(mfi); - Array4 const& by_fp = Bfield_fp[lev][1]->const_array(mfi); - Array4 const& bz_fp = Bfield_fp[lev][2]->const_array(mfi); - Array4 const& bx_cp = Bfield_cp[lev][0]->const_array(mfi); - Array4 const& by_cp = Bfield_cp[lev][1]->const_array(mfi); - Array4 const& bz_cp = Bfield_cp[lev][2]->const_array(mfi); + Array4 const& bx_fp = m_fields.get("Bfield_fp",Direction{0},lev)->const_array(mfi); + Array4 const& by_fp = m_fields.get("Bfield_fp",Direction{1},lev)->const_array(mfi); + Array4 const& bz_fp = m_fields.get("Bfield_fp",Direction{2},lev)->const_array(mfi); + Array4 const& bx_cp = m_fields.get("Bfield_cp",Direction{0},lev)->const_array(mfi); + Array4 const& by_cp = m_fields.get("Bfield_cp",Direction{1},lev)->const_array(mfi); + Array4 const& bz_cp = m_fields.get("Bfield_cp",Direction{2},lev)->const_array(mfi); Array4 const& bx_c = Btmp[0]->const_array(mfi); Array4 const& by_c = Btmp[1]->const_array(mfi); Array4 const& bz_c = Btmp[2]->const_array(mfi); From 9a6fdee084fa9e3c81b60978a1536f8c4497cf05 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 13 Sep 2024 10:54:09 -0700 Subject: [PATCH 221/314] Fix MR bug --- Source/Parallelization/WarpXComm.cpp | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index aee60a10ff7..4205950b10b 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -766,16 +766,20 @@ WarpX::FillBoundaryB (const int lev, const PatchType patch_type, const amrex::In std::array mf; amrex::Periodicity period; + using ablastr::fields::Direction; + if (patch_type == PatchType::fine) { - ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); - mf = {Bfield_fp[lev][0], Bfield_fp[lev][1], Bfield_fp[lev][2]}; + mf = {m_fields.get("Bfield_fp",Direction{0},lev), + m_fields.get("Bfield_fp",Direction{1},lev), + m_fields.get("Bfield_fp",Direction{2},lev)}; period = Geom(lev).periodicity(); } else // coarse patch { - ablastr::fields::MultiLevelVectorField const& Bfield_cp_new = m_fields.get_mr_levels_alldirs("Bfield_cp", finest_level); - mf = {Bfield_cp_new[lev][0], Bfield_cp_new[lev][1], Bfield_cp_new[lev][2]}; + mf = {m_fields.get("Bfield_cp",Direction{0},lev), + m_fields.get("Bfield_cp",Direction{1},lev), + m_fields.get("Bfield_cp",Direction{2},lev)}; period = Geom(lev-1).periodicity(); } @@ -1019,7 +1023,9 @@ void WarpX::FillBoundaryG (int lev, PatchType patch_type, IntVect ng, std::optio if (m_fields.has("pml_G_cp",lev) && m_fields.has("G_cp",lev)) { pml[lev]->Exchange(m_fields.get("pml_G_cp", lev), m_fields.get("G_cp", lev), patch_type, do_pml_in_domain); } - pml[lev]->FillBoundary(*m_fields.get("pml_G_cp", lev), patch_type, nodal_sync); + if (m_fields.has("pml_G_cp", lev)) { + pml[lev]->FillBoundary(*m_fields.get("pml_G_cp", lev), patch_type, nodal_sync); + } } if (m_fields.has("G_cp",lev)) From 376d16c2701e7e3329c9f61d340d0517bd1988d8 Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Fri, 13 Sep 2024 20:34:56 +0200 Subject: [PATCH 222/314] change vector_potential_fp to vector_potential_fp_nodal --- Source/Diagnostics/FullDiagnostics.cpp | 10 +++++----- Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index 493e996ae9f..95e1a04b40f 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -677,7 +677,7 @@ FullDiagnostics::InitializeFieldFunctors (int lev) } else if ( m_varnames[comp] == "jz_displacement" ) { m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, true); } else if ( m_varnames[comp] == "Az" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("vector_potential_fp", Direction{2}, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("vector_potential_fp_nodal", Direction{2}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "rho" ){ // Initialize rho functor to dump total rho m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true); @@ -726,9 +726,9 @@ FullDiagnostics::InitializeFieldFunctors (int lev) } else if (m_varnames[comp] == "jt_displacement" ){ m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, true); } else if ( m_varnames[comp] == "Ar" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("vector_potential_fp", Direction{0}, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("vector_potential_fp_nodal", Direction{0}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "At" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("vector_potential_fp", Direction{1}, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("vector_potential_fp_nodal", Direction{1}, lev), lev, m_crse_ratio); } else { WARPX_ABORT_WITH_MESSAGE(m_varnames[comp] + " is not a known field output type for RZ geometry"); } @@ -753,9 +753,9 @@ FullDiagnostics::InitializeFieldFunctors (int lev) } else if ( m_varnames[comp] == "jy_displacement" ){ m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio); } else if ( m_varnames[comp] == "Ax" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("vector_potential_fp", Direction{0}, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("vector_potential_fp_nodal", Direction{0}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Ay" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("vector_potential_fp", Direction{1}, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("vector_potential_fp_nodal", Direction{1}, lev), lev, m_crse_ratio); } else { std::cout << "Error on component " << m_varnames[comp] << std::endl; WARPX_ABORT_WITH_MESSAGE(m_varnames[comp] + " is not a known field output type for this geometry"); diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp index 0c502bb15a3..da359f35248 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp @@ -43,14 +43,14 @@ void WarpXSolverVec::Define ( WarpX* a_WarpX, else if (m_vector_type_name=="Bfield_fp") { m_array_type = FieldType::Efield_fp; } - else if (m_vector_type_name=="vector_potential_fp") { + else if (m_vector_type_name=="vector_potential_fp_nodal") { m_array_type = FieldType::vector_potential_fp; } else if (m_vector_type_name!="none") { WARPX_ABORT_WITH_MESSAGE(a_vector_type_name +"is not a valid option for array type used in Definining" +"a WarpXSolverVec. Valid array types are: Efield_fp, Bfield_fp," - +"and vector_potential_fp"); + +"and vector_potential_fp_nodal"); } if (m_scalar_type_name=="phi_fp") { From c75b2c107c6a753e0161b5698b84135e793bb1d2 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 13 Sep 2024 11:42:18 -0700 Subject: [PATCH 223/314] WarpX Bucket: Regex Raw String --- Python/pywarpx/WarpX.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/pywarpx/WarpX.py b/Python/pywarpx/WarpX.py index 9b3aaa27636..9ef7019cda9 100644 --- a/Python/pywarpx/WarpX.py +++ b/Python/pywarpx/WarpX.py @@ -137,7 +137,7 @@ def write_inputs(self, filename="inputs", **kw): for arg in argv: # This prints the name of the input group (prefix) as a header # before each group to make the input file more human readable - prefix_new = re.split(" |\.", arg)[0] + prefix_new = re.split(r" |\.", arg)[0] if prefix_new != prefix_old: if prefix_old != "": ff.write("\n") From fe8ba67d581940c4f4ff17a274fc701b2fd36d4a Mon Sep 17 00:00:00 2001 From: lucafedeli88 Date: Fri, 13 Sep 2024 20:51:38 +0200 Subject: [PATCH 224/314] avoid relying on nullptr MF pointers for Bfield_avg_fp and Efield_avg_fp --- Source/Parallelization/WarpXComm.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 4205950b10b..bba23752bdd 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -101,15 +101,19 @@ WarpX::UpdateAuxilaryDataStagToNodal () #endif using ablastr::fields::Direction; - ablastr::fields::MultiLevelVectorField const& Efield_fp = m_fields.get_mr_levels_alldirs("Efield_fp", finest_level); ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); - ablastr::fields::MultiLevelVectorField const& Efield_avg_fp = m_fields.get_mr_levels_alldirs("Efield_avg_fp", finest_level); - ablastr::fields::MultiLevelVectorField const& Bfield_avg_fp = m_fields.get_mr_levels_alldirs("Bfield_avg_fp", finest_level); + ablastr::fields::MultiLevelVectorField const& Efield_fp = m_fields.get_mr_levels_alldirs("Efield_fp", finest_level); ablastr::fields::MultiLevelVectorField const& Efield_aux = m_fields.get_mr_levels_alldirs("Efield_aux", finest_level); ablastr::fields::MultiLevelVectorField const& Bfield_aux = m_fields.get_mr_levels_alldirs("Bfield_aux", finest_level); - ablastr::fields::MultiLevelVectorField const & Bmf = WarpX::fft_do_time_averaging ? Bfield_avg_fp : Bfield_fp; - ablastr::fields::MultiLevelVectorField const & Emf = WarpX::fft_do_time_averaging ? Efield_avg_fp : Efield_fp; + ablastr::fields::MultiLevelVectorField const & Bmf = + WarpX::fft_do_time_averaging ? + m_fields.get_mr_levels_alldirs("Bfield_avg_fp", finest_level) : + Bfield_fp; + ablastr::fields::MultiLevelVectorField const & Emf = + WarpX::fft_do_time_averaging ? + m_fields.get_mr_levels_alldirs("Efield_avg_fp", finest_level) : + Efield_fp; const amrex::IntVect& Bx_stag = Bmf[0][0]->ixType().toIntVect(); const amrex::IntVect& By_stag = Bmf[0][1]->ixType().toIntVect(); From f1b448ed918fa09cfe0be86dff8e3a4c4e5e2a00 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 13 Sep 2024 12:17:23 -0700 Subject: [PATCH 225/314] PICMI Tests: A few Warnings --- .../magnetostatic_eb/inputs_test_3d_magnetostatic_eb_picmi.py | 2 +- .../magnetostatic_eb/inputs_test_rz_magnetostatic_eb_picmi.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Examples/Tests/magnetostatic_eb/inputs_test_3d_magnetostatic_eb_picmi.py b/Examples/Tests/magnetostatic_eb/inputs_test_3d_magnetostatic_eb_picmi.py index d3c35daf261..d34520b2ef2 100755 --- a/Examples/Tests/magnetostatic_eb/inputs_test_3d_magnetostatic_eb_picmi.py +++ b/Examples/Tests/magnetostatic_eb/inputs_test_3d_magnetostatic_eb_picmi.py @@ -298,7 +298,7 @@ def Bt_an(r): bt_err = np.abs(Bt_mean[r_idx] - Bt_an(r_sub)).max() / np.abs(Bt_an(r_sub)).max() -plt.ylabel("$B_{\Theta}$ (T)") +plt.ylabel(r"$B_{\Theta}$ (T)") plt.xlabel("r (m)") plt.title("Max % Error: {} %".format(bt_err * 100.0)) plt.tight_layout() diff --git a/Examples/Tests/magnetostatic_eb/inputs_test_rz_magnetostatic_eb_picmi.py b/Examples/Tests/magnetostatic_eb/inputs_test_rz_magnetostatic_eb_picmi.py index d0f1787a5a2..907c996a6d3 100755 --- a/Examples/Tests/magnetostatic_eb/inputs_test_rz_magnetostatic_eb_picmi.py +++ b/Examples/Tests/magnetostatic_eb/inputs_test_rz_magnetostatic_eb_picmi.py @@ -246,7 +246,7 @@ def Bth_an(r): bth_err = np.abs(Bth_mean[r_idx] - Bth_an(r_sub)).max() / np.abs(Bth_an(r_sub)).max() -plt.ylabel("$B_{\Theta}$ (T)") +plt.ylabel(r"$B_{\Theta}$ (T)") plt.xlabel("r (m)") plt.title("Max % Error: {} %".format(bth_err * 100.0)) plt.tight_layout() From e42b8c4a7cbec24617c9c9262fce01550b621f88 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 13 Sep 2024 12:17:53 -0700 Subject: [PATCH 226/314] PICMI debugging --- Python/pywarpx/particle_containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/pywarpx/particle_containers.py b/Python/pywarpx/particle_containers.py index 8af012f5e7b..bc6b2d74106 100644 --- a/Python/pywarpx/particle_containers.py +++ b/Python/pywarpx/particle_containers.py @@ -733,7 +733,7 @@ def deposit_charge_density(self, level, clear_rho=True, sync_rho=True): sync_rho : bool If True, perform MPI exchange and properly set boundary cells for rho_fp. """ - rho_fp = libwarpx.warpx.multifab(f"rho_fp[level={level}]") + rho_fp = libwarpx.warpx.multifab("rho_fp", level) if rho_fp is None: raise RuntimeError("Multifab `rho_fp` is not allocated.") From b6f274c366d8377289136994d9f98e06e2c05875 Mon Sep 17 00:00:00 2001 From: David Grote Date: Fri, 13 Sep 2024 12:33:38 -0700 Subject: [PATCH 227/314] Update for pml and remove old multifab_map reference --- Python/pywarpx/fields.py | 45 ++++++++++++++++++---------------------- Source/Python/WarpX.cpp | 2 -- 2 files changed, 20 insertions(+), 27 deletions(-) diff --git a/Python/pywarpx/fields.py b/Python/pywarpx/fields.py index 33cb2c7402a..356063739d1 100644 --- a/Python/pywarpx/fields.py +++ b/Python/pywarpx/fields.py @@ -120,16 +120,11 @@ def mf(self): else: # Always fetch this anew in case the C++ MultiFab is recreated warpx = libwarpx.libwarpx_so.get_instance() - if self.mf_name.startswith("pml"): - # Temporary until pml are updated to new method - name = f"{self.mf_name}[level={self.level}]" - else: - name = self.mf_name if self.idir is not None: direction = libwarpx.libwarpx_so.Direction(self.idir) - return warpx.multifab(name, direction, self.level) + return warpx.multifab(self.mf_name, direction, self.level) else: - return warpx.multifab(name, self.level) + return warpx.multifab(self.mf_name, self.level) @property def shape(self): @@ -898,55 +893,55 @@ def JzFPAmpereWrapper(level=0, include_ghosts=False): def ExFPPMLWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="pml_E_fp[x]", level=level, include_ghosts=include_ghosts + mf_name="pml_E_fp", idir=0, level=level, include_ghosts=include_ghosts ) def EyFPPMLWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="pml_E_fp[y]", level=level, include_ghosts=include_ghosts + mf_name="pml_E_fp", idir=1, level=level, include_ghosts=include_ghosts ) def EzFPPMLWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="pml_E_fp[z]", level=level, include_ghosts=include_ghosts + mf_name="pml_E_fp", idir=2, level=level, include_ghosts=include_ghosts ) def BxFPPMLWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="pml_B_fp[x]", level=level, include_ghosts=include_ghosts + mf_name="pml_B_fp", idir=0, level=level, include_ghosts=include_ghosts ) def ByFPPMLWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="pml_B_fp[y]", level=level, include_ghosts=include_ghosts + mf_name="pml_B_fp", idir=1, level=level, include_ghosts=include_ghosts ) def BzFPPMLWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="pml_B_fp[z]", level=level, include_ghosts=include_ghosts + mf_name="pml_B_fp", idir=2, level=level, include_ghosts=include_ghosts ) def JxFPPMLWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="pml_j_fp[x]", level=level, include_ghosts=include_ghosts + mf_name="pml_j_fp", idir=0, level=level, include_ghosts=include_ghosts ) def JyFPPMLWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="pml_j_fp[y]", level=level, include_ghosts=include_ghosts + mf_name="pml_j_fp", idir=1, level=level, include_ghosts=include_ghosts ) def JzFPPMLWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="pml_j_fp[z]", level=level, include_ghosts=include_ghosts + mf_name="pml_j_fp", idir=2, level=level, include_ghosts=include_ghosts ) @@ -964,55 +959,55 @@ def GFPPMLWrapper(level=0, include_ghosts=False): def ExCPPMLWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="pml_E_cp[x]", level=level, include_ghosts=include_ghosts + mf_name="pml_E_cp", idir=0, level=level, include_ghosts=include_ghosts ) def EyCPPMLWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="pml_E_cp[y]", level=level, include_ghosts=include_ghosts + mf_name="pml_E_cp", idir=1, level=level, include_ghosts=include_ghosts ) def EzCPPMLWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="pml_E_cp[z]", level=level, include_ghosts=include_ghosts + mf_name="pml_E_cp", idir=2, level=level, include_ghosts=include_ghosts ) def BxCPPMLWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="pml_B_cp[x]", level=level, include_ghosts=include_ghosts + mf_name="pml_B_cp", idir=0, level=level, include_ghosts=include_ghosts ) def ByCPPMLWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="pml_B_cp[y]", level=level, include_ghosts=include_ghosts + mf_name="pml_B_cp", idir=1, level=level, include_ghosts=include_ghosts ) def BzCPPMLWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="pml_B_cp[z]", level=level, include_ghosts=include_ghosts + mf_name="pml_B_cp", idir=2, level=level, include_ghosts=include_ghosts ) def JxCPPMLWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="pml_j_cp[x]", level=level, include_ghosts=include_ghosts + mf_name="pml_j_cp", idir=0, level=level, include_ghosts=include_ghosts ) def JyCPPMLWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="pml_j_cp[y]", level=level, include_ghosts=include_ghosts + mf_name="pml_j_cp", idir=1, level=level, include_ghosts=include_ghosts ) def JzCPPMLWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="pml_j_cp[z]", level=level, include_ghosts=include_ghosts + mf_name="pml_j_cp", idir=2, level=level, include_ghosts=include_ghosts ) diff --git a/Source/Python/WarpX.cpp b/Source/Python/WarpX.cpp index 311d3deee1e..c64e15043f9 100644 --- a/Source/Python/WarpX.cpp +++ b/Source/Python/WarpX.cpp @@ -123,8 +123,6 @@ void init_WarpX (py::module& m) [](WarpX & wx, std::string multifab_name, int level) { if (wx.m_fields.has(multifab_name, level)) { return wx.m_fields.get(multifab_name, level); - } else if (wx.multifab_map.count(multifab_name) > 0) { - return wx.multifab_map.at(multifab_name); } else { throw std::runtime_error("The MultiFab '" + multifab_name + "' is unknown or is not allocated!"); } From 616fb332fcd84f02f0da5bcd799bd819c52a86de Mon Sep 17 00:00:00 2001 From: Marco Acciarri Date: Fri, 13 Sep 2024 13:42:05 -0700 Subject: [PATCH 228/314] Add register to fluid container (#9) * modified MultiFluidContainer.H * modified MultiFluidContainer.cpp * modified WarpXFluidContainer.H * modified WarpXFluidContainer.cpp * modified WarpX.cpp * modified RhoFunctor * modified RhoFunctor * modified WarpXPushFieldsHybridPIC.cpp * modified WarpXEvolve.cpp and ElectrostaticSolver.cpp * modified WarpXFluidContainer.cpp * modified WarpXMovingWindow.cpp * modified WarpXMovingWindow.cpp * updated functions fluid container * updated functions fluid container --- .../ComputeDiagFunctors/RhoFunctor.cpp | 2 +- Source/Evolve/WarpXEvolve.cpp | 14 +- Source/FieldSolver/ElectrostaticSolver.cpp | 2 +- .../FieldSolver/WarpXPushFieldsHybridPIC.cpp | 7 +- Source/Fluids/MultiFluidContainer.H | 25 +- Source/Fluids/MultiFluidContainer.cpp | 32 +-- Source/Fluids/WarpXFluidContainer.H | 36 +-- Source/Fluids/WarpXFluidContainer.cpp | 237 ++++++++++-------- Source/Utils/WarpXMovingWindow.cpp | 12 +- Source/WarpX.cpp | 4 +- 10 files changed, 191 insertions(+), 180 deletions(-) diff --git a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp index 32e11903778..e7f572dd681 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp @@ -47,7 +47,7 @@ RhoFunctor::operator() ( amrex::MultiFab& mf_dst, const int dcomp, const int /*i rho = mypc.GetChargeDensity(m_lev, true); if (warpx.DoFluidSpecies()) { auto& myfl = warpx.GetFluidContainer(); - myfl.DepositCharge(m_lev, *rho); + myfl.DepositCharge(warpx.m_fields, *rho, m_lev); } } // Dump rho per species diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 16fe79c4ca6..a4df5ea73bd 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -1145,17 +1145,9 @@ WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, // Might this be related to issue #1943? #endif if (do_fluid_species) { - myfl->Evolve(lev, - *m_fields.get("Efield_aux", Direction{0}, lev), - *m_fields.get("Efield_aux", Direction{1}, lev), - *m_fields.get("Efield_aux", Direction{2}, lev), - *m_fields.get("Bfield_aux", Direction{0}, lev), - *m_fields.get("Bfield_aux", Direction{1}, lev), - *m_fields.get("Bfield_aux", Direction{2}, lev), - m_fields.get("rho_fp", lev), - *m_fields.get(current_fp_string, Direction{0}, lev), - *m_fields.get(current_fp_string, Direction{1}, lev), - *m_fields.get(current_fp_string, Direction{2}, lev), + myfl->Evolve(m_fields, + lev, + current_fp_string, cur_time, skip_current ); diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index e5ee4da6a6d..07dbc8c1424 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -262,7 +262,7 @@ WarpX::AddSpaceChargeFieldLabFrame () mypc->DepositCharge(m_fields.get_mr_levels("rho_fp", finest_level), 0.0_rt); if (do_fluid_species) { int const lev = 0; - myfl->DepositCharge( lev, *m_fields.get("rho_fp", lev)); + myfl->DepositCharge( m_fields, *m_fields.get("rho_fp", lev), lev ); } for (int lev = 0; lev <= max_level; lev++) { if (lev > 0) { diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index 7ba912d96ce..c23af2151db 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -40,11 +40,12 @@ void WarpX::HybridPICEvolveFields () // Deposit cold-relativistic fluid charge and current if (do_fluid_species) { int const lev = 0; - myfl->DepositCharge(lev, *m_fields.get("rho_fp", lev)); - myfl->DepositCurrent(lev, + myfl->DepositCharge(m_fields, *m_fields.get("rho_fp", lev), lev); + myfl->DepositCurrent(m_fields, *m_fields.get("current_fp", Direction{0}, lev), *m_fields.get("current_fp", Direction{1}, lev), - *m_fields.get("current_fp", Direction{2}, lev)); + *m_fields.get("current_fp", Direction{2}, lev), + lev); } // Synchronize J and rho: diff --git a/Source/Fluids/MultiFluidContainer.H b/Source/Fluids/MultiFluidContainer.H index 23f0c46590b..73b190a5d25 100644 --- a/Source/Fluids/MultiFluidContainer.H +++ b/Source/Fluids/MultiFluidContainer.H @@ -10,6 +10,8 @@ #include "WarpXFluidContainer_fwd.H" +#include + #include #include @@ -52,25 +54,26 @@ public: } #endif - void AllocateLevelMFs (int lev, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm); + void AllocateLevelMFs (ablastr::fields::MultiFabRegister& m_fields, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm, int lev); - void InitData (int lev, amrex::Box init_box, amrex::Real cur_time); + void InitData (ablastr::fields::MultiFabRegister& m_fields, amrex::Box init_box, amrex::Real cur_time, int lev); /// /// This evolves all the fluids by one PIC time step, including current deposition, the /// field solve, and pushing the fluids, for all the species in the MultiFluidContainer. /// - void Evolve (int lev, - const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, - const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, - amrex::MultiFab* rho, amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, - amrex::Real cur_time, bool skip_deposition=false); + void Evolve (ablastr::fields::MultiFabRegister& m_fields, + int lev, + std::string current_fp_string, + amrex::Real cur_time, + bool skip_deposition=false); [[nodiscard]] int nSpecies() const {return static_cast(species_names.size());} - void DepositCharge (int lev, amrex::MultiFab &rho); - void DepositCurrent (int lev, - amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz); + void DepositCharge (ablastr::fields::MultiFabRegister& m_fields, amrex::MultiFab &rho, int lev); + void DepositCurrent (ablastr::fields::MultiFabRegister& m_fields, + amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, + int lev); private: @@ -80,4 +83,4 @@ private: amrex::Vector> allcontainers; }; -#endif /*WARPX_MultiFluidContainer_H_*/ +#endif /*WARPX_MultiFluidContainer_H_*/ \ No newline at end of file diff --git a/Source/Fluids/MultiFluidContainer.cpp b/Source/Fluids/MultiFluidContainer.cpp index 234cefb4f07..e266b416632 100644 --- a/Source/Fluids/MultiFluidContainer.cpp +++ b/Source/Fluids/MultiFluidContainer.cpp @@ -27,47 +27,47 @@ MultiFluidContainer::MultiFluidContainer (int nlevs_max) } void -MultiFluidContainer::AllocateLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm) +MultiFluidContainer::AllocateLevelMFs (ablastr::fields::MultiFabRegister& m_fields, const BoxArray& ba, const DistributionMapping& dm, int lev) { for (auto& fl : allcontainers) { - fl->AllocateLevelMFs(lev, ba, dm); + fl->AllocateLevelMFs(m_fields, ba, dm, lev); } } void -MultiFluidContainer::InitData (int lev, amrex::Box init_box, amrex::Real cur_time) +MultiFluidContainer::InitData (ablastr::fields::MultiFabRegister& m_fields, amrex::Box init_box, amrex::Real cur_time, int lev) { for (auto& fl : allcontainers) { - fl->InitData(lev, init_box, cur_time); + fl->InitData(m_fields, init_box, cur_time, lev); } } void -MultiFluidContainer::DepositCharge (int lev, amrex::MultiFab &rho) +MultiFluidContainer::DepositCharge (ablastr::fields::MultiFabRegister& m_fields, amrex::MultiFab &rho, int lev) { for (auto& fl : allcontainers) { - fl->DepositCharge(lev,rho); + fl->DepositCharge(m_fields,rho,lev); } } void -MultiFluidContainer::DepositCurrent (int lev, - amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz) +MultiFluidContainer::DepositCurrent (ablastr::fields::MultiFabRegister& m_fields, + amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, int lev) { for (auto& fl : allcontainers) { - fl->DepositCurrent(lev,jx,jy,jz); + fl->DepositCurrent(m_fields,jx,jy,jz,lev); } } void -MultiFluidContainer::Evolve (int lev, - const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, - const MultiFab& Bx, const MultiFab& By, const MultiFab& Bz, - MultiFab* rho, MultiFab& jx, MultiFab& jy, MultiFab& jz, - amrex::Real cur_time, bool skip_deposition) +MultiFluidContainer::Evolve (ablastr::fields::MultiFabRegister& m_fields, + int lev, + std::string current_fp_string, + amrex::Real cur_time, + bool skip_deposition) { for (auto& fl : allcontainers) { - fl->Evolve(lev, Ex, Ey, Ez, Bx, By, Bz, rho, jx, jy, jz, cur_time, skip_deposition); + fl->Evolve(m_fields, lev, current_fp_string, cur_time, skip_deposition); } -} +} \ No newline at end of file diff --git a/Source/Fluids/WarpXFluidContainer.H b/Source/Fluids/WarpXFluidContainer.H index 04ec4d9e80d..3a05740247b 100644 --- a/Source/Fluids/WarpXFluidContainer.H +++ b/Source/Fluids/WarpXFluidContainer.H @@ -38,20 +38,20 @@ public: WarpXFluidContainer(WarpXFluidContainer&& ) = default; WarpXFluidContainer& operator=(WarpXFluidContainer&& ) = default; - void AllocateLevelMFs (int lev, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm); + void AllocateLevelMFs (ablastr::fields::MultiFabRegister& m_fields, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm, int lev); - void InitData (int lev, amrex::Box init_box, amrex::Real cur_time); + void InitData (ablastr::fields::MultiFabRegister& m_fields, amrex::Box init_box, amrex::Real cur_time, int lev); void ReadParameters (); /** * Evolve updates a single timestep (dt) of the cold relativistic fluid equations */ - void Evolve (int lev, - const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, - const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, - amrex::MultiFab* rho, amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, - amrex::Real cur_time, bool skip_deposition=false); + void Evolve (ablastr::fields::MultiFabRegister& m_fields, + int lev, + std::string current_fp_string, + amrex::Real cur_time, + bool skip_deposition=false); /** * AdvectivePush_Muscl takes a single timestep (dt) of the cold relativistic fluid equations @@ -61,7 +61,7 @@ public: * * \param[in] lev refinement level */ - void AdvectivePush_Muscl (int lev); + void AdvectivePush_Muscl (ablastr::fields::MultiFabRegister& m_fields, int lev); /** @@ -72,7 +72,7 @@ public: * * \param[in] lev refinement level */ - void ApplyBcFluidsAndComms (int lev); + void ApplyBcFluidsAndComms (ablastr::fields::MultiFabRegister& m_fields, int lev); #if defined(WARPX_DIM_RZ) /** @@ -83,7 +83,7 @@ public: * * \param[in] lev refinement level */ - void centrifugal_source_rz (int lev); + void centrifugal_source_rz (ablastr::fields::MultiFabRegister& m_fields, int lev); #endif /** @@ -101,10 +101,10 @@ public: * \param[in] Bz Yee magnetic field (z) * \param[in] t Current time */ - void GatherAndPush (int lev, + void GatherAndPush (ablastr::fields::MultiFabRegister& m_fields, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, - amrex::Real t); + amrex::Real t, int lev); /** * DepositCurrent interpolates the fluid current density comps. onto the Yee grid and @@ -117,8 +117,8 @@ public: * \param[in,out] jy current density MultiFab y comp. * \param[in,out] jz current density MultiFab z comp. */ - void DepositCurrent (int lev, - amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz); + void DepositCurrent (ablastr::fields::MultiFabRegister& m_fields, + amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, int lev); /** * DepositCharge interpolates the fluid charge density onto the Yee grid and @@ -129,7 +129,7 @@ public: * \param[in] lev refinement level * \param[in,out] rho charge density MultiFab. */ - void DepositCharge (int lev, amrex::MultiFab &rho, int icomp = 0); + void DepositCharge (ablastr::fields::MultiFabRegister& m_fields, amrex::MultiFab &rho, int lev, int icomp = 0); [[nodiscard]] amrex::Real getCharge () const {return charge;} [[nodiscard]] amrex::Real getMass () const {return mass;} @@ -185,9 +185,9 @@ protected: public: - // MultiFabs that contain the density (N) and momentum density (NU) of this fluid species, for each refinement level - amrex::Vector< std::unique_ptr > N; - amrex::Vector, 3 > > NU; + // Names of Multifabs that will be added to the mfs register + std::string name_mf_N = "fluid_density_"+species_name; + std::string name_mf_NU = "fluid_momentum_density_"+species_name; }; diff --git a/Source/Fluids/WarpXFluidContainer.cpp b/Source/Fluids/WarpXFluidContainer.cpp index 99a1212ac90..38544bf7bf9 100644 --- a/Source/Fluids/WarpXFluidContainer.cpp +++ b/Source/Fluids/WarpXFluidContainer.cpp @@ -50,9 +50,6 @@ WarpXFluidContainer::WarpXFluidContainer(int nlevs_max, int ispecies, const std: } amrex::Gpu::synchronize(); - // Resize the list of MultiFabs for the right number of levels - N.resize(nlevs_max); - NU.resize(nlevs_max); } void WarpXFluidContainer::ReadParameters() @@ -139,31 +136,33 @@ void WarpXFluidContainer::ReadParameters() } } -void WarpXFluidContainer::AllocateLevelMFs(int lev, const BoxArray &ba, const DistributionMapping &dm) +void WarpXFluidContainer::AllocateLevelMFs(ablastr::fields::MultiFabRegister& m_fields, const BoxArray &ba, const DistributionMapping &dm, int lev) { + using ablastr::fields::Direction; const int ncomps = 1; const amrex::IntVect nguards(AMREX_D_DECL(2, 2, 2)); - - // set human-readable tag for each MultiFab - auto const tag = [lev](std::string tagname) - { - tagname.append("[l=").append(std::to_string(lev)).append("]"); - return tagname; - }; - - WarpX::AllocInitMultiFab(N[lev], amrex::convert(ba, amrex::IntVect::TheNodeVector()), - dm, ncomps, nguards, lev, tag("fluid density"), 0.0_rt); - - WarpX::AllocInitMultiFab(NU[lev][0], amrex::convert(ba, amrex::IntVect::TheNodeVector()), - dm, ncomps, nguards, lev, tag("fluid momentum density [x]"), 0.0_rt); - WarpX::AllocInitMultiFab(NU[lev][1], amrex::convert(ba, amrex::IntVect::TheNodeVector()), - dm, ncomps, nguards, lev, tag("fluid momentum density [y]"), 0.0_rt); - WarpX::AllocInitMultiFab(NU[lev][2], amrex::convert(ba, amrex::IntVect::TheNodeVector()), - dm, ncomps, nguards, lev, tag("fluid momentum density [z]"), 0.0_rt); + + m_fields.alloc_init( + name_mf_N, lev, amrex::convert(ba, amrex::IntVect::TheNodeVector()), dm, + ncomps, nguards, 0.0_rt); + + m_fields.alloc_init( + name_mf_NU, Direction{0}, lev, amrex::convert(ba, amrex::IntVect::TheNodeVector()), dm, + ncomps, nguards, 0.0_rt); + + m_fields.alloc_init( + name_mf_NU, Direction{1}, lev, amrex::convert(ba, amrex::IntVect::TheNodeVector()), dm, + ncomps, nguards, 0.0_rt); + + m_fields.alloc_init( + name_mf_NU, Direction{2}, lev, amrex::convert(ba, amrex::IntVect::TheNodeVector()), dm, + ncomps, nguards, 0.0_rt); + } -void WarpXFluidContainer::InitData(int lev, amrex::Box init_box, amrex::Real cur_time) +void WarpXFluidContainer::InitData(ablastr::fields::MultiFabRegister& m_fields, amrex::Box init_box, amrex::Real cur_time, int lev) { + using ablastr::fields::Direction; WARPX_PROFILE("WarpXFluidContainer::InitData"); // Convert initialization box to nodal box @@ -186,14 +185,14 @@ void WarpXFluidContainer::InitData(int lev, amrex::Box init_box, amrex::Real cur #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(*m_fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) { - amrex::Box const tile_box = mfi.tilebox(N[lev]->ixType().toIntVect()); - amrex::Array4 const &N_arr = N[lev]->array(mfi); - amrex::Array4 const &NUx_arr = NU[lev][0]->array(mfi); - amrex::Array4 const &NUy_arr = NU[lev][1]->array(mfi); - amrex::Array4 const &NUz_arr = NU[lev][2]->array(mfi); + amrex::Box const tile_box = mfi.tilebox(m_fields.get(name_mf_N, lev)->ixType().toIntVect()); + amrex::Array4 const &N_arr = m_fields.get(name_mf_N, lev)->array(mfi); + amrex::Array4 const &NUx_arr = m_fields.get(name_mf_NU, Direction{0}, lev)->array(mfi); + amrex::Array4 const &NUy_arr = m_fields.get(name_mf_NU, Direction{1}, lev)->array(mfi); + amrex::Array4 const &NUz_arr = m_fields.get(name_mf_NU, Direction{2}, lev)->array(mfi); // Return the intersection of all cells and the ones we wish to update amrex::Box const init_box_intersection = init_box & tile_box; @@ -253,54 +252,66 @@ void WarpXFluidContainer::InitData(int lev, amrex::Box init_box, amrex::Real cur void WarpXFluidContainer::Evolve( + ablastr::fields::MultiFabRegister& m_fields, int lev, - const amrex::MultiFab &Ex, const amrex::MultiFab &Ey, const amrex::MultiFab &Ez, - const amrex::MultiFab &Bx, const amrex::MultiFab &By, const amrex::MultiFab &Bz, - amrex::MultiFab* rho, amrex::MultiFab &jx, amrex::MultiFab &jy, amrex::MultiFab &jz, - amrex::Real cur_time, bool skip_deposition) + std::string current_fp_string, + amrex::Real cur_time, + bool skip_deposition) { - + using ablastr::fields::Direction; WARPX_PROFILE("WarpXFluidContainer::Evolve"); - if (rho && ! skip_deposition && ! do_not_deposit) { + if (m_fields.get("rho_fp",lev) && ! skip_deposition && ! do_not_deposit) { // Deposit charge before particle push, in component 0 of MultiFab rho. - DepositCharge(lev, *rho, 0); + DepositCharge(m_fields, *m_fields.get("rho_fp",lev), lev, 0); } // Step the Lorentz Term if(!do_not_gather){ - GatherAndPush(lev, Ex, Ey, Ez, Bx, By, Bz, cur_time); + GatherAndPush(m_fields, + *m_fields.get("Efield_aux", Direction{0}, lev), + *m_fields.get("Efield_aux", Direction{1}, lev), + *m_fields.get("Efield_aux", Direction{2}, lev), + *m_fields.get("Bfield_aux", Direction{0}, lev), + *m_fields.get("Bfield_aux", Direction{1}, lev), + *m_fields.get("Bfield_aux", Direction{2}, lev), + cur_time, lev); } // Cylindrical centrifugal term if(!do_not_push){ #if defined(WARPX_DIM_RZ) - centrifugal_source_rz(lev); + centrifugal_source_rz(m_fields, lev); #endif // Apply (non-periodic) BC on the fluids (needed for spatial derivative), // and communicate N, NU at boundaries - ApplyBcFluidsAndComms(lev); + ApplyBcFluidsAndComms(m_fields, lev); // Step the Advective term - AdvectivePush_Muscl(lev); + AdvectivePush_Muscl(m_fields, lev); } // Deposit rho to the simulation mesh // Deposit charge (end of the step) - if (rho && ! skip_deposition && ! do_not_deposit) { - DepositCharge(lev, *rho, 1); + if (m_fields.get("rho_fp",lev) && ! skip_deposition && ! do_not_deposit) { + DepositCharge(m_fields, *m_fields.get("rho_fp",lev), lev, 1); } // Deposit J to the simulation mesh if (!skip_deposition && ! do_not_deposit) { - DepositCurrent(lev, jx, jy, jz); + DepositCurrent(m_fields, + *m_fields.get(current_fp_string, Direction{0}, lev), + *m_fields.get(current_fp_string, Direction{1}, lev), + *m_fields.get(current_fp_string, Direction{2}, lev), + lev); } } // Momentum source due to curvature -void WarpXFluidContainer::ApplyBcFluidsAndComms (int lev) +void WarpXFluidContainer::ApplyBcFluidsAndComms (ablastr::fields::MultiFabRegister& m_fields, int lev) { + using ablastr::fields::Direction; WARPX_PROFILE("WarpXFluidContainer::ApplyBcFluidsAndComms"); WarpX &warpx = WarpX::GetInstance(); @@ -315,15 +326,15 @@ void WarpXFluidContainer::ApplyBcFluidsAndComms (int lev) #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(*m_fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) { - amrex::Box tile_box = mfi.tilebox(N[lev]->ixType().toIntVect()); + amrex::Box tile_box = mfi.tilebox(m_fields.get(name_mf_N, lev)->ixType().toIntVect()); - const amrex::Array4 N_arr = N[lev]->array(mfi); - const amrex::Array4 NUx_arr = NU[lev][0]->array(mfi); - const amrex::Array4 NUy_arr = NU[lev][1]->array(mfi); - const amrex::Array4 NUz_arr = NU[lev][2]->array(mfi); + const amrex::Array4 N_arr = m_fields.get(name_mf_N, lev)->array(mfi); + const amrex::Array4 NUx_arr = m_fields.get(name_mf_NU, Direction{0}, lev)->array(mfi); + const amrex::Array4 NUy_arr = m_fields.get(name_mf_NU, Direction{1}, lev)->array(mfi); + const amrex::Array4 NUz_arr = m_fields.get(name_mf_NU, Direction{2}, lev)->array(mfi); //Grow the tilebox tile_box.grow(1); @@ -395,15 +406,16 @@ void WarpXFluidContainer::ApplyBcFluidsAndComms (int lev) } // Fill guard cells - FillBoundary(*N[lev], N[lev]->nGrowVect(), WarpX::do_single_precision_comms, period); - FillBoundary(*NU[lev][0], NU[lev][0]->nGrowVect(), WarpX::do_single_precision_comms, period); - FillBoundary(*NU[lev][1], NU[lev][1]->nGrowVect(), WarpX::do_single_precision_comms, period); - FillBoundary(*NU[lev][2], NU[lev][2]->nGrowVect(), WarpX::do_single_precision_comms, period); + FillBoundary(*m_fields.get(name_mf_N, lev), m_fields.get(name_mf_N, lev)->nGrowVect(), WarpX::do_single_precision_comms, period); + FillBoundary(*m_fields.get(name_mf_NU, Direction{0}, lev), m_fields.get(name_mf_NU, Direction{0}, lev)->nGrowVect(), WarpX::do_single_precision_comms, period); + FillBoundary(*m_fields.get(name_mf_NU, Direction{1}, lev), m_fields.get(name_mf_NU, Direction{1}, lev)->nGrowVect(), WarpX::do_single_precision_comms, period); + FillBoundary(*m_fields.get(name_mf_NU, Direction{2}, lev), m_fields.get(name_mf_NU, Direction{2}, lev)->nGrowVect(), WarpX::do_single_precision_comms, period); } // Muscl Advection Update -void WarpXFluidContainer::AdvectivePush_Muscl (int lev) +void WarpXFluidContainer::AdvectivePush_Muscl (ablastr::fields::MultiFabRegister& m_fields, int lev) { + using ablastr::fields::Direction; WARPX_PROFILE("WarpXFluidContainer::AdvectivePush_Muscl"); // Grab the grid spacing @@ -434,31 +446,31 @@ void WarpXFluidContainer::AdvectivePush_Muscl (int lev) const amrex::Real dt_over_dz_half = 0.5_rt*(dt/dx[0]); #endif - const amrex::BoxArray ba = N[lev]->boxArray(); + const amrex::BoxArray ba = m_fields.get(name_mf_N, lev)->boxArray(); // Temporary Half-step values #if defined(WARPX_DIM_3D) - amrex::MultiFab tmp_U_minus_x( amrex::convert(ba, IntVect(0,1,1)), N[lev]->DistributionMap(), 4, 1); - amrex::MultiFab tmp_U_plus_x( amrex::convert(ba, IntVect(0,1,1)), N[lev]->DistributionMap(), 4, 1); - amrex::MultiFab tmp_U_minus_y( amrex::convert(ba, IntVect(1,0,1)), N[lev]->DistributionMap(), 4, 1); - amrex::MultiFab tmp_U_plus_y( amrex::convert(ba, IntVect(1,0,1)), N[lev]->DistributionMap(), 4, 1); - amrex::MultiFab tmp_U_minus_z( amrex::convert(ba, IntVect(1,1,0)), N[lev]->DistributionMap(), 4, 1); - amrex::MultiFab tmp_U_plus_z( amrex::convert(ba, IntVect(1,1,0)), N[lev]->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_minus_x( amrex::convert(ba, IntVect(0,1,1)), m_fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_x( amrex::convert(ba, IntVect(0,1,1)), m_fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_minus_y( amrex::convert(ba, IntVect(1,0,1)), m_fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_y( amrex::convert(ba, IntVect(1,0,1)), m_fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_minus_z( amrex::convert(ba, IntVect(1,1,0)), m_fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_z( amrex::convert(ba, IntVect(1,1,0)), m_fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::MultiFab tmp_U_minus_x( amrex::convert(ba, IntVect(0,1)), N[lev]->DistributionMap(), 4, 1); - amrex::MultiFab tmp_U_plus_x( amrex::convert(ba, IntVect(0,1)), N[lev]->DistributionMap(), 4, 1); - amrex::MultiFab tmp_U_minus_z( amrex::convert(ba, IntVect(1,0)), N[lev]->DistributionMap(), 4, 1); - amrex::MultiFab tmp_U_plus_z( amrex::convert(ba, IntVect(1,0)), N[lev]->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_minus_x( amrex::convert(ba, IntVect(0,1)), m_fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_x( amrex::convert(ba, IntVect(0,1)), m_fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_minus_z( amrex::convert(ba, IntVect(1,0)), m_fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_z( amrex::convert(ba, IntVect(1,0)), m_fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); #else - amrex::MultiFab tmp_U_minus_z( amrex::convert(ba, IntVect(0)), N[lev]->DistributionMap(), 4, 1); - amrex::MultiFab tmp_U_plus_z( amrex::convert(ba, IntVect(0)), N[lev]->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_minus_z( amrex::convert(ba, IntVect(0)), m_fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_z( amrex::convert(ba, IntVect(0)), m_fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); #endif // Fill edge values of N and U at the half timestep for MUSCL #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(*m_fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) { // Loop over a box with one extra gridpoint in the ghost region to avoid @@ -476,10 +488,10 @@ void WarpXFluidContainer::AdvectivePush_Muscl (int lev) return tt; }(); - amrex::Array4 const &N_arr = N[lev]->array(mfi); - amrex::Array4 const &NUx_arr = NU[lev][0]->array(mfi); - amrex::Array4 const &NUy_arr = NU[lev][1]->array(mfi); - amrex::Array4 const &NUz_arr = NU[lev][2]->array(mfi); + amrex::Array4 const &N_arr = m_fields.get(name_mf_N, lev)->array(mfi); + amrex::Array4 const &NUx_arr = m_fields.get(name_mf_NU, Direction{0}, lev)->array(mfi); + amrex::Array4 const &NUy_arr = m_fields.get(name_mf_NU, Direction{1}, lev)->array(mfi); + amrex::Array4 const &NUz_arr = m_fields.get(name_mf_NU, Direction{2}, lev)->array(mfi); // Boxes are computed to avoid going out of bounds. // Grow the entire domain @@ -741,13 +753,13 @@ void WarpXFluidContainer::AdvectivePush_Muscl (int lev) #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(*m_fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const amrex::Box tile_box = mfi.tilebox(N[lev]->ixType().toIntVect()); - const amrex::Array4 N_arr = N[lev]->array(mfi); - const amrex::Array4 NUx_arr = NU[lev][0]->array(mfi); - const amrex::Array4 NUy_arr = NU[lev][1]->array(mfi); - const amrex::Array4 NUz_arr = NU[lev][2]->array(mfi); + const amrex::Box tile_box = mfi.tilebox(m_fields.get(name_mf_N, lev)->ixType().toIntVect()); + const amrex::Array4 N_arr = m_fields.get(name_mf_N, lev)->array(mfi); + const amrex::Array4 NUx_arr = m_fields.get(name_mf_NU, Direction{0}, lev)->array(mfi); + const amrex::Array4 NUy_arr = m_fields.get(name_mf_NU, Direction{1}, lev)->array(mfi); + const amrex::Array4 NUz_arr = m_fields.get(name_mf_NU, Direction{2}, lev)->array(mfi); #if defined(WARPX_DIM_3D) amrex::Array4 const &U_minus_x = tmp_U_minus_x.array(mfi); @@ -878,8 +890,9 @@ void WarpXFluidContainer::AdvectivePush_Muscl (int lev) // Momentum source due to curvature #if defined(WARPX_DIM_RZ) -void WarpXFluidContainer::centrifugal_source_rz (int lev) +void WarpXFluidContainer::centrifugal_source_rz (ablastr::fields::MultiFabRegister& m_fields, int lev) { + using ablastr::fields::Direction; WARPX_PROFILE("WarpXFluidContainer::centrifugal_source_rz"); WarpX &warpx = WarpX::GetInstance(); @@ -894,15 +907,15 @@ void WarpXFluidContainer::centrifugal_source_rz (int lev) #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(*m_fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) { - amrex::Box const &tile_box = mfi.tilebox(N[lev]->ixType().toIntVect()); + amrex::Box const &tile_box = mfi.tilebox(m_fields.get(name_mf_N, lev)->ixType().toIntVect()); - amrex::Array4 const &N_arr = N[lev]->array(mfi); - const amrex::Array4 NUx_arr = NU[lev][0]->array(mfi); - const amrex::Array4 NUy_arr = NU[lev][1]->array(mfi); - amrex::Array4 const &NUz_arr = NU[lev][2]->array(mfi); + amrex::Array4 const &N_arr = m_fields.get(name_mf_N, lev)->array(mfi); + const amrex::Array4 NUx_arr = m_fields.get(name_mf_NU, Direction{0}, lev)->array(mfi); + const amrex::Array4 NUy_arr = m_fields.get(name_mf_NU, Direction{1}, lev)->array(mfi); + amrex::Array4 const &NUz_arr = m_fields.get(name_mf_NU, Direction{2}, lev)->array(mfi); amrex::ParallelFor(tile_box, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept @@ -947,11 +960,13 @@ void WarpXFluidContainer::centrifugal_source_rz (int lev) // Momentum source from fields void WarpXFluidContainer::GatherAndPush ( - int lev, + ablastr::fields::MultiFabRegister& m_fields, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, - Real t) + Real t, + int lev) { + using ablastr::fields::Direction; WARPX_PROFILE("WarpXFluidContainer::GatherAndPush"); WarpX &warpx = WarpX::GetInstance(); @@ -978,7 +993,7 @@ void WarpXFluidContainer::GatherAndPush ( auto Bz_type = amrex::GpuArray{0, 0, 0}; for (int i = 0; i < AMREX_SPACEDIM; ++i) { - Nodal_type[i] = N[lev]->ixType()[i]; + Nodal_type[i] = m_fields.get(name_mf_N, lev)->ixType()[i]; Ex_type[i] = Ex.ixType()[i]; Ey_type[i] = Ey.ixType()[i]; Ez_type[i] = Ez.ixType()[i]; @@ -1015,15 +1030,15 @@ void WarpXFluidContainer::GatherAndPush ( #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(*m_fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) { - amrex::Box const &tile_box = mfi.tilebox(N[lev]->ixType().toIntVect()); + amrex::Box const &tile_box = mfi.tilebox(m_fields.get(name_mf_N, lev)->ixType().toIntVect()); - amrex::Array4 const &N_arr = N[lev]->array(mfi); - const amrex::Array4 NUx_arr = NU[lev][0]->array(mfi); - const amrex::Array4 NUy_arr = NU[lev][1]->array(mfi); - const amrex::Array4 NUz_arr = NU[lev][2]->array(mfi); + amrex::Array4 const &N_arr = m_fields.get(name_mf_N, lev)->array(mfi); + const amrex::Array4 NUx_arr = m_fields.get(name_mf_NU, Direction{0}, lev)->array(mfi); + const amrex::Array4 NUy_arr = m_fields.get(name_mf_NU, Direction{1}, lev)->array(mfi); + const amrex::Array4 NUz_arr = m_fields.get(name_mf_NU, Direction{2}, lev)->array(mfi); amrex::Array4 const& Ex_arr = Ex.array(mfi); amrex::Array4 const& Ey_arr = Ey.array(mfi); @@ -1218,7 +1233,7 @@ void WarpXFluidContainer::GatherAndPush ( } } -void WarpXFluidContainer::DepositCharge (int lev, amrex::MultiFab &rho, int icomp) +void WarpXFluidContainer::DepositCharge (ablastr::fields::MultiFabRegister& m_fields, amrex::MultiFab &rho, int lev, int icomp) { WARPX_PROFILE("WarpXFluidContainer::DepositCharge"); @@ -1235,11 +1250,11 @@ void WarpXFluidContainer::DepositCharge (int lev, amrex::MultiFab &rho, int icom #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(*m_fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) { - amrex::Box const &tile_box = mfi.tilebox(N[lev]->ixType().toIntVect()); - amrex::Array4 const &N_arr = N[lev]->array(mfi); + amrex::Box const &tile_box = mfi.tilebox(m_fields.get(name_mf_N, lev)->ixType().toIntVect()); + amrex::Array4 const &N_arr = m_fields.get(name_mf_N, lev)->array(mfi); const amrex::Array4 rho_arr = rho.array(mfi); const amrex::Array4 owner_mask_rho_arr = owner_mask_rho->array(mfi); @@ -1255,15 +1270,17 @@ void WarpXFluidContainer::DepositCharge (int lev, amrex::MultiFab &rho, int icom void WarpXFluidContainer::DepositCurrent( - int lev, - amrex::MultiFab &jx, amrex::MultiFab &jy, amrex::MultiFab &jz) + ablastr::fields::MultiFabRegister& m_fields, + amrex::MultiFab &jx, amrex::MultiFab &jy, amrex::MultiFab &jz, + int lev) { + using ablastr::fields::Direction; WARPX_PROFILE("WarpXFluidContainer::DepositCurrent"); // Temporary nodal currents - amrex::MultiFab tmp_jx_fluid(N[lev]->boxArray(), N[lev]->DistributionMap(), 1, 0); - amrex::MultiFab tmp_jy_fluid(N[lev]->boxArray(), N[lev]->DistributionMap(), 1, 0); - amrex::MultiFab tmp_jz_fluid(N[lev]->boxArray(), N[lev]->DistributionMap(), 1, 0); + amrex::MultiFab tmp_jx_fluid(m_fields.get(name_mf_N, lev)->boxArray(), m_fields.get(name_mf_N, lev)->DistributionMap(), 1, 0); + amrex::MultiFab tmp_jy_fluid(m_fields.get(name_mf_N, lev)->boxArray(), m_fields.get(name_mf_N, lev)->DistributionMap(), 1, 0); + amrex::MultiFab tmp_jz_fluid(m_fields.get(name_mf_N, lev)->boxArray(), m_fields.get(name_mf_N, lev)->DistributionMap(), 1, 0); const amrex::Real inv_clight_sq = 1.0_prt / PhysConst::c / PhysConst::c; const amrex::Real q = getCharge(); @@ -1293,14 +1310,14 @@ void WarpXFluidContainer::DepositCurrent( #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(*m_fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) { - amrex::Box const &tile_box = mfi.tilebox(N[lev]->ixType().toIntVect()); + amrex::Box const &tile_box = mfi.tilebox(m_fields.get(name_mf_N, lev)->ixType().toIntVect()); - amrex::Array4 const &N_arr = N[lev]->array(mfi); - amrex::Array4 const &NUx_arr = NU[lev][0]->array(mfi); - amrex::Array4 const &NUy_arr = NU[lev][1]->array(mfi); - amrex::Array4 const &NUz_arr = NU[lev][2]->array(mfi); + amrex::Array4 const &N_arr = m_fields.get(name_mf_N, lev)->array(mfi); + amrex::Array4 const &NUx_arr = m_fields.get(name_mf_NU, Direction{0}, lev)->array(mfi); + amrex::Array4 const &NUy_arr = m_fields.get(name_mf_NU, Direction{1}, lev)->array(mfi); + amrex::Array4 const &NUz_arr = m_fields.get(name_mf_NU, Direction{2}, lev)->array(mfi); const amrex::Array4 tmp_jx_fluid_arr = tmp_jx_fluid.array(mfi); const amrex::Array4 tmp_jy_fluid_arr = tmp_jy_fluid.array(mfi); @@ -1328,7 +1345,7 @@ void WarpXFluidContainer::DepositCurrent( #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(*m_fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) { amrex::Box const &tile_box_x = mfi.tilebox(jx.ixType().toIntVect()); amrex::Box const &tile_box_y = mfi.tilebox(jy.ixType().toIntVect()); diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 4e286a8e1e5..048bde253e3 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -137,8 +137,6 @@ WarpX::UpdateInjectionPosition (const amrex::Real a_dt) int WarpX::MoveWindow (const int step, bool move_j) { - using ablastr::fields::Direction; - WARPX_PROFILE("WarpX::MoveWindow"); using ablastr::fields::Direction; @@ -381,10 +379,10 @@ WarpX::MoveWindow (const int step, bool move_j) const int n_fluid_species = myfl->nSpecies(); for (int i=0; iGetFluidContainer(i); - shiftMF( *fl.N[lev], geom[lev], num_shift, dir, lev, do_update_cost ); - shiftMF( *fl.NU[lev][0], geom[lev], num_shift, dir, lev, do_update_cost ); - shiftMF( *fl.NU[lev][1], geom[lev], num_shift, dir, lev, do_update_cost ); - shiftMF( *fl.NU[lev][2], geom[lev], num_shift, dir, lev, do_update_cost ); + shiftMF( *m_fields.get(fl.name_mf_N, lev), geom[lev], num_shift, dir, lev, do_update_cost ); + shiftMF( *m_fields.get(fl.name_mf_NU, Direction{0}, lev), geom[lev], num_shift, dir, lev, do_update_cost ); + shiftMF( *m_fields.get(fl.name_mf_NU, Direction{1}, lev), geom[lev], num_shift, dir, lev, do_update_cost ); + shiftMF( *m_fields.get(fl.name_mf_NU, Direction{2}, lev), geom[lev], num_shift, dir, lev, do_update_cost ); } } } @@ -460,7 +458,7 @@ WarpX::MoveWindow (const int step, bool move_j) const amrex::Real cur_time = t_new[0]; for (int i=0; iGetFluidContainer(i); - fl.InitData( lev, injection_box, cur_time ); + fl.InitData( m_fields, injection_box, cur_time, lev ); } } diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 7ac8330e5c4..d4384d72f32 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2297,10 +2297,10 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm // Allocate extra multifabs needed for fluids if (do_fluid_species) { - myfl->AllocateLevelMFs(lev, ba, dm); + myfl->AllocateLevelMFs(m_fields, ba, dm, lev); auto & warpx = GetInstance(); const amrex::Real cur_time = warpx.gett_new(lev); - myfl->InitData(lev, geom[lev].Domain(),cur_time); + myfl->InitData(m_fields, geom[lev].Domain(), cur_time, lev); } // Allocate extra multifabs for macroscopic properties of the medium From bcd8a125f4d289cb622eb13c01c7bf90283236a0 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 13 Sep 2024 14:05:35 -0700 Subject: [PATCH 229/314] HybridPICModel: Register MultiFabs --- .../WarpXFieldBoundaries.cpp | 3 +- .../ComputeDiagFunctors/JdispFunctor.cpp | 6 +- .../FiniteDifferenceSolver.H | 34 ++--- .../HybridPICModel/HybridPICModel.H | 42 ++---- .../HybridPICModel/HybridPICModel.cpp | 135 ++++++++++-------- .../HybridPICSolveE.cpp | 46 +++--- .../FieldSolver/WarpXPushFieldsHybridPIC.cpp | 28 ++-- Source/Fluids/MultiFluidContainer.H | 2 +- Source/Fluids/MultiFluidContainer.cpp | 4 +- Source/Fluids/WarpXFluidContainer.H | 2 +- Source/Fluids/WarpXFluidContainer.cpp | 2 +- Source/WarpX.cpp | 7 +- 12 files changed, 151 insertions(+), 160 deletions(-) diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index 2121fce1d8c..cf832393c92 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -274,8 +274,9 @@ void WarpX::ApplyElectronPressureBoundary (const int lev, PatchType patch_type) { if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { if (patch_type == PatchType::fine) { + ablastr::fields::ScalarField electron_pressure_fp = m_fields.get("electron_pressure_fp", lev); PEC::ApplyPECtoElectronPressure( - m_hybrid_pic_model->get_pointer_electron_pressure_fp(lev), + electron_pressure_fp, field_boundary_lo, field_boundary_hi, Geom(lev), lev, patch_type, ref_ratio); } else { diff --git a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp index d7fae637106..f945c57386e 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp @@ -33,14 +33,14 @@ JdispFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buff auto* hybrid_pic_model = warpx.get_pointer_HybridPICModel(); /** pointer to total simulation current (J) multifab */ - amrex::MultiFab* mf_j = warpx.m_fields.get("current_fp",Direction{m_dir},m_lev); + amrex::MultiFab* mf_j = warpx.m_fields.get("current_fp", Direction{m_dir}, m_lev); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(hybrid_pic_model, "Displacement current diagnostic is only implemented for the HybridPICModel."); AMREX_ASSUME(hybrid_pic_model != nullptr); /** pointer to current calculated from Ampere's Law (Jamp) multifab */ - amrex::MultiFab* mf_curlB = hybrid_pic_model->get_pointer_current_fp_ampere(m_lev, m_dir);; + amrex::MultiFab* mf_curlB = warpx.m_fields.get("current_fp_ampere", Direction{m_dir}, m_lev); //if (!hybrid_pic_model) { // To finish this implementation, we need to implement a method to @@ -66,7 +66,7 @@ JdispFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buff if (hybrid_pic_model) { // Subtract the interpolated j_external value from j_displacement. /** pointer to external currents (Jext) multifab */ - amrex::MultiFab* mf_j_external = hybrid_pic_model->get_pointer_current_fp_external(m_lev, m_dir); + amrex::MultiFab* mf_j_external = warpx.m_fields.get("current_fp_external", Direction{m_dir}, m_lev); // Index type required for interpolating Jext from their respective // staggering (nodal) to the Jx_displacement, Jy_displacement, Jz_displacement diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index c98ea2b7d3f..e3a113136a2 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -153,12 +153,12 @@ class FiniteDifferenceSolver * \param[in] solve_for_Faraday boolean flag for whether the E-field is solved to be used in Faraday's equation */ void HybridPICSolveE ( ablastr::fields::VectorField const& Efield, - std::array< std::unique_ptr, 3 > & Jfield, + ablastr::fields::VectorField & Jfield, ablastr::fields::VectorField const& Jifield, - std::array< std::unique_ptr, 3 > const& Jextfield, + ablastr::fields::VectorField const& Jextfield, ablastr::fields::VectorField const& Bfield, - amrex::MultiFab* const rhofield, - std::unique_ptr const& Pefield, + amrex::MultiFab const& rhofield, + amrex::MultiFab const& Pefield, ablastr::fields::VectorField const& edge_lengths, int lev, HybridPICModel const* hybrid_model, bool solve_for_Faraday ); @@ -173,7 +173,7 @@ class FiniteDifferenceSolver * \param[in] lev level number for the calculation */ void CalculateCurrentAmpere ( - std::array< std::unique_ptr, 3>& Jfield, + ablastr::fields::VectorField& Jfield, ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& edge_lengths, int lev ); @@ -208,7 +208,7 @@ class FiniteDifferenceSolver template< typename T_Algo > void EvolveBCylindrical ( ablastr::fields::VectorField const& Bfield, - ablastr::fields::VectorField const & Efield, + ablastr::fields::VectorField const& Efield, int lev, amrex::Real dt ); @@ -239,19 +239,19 @@ class FiniteDifferenceSolver template void HybridPICSolveECylindrical ( ablastr::fields::VectorField const& Efield, - std::array< std::unique_ptr, 3> const& Jfield, + ablastr::fields::VectorField const& Jfield, ablastr::fields::VectorField const& Jifield, - std::array< std::unique_ptr, 3 > const& Jextfield, + ablastr::fields::VectorField const& Jextfield, ablastr::fields::VectorField const& Bfield, - amrex::MultiFab* const rhofield, - std::unique_ptr const& Pefield, + amrex::MultiFab const& rhofield, + amrex::MultiFab const& Pefield, ablastr::fields::VectorField const& edge_lengths, int lev, HybridPICModel const* hybrid_model, bool solve_for_Faraday ); template void CalculateCurrentAmpereCylindrical ( - std::array< std::unique_ptr, 3 >& Jfield, + ablastr::fields::VectorField& Jfield, ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& edge_lengths, int lev @@ -339,24 +339,24 @@ class FiniteDifferenceSolver template< typename T_Algo > void EvolveFPMLCartesian ( amrex::MultiFab* Ffield, ablastr::fields::VectorField Efield, - amrex::Real dt ); + amrex::Real dt ); template void HybridPICSolveECartesian ( ablastr::fields::VectorField const& Efield, - std::array< std::unique_ptr, 3 > const& Jfield, + ablastr::fields::VectorField const& Jfield, ablastr::fields::VectorField const& Jifield, - std::array< std::unique_ptr, 3 > const& Jextfield, + ablastr::fields::VectorField const& Jextfield, ablastr::fields::VectorField const& Bfield, - amrex::MultiFab* const rhofield, - std::unique_ptr const& Pefield, + amrex::MultiFab const& rhofield, + amrex::MultiFab const& Pefield, ablastr::fields::VectorField const& edge_lengths, int lev, HybridPICModel const* hybrid_model, bool solve_for_Faraday ); template void CalculateCurrentAmpereCartesian ( - std::array< std::unique_ptr, 3 >& Jfield, + ablastr::fields::VectorField& Jfield, ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& edge_lengths, int lev diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index 02f5b27db17..57ba40b4fb5 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -33,20 +33,20 @@ class HybridPICModel { public: - HybridPICModel (int nlevs_max); // constructor + HybridPICModel (); /** Read user-defined model parameters. Called in constructor. */ void ReadParameters (); /** Allocate hybrid-PIC specific multifabs. Called in constructor. */ - void AllocateMFs (int nlevs_max); - void AllocateLevelMFs (int lev, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm, + void AllocateLevelMFs (ablastr::fields::MultiFabRegister & fields, + int lev, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm, int ncomps, const amrex::IntVect& ngJ, const amrex::IntVect& ngRho, const amrex::IntVect& jx_nodal_flag, const amrex::IntVect& jy_nodal_flag, const amrex::IntVect& jz_nodal_flag, const amrex::IntVect& rho_nodal_flag); /** Helper function to clear values from hybrid-PIC specific multifabs. */ - void ClearLevel (int lev); + void ClearLevel (ablastr::fields::MultiFabRegister & fields, int lev); void InitData (); @@ -99,7 +99,7 @@ public: ablastr::fields::VectorField const& Efield, ablastr::fields::VectorField const& Jfield, ablastr::fields::VectorField const& Bfield, - amrex::MultiFab* const rhofield, + amrex::MultiFab const& rhofield, ablastr::fields::VectorField const& edge_lengths, int lev, bool solve_for_Faraday); @@ -107,7 +107,7 @@ public: ablastr::fields::VectorField const& Efield, ablastr::fields::VectorField const& Jfield, ablastr::fields::VectorField const& Bfield, - amrex::MultiFab* const rhofield, + amrex::MultiFab const& rhofield, ablastr::fields::VectorField const& edge_lengths, int lev, PatchType patch_type, bool solve_for_Faraday); @@ -155,8 +155,8 @@ public: * \param[in] rho_field scalar ion charge density Multifab at a given level */ void FillElectronPressureMF ( - std::unique_ptr const& Pe_field, - amrex::MultiFab* const& rho_field ) const; + amrex::MultiFab& Pe_field, + amrex::MultiFab const& rho_field ) const; // Declare variables to hold hybrid-PIC model parameters /** Number of substeps to take when evolving B */ @@ -189,32 +189,6 @@ public: std::array< amrex::ParserExecutor<4>, 3> m_J_external; bool m_external_field_has_time_dependence = false; - // Declare multifabs specifically needed for the hybrid-PIC model (FIXME: remove me) - amrex::Vector< std::unique_ptr > rho_fp_temp; - amrex::Vector, 3 > > current_fp_temp; - amrex::Vector, 3 > > current_fp_ampere; - amrex::Vector, 3 > > current_fp_external; - amrex::Vector< std::unique_ptr > electron_pressure_fp; - - // Helper functions to retrieve hybrid-PIC multifabs - [[nodiscard]] amrex::MultiFab* - get_pointer_current_fp_ampere (int lev, int direction) const - { - return current_fp_ampere[lev][direction].get(); - } - - [[nodiscard]] amrex::MultiFab* - get_pointer_current_fp_external (int lev, int direction) const - { - return current_fp_external[lev][direction].get(); - } - - [[nodiscard]] amrex::MultiFab* - get_pointer_electron_pressure_fp (int lev) const - { - return electron_pressure_fp[lev].get(); - } - /** Gpu Vector with index type of the Jx multifab */ amrex::GpuArray Jx_IndexType; /** Gpu Vector with index type of the Jy multifab */ diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 5b1cc787920..55c2e8ed447 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -16,10 +16,9 @@ using namespace amrex; using namespace warpx::fields; -HybridPICModel::HybridPICModel ( int nlevs_max ) +HybridPICModel::HybridPICModel () { ReadParameters(); - AllocateMFs(nlevs_max); } void HybridPICModel::ReadParameters () @@ -56,22 +55,16 @@ void HybridPICModel::ReadParameters () pp_hybrid.query("Jz_external_grid_function(x,y,z,t)", m_Jz_ext_grid_function); } -void HybridPICModel::AllocateMFs (int nlevs_max) -{ - electron_pressure_fp.resize(nlevs_max); - rho_fp_temp.resize(nlevs_max); - current_fp_temp.resize(nlevs_max); - current_fp_ampere.resize(nlevs_max); - current_fp_external.resize(nlevs_max); -} - -void HybridPICModel::AllocateLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm, +void HybridPICModel::AllocateLevelMFs (ablastr::fields::MultiFabRegister & fields, + int lev, const BoxArray& ba, const DistributionMapping& dm, const int ncomps, const IntVect& ngJ, const IntVect& ngRho, const IntVect& jx_nodal_flag, const IntVect& jy_nodal_flag, const IntVect& jz_nodal_flag, const IntVect& rho_nodal_flag) { + using ablastr::fields::Direction; + // The "electron_pressure_fp" multifab stores the electron pressure calculated // from the specified equation of state. // The "rho_fp_temp" multifab is used to store the ion charge density @@ -80,35 +73,44 @@ void HybridPICModel::AllocateLevelMFs (int lev, const BoxArray& ba, const Distri // interpolated or extrapolated to appropriate timesteps. // The "current_fp_ampere" multifab stores the total current calculated as // the curl of B. - WarpX::AllocInitMultiFab(electron_pressure_fp[lev], amrex::convert(ba, rho_nodal_flag), - dm, ncomps, ngRho, lev, "electron_pressure_fp", 0.0_rt); - - WarpX::AllocInitMultiFab(rho_fp_temp[lev], amrex::convert(ba, rho_nodal_flag), - dm, ncomps, ngRho, lev, "rho_fp_temp", 0.0_rt); - - WarpX::AllocInitMultiFab(current_fp_temp[lev][0], amrex::convert(ba, jx_nodal_flag), - dm, ncomps, ngJ, lev, "current_fp_temp[x]", 0.0_rt); - WarpX::AllocInitMultiFab(current_fp_temp[lev][1], amrex::convert(ba, jy_nodal_flag), - dm, ncomps, ngJ, lev, "current_fp_temp[y]", 0.0_rt); - WarpX::AllocInitMultiFab(current_fp_temp[lev][2], amrex::convert(ba, jz_nodal_flag), - dm, ncomps, ngJ, lev, "current_fp_temp[z]", 0.0_rt); - - WarpX::AllocInitMultiFab(current_fp_ampere[lev][0], amrex::convert(ba, jx_nodal_flag), - dm, ncomps, ngJ, lev, "current_fp_ampere[x]", 0.0_rt); - WarpX::AllocInitMultiFab(current_fp_ampere[lev][1], amrex::convert(ba, jy_nodal_flag), - dm, ncomps, ngJ, lev, "current_fp_ampere[y]", 0.0_rt); - WarpX::AllocInitMultiFab(current_fp_ampere[lev][2], amrex::convert(ba, jz_nodal_flag), - dm, ncomps, ngJ, lev, "current_fp_ampere[z]", 0.0_rt); + fields.alloc_init("electron_pressure_fp", + lev, amrex::convert(ba, rho_nodal_flag), + dm, ncomps, ngRho, 0.0_rt); + fields.alloc_init("rho_fp_temp", + lev, amrex::convert(ba, rho_nodal_flag), + dm, ncomps, ngRho, 0.0_rt); + fields.alloc_init("current_fp_temp", Direction{0}, + lev, amrex::convert(ba, jx_nodal_flag), + dm, ncomps, ngJ, 0.0_rt); + fields.alloc_init("current_fp_temp", Direction{1}, + lev, amrex::convert(ba, jy_nodal_flag), + dm, ncomps, ngJ, 0.0_rt); + fields.alloc_init("current_fp_temp", Direction{2}, + lev, amrex::convert(ba, jz_nodal_flag), + dm, ncomps, ngJ, 0.0_rt); + + fields.alloc_init("current_fp_ampere", Direction{0}, + lev, amrex::convert(ba, jx_nodal_flag), + dm, ncomps, ngJ, 0.0_rt); + fields.alloc_init("current_fp_ampere", Direction{1}, + lev, amrex::convert(ba, jy_nodal_flag), + dm, ncomps, ngJ, 0.0_rt); + fields.alloc_init("current_fp_ampere", Direction{2}, + lev, amrex::convert(ba, jz_nodal_flag), + dm, ncomps, ngJ, 0.0_rt); // the external current density multifab is made nodal to avoid needing to interpolate // to a nodal grid as has to be done for the ion and total current density multifabs // this also allows the external current multifab to not have any ghost cells - WarpX::AllocInitMultiFab(current_fp_external[lev][0], amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), - dm, ncomps, IntVect(AMREX_D_DECL(0,0,0)), lev, "current_fp_external[x]", 0.0_rt); - WarpX::AllocInitMultiFab(current_fp_external[lev][1], amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), - dm, ncomps, IntVect(AMREX_D_DECL(0,0,0)), lev, "current_fp_external[y]", 0.0_rt); - WarpX::AllocInitMultiFab(current_fp_external[lev][2], amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), - dm, ncomps, IntVect(AMREX_D_DECL(0,0,0)), lev, "current_fp_external[z]", 0.0_rt); + fields.alloc_init("current_fp_external", Direction{0}, + lev, amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), + dm, ncomps, IntVect(AMREX_D_DECL(0,0,0)), 0.0_rt); + fields.alloc_init("current_fp_external", Direction{1}, + lev, amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), + dm, ncomps, IntVect(AMREX_D_DECL(0,0,0)), 0.0_rt); + fields.alloc_init("current_fp_external", Direction{2}, + lev, amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), + dm, ncomps, IntVect(AMREX_D_DECL(0,0,0)), 0.0_rt); #ifdef WARPX_DIM_RZ WARPX_ALWAYS_ASSERT_WITH_MESSAGE( @@ -117,14 +119,16 @@ void HybridPICModel::AllocateLevelMFs (int lev, const BoxArray& ba, const Distri #endif } -void HybridPICModel::ClearLevel (int lev) +void HybridPICModel::ClearLevel (ablastr::fields::MultiFabRegister & fields, int lev) { - electron_pressure_fp[lev].reset(); - rho_fp_temp[lev].reset(); + using ablastr::fields::Direction; + + fields.erase("electron_pressure_fp", lev); + fields.erase("rho_fp_temp", lev); for (int i = 0; i < 3; ++i) { - current_fp_temp[lev][i].reset(); - current_fp_ampere[lev][i].reset(); - current_fp_external[lev][i].reset(); + fields.erase("current_fp_temp", Direction{i}, lev); + fields.erase("current_fp_ampere", Direction{i}, lev); + fields.erase("current_fp_external", Direction{i}, lev); } } @@ -277,9 +281,10 @@ void HybridPICModel::GetCurrentExternal ( auto dx_lev = warpx.Geom(lev).CellSizeArray(); const RealBox& real_box = warpx.Geom(lev).ProbDomain(); - auto& mfx = current_fp_external[lev][0]; - auto& mfy = current_fp_external[lev][1]; - auto& mfz = current_fp_external[lev][2]; + using ablastr::fields::Direction; + amrex::MultiFab * mfx = warpx.m_fields.get("current_fp_external", Direction{0}, lev); + amrex::MultiFab * mfy = warpx.m_fields.get("current_fp_external", Direction{1}, lev); + amrex::MultiFab * mfz = warpx.m_fields.get("current_fp_external", Direction{2}, lev); const amrex::IntVect x_nodal_flag = mfx->ixType().toIntVect(); const amrex::IntVect y_nodal_flag = mfy->ixType().toIntVect(); @@ -411,15 +416,16 @@ void HybridPICModel::CalculateCurrentAmpere ( WARPX_PROFILE("WarpX::CalculateCurrentAmpere()"); auto& warpx = WarpX::GetInstance(); + ablastr::fields::VectorField current_fp_ampere = warpx.m_fields.get_alldirs("current_fp_ampere", warpx.finestLevel()); warpx.get_pointer_fdtd_solver_fp(lev)->CalculateCurrentAmpere( - current_fp_ampere[lev], Bfield, edge_lengths, lev + current_fp_ampere, Bfield, edge_lengths, lev ); // we shouldn't apply the boundary condition to J since J = J_i - J_e but // the boundary correction was already applied to J_i and the B-field // boundary ensures that J itself complies with the boundary conditions, right? // ApplyJfieldBoundary(lev, Jfield[0].get(), Jfield[1].get(), Jfield[2].get()); - for (int i=0; i<3; i++) { current_fp_ampere[lev][i]->FillBoundary(warpx.Geom(lev).periodicity()); } + for (int i=0; i<3; i++) { current_fp_ampere[i]->FillBoundary(warpx.Geom(lev).periodicity()); } } void HybridPICModel::HybridPICSolveE ( @@ -434,7 +440,7 @@ void HybridPICModel::HybridPICSolveE ( for (int lev = 0; lev <= warpx.finestLevel(); ++lev) { HybridPICSolveE( - Efield[lev], Jfield[lev], Bfield[lev], rhofield[lev], + Efield[lev], Jfield[lev], Bfield[lev], *rhofield[lev], edge_lengths[lev], lev, solve_for_Faraday ); } @@ -444,7 +450,7 @@ void HybridPICModel::HybridPICSolveE ( ablastr::fields::VectorField const& Efield, ablastr::fields::VectorField const& Jfield, ablastr::fields::VectorField const& Bfield, - amrex::MultiFab* const rhofield, + amrex::MultiFab const& rhofield, ablastr::fields::VectorField const& edge_lengths, const int lev, const bool solve_for_Faraday) { @@ -465,7 +471,7 @@ void HybridPICModel::HybridPICSolveE ( ablastr::fields::VectorField const& Efield, ablastr::fields::VectorField const& Jfield, ablastr::fields::VectorField const& Bfield, - amrex::MultiFab* const rhofield, + amrex::MultiFab const& rhofield, ablastr::fields::VectorField const& edge_lengths, const int lev, PatchType patch_type, const bool solve_for_Faraday) @@ -474,11 +480,15 @@ void HybridPICModel::HybridPICSolveE ( auto& warpx = WarpX::GetInstance(); + ablastr::fields::VectorField current_fp_ampere = warpx.m_fields.get_alldirs("current_fp_ampere", lev); + ablastr::fields::VectorField current_fp_external = warpx.m_fields.get_alldirs("current_fp_external", lev); + ablastr::fields::ScalarField electron_pressure_fp = warpx.m_fields.get("electron_pressure_fp", lev); + // Solve E field in regular cells warpx.get_pointer_fdtd_solver_fp(lev)->HybridPICSolveE( - Efield, current_fp_ampere[lev], Jfield, current_fp_external[lev], + Efield, current_fp_ampere, Jfield, current_fp_external, Bfield, rhofield, - electron_pressure_fp[lev], + *electron_pressure_fp, edge_lengths, lev, this, solve_for_Faraday ); warpx.ApplyEfieldBoundary(lev, patch_type); @@ -498,17 +508,22 @@ void HybridPICModel::CalculateElectronPressure(const int lev) WARPX_PROFILE("WarpX::CalculateElectronPressure()"); auto& warpx = WarpX::GetInstance(); + ablastr::fields::ScalarField electron_pressure_fp = warpx.m_fields.get("electron_pressure_fp", lev); + ablastr::fields::ScalarField rho_fp = warpx.m_fields.get("rho_fp", lev); + // Calculate the electron pressure using rho^{n+1}. FillElectronPressureMF( - electron_pressure_fp[lev], warpx.m_fields.get("rho_fp", lev) + *electron_pressure_fp, + *rho_fp ); warpx.ApplyElectronPressureBoundary(lev, PatchType::fine); - electron_pressure_fp[lev]->FillBoundary(warpx.Geom(lev).periodicity()); + electron_pressure_fp->FillBoundary(warpx.Geom(lev).periodicity()); } void HybridPICModel::FillElectronPressureMF ( - std::unique_ptr const& Pe_field, - amrex::MultiFab* const& rho_field ) const + amrex::MultiFab& Pe_field, + amrex::MultiFab const& rho_field +) const { const auto n0_ref = m_n0_ref; const auto elec_temp = m_elec_temp; @@ -518,11 +533,11 @@ void HybridPICModel::FillElectronPressureMF ( #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for ( MFIter mfi(*Pe_field, TilingIfNotGPU()); mfi.isValid(); ++mfi ) + for ( MFIter mfi(Pe_field, TilingIfNotGPU()); mfi.isValid(); ++mfi ) { // Extract field data for this grid/tile - Array4 const& rho = rho_field->const_array(mfi); - Array4 const& Pe = Pe_field->array(mfi); + Array4 const& rho = rho_field.const_array(mfi); + Array4 const& Pe = Pe_field.array(mfi); // Extract tileboxes for which to loop const Box& tilebox = mfi.tilebox(); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp index 0d08e0f767f..34a84756203 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp @@ -24,7 +24,7 @@ using namespace amrex; void FiniteDifferenceSolver::CalculateCurrentAmpere ( - std::array< std::unique_ptr, 3>& Jfield, + ablastr::fields::VectorField & Jfield, ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& edge_lengths, int lev ) @@ -59,7 +59,7 @@ void FiniteDifferenceSolver::CalculateCurrentAmpere ( #ifdef WARPX_DIM_RZ template void FiniteDifferenceSolver::CalculateCurrentAmpereCylindrical ( - std::array< std::unique_ptr, 3 >& Jfield, + ablastr::fields::VectorField& Jfield, ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& edge_lengths, int lev @@ -242,7 +242,7 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCylindrical ( template void FiniteDifferenceSolver::CalculateCurrentAmpereCartesian ( - std::array< std::unique_ptr, 3 >& Jfield, + ablastr::fields::VectorField& Jfield, ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& edge_lengths, int lev @@ -352,12 +352,12 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCartesian ( void FiniteDifferenceSolver::HybridPICSolveE ( ablastr::fields::VectorField const& Efield, - std::array< std::unique_ptr, 3 >& Jfield, + ablastr::fields::VectorField& Jfield, ablastr::fields::VectorField const& Jifield, - std::array< std::unique_ptr, 3 > const& Jextfield, + ablastr::fields::VectorField const& Jextfield, ablastr::fields::VectorField const& Bfield, - amrex::MultiFab* const rhofield, - std::unique_ptr const& Pefield, + amrex::MultiFab const& rhofield, + amrex::MultiFab const& Pefield, ablastr::fields::VectorField const& edge_lengths, int lev, HybridPICModel const* hybrid_model, const bool solve_for_Faraday) @@ -390,12 +390,12 @@ void FiniteDifferenceSolver::HybridPICSolveE ( template void FiniteDifferenceSolver::HybridPICSolveECylindrical ( ablastr::fields::VectorField const& Efield, - std::array< std::unique_ptr, 3 > const& Jfield, + ablastr::fields::VectorField const& Jfield, ablastr::fields::VectorField const& Jifield, - std::array< std::unique_ptr, 3 > const& Jextfield, + ablastr::fields::VectorField const& Jextfield, ablastr::fields::VectorField const& Bfield, - amrex::MultiFab* const rhofield, - std::unique_ptr const& Pefield, + amrex::MultiFab const& rhofield, + amrex::MultiFab const& Pefield, ablastr::fields::VectorField const& edge_lengths, int lev, HybridPICModel const* hybrid_model, const bool solve_for_Faraday ) @@ -449,8 +449,8 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( // Also note that enE_nodal_mf does not need to have any guard cells since // these values will be interpolated to the Yee mesh which is contained // by the nodal mesh. - auto const& ba = convert(rhofield->boxArray(), IntVect::TheNodeVector()); - MultiFab enE_nodal_mf(ba, rhofield->DistributionMap(), 3, IntVect::TheZeroVector()); + auto const& ba = convert(rhofield.boxArray(), IntVect::TheNodeVector()); + MultiFab enE_nodal_mf(ba, rhofield.DistributionMap(), 3, IntVect::TheZeroVector()); // Loop through the grids, and over the tiles within each grid for the // initial, nodal calculation of E @@ -539,8 +539,8 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( Array4 const& Jt = Jfield[1]->const_array(mfi); Array4 const& Jz = Jfield[2]->const_array(mfi); Array4 const& enE = enE_nodal_mf.const_array(mfi); - Array4 const& rho = rhofield->const_array(mfi); - Array4 const& Pe = Pefield->array(mfi); + Array4 const& rho = rhofield.const_array(mfi); + Array4 const& Pe = Pefield.const_array(mfi); amrex::Array4 lr, lz; if (EB::enabled()) { @@ -705,12 +705,12 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( template void FiniteDifferenceSolver::HybridPICSolveECartesian ( ablastr::fields::VectorField const& Efield, - std::array< std::unique_ptr, 3 > const& Jfield, + ablastr::fields::VectorField const& Jfield, ablastr::fields::VectorField const& Jifield, - std::array< std::unique_ptr, 3 > const& Jextfield, + ablastr::fields::VectorField const& Jextfield, ablastr::fields::VectorField const& Bfield, - amrex::MultiFab* const rhofield, - std::unique_ptr const& Pefield, + amrex::MultiFab const& rhofield, + amrex::MultiFab const& Pefield, ablastr::fields::VectorField const& edge_lengths, int lev, HybridPICModel const* hybrid_model, const bool solve_for_Faraday ) @@ -758,8 +758,8 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( // Also note that enE_nodal_mf does not need to have any guard cells since // these values will be interpolated to the Yee mesh which is contained // by the nodal mesh. - auto const& ba = convert(rhofield->boxArray(), IntVect::TheNodeVector()); - MultiFab enE_nodal_mf(ba, rhofield->DistributionMap(), 3, IntVect::TheZeroVector()); + auto const& ba = convert(rhofield.boxArray(), IntVect::TheNodeVector()); + MultiFab enE_nodal_mf(ba, rhofield.DistributionMap(), 3, IntVect::TheZeroVector()); // Loop through the grids, and over the tiles within each grid for the // initial, nodal calculation of E @@ -848,8 +848,8 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( Array4 const& Jy = Jfield[1]->const_array(mfi); Array4 const& Jz = Jfield[2]->const_array(mfi); Array4 const& enE = enE_nodal_mf.const_array(mfi); - Array4 const& rho = rhofield->const_array(mfi); - Array4 const& Pe = Pefield->array(mfi); + Array4 const& rho = rhofield.const_array(mfi); + Array4 const& Pe = Pefield.array(mfi); amrex::Array4 lx, ly, lz; if (EB::enabled()) { diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index c23af2151db..9f76a74ffc9 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -70,8 +70,8 @@ void WarpX::HybridPICEvolveFields () m_fields.get_mr_levels_alldirs("edge_lengths", finest_level)); // Reference hybrid-PIC multifabs - auto& rho_fp_temp = m_hybrid_pic_model->rho_fp_temp; - auto& current_fp_temp = m_hybrid_pic_model->current_fp_temp; + ablastr::fields::MultiLevelScalarField rho_fp_temp = m_fields.get_mr_levels("rho_fp_temp", finest_level); + ablastr::fields::MultiLevelVectorField current_fp_temp = m_fields.get_mr_levels_alldirs("current_fp_temp", finest_level); // During the above deposition the charge and current density were updated // so that, at this time, we have rho^{n} in rho_fp_temp, rho{n+1} in the @@ -108,7 +108,7 @@ void WarpX::HybridPICEvolveFields () m_hybrid_pic_model->BfieldEvolveRK( m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level), m_fields.get_mr_levels_alldirs("Efield_fp", finest_level), - va2vm(current_fp_temp), amrex::GetVecOfPtrs(rho_fp_temp), + current_fp_temp, rho_fp_temp, m_fields.get_mr_levels_alldirs("edge_lenghts", finest_level), 0.5_rt/sub_steps*dt[0], DtType::FirstHalf, guard_cells.ng_FieldSolver, @@ -135,7 +135,7 @@ void WarpX::HybridPICEvolveFields () m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level), m_fields.get_mr_levels_alldirs("Efield_fp", finest_level), m_fields.get_mr_levels_alldirs("current_fp", finest_level), - amrex::GetVecOfPtrs(rho_fp_temp), + rho_fp_temp, m_fields.get_mr_levels_alldirs("edge_lenghts", finest_level), 0.5_rt/sub_steps*dt[0], DtType::SecondHalf, guard_cells.ng_FieldSolver, @@ -170,7 +170,7 @@ void WarpX::HybridPICEvolveFields () m_fields.get_mr_levels_alldirs("edge_lengths", finest_level)); m_hybrid_pic_model->HybridPICSolveE( m_fields.get_mr_levels_alldirs("Efield_fp", finest_level), - va2vm(current_fp_temp), + current_fp_temp, m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level), m_fields.get_mr_levels("rho_fp", finest_level), m_fields.get_mr_levels_alldirs("edge_lengths", finest_level), false @@ -196,11 +196,11 @@ void WarpX::HybridPICDepositInitialRhoAndJ () { using ablastr::fields::va2vm; - auto& rho_fp_temp = m_hybrid_pic_model->rho_fp_temp; - auto& current_fp_temp = m_hybrid_pic_model->current_fp_temp; - mypc->DepositCharge(amrex::GetVecOfPtrs(rho_fp_temp), 0._rt); - mypc->DepositCurrent(va2vm(current_fp_temp), dt[0], 0._rt); - SyncRho(amrex::GetVecOfPtrs(rho_fp_temp), m_fields.get_mr_levels("rho_cp", finest_level), m_fields.get_mr_levels("rho_buf", finest_level)); + ablastr::fields::MultiLevelScalarField rho_fp_temp = m_fields.get_mr_levels("rho_fp_temp", finest_level); + ablastr::fields::MultiLevelVectorField current_fp_temp = m_fields.get_mr_levels_alldirs("current_fp_temp", finest_level); + mypc->DepositCharge(rho_fp_temp, 0._rt); + mypc->DepositCurrent(current_fp_temp, dt[0], 0._rt); + SyncRho(rho_fp_temp, m_fields.get_mr_levels("rho_cp", finest_level), m_fields.get_mr_levels("rho_buf", finest_level)); SyncCurrent("current_fp"); for (int lev=0; lev <= finest_level; ++lev) { // SyncCurrent does not include a call to FillBoundary, but it is needed @@ -210,12 +210,12 @@ void WarpX::HybridPICDepositInitialRhoAndJ () current_fp_temp[lev][1]->FillBoundary(Geom(lev).periodicity()); current_fp_temp[lev][2]->FillBoundary(Geom(lev).periodicity()); - ApplyRhofieldBoundary(lev, rho_fp_temp[lev].get(), PatchType::fine); + ApplyRhofieldBoundary(lev, rho_fp_temp[lev], PatchType::fine); // Set current density at PEC boundaries, if needed. ApplyJfieldBoundary( - lev, current_fp_temp[lev][0].get(), - current_fp_temp[lev][1].get(), - current_fp_temp[lev][2].get(), + lev, current_fp_temp[lev][0], + current_fp_temp[lev][1], + current_fp_temp[lev][2], PatchType::fine ); } diff --git a/Source/Fluids/MultiFluidContainer.H b/Source/Fluids/MultiFluidContainer.H index 73b190a5d25..21ae535a73a 100644 --- a/Source/Fluids/MultiFluidContainer.H +++ b/Source/Fluids/MultiFluidContainer.H @@ -36,7 +36,7 @@ class MultiFluidContainer public: - MultiFluidContainer (int nlevs_max); + MultiFluidContainer (); ~MultiFluidContainer() = default; diff --git a/Source/Fluids/MultiFluidContainer.cpp b/Source/Fluids/MultiFluidContainer.cpp index e266b416632..4e935e66915 100644 --- a/Source/Fluids/MultiFluidContainer.cpp +++ b/Source/Fluids/MultiFluidContainer.cpp @@ -13,7 +13,7 @@ using namespace amrex; -MultiFluidContainer::MultiFluidContainer (int nlevs_max) +MultiFluidContainer::MultiFluidContainer () { const ParmParse pp_fluids("fluids"); pp_fluids.queryarr("species_names", species_names); @@ -22,7 +22,7 @@ MultiFluidContainer::MultiFluidContainer (int nlevs_max) allcontainers.resize(nspecies); for (int i = 0; i < nspecies; ++i) { - allcontainers[i] = std::make_unique(nlevs_max, i, species_names[i]); + allcontainers[i] = std::make_unique(i, species_names[i]); } } diff --git a/Source/Fluids/WarpXFluidContainer.H b/Source/Fluids/WarpXFluidContainer.H index 3a05740247b..d66c4af896c 100644 --- a/Source/Fluids/WarpXFluidContainer.H +++ b/Source/Fluids/WarpXFluidContainer.H @@ -30,7 +30,7 @@ class WarpXFluidContainer public: friend MultiFluidContainer; - WarpXFluidContainer (int nlevs_max, int ispecies, const std::string& name); + WarpXFluidContainer (int ispecies, const std::string &name); ~WarpXFluidContainer() = default; WarpXFluidContainer (WarpXFluidContainer const &) = delete; diff --git a/Source/Fluids/WarpXFluidContainer.cpp b/Source/Fluids/WarpXFluidContainer.cpp index 38544bf7bf9..37f7a1a2d31 100644 --- a/Source/Fluids/WarpXFluidContainer.cpp +++ b/Source/Fluids/WarpXFluidContainer.cpp @@ -19,7 +19,7 @@ using namespace ablastr::utils::communication; using namespace amrex; -WarpXFluidContainer::WarpXFluidContainer(int nlevs_max, int ispecies, const std::string &name): +WarpXFluidContainer::WarpXFluidContainer(int ispecies, const std::string &name) : species_id{ispecies}, species_name{name} { diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index d4384d72f32..d76b2bbff70 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -314,7 +314,7 @@ WarpX::WarpX () // Fluid Container if (do_fluid_species) { - myfl = std::make_unique(nlevs_max); + myfl = std::make_unique(); } Efield_dotMask.resize(nlevs_max); @@ -329,7 +329,7 @@ WarpX::WarpX () if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { // Create hybrid-PIC model object if needed - m_hybrid_pic_model = std::make_unique(nlevs_max); + m_hybrid_pic_model = std::make_unique(); } current_buffer_masks.resize(nlevs_max); @@ -2035,7 +2035,7 @@ WarpX::ClearLevel (int lev) if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { - m_hybrid_pic_model->ClearLevel(lev); + m_hybrid_pic_model->ClearLevel(m_fields, lev); } for (int i = 0; i < 3; ++i) { @@ -2290,6 +2290,7 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { m_hybrid_pic_model->AllocateLevelMFs( + m_fields, lev, ba, dm, ncomps, ngJ, ngRho, jx_nodal_flag, jy_nodal_flag, jz_nodal_flag, rho_nodal_flag ); From 019e4edc94a6072cf7be436d471a771abc198d90 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 13 Sep 2024 14:21:42 -0700 Subject: [PATCH 230/314] HybridPICModel: Rename MultiFabs --- Python/pywarpx/fields.py | 6 +- .../WarpXFieldBoundaries.cpp | 2 +- .../ComputeDiagFunctors/JdispFunctor.cpp | 4 +- .../HybridPICModel/HybridPICModel.cpp | 56 +++++++++---------- .../FieldSolver/WarpXPushFieldsHybridPIC.cpp | 8 +-- 5 files changed, 38 insertions(+), 38 deletions(-) diff --git a/Python/pywarpx/fields.py b/Python/pywarpx/fields.py index 356063739d1..94603d9af40 100644 --- a/Python/pywarpx/fields.py +++ b/Python/pywarpx/fields.py @@ -875,19 +875,19 @@ def FaceAreaszWrapper(level=0, include_ghosts=False): def JxFPAmpereWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="current_fp_ampere", idir=0, level=level, include_ghosts=include_ghosts + mf_name="hybrid_current_fp_ampere", idir=0, level=level, include_ghosts=include_ghosts ) def JyFPAmpereWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="current_fp_ampere", idir=1, level=level, include_ghosts=include_ghosts + mf_name="hybrid_current_fp_ampere", idir=1, level=level, include_ghosts=include_ghosts ) def JzFPAmpereWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="current_fp_ampere", idir=2, level=level, include_ghosts=include_ghosts + mf_name="hybrid_current_fp_ampere", idir=2, level=level, include_ghosts=include_ghosts ) diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index cf832393c92..9de01d20adc 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -274,7 +274,7 @@ void WarpX::ApplyElectronPressureBoundary (const int lev, PatchType patch_type) { if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { if (patch_type == PatchType::fine) { - ablastr::fields::ScalarField electron_pressure_fp = m_fields.get("electron_pressure_fp", lev); + ablastr::fields::ScalarField electron_pressure_fp = m_fields.get("hybrid_electron_pressure_fp", lev); PEC::ApplyPECtoElectronPressure( electron_pressure_fp, field_boundary_lo, field_boundary_hi, diff --git a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp index f945c57386e..3c78bdf6d31 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp @@ -40,7 +40,7 @@ JdispFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buff AMREX_ASSUME(hybrid_pic_model != nullptr); /** pointer to current calculated from Ampere's Law (Jamp) multifab */ - amrex::MultiFab* mf_curlB = warpx.m_fields.get("current_fp_ampere", Direction{m_dir}, m_lev); + amrex::MultiFab* mf_curlB = warpx.m_fields.get("hybrid_current_fp_ampere", Direction{m_dir}, m_lev); //if (!hybrid_pic_model) { // To finish this implementation, we need to implement a method to @@ -66,7 +66,7 @@ JdispFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buff if (hybrid_pic_model) { // Subtract the interpolated j_external value from j_displacement. /** pointer to external currents (Jext) multifab */ - amrex::MultiFab* mf_j_external = warpx.m_fields.get("current_fp_external", Direction{m_dir}, m_lev); + amrex::MultiFab* mf_j_external = warpx.m_fields.get("hybrid_current_fp_external", Direction{m_dir}, m_lev); // Index type required for interpolating Jext from their respective // staggering (nodal) to the Jx_displacement, Jy_displacement, Jz_displacement diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 55c2e8ed447..8cc8b2ad170 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -65,50 +65,50 @@ void HybridPICModel::AllocateLevelMFs (ablastr::fields::MultiFabRegister & field { using ablastr::fields::Direction; - // The "electron_pressure_fp" multifab stores the electron pressure calculated + // The "hybrid_electron_pressure_fp" multifab stores the electron pressure calculated // from the specified equation of state. - // The "rho_fp_temp" multifab is used to store the ion charge density + // The "hybrid_rho_fp_temp" multifab is used to store the ion charge density // interpolated or extrapolated to appropriate timesteps. - // The "current_fp_temp" multifab is used to store the ion current density + // The "hybrid_current_fp_temp" multifab is used to store the ion current density // interpolated or extrapolated to appropriate timesteps. - // The "current_fp_ampere" multifab stores the total current calculated as + // The "hybrid_current_fp_ampere" multifab stores the total current calculated as // the curl of B. - fields.alloc_init("electron_pressure_fp", + fields.alloc_init("hybrid_electron_pressure_fp", lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngRho, 0.0_rt); - fields.alloc_init("rho_fp_temp", + fields.alloc_init("hybrid_rho_fp_temp", lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngRho, 0.0_rt); - fields.alloc_init("current_fp_temp", Direction{0}, + fields.alloc_init("hybrid_current_fp_temp", Direction{0}, lev, amrex::convert(ba, jx_nodal_flag), dm, ncomps, ngJ, 0.0_rt); - fields.alloc_init("current_fp_temp", Direction{1}, + fields.alloc_init("hybrid_current_fp_temp", Direction{1}, lev, amrex::convert(ba, jy_nodal_flag), dm, ncomps, ngJ, 0.0_rt); - fields.alloc_init("current_fp_temp", Direction{2}, + fields.alloc_init("hybrid_current_fp_temp", Direction{2}, lev, amrex::convert(ba, jz_nodal_flag), dm, ncomps, ngJ, 0.0_rt); - fields.alloc_init("current_fp_ampere", Direction{0}, + fields.alloc_init("hybrid_current_fp_ampere", Direction{0}, lev, amrex::convert(ba, jx_nodal_flag), dm, ncomps, ngJ, 0.0_rt); - fields.alloc_init("current_fp_ampere", Direction{1}, + fields.alloc_init("hybrid_current_fp_ampere", Direction{1}, lev, amrex::convert(ba, jy_nodal_flag), dm, ncomps, ngJ, 0.0_rt); - fields.alloc_init("current_fp_ampere", Direction{2}, + fields.alloc_init("hybrid_current_fp_ampere", Direction{2}, lev, amrex::convert(ba, jz_nodal_flag), dm, ncomps, ngJ, 0.0_rt); // the external current density multifab is made nodal to avoid needing to interpolate // to a nodal grid as has to be done for the ion and total current density multifabs // this also allows the external current multifab to not have any ghost cells - fields.alloc_init("current_fp_external", Direction{0}, + fields.alloc_init("hybrid_current_fp_external", Direction{0}, lev, amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), dm, ncomps, IntVect(AMREX_D_DECL(0,0,0)), 0.0_rt); - fields.alloc_init("current_fp_external", Direction{1}, + fields.alloc_init("hybrid_current_fp_external", Direction{1}, lev, amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), dm, ncomps, IntVect(AMREX_D_DECL(0,0,0)), 0.0_rt); - fields.alloc_init("current_fp_external", Direction{2}, + fields.alloc_init("hybrid_current_fp_external", Direction{2}, lev, amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), dm, ncomps, IntVect(AMREX_D_DECL(0,0,0)), 0.0_rt); @@ -123,12 +123,12 @@ void HybridPICModel::ClearLevel (ablastr::fields::MultiFabRegister & fields, int { using ablastr::fields::Direction; - fields.erase("electron_pressure_fp", lev); - fields.erase("rho_fp_temp", lev); + fields.erase("hybrid_electron_pressure_fp", lev); + fields.erase("hybrid_rho_fp_temp", lev); for (int i = 0; i < 3; ++i) { - fields.erase("current_fp_temp", Direction{i}, lev); - fields.erase("current_fp_ampere", Direction{i}, lev); - fields.erase("current_fp_external", Direction{i}, lev); + fields.erase("hybrid_current_fp_temp", Direction{i}, lev); + fields.erase("hybrid_current_fp_ampere", Direction{i}, lev); + fields.erase("hybrid_current_fp_external", Direction{i}, lev); } } @@ -282,9 +282,9 @@ void HybridPICModel::GetCurrentExternal ( const RealBox& real_box = warpx.Geom(lev).ProbDomain(); using ablastr::fields::Direction; - amrex::MultiFab * mfx = warpx.m_fields.get("current_fp_external", Direction{0}, lev); - amrex::MultiFab * mfy = warpx.m_fields.get("current_fp_external", Direction{1}, lev); - amrex::MultiFab * mfz = warpx.m_fields.get("current_fp_external", Direction{2}, lev); + amrex::MultiFab * mfx = warpx.m_fields.get("hybrid_current_fp_external", Direction{0}, lev); + amrex::MultiFab * mfy = warpx.m_fields.get("hybrid_current_fp_external", Direction{1}, lev); + amrex::MultiFab * mfz = warpx.m_fields.get("hybrid_current_fp_external", Direction{2}, lev); const amrex::IntVect x_nodal_flag = mfx->ixType().toIntVect(); const amrex::IntVect y_nodal_flag = mfy->ixType().toIntVect(); @@ -416,7 +416,7 @@ void HybridPICModel::CalculateCurrentAmpere ( WARPX_PROFILE("WarpX::CalculateCurrentAmpere()"); auto& warpx = WarpX::GetInstance(); - ablastr::fields::VectorField current_fp_ampere = warpx.m_fields.get_alldirs("current_fp_ampere", warpx.finestLevel()); + ablastr::fields::VectorField current_fp_ampere = warpx.m_fields.get_alldirs("hybrid_current_fp_ampere", warpx.finestLevel()); warpx.get_pointer_fdtd_solver_fp(lev)->CalculateCurrentAmpere( current_fp_ampere, Bfield, edge_lengths, lev ); @@ -480,9 +480,9 @@ void HybridPICModel::HybridPICSolveE ( auto& warpx = WarpX::GetInstance(); - ablastr::fields::VectorField current_fp_ampere = warpx.m_fields.get_alldirs("current_fp_ampere", lev); - ablastr::fields::VectorField current_fp_external = warpx.m_fields.get_alldirs("current_fp_external", lev); - ablastr::fields::ScalarField electron_pressure_fp = warpx.m_fields.get("electron_pressure_fp", lev); + ablastr::fields::VectorField current_fp_ampere = warpx.m_fields.get_alldirs("hybrid_current_fp_ampere", lev); + ablastr::fields::VectorField current_fp_external = warpx.m_fields.get_alldirs("hybrid_current_fp_external", lev); + ablastr::fields::ScalarField electron_pressure_fp = warpx.m_fields.get("hybrid_electron_pressure_fp", lev); // Solve E field in regular cells warpx.get_pointer_fdtd_solver_fp(lev)->HybridPICSolveE( @@ -508,7 +508,7 @@ void HybridPICModel::CalculateElectronPressure(const int lev) WARPX_PROFILE("WarpX::CalculateElectronPressure()"); auto& warpx = WarpX::GetInstance(); - ablastr::fields::ScalarField electron_pressure_fp = warpx.m_fields.get("electron_pressure_fp", lev); + ablastr::fields::ScalarField electron_pressure_fp = warpx.m_fields.get("hybrid_electron_pressure_fp", lev); ablastr::fields::ScalarField rho_fp = warpx.m_fields.get("rho_fp", lev); // Calculate the electron pressure using rho^{n+1}. diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index 9f76a74ffc9..c1f2e36f49d 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -70,8 +70,8 @@ void WarpX::HybridPICEvolveFields () m_fields.get_mr_levels_alldirs("edge_lengths", finest_level)); // Reference hybrid-PIC multifabs - ablastr::fields::MultiLevelScalarField rho_fp_temp = m_fields.get_mr_levels("rho_fp_temp", finest_level); - ablastr::fields::MultiLevelVectorField current_fp_temp = m_fields.get_mr_levels_alldirs("current_fp_temp", finest_level); + ablastr::fields::MultiLevelScalarField rho_fp_temp = m_fields.get_mr_levels("hybrid_rho_fp_temp", finest_level); + ablastr::fields::MultiLevelVectorField current_fp_temp = m_fields.get_mr_levels_alldirs("hybrid_current_fp_temp", finest_level); // During the above deposition the charge and current density were updated // so that, at this time, we have rho^{n} in rho_fp_temp, rho{n+1} in the @@ -196,8 +196,8 @@ void WarpX::HybridPICDepositInitialRhoAndJ () { using ablastr::fields::va2vm; - ablastr::fields::MultiLevelScalarField rho_fp_temp = m_fields.get_mr_levels("rho_fp_temp", finest_level); - ablastr::fields::MultiLevelVectorField current_fp_temp = m_fields.get_mr_levels_alldirs("current_fp_temp", finest_level); + ablastr::fields::MultiLevelScalarField rho_fp_temp = m_fields.get_mr_levels("hybrid_rho_fp_temp", finest_level); + ablastr::fields::MultiLevelVectorField current_fp_temp = m_fields.get_mr_levels_alldirs("hybrid_current_fp_temp", finest_level); mypc->DepositCharge(rho_fp_temp, 0._rt); mypc->DepositCurrent(current_fp_temp, dt[0], 0._rt); SyncRho(rho_fp_temp, m_fields.get_mr_levels("rho_cp", finest_level), m_fields.get_mr_levels("rho_buf", finest_level)); From a3525637e2c3f36f6a11b45ef348daf815d07944 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 13 Sep 2024 14:31:13 -0700 Subject: [PATCH 231/314] HybridPIC: Remove ClearLevel Centrally managed. --- .../HybridPICModel/HybridPICModel.H | 3 --- .../HybridPICModel/HybridPICModel.cpp | 13 ------------- Source/WarpX.cpp | 5 ----- 3 files changed, 21 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index 57ba40b4fb5..f59210315f6 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -45,9 +45,6 @@ public: const amrex::IntVect& jx_nodal_flag, const amrex::IntVect& jy_nodal_flag, const amrex::IntVect& jz_nodal_flag, const amrex::IntVect& rho_nodal_flag); - /** Helper function to clear values from hybrid-PIC specific multifabs. */ - void ClearLevel (ablastr::fields::MultiFabRegister & fields, int lev); - void InitData (); /** diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 8cc8b2ad170..b05b9ae1562 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -119,19 +119,6 @@ void HybridPICModel::AllocateLevelMFs (ablastr::fields::MultiFabRegister & field #endif } -void HybridPICModel::ClearLevel (ablastr::fields::MultiFabRegister & fields, int lev) -{ - using ablastr::fields::Direction; - - fields.erase("hybrid_electron_pressure_fp", lev); - fields.erase("hybrid_rho_fp_temp", lev); - for (int i = 0; i < 3; ++i) { - fields.erase("hybrid_current_fp_temp", Direction{i}, lev); - fields.erase("hybrid_current_fp_ampere", Direction{i}, lev); - fields.erase("hybrid_current_fp_external", Direction{i}, lev); - } -} - void HybridPICModel::InitData () { m_resistivity_parser = std::make_unique( diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index d76b2bbff70..f62dcde1e0d 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2033,11 +2033,6 @@ WarpX::ClearLevel (int lev) { m_fields.clear_level(lev); - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) - { - m_hybrid_pic_model->ClearLevel(m_fields, lev); - } - for (int i = 0; i < 3; ++i) { Efield_dotMask [lev][i].reset(); Bfield_dotMask [lev][i].reset(); From c24fb1fe57dd43e8655cb914fe7b51e1786966ec Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 13 Sep 2024 14:31:39 -0700 Subject: [PATCH 232/314] Register: Update name in erase error message --- Source/ablastr/fields/MultiFabRegister.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index b42ead54022..7530eec0509 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -548,7 +548,7 @@ namespace ablastr::fields name = mf_name(name, level); if (m_mf_register.count(name) != 1) { - throw std::runtime_error("MultiFabRegister::remove name does not exist in register: " + name); + throw std::runtime_error("MultiFabRegister::erase name does not exist in register: " + name); } m_mf_register.erase(name); } @@ -563,7 +563,7 @@ namespace ablastr::fields name = mf_name(name, dir, level); if (m_mf_register.count(name) != 1) { - throw std::runtime_error("MultiFabRegister::remove name does not exist in register: " + name); + throw std::runtime_error("MultiFabRegister::erase name does not exist in register: " + name); } m_mf_register.erase(name); } From c595a39f646c420a34090c52ed438fb49628402d Mon Sep 17 00:00:00 2001 From: David Grote Date: Fri, 13 Sep 2024 14:35:23 -0700 Subject: [PATCH 233/314] Remove end-of-line spaces in fluid code --- Source/Fluids/MultiFluidContainer.H | 6 +++--- Source/Fluids/MultiFluidContainer.cpp | 6 +++--- Source/Fluids/WarpXFluidContainer.H | 8 ++++---- Source/Fluids/WarpXFluidContainer.cpp | 22 +++++++++++----------- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/Source/Fluids/MultiFluidContainer.H b/Source/Fluids/MultiFluidContainer.H index 21ae535a73a..9d55f975754 100644 --- a/Source/Fluids/MultiFluidContainer.H +++ b/Source/Fluids/MultiFluidContainer.H @@ -62,10 +62,10 @@ public: /// This evolves all the fluids by one PIC time step, including current deposition, the /// field solve, and pushing the fluids, for all the species in the MultiFluidContainer. /// - void Evolve (ablastr::fields::MultiFabRegister& m_fields, + void Evolve (ablastr::fields::MultiFabRegister& m_fields, int lev, std::string current_fp_string, - amrex::Real cur_time, + amrex::Real cur_time, bool skip_deposition=false); [[nodiscard]] int nSpecies() const {return static_cast(species_names.size());} @@ -83,4 +83,4 @@ private: amrex::Vector> allcontainers; }; -#endif /*WARPX_MultiFluidContainer_H_*/ \ No newline at end of file +#endif /*WARPX_MultiFluidContainer_H_*/ diff --git a/Source/Fluids/MultiFluidContainer.cpp b/Source/Fluids/MultiFluidContainer.cpp index 4e935e66915..4b0f45341c1 100644 --- a/Source/Fluids/MultiFluidContainer.cpp +++ b/Source/Fluids/MultiFluidContainer.cpp @@ -61,13 +61,13 @@ MultiFluidContainer::DepositCurrent (ablastr::fields::MultiFabRegister& m_fields } void -MultiFluidContainer::Evolve (ablastr::fields::MultiFabRegister& m_fields, +MultiFluidContainer::Evolve (ablastr::fields::MultiFabRegister& m_fields, int lev, std::string current_fp_string, - amrex::Real cur_time, + amrex::Real cur_time, bool skip_deposition) { for (auto& fl : allcontainers) { fl->Evolve(m_fields, lev, current_fp_string, cur_time, skip_deposition); } -} \ No newline at end of file +} diff --git a/Source/Fluids/WarpXFluidContainer.H b/Source/Fluids/WarpXFluidContainer.H index d66c4af896c..58342f5aca0 100644 --- a/Source/Fluids/WarpXFluidContainer.H +++ b/Source/Fluids/WarpXFluidContainer.H @@ -47,10 +47,10 @@ public: /** * Evolve updates a single timestep (dt) of the cold relativistic fluid equations */ - void Evolve (ablastr::fields::MultiFabRegister& m_fields, + void Evolve (ablastr::fields::MultiFabRegister& m_fields, int lev, std::string current_fp_string, - amrex::Real cur_time, + amrex::Real cur_time, bool skip_deposition=false); /** @@ -186,8 +186,8 @@ protected: public: // Names of Multifabs that will be added to the mfs register - std::string name_mf_N = "fluid_density_"+species_name; - std::string name_mf_NU = "fluid_momentum_density_"+species_name; + std::string name_mf_N = "fluid_density_"+species_name; + std::string name_mf_NU = "fluid_momentum_density_"+species_name; }; diff --git a/Source/Fluids/WarpXFluidContainer.cpp b/Source/Fluids/WarpXFluidContainer.cpp index 37f7a1a2d31..18bab9d38d6 100644 --- a/Source/Fluids/WarpXFluidContainer.cpp +++ b/Source/Fluids/WarpXFluidContainer.cpp @@ -19,7 +19,7 @@ using namespace ablastr::utils::communication; using namespace amrex; -WarpXFluidContainer::WarpXFluidContainer(int ispecies, const std::string &name) : +WarpXFluidContainer::WarpXFluidContainer(int ispecies, const std::string &name): species_id{ispecies}, species_name{name} { @@ -141,7 +141,7 @@ void WarpXFluidContainer::AllocateLevelMFs(ablastr::fields::MultiFabRegister& m_ using ablastr::fields::Direction; const int ncomps = 1; const amrex::IntVect nguards(AMREX_D_DECL(2, 2, 2)); - + m_fields.alloc_init( name_mf_N, lev, amrex::convert(ba, amrex::IntVect::TheNodeVector()), dm, ncomps, nguards, 0.0_rt); @@ -157,7 +157,7 @@ void WarpXFluidContainer::AllocateLevelMFs(ablastr::fields::MultiFabRegister& m_ m_fields.alloc_init( name_mf_NU, Direction{2}, lev, amrex::convert(ba, amrex::IntVect::TheNodeVector()), dm, ncomps, nguards, 0.0_rt); - + } void WarpXFluidContainer::InitData(ablastr::fields::MultiFabRegister& m_fields, amrex::Box init_box, amrex::Real cur_time, int lev) @@ -252,10 +252,10 @@ void WarpXFluidContainer::InitData(ablastr::fields::MultiFabRegister& m_fields, void WarpXFluidContainer::Evolve( - ablastr::fields::MultiFabRegister& m_fields, + ablastr::fields::MultiFabRegister& m_fields, int lev, std::string current_fp_string, - amrex::Real cur_time, + amrex::Real cur_time, bool skip_deposition) { using ablastr::fields::Direction; @@ -268,9 +268,9 @@ void WarpXFluidContainer::Evolve( // Step the Lorentz Term if(!do_not_gather){ - GatherAndPush(m_fields, + GatherAndPush(m_fields, *m_fields.get("Efield_aux", Direction{0}, lev), - *m_fields.get("Efield_aux", Direction{1}, lev), + *m_fields.get("Efield_aux", Direction{1}, lev), *m_fields.get("Efield_aux", Direction{2}, lev), *m_fields.get("Bfield_aux", Direction{0}, lev), *m_fields.get("Bfield_aux", Direction{1}, lev), @@ -300,10 +300,10 @@ void WarpXFluidContainer::Evolve( // Deposit J to the simulation mesh if (!skip_deposition && ! do_not_deposit) { - DepositCurrent(m_fields, - *m_fields.get(current_fp_string, Direction{0}, lev), - *m_fields.get(current_fp_string, Direction{1}, lev), - *m_fields.get(current_fp_string, Direction{2}, lev), + DepositCurrent(m_fields, + *m_fields.get(current_fp_string, Direction{0}, lev), + *m_fields.get(current_fp_string, Direction{1}, lev), + *m_fields.get(current_fp_string, Direction{2}, lev), lev); } } From d63c10f288f1b414c716fdcd720c752a07b32417 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 13 Sep 2024 22:34:57 +0000 Subject: [PATCH 234/314] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- Python/pywarpx/fields.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/Python/pywarpx/fields.py b/Python/pywarpx/fields.py index 94603d9af40..0100f64f261 100644 --- a/Python/pywarpx/fields.py +++ b/Python/pywarpx/fields.py @@ -875,19 +875,28 @@ def FaceAreaszWrapper(level=0, include_ghosts=False): def JxFPAmpereWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="hybrid_current_fp_ampere", idir=0, level=level, include_ghosts=include_ghosts + mf_name="hybrid_current_fp_ampere", + idir=0, + level=level, + include_ghosts=include_ghosts, ) def JyFPAmpereWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="hybrid_current_fp_ampere", idir=1, level=level, include_ghosts=include_ghosts + mf_name="hybrid_current_fp_ampere", + idir=1, + level=level, + include_ghosts=include_ghosts, ) def JzFPAmpereWrapper(level=0, include_ghosts=False): return _MultiFABWrapper( - mf_name="hybrid_current_fp_ampere", idir=2, level=level, include_ghosts=include_ghosts + mf_name="hybrid_current_fp_ampere", + idir=2, + level=level, + include_ghosts=include_ghosts, ) From aae5faa04c47d152b67dae3a000167ac406b50f3 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 13 Sep 2024 15:52:26 -0700 Subject: [PATCH 235/314] `WarpXPushFieldsEM`: Clean Unused Vars --- Source/FieldSolver/WarpXPushFieldsEM.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index ca8288f8712..ede04b39632 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -781,11 +781,7 @@ WarpX::PushPSATD () auto Bfield_cp = m_fields.get_mr_levels_alldirs("Bfield_cp", finest_level); // FFT of E and B - PSATDForwardTransformEB( - m_fields.get_mr_levels_alldirs("Efield_fp", finest_level), - m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level), - m_fields.get_mr_levels_alldirs("Efield_cp", finest_level), - m_fields.get_mr_levels_alldirs("Bfield_cp", finest_level) ); + PSATDForwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp ); #ifdef WARPX_DIM_RZ if (pml_rz[0]) { pml_rz[0]->PushPSATD(0); } @@ -1100,7 +1096,6 @@ WarpX::MacroscopicEvolveE (int lev, PatchType patch_type, amrex::Real a_dt) { "Macroscopic EvolveE is not implemented for lev>0, yet." ); - ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); m_fdtd_solver_fp[lev]->MacroscopicEvolveE( m_fields.get_alldirs("Efield_fp", lev), m_fields.get_alldirs("Bfield_fp", lev), From 66009eedd15974b97ff12bdae5a585a12223fed7 Mon Sep 17 00:00:00 2001 From: David Grote Date: Fri, 13 Sep 2024 16:57:38 -0700 Subject: [PATCH 236/314] Remove unused variable --- Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index c1f2e36f49d..ea5517f31aa 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -98,8 +98,6 @@ void WarpX::HybridPICEvolveFields () } } - ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); - // Push the B field from t=n to t=n+1/2 using the current and density // at t=n, while updating the E field along with B using the electron // momentum equation From 9c7a3cfb4744826bd320e29517e23c1f7f78e2d5 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 13 Sep 2024 19:50:47 -0700 Subject: [PATCH 237/314] Reconcile ES code with register code --- .../ElectrostaticSolver.H | 27 ++++---- .../ElectrostaticSolver.cpp | 50 +++++++------- .../ElectrostaticSolvers/LabFrameExplicitES.H | 13 ++-- .../LabFrameExplicitES.cpp | 37 +++++------ .../RelativisticExplicitES.H | 20 ++---- .../RelativisticExplicitES.cpp | 66 ++++++++++--------- Source/FieldSolver/WarpXSolveFieldsES.cpp | 9 +-- Source/Initialization/WarpXInitData.cpp | 22 +++---- Source/WarpX.H | 24 ------- Source/ablastr/fields/PoissonSolver.H | 44 +++++++------ 10 files changed, 143 insertions(+), 169 deletions(-) diff --git a/Source/FieldSolver/ElectrostaticSolvers/ElectrostaticSolver.H b/Source/FieldSolver/ElectrostaticSolvers/ElectrostaticSolver.H index 8d23088799f..e58af394a7a 100644 --- a/Source/FieldSolver/ElectrostaticSolvers/ElectrostaticSolver.H +++ b/Source/FieldSolver/ElectrostaticSolvers/ElectrostaticSolver.H @@ -50,15 +50,10 @@ public: * This function must be defined in the derived classes. */ virtual void ComputeSpaceChargeField ( - [[maybe_unused]] amrex::Vector< std::unique_ptr >& rho_fp, - [[maybe_unused]] amrex::Vector< std::unique_ptr >& rho_cp, - [[maybe_unused]] amrex::Vector< std::unique_ptr >& charge_buf, - [[maybe_unused]] amrex::Vector< std::unique_ptr >& phi_fp, - [[maybe_unused]] MultiParticleContainer& mpc, - [[maybe_unused]] MultiFluidContainer* mfl, - [[maybe_unused]] amrex::Vector< std::array< std::unique_ptr, 3> >& Efield_fp, - [[maybe_unused]] amrex::Vector< std::array< std::unique_ptr, 3> >& Bfield_fp - ) = 0; + ablastr::fields::MultiFabRegister& fields, + MultiParticleContainer& mpc, + MultiFluidContainer* mfl, + int max_level) = 0; /** * \brief Set Dirichlet boundary conditions for the electrostatic solver. @@ -69,7 +64,7 @@ public: * \param[in] idim The dimension for which the Dirichlet boundary condition is set */ void setPhiBC ( - amrex::Vector>& phi, + ablastr::fields::MultiLevelScalarField const& phi, amrex::Real t ) const; @@ -91,8 +86,8 @@ public: * \param[in] verbosity The verbosity setting for the MLMG solver */ void computePhi ( - const amrex::Vector >& rho, - amrex::Vector >& phi, + ablastr::fields::MultiLevelScalarField const& rho, + ablastr::fields::MultiLevelScalarField const& phi, std::array beta, amrex::Real required_precision, amrex::Real absolute_tolerance, @@ -116,8 +111,8 @@ public: * \param[in] beta Represents the velocity of the source of `phi` */ void computeE ( - amrex::Vector, 3> >& E, - const amrex::Vector >& phi, + ablastr::fields::MultiLevelVectorField const& E, + ablastr::fields::MultiLevelScalarField const& phi, std::array beta ) const; @@ -136,8 +131,8 @@ public: *\param[in] beta Represents the velocity of the source of `phi` */ void computeB ( - amrex::Vector, 3> >& B, - const amrex::Vector >& phi, + ablastr::fields::MultiLevelVectorField const& B, + ablastr::fields::MultiLevelScalarField const& phi, std::array beta ) const; diff --git a/Source/FieldSolver/ElectrostaticSolvers/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolvers/ElectrostaticSolver.cpp index 895615a5b21..50a9aab4980 100644 --- a/Source/FieldSolver/ElectrostaticSolvers/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolvers/ElectrostaticSolver.cpp @@ -39,7 +39,7 @@ void ElectrostaticSolver::ReadParameters () { void ElectrostaticSolver::setPhiBC ( - amrex::Vector>& phi, + ablastr::fields::MultiLevelScalarField const& phi, amrex::Real t ) const { @@ -110,19 +110,23 @@ ElectrostaticSolver::setPhiBC ( void -ElectrostaticSolver::computePhi (const amrex::Vector >& rho, - amrex::Vector >& phi, - std::array const beta, - Real const required_precision, - Real absolute_tolerance, - int const max_iters, - int const verbosity) const { +ElectrostaticSolver::computePhi ( + ablastr::fields::MultiLevelScalarField const& rho, + ablastr::fields::MultiLevelScalarField const& phi, + std::array const beta, + Real const required_precision, + Real absolute_tolerance, + int const max_iters, + int const verbosity) const +{ + using ablastr::fields::Direction; + // create a vector to our fields, sorted by level amrex::Vector sorted_rho; amrex::Vector sorted_phi; for (int lev = 0; lev < num_levels; ++lev) { - sorted_rho.emplace_back(rho[lev].get()); - sorted_phi.emplace_back(phi[lev].get()); + sorted_rho.emplace_back(rho[lev]); + sorted_phi.emplace_back(phi[lev]); } std::optional post_phi_calculation; @@ -149,18 +153,18 @@ ElectrostaticSolver::computePhi (const amrex::Vector{ - warpx.getFieldPointer(warpx::fields::FieldType::Efield_fp, lev, 2) + warpx.m_fields.get("Efield_fp", Direction{2}, lev) } #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) amrex::Array{ - warpx.getFieldPointer(warpx::fields::FieldType::Efield_fp, lev, 0), - warpx.getFieldPointer(warpx::fields::FieldType::Efield_fp, lev, 2) + warpx.m_fields.get("Efield_fp", Direction{0}, lev), + warpx.m_fields.get("Efield_fp", Direction{2}, lev) } #elif defined(WARPX_DIM_3D) amrex::Array{ - warpx.getFieldPointer(warpx::fields::FieldType::Efield_fp, lev, 0), - warpx.getFieldPointer(warpx::fields::FieldType::Efield_fp, lev, 1), - warpx.getFieldPointer(warpx::fields::FieldType::Efield_fp, lev, 2) + warpx.m_fields.get("Efield_fp", Direction{0}, lev), + warpx.m_fields.get("Efield_fp", Direction{1}, lev), + warpx.m_fields.get("Efield_fp", Direction{2}, lev) } #endif ); @@ -206,9 +210,10 @@ ElectrostaticSolver::computePhi (const amrex::Vector, 3> >& E, - const amrex::Vector >& phi, - std::array const beta ) const +ElectrostaticSolver::computeE ( + ablastr::fields::MultiLevelVectorField const& E, + ablastr::fields::MultiLevelScalarField const& phi, + std::array beta ) const { auto & warpx = WarpX::GetInstance(); for (int lev = 0; lev < num_levels; lev++) { @@ -369,9 +374,10 @@ ElectrostaticSolver::computeE (amrex::Vector, 3> >& B, - const amrex::Vector >& phi, - std::array const beta ) const +void ElectrostaticSolver::computeB ( + ablastr::fields::MultiLevelVectorField const& B, + ablastr::fields::MultiLevelScalarField const& phi, + std::array beta) const { // return early if beta is 0 since there will be no B-field if ((beta[0] == 0._rt) && (beta[1] == 0._rt) && (beta[2] == 0._rt)) { return; } diff --git a/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.H b/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.H index 7dc41f0a056..c919b138c00 100644 --- a/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.H +++ b/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.H @@ -22,19 +22,14 @@ public: void InitData () override; void ComputeSpaceChargeField ( - amrex::Vector< std::unique_ptr >& rho_fp, - amrex::Vector< std::unique_ptr >& rho_cp, - amrex::Vector< std::unique_ptr >& charge_buf, - amrex::Vector< std::unique_ptr >& phi_fp, + ablastr::fields::MultiFabRegister& fields, MultiParticleContainer& mpc, MultiFluidContainer* mfl, - amrex::Vector< std::array< std::unique_ptr, 3> >& Efield_fp, - amrex::Vector< std::array< std::unique_ptr, 3> >& Bfield_fp - ) override; + int max_level) override; void computePhiTriDiagonal ( - const amrex::Vector >& rho, - amrex::Vector >& phi + const ablastr::fields::MultiLevelScalarField& rho, + ablastr::fields::MultiLevelScalarField& phi ); }; diff --git a/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp b/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp index d14abd1848a..de9f20a54aa 100644 --- a/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp +++ b/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp @@ -21,35 +21,34 @@ void LabFrameExplicitES::InitData() { } void LabFrameExplicitES::ComputeSpaceChargeField ( - amrex::Vector< std::unique_ptr >& rho_fp, - amrex::Vector< std::unique_ptr >& rho_cp, - amrex::Vector< std::unique_ptr >& charge_buf, - amrex::Vector< std::unique_ptr >& phi_fp, + ablastr::fields::MultiFabRegister& fields, MultiParticleContainer& mpc, MultiFluidContainer* mfl, - amrex::Vector< std::array< std::unique_ptr, 3> >& Efield_fp, - amrex::Vector< std::array< std::unique_ptr, 3> >& /*Bfield_fp*/ -) { + int max_level) +{ + using ablastr::fields::MultiLevelScalarField; + using ablastr::fields::MultiLevelVectorField; + + MultiLevelScalarField rho_fp = fields.get_mr_levels("rho_fp", max_level); + MultiLevelScalarField rho_cp = fields.get_mr_levels("rho_cp", max_level); + MultiLevelScalarField phi_fp = fields.get_mr_levels("phi_fp", max_level); + MultiLevelVectorField Efield_fp = fields.get_mr_levels_alldirs("Efield_fp", max_level); + mpc.DepositCharge(rho_fp, 0.0_rt); if (mfl) { const int lev = 0; - mfl->DepositCharge(lev, *rho_fp[lev]); + mfl->DepositCharge(fields, *rho_fp[lev], lev); } + // Apply filter, perform MPI exchange, interpolate across levels + Vector > rho_buf(num_levels); auto & warpx = WarpX::GetInstance(); - for (int lev = 0; lev < num_levels; lev++) { - if (lev > 0) { - if (charge_buf[lev]) { - charge_buf[lev]->setVal(0.); - } - } - } - warpx.SyncRho(rho_fp, rho_cp, charge_buf); // Apply filter, perform MPI exchange, interpolate across levels + warpx.SyncRho( rho_fp, rho_cp, amrex::GetVecOfPtrs(rho_buf) ); #ifndef WARPX_DIM_RZ for (int lev = 0; lev < num_levels; lev++) { // Reflect density over PEC boundaries, if needed. - warpx.ApplyRhofieldBoundary(lev, rho_fp[lev].get(), PatchType::fine); + warpx.ApplyRhofieldBoundary(lev, rho_fp[lev], PatchType::fine); } #endif // beta is zero in lab frame @@ -94,8 +93,8 @@ void LabFrameExplicitES::ComputeSpaceChargeField ( \param[out] phi The potential to be computed by this function */ void LabFrameExplicitES::computePhiTriDiagonal ( - const amrex::Vector >& rho, - amrex::Vector >& phi) + const ablastr::fields::MultiLevelScalarField& rho, + ablastr::fields::MultiLevelScalarField& phi) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE(num_levels == 1, "The tridiagonal solver cannot be used with mesh refinement"); diff --git a/Source/FieldSolver/ElectrostaticSolvers/RelativisticExplicitES.H b/Source/FieldSolver/ElectrostaticSolvers/RelativisticExplicitES.H index 70382d7ced5..cf831a7ab10 100644 --- a/Source/FieldSolver/ElectrostaticSolvers/RelativisticExplicitES.H +++ b/Source/FieldSolver/ElectrostaticSolvers/RelativisticExplicitES.H @@ -44,15 +44,10 @@ public: * \param[in,out] Bfield Field contribution from phi computed from each species' charge density is added */ void ComputeSpaceChargeField ( - [[maybe_unused]] amrex::Vector< std::unique_ptr >& rho_fp, - [[maybe_unused]] amrex::Vector< std::unique_ptr >& rho_cp, - amrex::Vector< std::unique_ptr >& charge_buf, - amrex::Vector< std::unique_ptr >& phi_fp, - MultiParticleContainer& mpc, - [[maybe_unused]] MultiFluidContainer* mfl, - amrex::Vector< std::array< std::unique_ptr, 3> >& Efield_fp, - amrex::Vector< std::array< std::unique_ptr, 3> >& Bfield_fp - ) override; + ablastr::fields::MultiFabRegister& fields, + MultiParticleContainer& mpc, + MultiFluidContainer* mfl, + int max_level) override; /** * Compute the charge density of the species paricle container, pc, @@ -65,10 +60,9 @@ public: * \param[in] Bfield Bfield updated to include potential computed for selected species charge density as source */ void AddSpaceChargeField ( - amrex::Vector >& charge_buf, WarpXParticleContainer& pc, - amrex::Vector, 3>>& Efield, - amrex::Vector, 3>>& Bfield + ablastr::fields::MultiLevelVectorField& Efield_fp, + ablastr::fields::MultiLevelVectorField& Bfield_fp ); /** Compute the potential `phi` by solving the Poisson equation with the @@ -77,7 +71,7 @@ public: * \param[in] Efield Efield updated to include potential gradient from boundary condition */ void AddBoundaryField ( - amrex::Vector, 3>>& Efield + ablastr::fields::MultiLevelVectorField& Efield ); }; diff --git a/Source/FieldSolver/ElectrostaticSolvers/RelativisticExplicitES.cpp b/Source/FieldSolver/ElectrostaticSolvers/RelativisticExplicitES.cpp index 1660efd48c2..4e0b0fa0d4b 100644 --- a/Source/FieldSolver/ElectrostaticSolvers/RelativisticExplicitES.cpp +++ b/Source/FieldSolver/ElectrostaticSolvers/RelativisticExplicitES.cpp @@ -30,28 +30,29 @@ void RelativisticExplicitES::InitData () { } void RelativisticExplicitES::ComputeSpaceChargeField ( - amrex::Vector< std::unique_ptr >& rho_fp, - amrex::Vector< std::unique_ptr >& rho_cp, - amrex::Vector< std::unique_ptr >& charge_buf, - amrex::Vector< std::unique_ptr >& phi_fp, + ablastr::fields::MultiFabRegister& fields, MultiParticleContainer& mpc, MultiFluidContainer* mfl, - amrex::Vector< std::array< std::unique_ptr, 3> >& Efield_fp, - amrex::Vector< std::array< std::unique_ptr, 3> >& Bfield_fp -) { + int max_level) +{ + using ablastr::fields::MultiLevelVectorField; + WARPX_PROFILE("RelativisticExplicitES::ComputeSpaceChargeField"); - amrex::ignore_unused(rho_fp, rho_cp, phi_fp, mfl); const bool always_run_solve = (WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::Relativistic); + MultiLevelVectorField Efield_fp = fields.get_mr_levels_alldirs("Efield_fp", max_level); + MultiLevelVectorField Bfield_fp = fields.get_mr_levels_alldirs("Bfield_fp", max_level); + // Loop over the species and add their space-charge contribution to E and B. // Note that the fields calculated here does not include the E field // due to simulation boundary potentials for (auto const& species : mpc) { if (always_run_solve || (species->initialize_self_fields)) { - AddSpaceChargeField(charge_buf, *species, Efield_fp, Bfield_fp); + AddSpaceChargeField(*species, Efield_fp, Bfield_fp); } } + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( !(mfl), "Fluid species are not supported with the relativistic electrostatic solver"); // Add the field due to the boundary potentials if (always_run_solve || (m_poisson_boundary_handler->m_boundary_potential_specified)) @@ -61,10 +62,9 @@ void RelativisticExplicitES::ComputeSpaceChargeField ( } void RelativisticExplicitES::AddSpaceChargeField ( - amrex::Vector >& charge_buf, WarpXParticleContainer& pc, - amrex::Vector, 3>>& Efield_fp, - amrex::Vector, 3>>& Bfield_fp) + ablastr::fields::MultiLevelVectorField& Efield_fp, + ablastr::fields::MultiLevelVectorField& Bfield_fp) { WARPX_PROFILE("RelativisticExplicitES::AddSpaceChargeField"); @@ -78,9 +78,9 @@ void RelativisticExplicitES::AddSpaceChargeField ( auto & warpx = WarpX::GetInstance(); // Allocate fields for charge and potential - Vector > rho(num_levels); - Vector > rho_coarse(num_levels); // Used in order to interpolate between levels - Vector > phi(num_levels); + Vector> rho(num_levels); + Vector> rho_coarse(num_levels); // Used in order to interpolate between levels + Vector> phi(num_levels); // Use number of guard cells used for local deposition of rho const amrex::IntVect ng = warpx.get_ng_depos_rho(); for (int lev = 0; lev < num_levels; lev++) { @@ -96,9 +96,6 @@ void RelativisticExplicitES::AddSpaceChargeField ( cba.coarsen(warpx.refRatio(lev-1)); rho_coarse[lev] = std::make_unique(cba, warpx.DistributionMap(lev), 1, ng); rho_coarse[lev]->setVal(0.); - if (charge_buf[lev]) { - charge_buf[lev]->setVal(0.); - } } } // Deposit particle charge density (source of Poisson solver) @@ -108,10 +105,17 @@ void RelativisticExplicitES::AddSpaceChargeField ( bool const apply_boundary_and_scale_volume = true; bool const interpolate_across_levels = false; if ( !pc.do_not_deposit) { - pc.DepositCharge(rho, local, reset, apply_boundary_and_scale_volume, - interpolate_across_levels); + pc.DepositCharge(amrex::GetVecOfPtrs(rho), + local, reset, apply_boundary_and_scale_volume, + interpolate_across_levels); } - warpx.SyncRho(rho, rho_coarse, charge_buf); // Apply filter, perform MPI exchange, interpolate across levels + + // Apply filter, perform MPI exchange, interpolate across levels + Vector> rho_buf(num_levels); + warpx.SyncRho( + amrex::GetVecOfPtrs(rho), + amrex::GetVecOfPtrs(rho_coarse), + amrex::GetVecOfPtrs(rho_buf)); // Get the particle beta vector bool const local_average = false; // Average across all MPI ranks @@ -122,25 +126,26 @@ void RelativisticExplicitES::AddSpaceChargeField ( } // Compute the potential phi, by solving the Poisson equation - computePhi( rho, phi, beta, pc.self_fields_required_precision, + computePhi( amrex::GetVecOfPtrs(rho), amrex::GetVecOfPtrs(phi), + beta, pc.self_fields_required_precision, pc.self_fields_absolute_tolerance, pc.self_fields_max_iters, pc.self_fields_verbosity ); // Compute the corresponding electric and magnetic field, from the potential phi - computeE( Efield_fp, phi, beta ); - computeB( Bfield_fp, phi, beta ); + computeE( Efield_fp, amrex::GetVecOfPtrs(phi), beta ); + computeB( Bfield_fp, amrex::GetVecOfPtrs(phi), beta ); } -void RelativisticExplicitES::AddBoundaryField (amrex::Vector, 3>>& Efield_fp) +void RelativisticExplicitES::AddBoundaryField (ablastr::fields::MultiLevelVectorField& Efield_fp) { WARPX_PROFILE("RelativisticExplicitES::AddBoundaryField"); auto & warpx = WarpX::GetInstance(); // Allocate fields for charge and potential - amrex::Vector > rho(num_levels); - amrex::Vector > phi(num_levels); + Vector> rho(num_levels); + Vector> phi(num_levels); // Use number of guard cells used for local deposition of rho const amrex::IntVect ng = warpx.get_ng_depos_rho(); for (int lev = 0; lev < num_levels; lev++) { @@ -153,16 +158,17 @@ void RelativisticExplicitES::AddBoundaryField (amrex::Vector beta = {0._rt}; // Compute the potential phi, by solving the Poisson equation - computePhi( rho, phi, beta, self_fields_required_precision, + computePhi( amrex::GetVecOfPtrs(rho), amrex::GetVecOfPtrs(phi), + beta, self_fields_required_precision, self_fields_absolute_tolerance, self_fields_max_iters, self_fields_verbosity ); // Compute the corresponding electric field, from the potential phi. - computeE( Efield_fp, phi, beta ); + computeE( Efield_fp, amrex::GetVecOfPtrs(phi), beta ); } diff --git a/Source/FieldSolver/WarpXSolveFieldsES.cpp b/Source/FieldSolver/WarpXSolveFieldsES.cpp index 42a537b5c2a..32362204b4a 100644 --- a/Source/FieldSolver/WarpXSolveFieldsES.cpp +++ b/Source/FieldSolver/WarpXSolveFieldsES.cpp @@ -13,18 +13,19 @@ void WarpX::ComputeSpaceChargeField (bool const reset_fields) { WARPX_PROFILE("WarpX::ComputeSpaceChargeField"); + using ablastr::fields::Direction; + if (reset_fields) { // Reset all E and B fields to 0, before calculating space-charge fields WARPX_PROFILE("WarpX::ComputeSpaceChargeField::reset_fields"); for (int lev = 0; lev <= max_level; lev++) { for (int comp=0; comp<3; comp++) { - Efield_fp[lev][comp]->setVal(0); - Bfield_fp[lev][comp]->setVal(0); + m_fields.get("Efield_fp",Direction{comp},lev)->setVal(0); + m_fields.get("Bfield_fp",Direction{comp},lev)->setVal(0); } } } m_electrostatic_solver->ComputeSpaceChargeField( - rho_fp, rho_cp, charge_buf, phi_fp, *mypc, myfl.get(), Efield_fp, Bfield_fp - ); + m_fields, *mypc, myfl.get(), max_level ); } diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index d2c1a8ddf8f..56f4de1a561 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -634,7 +634,7 @@ WarpX::AddExternalFields (int const lev) { // FIXME: RZ multimode has more than one component for all these if (m_p_ext_field_params->E_ext_grid_type != ExternalFieldType::default_zero) { - ablastr::fields::MultiLevelVectorField Efield_fp = m_fields.get_mr_levels_alldirs("Efield_fp",finest_level); + ablastr::fields::MultiLevelVectorField Efield_fp = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::constant) { Efield_fp[lev][0]->plus(m_p_ext_field_params->E_external_grid[0], guard_cells.ng_alloc_EB.min()); Efield_fp[lev][1]->plus(m_p_ext_field_params->E_external_grid[1], guard_cells.ng_alloc_EB.min()); @@ -647,7 +647,7 @@ WarpX::AddExternalFields (int const lev) { } } if (m_p_ext_field_params->B_ext_grid_type != ExternalFieldType::default_zero) { - ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); + ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", max_level); if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::constant) { Bfield_fp[lev][0]->plus(m_p_ext_field_params->B_external_grid[0], guard_cells.ng_alloc_EB.min()); Bfield_fp[lev][1]->plus(m_p_ext_field_params->B_external_grid[1], guard_cells.ng_alloc_EB.min()); @@ -716,7 +716,7 @@ WarpX::InitPML () do_pml_Hi[0][idim] = 1; // on level 0 } } - if (finest_level > 0) { do_pml = 1; } + if (max_level > 0) { do_pml = 1; } if (do_pml) { bool const eb_enabled = EB::enabled(); @@ -741,7 +741,7 @@ WarpX::InitPML () do_pml_Lo[0], do_pml_Hi[0]); #endif - for (int lev = 1; lev <= finest_level; ++lev) + for (int lev = 1; lev <= max_level; ++lev) { do_pml_Lo[lev] = amrex::IntVect::TheUnitVector(); do_pml_Hi[lev] = amrex::IntVect::TheUnitVector(); @@ -788,7 +788,7 @@ WarpX::ComputePMLFactors () { if (do_pml) { - for (int lev = 0; lev <= finest_level; ++lev) + for (int lev = 0; lev <= max_level; ++lev) { if (pml[lev]) { pml[lev]->ComputePMLFactors(dt[lev]); @@ -907,8 +907,8 @@ WarpX::InitLevelData (int lev, Real /*time*/) { using ablastr::fields::Direction; - ablastr::fields::MultiLevelVectorField Efield_aux = m_fields.get_mr_levels_alldirs("Efield_aux", finest_level); - ablastr::fields::MultiLevelVectorField Bfield_aux = m_fields.get_mr_levels_alldirs("Bfield_aux", finest_level); + ablastr::fields::MultiLevelVectorField Efield_aux = m_fields.get_mr_levels_alldirs("Efield_aux", max_level); + ablastr::fields::MultiLevelVectorField Bfield_aux = m_fields.get_mr_levels_alldirs("Bfield_aux", max_level); // initialize the averaged fields only if the averaged algorithm // is activated ('psatd.do_time_averaging=1') @@ -990,7 +990,7 @@ WarpX::InitLevelData (int lev, Real /*time*/) m_p_ext_field_params->Byfield_parser->compile<3>(), m_p_ext_field_params->Bzfield_parser->compile<3>(), m_fields.get_alldirs("edge_lengths", lev), - m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev], + m_fields.get_mr_levels_alldirs("face_areas", max_level)[lev], 'B', lev, PatchType::coarse); } @@ -1010,7 +1010,7 @@ WarpX::InitLevelData (int lev, Real /*time*/) m_fdtd_solver_fp[lev]->EvolveECTRho( m_fields.get_alldirs("Efield_fp",lev), m_fields.get_alldirs("edge_lengths", lev), - m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev], + m_fields.get_mr_levels_alldirs("face_areas", max_level)[lev], m_fields.get_alldirs("ECTRhofield", lev), lev); } @@ -1048,7 +1048,7 @@ WarpX::InitLevelData (int lev, Real /*time*/) m_fdtd_solver_cp[lev]->EvolveECTRho( m_fields.get_alldirs("Efield_cp",lev), m_fields.get_alldirs("edge_lengths", lev), - m_fields.get_mr_levels_alldirs("face_areas", finest_level)[lev], + m_fields.get_mr_levels_alldirs("face_areas", max_level)[lev], m_fields.get_alldirs("ECTRhofield", lev), lev); } @@ -1232,7 +1232,7 @@ WarpX::InitializeExternalFieldsOnGridUsingParser ( void WarpX::CheckGuardCells() { - for (int lev = 0; lev <= finest_level; ++lev) + for (int lev = 0; lev <= max_level; ++lev) { for (int dim = 0; dim < 3; ++dim) { diff --git a/Source/WarpX.H b/Source/WarpX.H index 9b475177de9..40ad6f031c3 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -922,30 +922,6 @@ public: /** Electrostatic solve call */ void ComputeSpaceChargeField (bool reset_fields); -<<<<<<< HEAD - void AddBoundaryField (); - void AddSpaceChargeField (WarpXParticleContainer& pc); - void AddSpaceChargeFieldLabFrame (); - void computePhi (const ablastr::fields::MultiLevelScalarField& rho, - ablastr::fields::MultiLevelScalarField& phi, - std::array beta = {{0,0,0}}, - amrex::Real required_precision=amrex::Real(1.e-11), - amrex::Real absolute_tolerance=amrex::Real(0.0), - int max_iters=200, - int verbosity=2); - - void setPhiBC (ablastr::fields::MultiLevelScalarField& phi ) const; - - void computeE (ablastr::fields::MultiLevelVectorField& E, - const ablastr::fields::MultiLevelScalarField& phi, - std::array beta = {{0,0,0}} ) const; - void computeB (ablastr::fields::MultiLevelVectorField& B, - const ablastr::fields::MultiLevelScalarField& phi, - std::array beta = {{0,0,0}} ) const; - void computePhiTriDiagonal (const ablastr::fields::MultiLevelScalarField& rho, - ablastr::fields::MultiLevelScalarField& phi) const; -======= ->>>>>>> development // Magnetostatic Solver Interface MagnetostaticSolver::VectorPoissonBoundaryHandler m_vector_poisson_boundary_handler; diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index ca4c3bfeccb..727280d630b 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -66,7 +66,7 @@ namespace ablastr::fields { * \param[out] max_norm_rho The maximum L-infinity norm of `rho` across all levels */ inline amrex::Real getMaxNormRho ( - amrex::Vector const & rho, + ablastr::fields::ConstMultiLevelScalarField const& rho, int finest_level, amrex::Real & absolute_tolerance) { @@ -191,25 +191,26 @@ template< typename T_FArrayBoxFactory = void > void -computePhi (amrex::Vector const & rho, - ablastr::fields::MultiLevelScalarField & phi, - std::array const beta, - amrex::Real relative_tolerance, - amrex::Real absolute_tolerance, - int max_iters, - int verbosity, - amrex::Vector const& geom, - amrex::Vector const& dmap, - amrex::Vector const& grids, - utils::enums::GridType grid_type, - T_BoundaryHandler const boundary_handler, - bool is_solver_igf_on_lev0, - bool eb_enabled = false, - bool do_single_precision_comms = false, - std::optional > rel_ref_ratio = std::nullopt, - [[maybe_unused]] T_PostPhiCalculationFunctor post_phi_calculation = std::nullopt, - [[maybe_unused]] std::optional current_time = std::nullopt, // only used for EB - [[maybe_unused]] std::optional > eb_farray_box_factory = std::nullopt // only used for EB +computePhi ( + ablastr::fields::MultiLevelScalarField const& rho, + ablastr::fields::MultiLevelScalarField const& phi, + std::array const beta, + amrex::Real relative_tolerance, + amrex::Real absolute_tolerance, + int max_iters, + int verbosity, + amrex::Vector const& geom, + amrex::Vector const& dmap, + amrex::Vector const& grids, + utils::enums::GridType grid_type, + T_BoundaryHandler const boundary_handler, + bool is_solver_igf_on_lev0, + bool eb_enabled = false, + bool do_single_precision_comms = false, + std::optional > rel_ref_ratio = std::nullopt, + [[maybe_unused]] T_PostPhiCalculationFunctor post_phi_calculation = std::nullopt, + [[maybe_unused]] std::optional current_time = std::nullopt, // only used for EB + [[maybe_unused]] std::optional > eb_farray_box_factory = std::nullopt // only used for EB ) { using namespace amrex::literals; @@ -250,7 +251,8 @@ computePhi (amrex::Vector const & rho, #endif // determine if rho is zero everywhere - const amrex::Real max_norm_b = getMaxNormRho(rho, finest_level, absolute_tolerance); + const amrex::Real max_norm_b = getMaxNormRho( + amrex::GetVecOfConstPtrs(rho), finest_level, absolute_tolerance); amrex::LPInfo info; From 8f3b973a7921a3e77907b871d35fb47bb0823a10 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Sat, 14 Sep 2024 06:13:30 -0700 Subject: [PATCH 238/314] Rename m_fluids to fluids in FluidContainer functions --- .../RelativisticExplicitES.cpp | 3 +- Source/Fluids/MultiFluidContainer.H | 2 +- Source/Fluids/MultiFluidContainer.cpp | 4 +- Source/Fluids/WarpXFluidContainer.H | 2 +- Source/Fluids/WarpXFluidContainer.cpp | 194 +++++++++--------- 5 files changed, 102 insertions(+), 103 deletions(-) diff --git a/Source/FieldSolver/ElectrostaticSolvers/RelativisticExplicitES.cpp b/Source/FieldSolver/ElectrostaticSolvers/RelativisticExplicitES.cpp index 4e0b0fa0d4b..3c9a15c931b 100644 --- a/Source/FieldSolver/ElectrostaticSolvers/RelativisticExplicitES.cpp +++ b/Source/FieldSolver/ElectrostaticSolvers/RelativisticExplicitES.cpp @@ -32,7 +32,7 @@ void RelativisticExplicitES::InitData () { void RelativisticExplicitES::ComputeSpaceChargeField ( ablastr::fields::MultiFabRegister& fields, MultiParticleContainer& mpc, - MultiFluidContainer* mfl, + [[maybe_unused]] MultiFluidContainer* mfl, int max_level) { using ablastr::fields::MultiLevelVectorField; @@ -52,7 +52,6 @@ void RelativisticExplicitES::ComputeSpaceChargeField ( AddSpaceChargeField(*species, Efield_fp, Bfield_fp); } } - AMREX_ALWAYS_ASSERT_WITH_MESSAGE( !(mfl), "Fluid species are not supported with the relativistic electrostatic solver"); // Add the field due to the boundary potentials if (always_run_solve || (m_poisson_boundary_handler->m_boundary_potential_specified)) diff --git a/Source/Fluids/MultiFluidContainer.H b/Source/Fluids/MultiFluidContainer.H index 9d55f975754..3bc9f1488c8 100644 --- a/Source/Fluids/MultiFluidContainer.H +++ b/Source/Fluids/MultiFluidContainer.H @@ -62,7 +62,7 @@ public: /// This evolves all the fluids by one PIC time step, including current deposition, the /// field solve, and pushing the fluids, for all the species in the MultiFluidContainer. /// - void Evolve (ablastr::fields::MultiFabRegister& m_fields, + void Evolve (ablastr::fields::MultiFabRegister& fields, int lev, std::string current_fp_string, amrex::Real cur_time, diff --git a/Source/Fluids/MultiFluidContainer.cpp b/Source/Fluids/MultiFluidContainer.cpp index 4b0f45341c1..a7a00416ad1 100644 --- a/Source/Fluids/MultiFluidContainer.cpp +++ b/Source/Fluids/MultiFluidContainer.cpp @@ -61,13 +61,13 @@ MultiFluidContainer::DepositCurrent (ablastr::fields::MultiFabRegister& m_fields } void -MultiFluidContainer::Evolve (ablastr::fields::MultiFabRegister& m_fields, +MultiFluidContainer::Evolve (ablastr::fields::MultiFabRegister& fields, int lev, std::string current_fp_string, amrex::Real cur_time, bool skip_deposition) { for (auto& fl : allcontainers) { - fl->Evolve(m_fields, lev, current_fp_string, cur_time, skip_deposition); + fl->Evolve(fields, lev, current_fp_string, cur_time, skip_deposition); } } diff --git a/Source/Fluids/WarpXFluidContainer.H b/Source/Fluids/WarpXFluidContainer.H index 58342f5aca0..dbbe8267542 100644 --- a/Source/Fluids/WarpXFluidContainer.H +++ b/Source/Fluids/WarpXFluidContainer.H @@ -47,7 +47,7 @@ public: /** * Evolve updates a single timestep (dt) of the cold relativistic fluid equations */ - void Evolve (ablastr::fields::MultiFabRegister& m_fields, + void Evolve (ablastr::fields::MultiFabRegister& fields, int lev, std::string current_fp_string, amrex::Real cur_time, diff --git a/Source/Fluids/WarpXFluidContainer.cpp b/Source/Fluids/WarpXFluidContainer.cpp index 18bab9d38d6..7d7d56ef62b 100644 --- a/Source/Fluids/WarpXFluidContainer.cpp +++ b/Source/Fluids/WarpXFluidContainer.cpp @@ -136,31 +136,31 @@ void WarpXFluidContainer::ReadParameters() } } -void WarpXFluidContainer::AllocateLevelMFs(ablastr::fields::MultiFabRegister& m_fields, const BoxArray &ba, const DistributionMapping &dm, int lev) +void WarpXFluidContainer::AllocateLevelMFs(ablastr::fields::MultiFabRegister& fields, const BoxArray &ba, const DistributionMapping &dm, int lev) { using ablastr::fields::Direction; const int ncomps = 1; const amrex::IntVect nguards(AMREX_D_DECL(2, 2, 2)); - m_fields.alloc_init( + fields.alloc_init( name_mf_N, lev, amrex::convert(ba, amrex::IntVect::TheNodeVector()), dm, ncomps, nguards, 0.0_rt); - m_fields.alloc_init( + fields.alloc_init( name_mf_NU, Direction{0}, lev, amrex::convert(ba, amrex::IntVect::TheNodeVector()), dm, ncomps, nguards, 0.0_rt); - m_fields.alloc_init( + fields.alloc_init( name_mf_NU, Direction{1}, lev, amrex::convert(ba, amrex::IntVect::TheNodeVector()), dm, ncomps, nguards, 0.0_rt); - m_fields.alloc_init( + fields.alloc_init( name_mf_NU, Direction{2}, lev, amrex::convert(ba, amrex::IntVect::TheNodeVector()), dm, ncomps, nguards, 0.0_rt); } -void WarpXFluidContainer::InitData(ablastr::fields::MultiFabRegister& m_fields, amrex::Box init_box, amrex::Real cur_time, int lev) +void WarpXFluidContainer::InitData(ablastr::fields::MultiFabRegister& fields, amrex::Box init_box, amrex::Real cur_time, int lev) { using ablastr::fields::Direction; WARPX_PROFILE("WarpXFluidContainer::InitData"); @@ -185,14 +185,14 @@ void WarpXFluidContainer::InitData(ablastr::fields::MultiFabRegister& m_fields, #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*m_fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(*fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) { - amrex::Box const tile_box = mfi.tilebox(m_fields.get(name_mf_N, lev)->ixType().toIntVect()); - amrex::Array4 const &N_arr = m_fields.get(name_mf_N, lev)->array(mfi); - amrex::Array4 const &NUx_arr = m_fields.get(name_mf_NU, Direction{0}, lev)->array(mfi); - amrex::Array4 const &NUy_arr = m_fields.get(name_mf_NU, Direction{1}, lev)->array(mfi); - amrex::Array4 const &NUz_arr = m_fields.get(name_mf_NU, Direction{2}, lev)->array(mfi); + amrex::Box const tile_box = mfi.tilebox(fields.get(name_mf_N, lev)->ixType().toIntVect()); + amrex::Array4 const &N_arr = fields.get(name_mf_N, lev)->array(mfi); + amrex::Array4 const &NUx_arr = fields.get(name_mf_NU, Direction{0}, lev)->array(mfi); + amrex::Array4 const &NUy_arr = fields.get(name_mf_NU, Direction{1}, lev)->array(mfi); + amrex::Array4 const &NUz_arr = fields.get(name_mf_NU, Direction{2}, lev)->array(mfi); // Return the intersection of all cells and the ones we wish to update amrex::Box const init_box_intersection = init_box & tile_box; @@ -252,7 +252,7 @@ void WarpXFluidContainer::InitData(ablastr::fields::MultiFabRegister& m_fields, void WarpXFluidContainer::Evolve( - ablastr::fields::MultiFabRegister& m_fields, + ablastr::fields::MultiFabRegister& fields, int lev, std::string current_fp_string, amrex::Real cur_time, @@ -261,55 +261,55 @@ void WarpXFluidContainer::Evolve( using ablastr::fields::Direction; WARPX_PROFILE("WarpXFluidContainer::Evolve"); - if (m_fields.get("rho_fp",lev) && ! skip_deposition && ! do_not_deposit) { + if (fields.get("rho_fp",lev) && ! skip_deposition && ! do_not_deposit) { // Deposit charge before particle push, in component 0 of MultiFab rho. - DepositCharge(m_fields, *m_fields.get("rho_fp",lev), lev, 0); + DepositCharge(fields, *fields.get("rho_fp",lev), lev, 0); } // Step the Lorentz Term if(!do_not_gather){ - GatherAndPush(m_fields, - *m_fields.get("Efield_aux", Direction{0}, lev), - *m_fields.get("Efield_aux", Direction{1}, lev), - *m_fields.get("Efield_aux", Direction{2}, lev), - *m_fields.get("Bfield_aux", Direction{0}, lev), - *m_fields.get("Bfield_aux", Direction{1}, lev), - *m_fields.get("Bfield_aux", Direction{2}, lev), + GatherAndPush(fields, + *fields.get("Efield_aux", Direction{0}, lev), + *fields.get("Efield_aux", Direction{1}, lev), + *fields.get("Efield_aux", Direction{2}, lev), + *fields.get("Bfield_aux", Direction{0}, lev), + *fields.get("Bfield_aux", Direction{1}, lev), + *fields.get("Bfield_aux", Direction{2}, lev), cur_time, lev); } // Cylindrical centrifugal term if(!do_not_push){ #if defined(WARPX_DIM_RZ) - centrifugal_source_rz(m_fields, lev); + centrifugal_source_rz(fields, lev); #endif // Apply (non-periodic) BC on the fluids (needed for spatial derivative), // and communicate N, NU at boundaries - ApplyBcFluidsAndComms(m_fields, lev); + ApplyBcFluidsAndComms(fields, lev); // Step the Advective term - AdvectivePush_Muscl(m_fields, lev); + AdvectivePush_Muscl(fields, lev); } // Deposit rho to the simulation mesh // Deposit charge (end of the step) - if (m_fields.get("rho_fp",lev) && ! skip_deposition && ! do_not_deposit) { - DepositCharge(m_fields, *m_fields.get("rho_fp",lev), lev, 1); + if (fields.get("rho_fp",lev) && ! skip_deposition && ! do_not_deposit) { + DepositCharge(fields, *fields.get("rho_fp",lev), lev, 1); } // Deposit J to the simulation mesh if (!skip_deposition && ! do_not_deposit) { - DepositCurrent(m_fields, - *m_fields.get(current_fp_string, Direction{0}, lev), - *m_fields.get(current_fp_string, Direction{1}, lev), - *m_fields.get(current_fp_string, Direction{2}, lev), + DepositCurrent(fields, + *fields.get(current_fp_string, Direction{0}, lev), + *fields.get(current_fp_string, Direction{1}, lev), + *fields.get(current_fp_string, Direction{2}, lev), lev); } } // Momentum source due to curvature -void WarpXFluidContainer::ApplyBcFluidsAndComms (ablastr::fields::MultiFabRegister& m_fields, int lev) +void WarpXFluidContainer::ApplyBcFluidsAndComms (ablastr::fields::MultiFabRegister& fields, int lev) { using ablastr::fields::Direction; WARPX_PROFILE("WarpXFluidContainer::ApplyBcFluidsAndComms"); @@ -326,15 +326,15 @@ void WarpXFluidContainer::ApplyBcFluidsAndComms (ablastr::fields::MultiFabRegist #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*m_fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(*fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) { - amrex::Box tile_box = mfi.tilebox(m_fields.get(name_mf_N, lev)->ixType().toIntVect()); + amrex::Box tile_box = mfi.tilebox(fields.get(name_mf_N, lev)->ixType().toIntVect()); - const amrex::Array4 N_arr = m_fields.get(name_mf_N, lev)->array(mfi); - const amrex::Array4 NUx_arr = m_fields.get(name_mf_NU, Direction{0}, lev)->array(mfi); - const amrex::Array4 NUy_arr = m_fields.get(name_mf_NU, Direction{1}, lev)->array(mfi); - const amrex::Array4 NUz_arr = m_fields.get(name_mf_NU, Direction{2}, lev)->array(mfi); + const amrex::Array4 N_arr = fields.get(name_mf_N, lev)->array(mfi); + const amrex::Array4 NUx_arr = fields.get(name_mf_NU, Direction{0}, lev)->array(mfi); + const amrex::Array4 NUy_arr = fields.get(name_mf_NU, Direction{1}, lev)->array(mfi); + const amrex::Array4 NUz_arr = fields.get(name_mf_NU, Direction{2}, lev)->array(mfi); //Grow the tilebox tile_box.grow(1); @@ -406,14 +406,14 @@ void WarpXFluidContainer::ApplyBcFluidsAndComms (ablastr::fields::MultiFabRegist } // Fill guard cells - FillBoundary(*m_fields.get(name_mf_N, lev), m_fields.get(name_mf_N, lev)->nGrowVect(), WarpX::do_single_precision_comms, period); - FillBoundary(*m_fields.get(name_mf_NU, Direction{0}, lev), m_fields.get(name_mf_NU, Direction{0}, lev)->nGrowVect(), WarpX::do_single_precision_comms, period); - FillBoundary(*m_fields.get(name_mf_NU, Direction{1}, lev), m_fields.get(name_mf_NU, Direction{1}, lev)->nGrowVect(), WarpX::do_single_precision_comms, period); - FillBoundary(*m_fields.get(name_mf_NU, Direction{2}, lev), m_fields.get(name_mf_NU, Direction{2}, lev)->nGrowVect(), WarpX::do_single_precision_comms, period); + FillBoundary(*fields.get(name_mf_N, lev), fields.get(name_mf_N, lev)->nGrowVect(), WarpX::do_single_precision_comms, period); + FillBoundary(*fields.get(name_mf_NU, Direction{0}, lev), fields.get(name_mf_NU, Direction{0}, lev)->nGrowVect(), WarpX::do_single_precision_comms, period); + FillBoundary(*fields.get(name_mf_NU, Direction{1}, lev), fields.get(name_mf_NU, Direction{1}, lev)->nGrowVect(), WarpX::do_single_precision_comms, period); + FillBoundary(*fields.get(name_mf_NU, Direction{2}, lev), fields.get(name_mf_NU, Direction{2}, lev)->nGrowVect(), WarpX::do_single_precision_comms, period); } // Muscl Advection Update -void WarpXFluidContainer::AdvectivePush_Muscl (ablastr::fields::MultiFabRegister& m_fields, int lev) +void WarpXFluidContainer::AdvectivePush_Muscl (ablastr::fields::MultiFabRegister& fields, int lev) { using ablastr::fields::Direction; WARPX_PROFILE("WarpXFluidContainer::AdvectivePush_Muscl"); @@ -446,31 +446,31 @@ void WarpXFluidContainer::AdvectivePush_Muscl (ablastr::fields::MultiFabRegister const amrex::Real dt_over_dz_half = 0.5_rt*(dt/dx[0]); #endif - const amrex::BoxArray ba = m_fields.get(name_mf_N, lev)->boxArray(); + const amrex::BoxArray ba = fields.get(name_mf_N, lev)->boxArray(); // Temporary Half-step values #if defined(WARPX_DIM_3D) - amrex::MultiFab tmp_U_minus_x( amrex::convert(ba, IntVect(0,1,1)), m_fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); - amrex::MultiFab tmp_U_plus_x( amrex::convert(ba, IntVect(0,1,1)), m_fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); - amrex::MultiFab tmp_U_minus_y( amrex::convert(ba, IntVect(1,0,1)), m_fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); - amrex::MultiFab tmp_U_plus_y( amrex::convert(ba, IntVect(1,0,1)), m_fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); - amrex::MultiFab tmp_U_minus_z( amrex::convert(ba, IntVect(1,1,0)), m_fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); - amrex::MultiFab tmp_U_plus_z( amrex::convert(ba, IntVect(1,1,0)), m_fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_minus_x( amrex::convert(ba, IntVect(0,1,1)), fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_x( amrex::convert(ba, IntVect(0,1,1)), fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_minus_y( amrex::convert(ba, IntVect(1,0,1)), fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_y( amrex::convert(ba, IntVect(1,0,1)), fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_minus_z( amrex::convert(ba, IntVect(1,1,0)), fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_z( amrex::convert(ba, IntVect(1,1,0)), fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::MultiFab tmp_U_minus_x( amrex::convert(ba, IntVect(0,1)), m_fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); - amrex::MultiFab tmp_U_plus_x( amrex::convert(ba, IntVect(0,1)), m_fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); - amrex::MultiFab tmp_U_minus_z( amrex::convert(ba, IntVect(1,0)), m_fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); - amrex::MultiFab tmp_U_plus_z( amrex::convert(ba, IntVect(1,0)), m_fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_minus_x( amrex::convert(ba, IntVect(0,1)), fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_x( amrex::convert(ba, IntVect(0,1)), fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_minus_z( amrex::convert(ba, IntVect(1,0)), fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_z( amrex::convert(ba, IntVect(1,0)), fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); #else - amrex::MultiFab tmp_U_minus_z( amrex::convert(ba, IntVect(0)), m_fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); - amrex::MultiFab tmp_U_plus_z( amrex::convert(ba, IntVect(0)), m_fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_minus_z( amrex::convert(ba, IntVect(0)), fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_z( amrex::convert(ba, IntVect(0)), fields.get(name_mf_N, lev)->DistributionMap(), 4, 1); #endif // Fill edge values of N and U at the half timestep for MUSCL #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*m_fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(*fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) { // Loop over a box with one extra gridpoint in the ghost region to avoid @@ -488,10 +488,10 @@ void WarpXFluidContainer::AdvectivePush_Muscl (ablastr::fields::MultiFabRegister return tt; }(); - amrex::Array4 const &N_arr = m_fields.get(name_mf_N, lev)->array(mfi); - amrex::Array4 const &NUx_arr = m_fields.get(name_mf_NU, Direction{0}, lev)->array(mfi); - amrex::Array4 const &NUy_arr = m_fields.get(name_mf_NU, Direction{1}, lev)->array(mfi); - amrex::Array4 const &NUz_arr = m_fields.get(name_mf_NU, Direction{2}, lev)->array(mfi); + amrex::Array4 const &N_arr = fields.get(name_mf_N, lev)->array(mfi); + amrex::Array4 const &NUx_arr = fields.get(name_mf_NU, Direction{0}, lev)->array(mfi); + amrex::Array4 const &NUy_arr = fields.get(name_mf_NU, Direction{1}, lev)->array(mfi); + amrex::Array4 const &NUz_arr = fields.get(name_mf_NU, Direction{2}, lev)->array(mfi); // Boxes are computed to avoid going out of bounds. // Grow the entire domain @@ -753,13 +753,13 @@ void WarpXFluidContainer::AdvectivePush_Muscl (ablastr::fields::MultiFabRegister #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*m_fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(*fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const amrex::Box tile_box = mfi.tilebox(m_fields.get(name_mf_N, lev)->ixType().toIntVect()); - const amrex::Array4 N_arr = m_fields.get(name_mf_N, lev)->array(mfi); - const amrex::Array4 NUx_arr = m_fields.get(name_mf_NU, Direction{0}, lev)->array(mfi); - const amrex::Array4 NUy_arr = m_fields.get(name_mf_NU, Direction{1}, lev)->array(mfi); - const amrex::Array4 NUz_arr = m_fields.get(name_mf_NU, Direction{2}, lev)->array(mfi); + const amrex::Box tile_box = mfi.tilebox(fields.get(name_mf_N, lev)->ixType().toIntVect()); + const amrex::Array4 N_arr = fields.get(name_mf_N, lev)->array(mfi); + const amrex::Array4 NUx_arr = fields.get(name_mf_NU, Direction{0}, lev)->array(mfi); + const amrex::Array4 NUy_arr = fields.get(name_mf_NU, Direction{1}, lev)->array(mfi); + const amrex::Array4 NUz_arr = fields.get(name_mf_NU, Direction{2}, lev)->array(mfi); #if defined(WARPX_DIM_3D) amrex::Array4 const &U_minus_x = tmp_U_minus_x.array(mfi); @@ -890,7 +890,7 @@ void WarpXFluidContainer::AdvectivePush_Muscl (ablastr::fields::MultiFabRegister // Momentum source due to curvature #if defined(WARPX_DIM_RZ) -void WarpXFluidContainer::centrifugal_source_rz (ablastr::fields::MultiFabRegister& m_fields, int lev) +void WarpXFluidContainer::centrifugal_source_rz (ablastr::fields::MultiFabRegister& fields, int lev) { using ablastr::fields::Direction; WARPX_PROFILE("WarpXFluidContainer::centrifugal_source_rz"); @@ -907,15 +907,15 @@ void WarpXFluidContainer::centrifugal_source_rz (ablastr::fields::MultiFabRegist #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*m_fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(*fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) { - amrex::Box const &tile_box = mfi.tilebox(m_fields.get(name_mf_N, lev)->ixType().toIntVect()); + amrex::Box const &tile_box = mfi.tilebox(fields.get(name_mf_N, lev)->ixType().toIntVect()); - amrex::Array4 const &N_arr = m_fields.get(name_mf_N, lev)->array(mfi); - const amrex::Array4 NUx_arr = m_fields.get(name_mf_NU, Direction{0}, lev)->array(mfi); - const amrex::Array4 NUy_arr = m_fields.get(name_mf_NU, Direction{1}, lev)->array(mfi); - amrex::Array4 const &NUz_arr = m_fields.get(name_mf_NU, Direction{2}, lev)->array(mfi); + amrex::Array4 const &N_arr = fields.get(name_mf_N, lev)->array(mfi); + const amrex::Array4 NUx_arr = fields.get(name_mf_NU, Direction{0}, lev)->array(mfi); + const amrex::Array4 NUy_arr = fields.get(name_mf_NU, Direction{1}, lev)->array(mfi); + amrex::Array4 const &NUz_arr = fields.get(name_mf_NU, Direction{2}, lev)->array(mfi); amrex::ParallelFor(tile_box, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept @@ -960,7 +960,7 @@ void WarpXFluidContainer::centrifugal_source_rz (ablastr::fields::MultiFabRegist // Momentum source from fields void WarpXFluidContainer::GatherAndPush ( - ablastr::fields::MultiFabRegister& m_fields, + ablastr::fields::MultiFabRegister& fields, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, Real t, @@ -993,7 +993,7 @@ void WarpXFluidContainer::GatherAndPush ( auto Bz_type = amrex::GpuArray{0, 0, 0}; for (int i = 0; i < AMREX_SPACEDIM; ++i) { - Nodal_type[i] = m_fields.get(name_mf_N, lev)->ixType()[i]; + Nodal_type[i] = fields.get(name_mf_N, lev)->ixType()[i]; Ex_type[i] = Ex.ixType()[i]; Ey_type[i] = Ey.ixType()[i]; Ez_type[i] = Ez.ixType()[i]; @@ -1030,15 +1030,15 @@ void WarpXFluidContainer::GatherAndPush ( #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*m_fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(*fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) { - amrex::Box const &tile_box = mfi.tilebox(m_fields.get(name_mf_N, lev)->ixType().toIntVect()); + amrex::Box const &tile_box = mfi.tilebox(fields.get(name_mf_N, lev)->ixType().toIntVect()); - amrex::Array4 const &N_arr = m_fields.get(name_mf_N, lev)->array(mfi); - const amrex::Array4 NUx_arr = m_fields.get(name_mf_NU, Direction{0}, lev)->array(mfi); - const amrex::Array4 NUy_arr = m_fields.get(name_mf_NU, Direction{1}, lev)->array(mfi); - const amrex::Array4 NUz_arr = m_fields.get(name_mf_NU, Direction{2}, lev)->array(mfi); + amrex::Array4 const &N_arr = fields.get(name_mf_N, lev)->array(mfi); + const amrex::Array4 NUx_arr = fields.get(name_mf_NU, Direction{0}, lev)->array(mfi); + const amrex::Array4 NUy_arr = fields.get(name_mf_NU, Direction{1}, lev)->array(mfi); + const amrex::Array4 NUz_arr = fields.get(name_mf_NU, Direction{2}, lev)->array(mfi); amrex::Array4 const& Ex_arr = Ex.array(mfi); amrex::Array4 const& Ey_arr = Ey.array(mfi); @@ -1233,7 +1233,7 @@ void WarpXFluidContainer::GatherAndPush ( } } -void WarpXFluidContainer::DepositCharge (ablastr::fields::MultiFabRegister& m_fields, amrex::MultiFab &rho, int lev, int icomp) +void WarpXFluidContainer::DepositCharge (ablastr::fields::MultiFabRegister& fields, amrex::MultiFab &rho, int lev, int icomp) { WARPX_PROFILE("WarpXFluidContainer::DepositCharge"); @@ -1250,11 +1250,11 @@ void WarpXFluidContainer::DepositCharge (ablastr::fields::MultiFabRegister& m_fi #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*m_fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(*fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) { - amrex::Box const &tile_box = mfi.tilebox(m_fields.get(name_mf_N, lev)->ixType().toIntVect()); - amrex::Array4 const &N_arr = m_fields.get(name_mf_N, lev)->array(mfi); + amrex::Box const &tile_box = mfi.tilebox(fields.get(name_mf_N, lev)->ixType().toIntVect()); + amrex::Array4 const &N_arr = fields.get(name_mf_N, lev)->array(mfi); const amrex::Array4 rho_arr = rho.array(mfi); const amrex::Array4 owner_mask_rho_arr = owner_mask_rho->array(mfi); @@ -1270,7 +1270,7 @@ void WarpXFluidContainer::DepositCharge (ablastr::fields::MultiFabRegister& m_fi void WarpXFluidContainer::DepositCurrent( - ablastr::fields::MultiFabRegister& m_fields, + ablastr::fields::MultiFabRegister& fields, amrex::MultiFab &jx, amrex::MultiFab &jy, amrex::MultiFab &jz, int lev) { @@ -1278,9 +1278,9 @@ void WarpXFluidContainer::DepositCurrent( WARPX_PROFILE("WarpXFluidContainer::DepositCurrent"); // Temporary nodal currents - amrex::MultiFab tmp_jx_fluid(m_fields.get(name_mf_N, lev)->boxArray(), m_fields.get(name_mf_N, lev)->DistributionMap(), 1, 0); - amrex::MultiFab tmp_jy_fluid(m_fields.get(name_mf_N, lev)->boxArray(), m_fields.get(name_mf_N, lev)->DistributionMap(), 1, 0); - amrex::MultiFab tmp_jz_fluid(m_fields.get(name_mf_N, lev)->boxArray(), m_fields.get(name_mf_N, lev)->DistributionMap(), 1, 0); + amrex::MultiFab tmp_jx_fluid(fields.get(name_mf_N, lev)->boxArray(), fields.get(name_mf_N, lev)->DistributionMap(), 1, 0); + amrex::MultiFab tmp_jy_fluid(fields.get(name_mf_N, lev)->boxArray(), fields.get(name_mf_N, lev)->DistributionMap(), 1, 0); + amrex::MultiFab tmp_jz_fluid(fields.get(name_mf_N, lev)->boxArray(), fields.get(name_mf_N, lev)->DistributionMap(), 1, 0); const amrex::Real inv_clight_sq = 1.0_prt / PhysConst::c / PhysConst::c; const amrex::Real q = getCharge(); @@ -1310,14 +1310,14 @@ void WarpXFluidContainer::DepositCurrent( #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*m_fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(*fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) { - amrex::Box const &tile_box = mfi.tilebox(m_fields.get(name_mf_N, lev)->ixType().toIntVect()); + amrex::Box const &tile_box = mfi.tilebox(fields.get(name_mf_N, lev)->ixType().toIntVect()); - amrex::Array4 const &N_arr = m_fields.get(name_mf_N, lev)->array(mfi); - amrex::Array4 const &NUx_arr = m_fields.get(name_mf_NU, Direction{0}, lev)->array(mfi); - amrex::Array4 const &NUy_arr = m_fields.get(name_mf_NU, Direction{1}, lev)->array(mfi); - amrex::Array4 const &NUz_arr = m_fields.get(name_mf_NU, Direction{2}, lev)->array(mfi); + amrex::Array4 const &N_arr = fields.get(name_mf_N, lev)->array(mfi); + amrex::Array4 const &NUx_arr = fields.get(name_mf_NU, Direction{0}, lev)->array(mfi); + amrex::Array4 const &NUy_arr = fields.get(name_mf_NU, Direction{1}, lev)->array(mfi); + amrex::Array4 const &NUz_arr = fields.get(name_mf_NU, Direction{2}, lev)->array(mfi); const amrex::Array4 tmp_jx_fluid_arr = tmp_jx_fluid.array(mfi); const amrex::Array4 tmp_jy_fluid_arr = tmp_jy_fluid.array(mfi); @@ -1345,7 +1345,7 @@ void WarpXFluidContainer::DepositCurrent( #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*m_fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(*fields.get(name_mf_N, lev), TilingIfNotGPU()); mfi.isValid(); ++mfi) { amrex::Box const &tile_box_x = mfi.tilebox(jx.ixType().toIntVect()); amrex::Box const &tile_box_y = mfi.tilebox(jy.ixType().toIntVect()); From 1886812ec269503aaac254e09f40d6d90a363318 Mon Sep 17 00:00:00 2001 From: David Grote Date: Mon, 16 Sep 2024 10:51:58 -0700 Subject: [PATCH 239/314] Convert PML_RZ multifabs --- Source/BoundaryConditions/PML_RZ.H | 33 ++-- Source/BoundaryConditions/PML_RZ.cpp | 167 +++++++++--------- Source/BoundaryConditions/WarpXEvolvePML.cpp | 2 +- .../FlushFormats/FlushFormatCheckpoint.cpp | 1 + Source/Diagnostics/WarpXIO.cpp | 2 +- Source/Parallelization/WarpXComm.cpp | 4 +- Source/Utils/WarpXMovingWindow.cpp | 8 +- 7 files changed, 104 insertions(+), 113 deletions(-) diff --git a/Source/BoundaryConditions/PML_RZ.H b/Source/BoundaryConditions/PML_RZ.H index c908681d8e5..20c7d360fc7 100644 --- a/Source/BoundaryConditions/PML_RZ.H +++ b/Source/BoundaryConditions/PML_RZ.H @@ -16,6 +16,8 @@ # include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" #endif +#include + #include #include #include @@ -30,27 +32,24 @@ class PML_RZ { public: - PML_RZ (int lev, const amrex::BoxArray& grid_ba, const amrex::DistributionMapping& grid_dm, - const amrex::Geometry* geom, int ncell, int do_pml_in_domain); + PML_RZ (int lev, amrex::BoxArray const& grid_ba, amrex::DistributionMapping const& grid_dm, + amrex::Geometry const* geom, int ncell, int do_pml_in_domain); void ApplyDamping(amrex::MultiFab* Et_fp, amrex::MultiFab* Ez_fp, amrex::MultiFab* Bt_fp, amrex::MultiFab* Bz_fp, - amrex::Real dt); - - std::array GetE_fp (); - std::array GetB_fp (); + amrex::Real dt, ablastr::fields::MultiFabRegister& fields); #ifdef WARPX_USE_FFT void PushPSATD (int lev); #endif - void FillBoundaryE (); - void FillBoundaryB (); - void FillBoundaryE (PatchType patch_type, std::optional nodal_sync=std::nullopt); - void FillBoundaryB (PatchType patch_type, std::optional nodal_sync=std::nullopt); + void FillBoundaryE (ablastr::fields::MultiFabRegister& fields, + PatchType patch_type, std::optional nodal_sync=std::nullopt); + void FillBoundaryB (ablastr::fields::MultiFabRegister& fields, + PatchType patch_type, std::optional nodal_sync=std::nullopt); - void CheckPoint (const std::string& dir) const; - void Restart (const std::string& dir); + void CheckPoint (ablastr::fields::MultiFabRegister& fields, std::string const& dir) const; + void Restart (ablastr::fields::MultiFabRegister& fields, std::string const& dir); private: @@ -58,15 +57,13 @@ private: const int m_do_pml_in_domain; const amrex::Geometry* m_geom; - // Only contains Er and Et, and Br and Bt - std::array,2> pml_E_fp; - std::array,2> pml_B_fp; + // The MultiFabs pml_E_fp and pml_B_fp are setup using the registry. + // They hold Er, Et, and Br, Bt. #ifdef WARPX_USE_FFT - void PushPMLPSATDSinglePatchRZ ( int lev, + void PushPMLPSATDSinglePatchRZ (int lev, SpectralSolverRZ& solver, - std::array,2>& pml_E, - std::array,2>& pml_B); + ablastr::fields::MultiFabRegister& fields); #endif }; diff --git a/Source/BoundaryConditions/PML_RZ.cpp b/Source/BoundaryConditions/PML_RZ.cpp index d9982a4bb85..cc05c126a55 100644 --- a/Source/BoundaryConditions/PML_RZ.cpp +++ b/Source/BoundaryConditions/PML_RZ.cpp @@ -33,45 +33,55 @@ #include #include -using namespace amrex; +using namespace amrex::literals; using namespace warpx::fields; +using ablastr::fields::Direction; -PML_RZ::PML_RZ (const int lev, const amrex::BoxArray& grid_ba, const amrex::DistributionMapping& grid_dm, - const amrex::Geometry* geom, const int ncell, const int do_pml_in_domain) +PML_RZ::PML_RZ (int lev, amrex::BoxArray const& grid_ba, amrex::DistributionMapping const& grid_dm, + amrex::Geometry const* geom, int ncell, int do_pml_in_domain) : m_ncell(ncell), m_do_pml_in_domain(do_pml_in_domain), m_geom(geom) { - auto& warpx = WarpX::GetInstance(); - using ablastr::fields::Direction; - - const amrex::MultiFab & Er_fp = *warpx.m_fields.get("Efield_fp",Direction{0},lev); - const amrex::MultiFab & Et_fp = *warpx.m_fields.get("Efield_fp",Direction{1},lev); - const amrex::BoxArray ba_Er = amrex::convert(grid_ba, Er_fp.ixType().toIntVect()); - const amrex::BoxArray ba_Et = amrex::convert(grid_ba, Et_fp.ixType().toIntVect()); - WarpX::AllocInitMultiFab(pml_E_fp[0], ba_Er, grid_dm, Er_fp.nComp(), Er_fp.nGrowVect(), lev, "pml_E_fp[0]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_E_fp[1], ba_Et, grid_dm, Et_fp.nComp(), Et_fp.nGrowVect(), lev, "pml_E_fp[1]", 0.0_rt); - - const amrex::MultiFab & Br_fp = *warpx.m_fields.get("Bfield_fp",Direction{0},lev); - const amrex::MultiFab & Bt_fp = *warpx.m_fields.get("Bfield_fp",Direction{1},lev); - const amrex::BoxArray ba_Br = amrex::convert(grid_ba, Br_fp.ixType().toIntVect()); - const amrex::BoxArray ba_Bt = amrex::convert(grid_ba, Bt_fp.ixType().toIntVect()); - WarpX::AllocInitMultiFab(pml_B_fp[0], ba_Br, grid_dm, Br_fp.nComp(), Br_fp.nGrowVect(), lev, "pml_B_fp[0]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_B_fp[1], ba_Bt, grid_dm, Bt_fp.nComp(), Bt_fp.nGrowVect(), lev, "pml_B_fp[1]", 0.0_rt); + auto & warpx = WarpX::GetInstance(); + + bool const remake = false; + bool const redistribute_on_remake = false; + + amrex::MultiFab const& Er_fp = *warpx.m_fields.get("Efield_fp",Direction{0},lev); + amrex::MultiFab const& Et_fp = *warpx.m_fields.get("Efield_fp",Direction{1},lev); + amrex::BoxArray const ba_Er = amrex::convert(grid_ba, Er_fp.ixType().toIntVect()); + amrex::BoxArray const ba_Et = amrex::convert(grid_ba, Et_fp.ixType().toIntVect()); + warpx.m_fields.alloc_init("pml_E_fp", Direction{0}, lev, ba_Er, grid_dm, Er_fp.nComp(), Er_fp.nGrowVect(), 0.0_rt, + remake, redistribute_on_remake); + warpx.m_fields.alloc_init("pml_E_fp", Direction{1}, lev, ba_Et, grid_dm, Et_fp.nComp(), Et_fp.nGrowVect(), 0.0_rt, + remake, redistribute_on_remake); + + amrex::MultiFab const& Br_fp = *warpx.m_fields.get("Bfield_fp",Direction{0},lev); + amrex::MultiFab const& Bt_fp = *warpx.m_fields.get("Bfield_fp",Direction{1},lev); + amrex::BoxArray const ba_Br = amrex::convert(grid_ba, Br_fp.ixType().toIntVect()); + amrex::BoxArray const ba_Bt = amrex::convert(grid_ba, Bt_fp.ixType().toIntVect()); + warpx.m_fields.alloc_init("pml_B_fp", Direction{0}, lev, ba_Br, grid_dm, Br_fp.nComp(), Br_fp.nGrowVect(), 0.0_rt, + remake, redistribute_on_remake); + warpx.m_fields.alloc_init("pml_B_fp", Direction{1}, lev, ba_Bt, grid_dm, Bt_fp.nComp(), Bt_fp.nGrowVect(), 0.0_rt, + remake, redistribute_on_remake); } void PML_RZ::ApplyDamping (amrex::MultiFab* Et_fp, amrex::MultiFab* Ez_fp, amrex::MultiFab* Bt_fp, amrex::MultiFab* Bz_fp, - amrex::Real dt) + amrex::Real dt, ablastr::fields::MultiFabRegister& fields) { - const amrex::Real dr = m_geom->CellSize(0); - const amrex::Real cdt_over_dr = PhysConst::c*dt/dr; + amrex::Real const dr = m_geom->CellSize(0); + amrex::Real const cdt_over_dr = PhysConst::c*dt/dr; + + amrex::MultiFab* pml_Et = fields.get("pml_E_fp", Direction{1}, 0); + amrex::MultiFab* pml_Bt = fields.get("pml_B_fp", Direction{1}, 0); #ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif for ( amrex::MFIter mfi(*Et_fp, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi ) { @@ -80,8 +90,8 @@ PML_RZ::ApplyDamping (amrex::MultiFab* Et_fp, amrex::MultiFab* Ez_fp, amrex::Array4 const& Bt_arr = Bt_fp->array(mfi); amrex::Array4 const& Bz_arr = Bz_fp->array(mfi); - amrex::Array4 const& pml_Et_arr = pml_E_fp[1]->array(mfi); - amrex::Array4 const& pml_Bt_arr = pml_B_fp[1]->array(mfi); + amrex::Array4 const& pml_Et_arr = pml_Et->array(mfi); + amrex::Array4 const& pml_Bt_arr = pml_Bt->array(mfi); // Get the tileboxes from Efield and Bfield so that they include the guard cells // They are all the same, cell centered @@ -89,19 +99,19 @@ PML_RZ::ApplyDamping (amrex::MultiFab* Et_fp, amrex::MultiFab* Ez_fp, // Box for the whole simulation domain amrex::Box const& domain = m_geom->Domain(); - const int nr_domain = domain.bigEnd(0); + int const nr_domain = domain.bigEnd(0); // Set tilebox to only include the upper radial cells - const int nr_damp = m_ncell; - const int nr_damp_min = (m_do_pml_in_domain)?(nr_domain - nr_damp):(nr_domain); + int const nr_damp = m_ncell; + int const nr_damp_min = (m_do_pml_in_domain)?(nr_domain - nr_damp):(nr_domain); tilebox.setSmall(0, nr_damp_min + 1); amrex::ParallelFor( tilebox, Et_fp->nComp(), [=] AMREX_GPU_DEVICE (int i, int j, int k, int icomp) { - const auto rr = static_cast(i - nr_damp_min); - const amrex::Real wr = rr/nr_damp; - const amrex::Real damp_factor = std::exp( -4._rt * cdt_over_dr * wr*wr ); + auto const rr = static_cast(i - nr_damp_min); + amrex::Real const wr = rr/nr_damp; + amrex::Real const damp_factor = std::exp( -4._rt * cdt_over_dr * wr*wr ); // Substract the theta PML fields from the regular theta fields Et_arr(i,j,k,icomp) -= pml_Et_arr(i,j,k,icomp); @@ -119,105 +129,88 @@ PML_RZ::ApplyDamping (amrex::MultiFab* Et_fp, amrex::MultiFab* Ez_fp, } } -std::array -PML_RZ::GetE_fp () -{ - return {pml_E_fp[0].get(), pml_E_fp[1].get()}; -} - -std::array -PML_RZ::GetB_fp () -{ - return {pml_B_fp[0].get(), pml_B_fp[1].get()}; -} - void -PML_RZ::FillBoundaryE () +PML_RZ::FillBoundaryE (ablastr::fields::MultiFabRegister& fields, PatchType patch_type, std::optional nodal_sync) { - FillBoundaryE(PatchType::fine); -} + amrex::MultiFab * pml_Er = fields.get("pml_E_fp", Direction{0}, 0); + amrex::MultiFab * pml_Et = fields.get("pml_E_fp", Direction{1}, 0); -void -PML_RZ::FillBoundaryE (PatchType patch_type, std::optional nodal_sync) -{ - if (patch_type == PatchType::fine && pml_E_fp[0] && pml_E_fp[0]->nGrowVect().max() > 0) + if (patch_type == PatchType::fine && pml_Er->nGrowVect().max() > 0) { - const amrex::Periodicity& period = m_geom->periodicity(); - const Vector mf{pml_E_fp[0].get(),pml_E_fp[1].get()}; + amrex::Periodicity const& period = m_geom->periodicity(); + amrex::Vector mf = {pml_Er, pml_Et}; ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period, nodal_sync); } } void -PML_RZ::FillBoundaryB () -{ - FillBoundaryB(PatchType::fine); -} - -void -PML_RZ::FillBoundaryB (PatchType patch_type, std::optional nodal_sync) +PML_RZ::FillBoundaryB (ablastr::fields::MultiFabRegister& fields, PatchType patch_type, std::optional nodal_sync) { - if (patch_type == PatchType::fine && pml_B_fp[0]) + if (patch_type == PatchType::fine) { - const amrex::Periodicity& period = m_geom->periodicity(); - const Vector mf{pml_B_fp[0].get(),pml_B_fp[1].get()}; + amrex::MultiFab * pml_Br = fields.get("pml_B_fp", Direction{0}, 0); + amrex::MultiFab * pml_Bt = fields.get("pml_B_fp", Direction{1}, 0); + + amrex::Periodicity const& period = m_geom->periodicity(); + amrex::Vector mf = {pml_Br, pml_Bt}; ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period, nodal_sync); } } void -PML_RZ::CheckPoint (const std::string& dir) const +PML_RZ::CheckPoint (ablastr::fields::MultiFabRegister& fields, std::string const& dir) const { - if (pml_E_fp[0]) - { - VisMF::AsyncWrite(*pml_E_fp[0], dir+"_Er_fp"); - VisMF::AsyncWrite(*pml_E_fp[1], dir+"_Et_fp"); - VisMF::AsyncWrite(*pml_B_fp[0], dir+"_Br_fp"); - VisMF::AsyncWrite(*pml_B_fp[1], dir+"_Bt_fp"); + if (fields.has("pml_E_fp", Direction{0}, 0)) { + amrex::VisMF::AsyncWrite(*fields.get("pml_E_fp", Direction{0}, 0), dir+"_Er_fp"); + amrex::VisMF::AsyncWrite(*fields.get("pml_E_fp", Direction{1}, 0), dir+"_Et_fp"); + amrex::VisMF::AsyncWrite(*fields.get("pml_B_fp", Direction{0}, 0), dir+"_Br_fp"); + amrex::VisMF::AsyncWrite(*fields.get("pml_B_fp", Direction{1}, 0), dir+"_Bt_fp"); } } void -PML_RZ::Restart (const std::string& dir) +PML_RZ::Restart (ablastr::fields::MultiFabRegister& fields, std::string const& dir) { - if (pml_E_fp[0]) - { - VisMF::Read(*pml_E_fp[0], dir+"_Er_fp"); - VisMF::Read(*pml_E_fp[1], dir+"_Et_fp"); - VisMF::Read(*pml_B_fp[0], dir+"_Br_fp"); - VisMF::Read(*pml_B_fp[1], dir+"_Bt_fp"); + if (fields.has("pml_E_fp", Direction{0}, 0)) { + amrex::VisMF::Read(*fields.get("pml_E_fp", Direction{0}, 0), dir+"_Er_fp"); + amrex::VisMF::Read(*fields.get("pml_E_fp", Direction{1}, 0), dir+"_Et_fp"); + amrex::VisMF::Read(*fields.get("pml_B_fp", Direction{0}, 0), dir+"_Br_fp"); + amrex::VisMF::Read(*fields.get("pml_B_fp", Direction{1}, 0), dir+"_Bt_fp"); } } #ifdef WARPX_USE_FFT void -PML_RZ::PushPSATD (const int lev) +PML_RZ::PushPSATD (int lev) { // Update the fields on the fine and coarse patch WarpX& warpx = WarpX::GetInstance(); SpectralSolverRZ& solver = warpx.get_spectral_solver_fp(lev); - PushPMLPSATDSinglePatchRZ(lev, solver, pml_E_fp, pml_B_fp); + PushPMLPSATDSinglePatchRZ(lev, solver, warpx.m_fields); } void PML_RZ::PushPMLPSATDSinglePatchRZ ( - const int lev, + int lev, SpectralSolverRZ& solver, - std::array,2>& pml_E, - std::array,2>& pml_B) + ablastr::fields::MultiFabRegister& fields) { - const SpectralFieldIndex& Idx = solver.m_spectral_index; + SpectralFieldIndex const& Idx = solver.m_spectral_index; + amrex::MultiFab * pml_Er = fields.get("pml_E_fp", Direction{0}, 0); + amrex::MultiFab * pml_Et = fields.get("pml_E_fp", Direction{1}, 0); + amrex::MultiFab * pml_Br = fields.get("pml_B_fp", Direction{0}, 0); + amrex::MultiFab * pml_Bt = fields.get("pml_B_fp", Direction{1}, 0); // Perform forward Fourier transforms - solver.ForwardTransform(lev, *pml_E[0], Idx.Er_pml, *pml_E[1], Idx.Et_pml); - solver.ForwardTransform(lev, *pml_B[0], Idx.Br_pml, *pml_B[1], Idx.Bt_pml); + solver.ForwardTransform(lev, *pml_Er, Idx.Er_pml, *pml_Et, Idx.Et_pml); + solver.ForwardTransform(lev, *pml_Br, Idx.Br_pml, *pml_Bt, Idx.Bt_pml); // Advance fields in spectral space - const bool doing_pml = true; + bool const doing_pml = true; solver.pushSpectralFields(doing_pml); // Perform backward Fourier transforms - solver.BackwardTransform(lev, *pml_E[0], Idx.Er_pml, *pml_E[1], Idx.Et_pml); - solver.BackwardTransform(lev, *pml_B[0], Idx.Br_pml, *pml_B[1], Idx.Bt_pml); + solver.BackwardTransform(lev, *pml_Er, Idx.Er_pml, *pml_Et, Idx.Et_pml); + solver.BackwardTransform(lev, *pml_Br, Idx.Br_pml, *pml_Bt, Idx.Bt_pml); } #endif diff --git a/Source/BoundaryConditions/WarpXEvolvePML.cpp b/Source/BoundaryConditions/WarpXEvolvePML.cpp index 087cf36a2a9..b50010d83f6 100644 --- a/Source/BoundaryConditions/WarpXEvolvePML.cpp +++ b/Source/BoundaryConditions/WarpXEvolvePML.cpp @@ -70,7 +70,7 @@ WarpX::DampPML (const int lev, PatchType patch_type) m_fields.get("Efield_fp",Direction{2},lev), m_fields.get("Bfield_fp",Direction{1},lev), m_fields.get("Bfield_fp",Direction{2},lev), - dt[lev]); + dt[lev], m_fields); } #endif if (pml[lev]) { diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index 8a9c3c13ccc..49d1e7e1ad0 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -160,6 +160,7 @@ FlushFormatCheckpoint::WriteToFile ( #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (warpx.GetPML_RZ(lev)) { warpx.GetPML_RZ(lev)->CheckPoint( + warpx.m_fields, amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "pml_rz")); } #endif diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp index 47583ec99de..a572eaa2ed3 100644 --- a/Source/Diagnostics/WarpXIO.cpp +++ b/Source/Diagnostics/WarpXIO.cpp @@ -390,7 +390,7 @@ WarpX::InitFromCheckpoint () } #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev]) { - pml_rz[lev]->Restart(amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "pml_rz")); + pml_rz[lev]->Restart(m_fields, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "pml_rz")); } #endif } diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index bba23752bdd..d73eb4044fb 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -740,7 +740,7 @@ WarpX::FillBoundaryE (const int lev, const PatchType patch_type, const amrex::In #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev]) { - pml_rz[lev]->FillBoundaryE(patch_type, nodal_sync); + pml_rz[lev]->FillBoundaryE(m_fields, patch_type, nodal_sync); } #endif } @@ -805,7 +805,7 @@ WarpX::FillBoundaryB (const int lev, const PatchType patch_type, const amrex::In #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev]) { - pml_rz[lev]->FillBoundaryB(patch_type, nodal_sync); + pml_rz[lev]->FillBoundaryB(m_fields, patch_type, nodal_sync); } #endif } diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 048bde253e3..45b61b256d7 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -262,10 +262,10 @@ WarpX::MoveWindow (const int step, bool move_j) } #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev] && dim < 2) { - const std::array& pml_rz_B = pml_rz[lev]->GetB_fp(); - const std::array& pml_rz_E = pml_rz[lev]->GetE_fp(); - shiftMF(*pml_rz_B[dim], geom[lev], num_shift, dir, lev, dont_update_cost); - shiftMF(*pml_rz_E[dim], geom[lev], num_shift, dir, lev, dont_update_cost); + amrex::MultiFab* pml_rz_B = m_fields.get("pml_B_fp", Direction{dim}, lev); + amrex::MultiFab* pml_rz_E = m_fields.get("pml_E_fp", Direction{dim}, lev); + shiftMF(*pml_rz_B, geom[lev], num_shift, dir, lev, dont_update_cost); + shiftMF(*pml_rz_E, geom[lev], num_shift, dir, lev, dont_update_cost); } #endif if (lev > 0) { From 1a42aa5c7afcd0d391a835f8a55cad70bf91cbe0 Mon Sep 17 00:00:00 2001 From: David Grote Date: Mon, 16 Sep 2024 10:57:42 -0700 Subject: [PATCH 240/314] Small cleanup in PML with moving window --- Source/Utils/WarpXMovingWindow.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 45b61b256d7..4fa9ad4f7c1 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -255,10 +255,10 @@ WarpX::MoveWindow (const int step, bool move_j) shiftMF(*m_fields.get("current_fp", Direction{dim}, lev), geom[lev], num_shift, dir, lev, do_update_cost); } if (pml[lev] && pml[lev]->ok()) { - const std::array& pml_B = m_fields.get_alldirs("pml_B_fp", lev); - const std::array& pml_E = m_fields.get_alldirs("pml_E_fp", lev); - shiftMF(*pml_B[dim], geom[lev], num_shift, dir, lev, dont_update_cost); - shiftMF(*pml_E[dim], geom[lev], num_shift, dir, lev, dont_update_cost); + amrex::MultiFab* pml_B = m_fields.get("pml_B_fp", Direction{dim}, lev); + amrex::MultiFab* pml_E = m_fields.get("pml_E_fp", Direction{dim}, lev); + shiftMF(*pml_B, geom[lev], num_shift, dir, lev, dont_update_cost); + shiftMF(*pml_E, geom[lev], num_shift, dir, lev, dont_update_cost); } #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev] && dim < 2) { @@ -288,10 +288,10 @@ WarpX::MoveWindow (const int step, bool move_j) shiftMF(*m_fields.get("current_cp", Direction{dim}, lev), geom[lev-1], num_shift_crse, dir, lev, do_update_cost); } if (do_pml && pml[lev]->ok()) { - const std::array& pml_B = m_fields.get_alldirs("pml_B_cp", lev); - const std::array& pml_E = m_fields.get_alldirs("pml_E_cp", lev); - shiftMF(*pml_B[dim], geom[lev-1], num_shift_crse, dir, lev, dont_update_cost); - shiftMF(*pml_E[dim], geom[lev-1], num_shift_crse, dir, lev, dont_update_cost); + amrex::MultiFab* pml_B_cp = m_fields.get("pml_B_cp", Direction{dim}, lev); + amrex::MultiFab* pml_E_cp = m_fields.get("pml_E_cp", Direction{dim}, lev); + shiftMF(*pml_B_cp, geom[lev-1], num_shift_crse, dir, lev, dont_update_cost); + shiftMF(*pml_E_cp, geom[lev-1], num_shift_crse, dir, lev, dont_update_cost); } } } From ac4c6e43c632908845677a02f1e750c7db02e044 Mon Sep 17 00:00:00 2001 From: David Grote Date: Mon, 16 Sep 2024 13:41:49 -0700 Subject: [PATCH 241/314] Use [[nodiscard]] in MultiFabRegister.H --- Source/ablastr/fields/MultiFabRegister.H | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index e4313959f29..4a4184e64d5 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -241,7 +241,7 @@ namespace ablastr::fields * @param level ... * @return true if contained, otherwise false */ - bool + [[nodiscard]] bool has ( std::string name, int level @@ -257,7 +257,7 @@ namespace ablastr::fields * @param level ... * @return true if contained, otherwise false */ - bool + [[nodiscard]] bool has ( std::string name, Direction dir, @@ -305,7 +305,7 @@ namespace ablastr::fields * @param level ... * @return ... */ - amrex::MultiFab const * + [[nodiscard]] amrex::MultiFab const * get ( std::string name, int level @@ -321,7 +321,7 @@ namespace ablastr::fields * @param level ... * @return ... */ - amrex::MultiFab const * + [[nodiscard]] amrex::MultiFab const * get ( std::string name, Direction dir, @@ -341,7 +341,7 @@ namespace ablastr::fields std::string name, int finest_level ); - ConstMultiLevelScalarField + [[nodiscard]] ConstMultiLevelScalarField get_mr_levels ( std::string name, int finest_level @@ -361,7 +361,7 @@ namespace ablastr::fields int level ); ConstVectorField - get_alldirs ( + [[nodiscard]] get_alldirs ( std::string name, int level ) const; @@ -380,7 +380,7 @@ namespace ablastr::fields std::string name, int finest_level ); - ConstMultiLevelVectorField + [[nodiscard]] ConstMultiLevelVectorField get_mr_levels_alldirs ( std::string name, int finest_level @@ -393,7 +393,7 @@ namespace ablastr::fields * * @return ... */ - std::vector + [[nodiscard]] std::vector list () const; /** title @@ -459,7 +459,7 @@ namespace ablastr::fields * @param level ... * @return ... */ - std::string + [[nodiscard]] std::string mf_name ( std::string name, int level @@ -475,7 +475,7 @@ namespace ablastr::fields * @param level ... * @return ... */ - std::string + [[nodiscard]] std::string mf_name ( std::string name, Direction dir, @@ -487,7 +487,7 @@ namespace ablastr::fields internal_get ( std::string key ); - amrex::MultiFab const * + [[nodiscard]] amrex::MultiFab const * internal_get ( std::string key ) const; From ead8a5798c5853c187e37fc0e0430226a02757ef Mon Sep 17 00:00:00 2001 From: David Grote Date: Mon, 16 Sep 2024 13:43:01 -0700 Subject: [PATCH 242/314] Fix for nodiscard --- Source/ablastr/fields/MultiFabRegister.H | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 4a4184e64d5..78917613396 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -360,8 +360,8 @@ namespace ablastr::fields std::string name, int level ); - ConstVectorField - [[nodiscard]] get_alldirs ( + [[nodiscard]] ConstVectorField + get_alldirs ( std::string name, int level ) const; From 4fe61279120c48f45ad5ceed343fe4909c64c37a Mon Sep 17 00:00:00 2001 From: David Grote Date: Mon, 16 Sep 2024 15:55:39 -0700 Subject: [PATCH 243/314] Clean up errors from clang tidy --- .../FiniteDifferenceSolver/EvolveECTRho.cpp | 4 ++-- .../FiniteDifferenceSolver.H | 8 ++++---- Source/Fluids/MultiFluidContainer.H | 2 +- Source/Fluids/MultiFluidContainer.cpp | 2 +- .../DivCleaner/ProjectionDivCleaner.H | 2 +- .../DivCleaner/ProjectionDivCleaner.cpp | 2 +- Source/Parallelization/WarpXComm.cpp | 18 +++++++++--------- Source/Particles/MultiParticleContainer.H | 2 +- Source/Particles/MultiParticleContainer.cpp | 6 +++--- Source/Utils/WarpXMovingWindow.cpp | 2 +- Source/ablastr/fields/MultiFabRegister.H | 2 +- 11 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp index b2102a93130..0740a190bec 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp @@ -48,7 +48,7 @@ using namespace ablastr::fields; * \brief Update the B field, over one timestep */ void FiniteDifferenceSolver::EvolveECTRho ( - ablastr::fields::VectorField const Efield, + ablastr::fields::VectorField const& Efield, ablastr::fields::VectorField const& edge_lengths, ablastr::fields::VectorField const& face_areas, ablastr::fields::VectorField const& ECTRhofield, @@ -68,7 +68,7 @@ void FiniteDifferenceSolver::EvolveECTRho ( // If we implement ECT in 1D we will need to take care of this #ifndef differently #ifndef WARPX_DIM_RZ void FiniteDifferenceSolver::EvolveRhoCartesianECT ( - ablastr::fields::VectorField const Efield, + ablastr::fields::VectorField const& Efield, ablastr::fields::VectorField const& edge_lengths, ablastr::fields::VectorField const& face_areas, ablastr::fields::VectorField const& ECTRhofield, const int lev ) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 250f6280f92..49d187431ce 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -67,7 +67,7 @@ class FiniteDifferenceSolver void EvolveF ( amrex::MultiFab* Ffield, ablastr::fields::VectorField const& Efield, - amrex::MultiFab* const rhofield, + amrex::MultiFab* rhofield, int rhocomp, amrex::Real dt ); @@ -75,7 +75,7 @@ class FiniteDifferenceSolver ablastr::fields::VectorField const& Bfield, amrex::Real dt); - void EvolveECTRho ( ablastr::fields::VectorField const Efield, + void EvolveECTRho ( ablastr::fields::VectorField const& Efield, ablastr::fields::VectorField const& edge_lengths, ablastr::fields::VectorField const& face_areas, ablastr::fields::VectorField const& ECTRhofield, @@ -278,7 +278,7 @@ class FiniteDifferenceSolver void EvolveFCartesian ( amrex::MultiFab* Ffield, ablastr::fields::VectorField Efield, - amrex::MultiFab* const rhofield, + amrex::MultiFab* rhofield, int rhocomp, amrex::Real dt ); @@ -289,7 +289,7 @@ class FiniteDifferenceSolver amrex::Real dt); void EvolveRhoCartesianECT ( - ablastr::fields::VectorField const Efield, + ablastr::fields::VectorField const& Efield, ablastr::fields::VectorField const& edge_lengths, ablastr::fields::VectorField const& face_areas, ablastr::fields::VectorField const& ECTRhofield, int lev); diff --git a/Source/Fluids/MultiFluidContainer.H b/Source/Fluids/MultiFluidContainer.H index 3bc9f1488c8..c2cdfc3e19f 100644 --- a/Source/Fluids/MultiFluidContainer.H +++ b/Source/Fluids/MultiFluidContainer.H @@ -64,7 +64,7 @@ public: /// void Evolve (ablastr::fields::MultiFabRegister& fields, int lev, - std::string current_fp_string, + std::string const& current_fp_string, amrex::Real cur_time, bool skip_deposition=false); diff --git a/Source/Fluids/MultiFluidContainer.cpp b/Source/Fluids/MultiFluidContainer.cpp index a7a00416ad1..b160817d886 100644 --- a/Source/Fluids/MultiFluidContainer.cpp +++ b/Source/Fluids/MultiFluidContainer.cpp @@ -63,7 +63,7 @@ MultiFluidContainer::DepositCurrent (ablastr::fields::MultiFabRegister& m_fields void MultiFluidContainer::Evolve (ablastr::fields::MultiFabRegister& fields, int lev, - std::string current_fp_string, + std::string const& current_fp_string, amrex::Real cur_time, bool skip_deposition) { diff --git a/Source/Initialization/DivCleaner/ProjectionDivCleaner.H b/Source/Initialization/DivCleaner/ProjectionDivCleaner.H index cde1c98bed2..f2414323f25 100644 --- a/Source/Initialization/DivCleaner/ProjectionDivCleaner.H +++ b/Source/Initialization/DivCleaner/ProjectionDivCleaner.H @@ -83,7 +83,7 @@ public: amrex::Gpu::DeviceVector m_stencil_coefs_z; // Default Constructor - ProjectionDivCleaner (std::string a_field_name); + ProjectionDivCleaner (std::string const& a_field_name); void ReadParameters (); diff --git a/Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp b/Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp index e3d419dbae9..9fc6fab656a 100644 --- a/Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp +++ b/Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp @@ -30,7 +30,7 @@ using namespace amrex; namespace warpx::initialization { -ProjectionDivCleaner::ProjectionDivCleaner(std::string a_field_name) : +ProjectionDivCleaner::ProjectionDivCleaner(std::string const& a_field_name) : m_field_name(a_field_name) { using ablastr::fields::Direction; diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index d73eb4044fb..7069982d0d7 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -57,7 +57,7 @@ WarpX::UpdateAuxilaryData () using ablastr::fields::Direction; - auto Bfield_aux_lvl0_0 = m_fields.get("Bfield_aux", Direction{0}, 0); + amrex::MultiFab *Bfield_aux_lvl0_0 = m_fields.get("Bfield_aux", Direction{0}, 0); ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); @@ -69,9 +69,9 @@ WarpX::UpdateAuxilaryData () // When loading particle fields from file: add the external fields: for (int lev = 0; lev <= finest_level; ++lev) { - auto Bfield_aux_lvl_0 = m_fields.get("Bfield_aux", Direction{0}, lev); - auto Bfield_aux_lvl_1 = m_fields.get("Bfield_aux", Direction{1}, lev); - auto Bfield_aux_lvl_2 = m_fields.get("Bfield_aux", Direction{2}, lev); + amrex::MultiFab *Bfield_aux_lvl_0 = m_fields.get("Bfield_aux", Direction{0}, lev); + amrex::MultiFab *Bfield_aux_lvl_1 = m_fields.get("Bfield_aux", Direction{1}, lev); + amrex::MultiFab *Bfield_aux_lvl_2 = m_fields.get("Bfield_aux", Direction{2}, lev); if (mypc->m_E_ext_particle_s == "read_from_file") { const auto& E_ext_lev = m_fields.get_alldirs("E_external_particle_field", lev); @@ -1073,12 +1073,12 @@ WarpX::SyncCurrent (std::string current_fp_string) WARPX_PROFILE("WarpX::SyncCurrent()"); - ablastr::fields::MultiLevelVectorField J_fp = m_fields.get_mr_levels_alldirs(current_fp_string, finest_level); + ablastr::fields::MultiLevelVectorField const& J_fp = m_fields.get_mr_levels_alldirs(current_fp_string, finest_level); // If warpx.do_current_centering = 1, center currents from nodal grid to staggered grid if (do_current_centering) { - ablastr::fields::MultiLevelVectorField J_fp_nodal = m_fields.get_mr_levels_alldirs("current_fp_nodal", finest_level+1); + ablastr::fields::MultiLevelVectorField const& J_fp_nodal = m_fields.get_mr_levels_alldirs("current_fp_nodal", finest_level+1); AMREX_ALWAYS_ASSERT_WITH_MESSAGE(finest_level <= 1, "warpx.do_current_centering=1 not supported with more than one fine levels"); @@ -1188,7 +1188,7 @@ WarpX::SyncCurrent (std::string current_fp_string) } }); // Now it's safe to apply filter and sumboundary on J_cp - ablastr::fields::MultiLevelVectorField J_cp = m_fields.get_mr_levels_alldirs("current_cp", finest_level); + ablastr::fields::MultiLevelVectorField const& J_cp = m_fields.get_mr_levels_alldirs("current_cp", finest_level); if (use_filter) { ApplyFilterJ(J_cp, lev+1, idim); @@ -1203,14 +1203,14 @@ WarpX::SyncCurrent (std::string current_fp_string) // filtering depends on the level. This is also done before any // same-level communication because it's easier this way to // avoid double counting. - ablastr::fields::MultiLevelVectorField J_cp = m_fields.get_mr_levels_alldirs("current_cp", finest_level); + ablastr::fields::MultiLevelVectorField const& J_cp = m_fields.get_mr_levels_alldirs("current_cp", finest_level); J_cp[lev][Direction{idim}]->setVal(0.0); ablastr::coarsen::average::Coarsen(*J_cp[lev][Direction{idim}], *J_fp[lev][Direction{idim}], refRatio(lev-1)); if (m_fields.has("current_buf", Direction{idim}, lev)) { - ablastr::fields::MultiLevelVectorField J_buffer = m_fields.get_mr_levels_alldirs("current_buf", finest_level); + ablastr::fields::MultiLevelVectorField const& J_buffer = m_fields.get_mr_levels_alldirs("current_buf", finest_level); IntVect const& ng = J_cp[lev][Direction{idim}]->nGrowVect(); AMREX_ASSERT(ng.allLE(J_buffer[lev][Direction{idim}]->nGrowVect())); diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index 3334a5233cc..0e33b6bac3c 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -107,7 +107,7 @@ public: void Evolve ( ablastr::fields::MultiFabRegister& fields, int lev, - std::string current_fp_string, + std::string const& current_fp_string, amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index d367468c36a..a44808195e0 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -460,7 +460,7 @@ MultiParticleContainer::InitMultiPhysicsModules () void MultiParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, int lev, - std::string current_fp_string, + std::string const& current_fp_string, Real t, Real dt, DtType a_dt_type, bool skip_deposition, PushType push_type) { @@ -530,7 +530,7 @@ MultiParticleContainer::DepositCurrent ( const amrex::Real dt, const amrex::Real relative_time) { // Reset the J arrays - for (auto& J_lev : J) + for (const auto& J_lev : J) { J_lev[0]->setVal(0.0_rt); J_lev[1]->setVal(0.0_rt); @@ -558,7 +558,7 @@ MultiParticleContainer::DepositCharge ( const amrex::Real relative_time) { // Reset the rho array - for (auto& rho_lev : rho) + for (const auto& rho_lev : rho) { rho_lev->setVal(0.0_rt); } diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 4fa9ad4f7c1..ec1d2bffc5f 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -378,7 +378,7 @@ WarpX::MoveWindow (const int step, bool move_j) if (do_fluid_species) { const int n_fluid_species = myfl->nSpecies(); for (int i=0; iGetFluidContainer(i); + WarpXFluidContainer const& fl = myfl->GetFluidContainer(i); shiftMF( *m_fields.get(fl.name_mf_N, lev), geom[lev], num_shift, dir, lev, do_update_cost ); shiftMF( *m_fields.get(fl.name_mf_NU, Direction{0}, lev), geom[lev], num_shift, dir, lev, do_update_cost ); shiftMF( *m_fields.get(fl.name_mf_NU, Direction{1}, lev), geom[lev], num_shift, dir, lev, do_update_cost ); diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 78917613396..43e87a21846 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -43,7 +43,7 @@ namespace ablastr::fields } /* TODO: just temporary int compatibility */ - operator int() { return dir; } + operator int() const { return dir; } }; /** A scalar field (a MultiFab) From f4bcea5270eb015a2dab8cd290ea1335548d49f9 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 16 Sep 2024 20:51:36 -0700 Subject: [PATCH 244/314] `MultiFabRegister`: More Doxygen Strings --- Source/ablastr/fields/MultiFabRegister.H | 81 +++++++++++++++--------- 1 file changed, 52 insertions(+), 29 deletions(-) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 43e87a21846..9d1713849ca 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -141,19 +141,18 @@ namespace ablastr::fields MultiFabRegister& operator=(MultiFabRegister&&) = delete; ~MultiFabRegister() = default; - /** title - * - * body body - * body - * body - * - * @param name ... - * @param level ... - * @param ba ... - * @param dm ... - * @param ncomp ... - * @param ngrow ... - * @param initial_value ... + /** Allocate and optionally initialize a MultiFab (field) + * + * This registers a new MultiFab under a unique name, allocates it and + * optionally assigns it an initial value. + * + * @param name a unique name for this field + * @param level the MR level to represent + * @param ba the list of boxes to cover the field + * @param dm the distribution mapping for load balancing with MPI + * @param ncomp the number of components of the field (all with the same staggering) + * @param ngrow the number of guard (ghost, halo) cells + * @param initial_value the optional initial value * @param remake follow the default domain decomposition of the simulation * @param redistribute_on_remake redistribute on @see amrex::AmrCore::RemakeLevel * @return pointer to newly allocated MultiFab @@ -170,6 +169,24 @@ namespace ablastr::fields bool remake = true, bool redistribute_on_remake = true ); + + /** Allocate and optionally initialize a MultiFab (field) + * + * This registers a new MultiFab under a unique name, allocates it and + * optionally assigns it an initial value. + * + * @param name a unique name for this field + * @param dir the field component for vector fields ("direction" of the unit vector) + * @param level the MR level to represent + * @param ba the list of boxes to cover the field + * @param dm the distribution mapping for load balancing with MPI + * @param ncomp the number of components of the field (all with the same staggering) + * @param ngrow the number of guard (ghost, halo) cells + * @param initial_value the optional initial value + * @param remake follow the default domain decomposition of the simulation + * @param redistribute_on_remake redistribute on @see amrex::AmrCore::RemakeLevel + * @return pointer to newly allocated MultiFab + */ amrex::MultiFab* alloc_init ( std::string name, @@ -184,13 +201,16 @@ namespace ablastr::fields bool redistribute_on_remake = true ); - /** Create an alias + /** Create an alias of a MultiFab (field) + * + * Registers a new name for an existing MultiFab (field) and optionally assigning + * a value. * * @param new_name new name - * @param alias_name owner to alias to - * @param level ... - * @param initial_value ... - * @return + * @param alias_name owner name to alias + * @param level the MR level to represent + * @param initial_value the optional value to assign + * @return the newly aliased MultiFab */ amrex::MultiFab* alias_init ( @@ -200,22 +220,25 @@ namespace ablastr::fields std::optional initial_value = std::nullopt ); - /** Create an alias + /** Create an alias of a MultiFab (field) + * + * Registers a new name for an existing MultiFab (field) and optionally assigning + * a value. * * @param new_name new name - * @param alias_name owner to alias to - * @param dir ... - * @param level ... - * @param initial_value ... - * @return + * @param alias_name owner name to alias + * @param dir the field component for vector fields ("direction" of the unit vector) both in the alias and aliased + * @param level the MR level to represent + * @param initial_value the optional value to assign + * @return the newly aliased MultiFab */ amrex::MultiFab* alias_init ( - std::string new_name, - std::string alias_name, - Direction dir, - int level, - std::optional initial_value = std::nullopt + std::string new_name, + std::string alias_name, + Direction dir, + int level, + std::optional initial_value = std::nullopt ); /** title From fc9cc97b738cc4554c69d35a7bc970097b935f9b Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 16 Sep 2024 20:51:59 -0700 Subject: [PATCH 245/314] `MultiFabRegister`: Clang-Tidy --- Source/ablastr/fields/MultiFabRegister.cpp | 81 ++++++++++------------ 1 file changed, 37 insertions(+), 44 deletions(-) diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 7530eec0509..c043c044cd4 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -38,17 +38,15 @@ namespace ablastr::fields // allocate const auto tag = amrex::MFInfo().SetTag(name); auto [it, success] = m_mf_register.emplace( - std::make_pair( - name, - MultiFabOwner{ - {ba, dm, ncomp, ngrow, tag}, - std::nullopt, // scalar: no direction - level, - remake, - redistribute_on_remake, - "" // we own the memory - } - ) + name, + MultiFabOwner{ + {ba, dm, ncomp, ngrow, tag}, + std::nullopt, // scalar: no direction + level, + remake, + redistribute_on_remake, + "" // we own the memory + } ); if (!success) { throw std::runtime_error("MultiFabRegister::alloc_init failed for " + name); @@ -94,17 +92,15 @@ namespace ablastr::fields // allocate const auto tag = amrex::MFInfo().SetTag(name); auto [it, success] = m_mf_register.emplace( - std::make_pair( - name, - MultiFabOwner{ - {ba, dm, ncomp, ngrow, tag}, - dir, - level, - remake, - redistribute_on_remake, - "" // we own the memory - } - ) + name, + MultiFabOwner{ + {ba, dm, ncomp, ngrow, tag}, + dir, + level, + remake, + redistribute_on_remake, + "" // we own the memory + } ); if (!success) { throw std::runtime_error("MultiFabRegister::alloc_init failed for " + name); @@ -168,17 +164,16 @@ namespace ablastr::fields // allocate auto [it, success] = m_mf_register.emplace( - std::make_pair( - new_name, - MultiFabOwner{ - {mf_alias, amrex::make_alias, 0, mf_alias.nComp()}, - std::nullopt, // scalar: no direction - level, - alias.m_remake, - alias.m_redistribute_on_remake, - alias_name - } - ) + new_name, + MultiFabOwner{ + {mf_alias, amrex::make_alias, 0, mf_alias.nComp()}, + std::nullopt, // scalar: no direction + level, + alias.m_remake, + alias.m_redistribute_on_remake, + alias_name + } + ); if (!success) { throw std::runtime_error("MultiFabRegister::alias_init failed for " + new_name); @@ -229,17 +224,15 @@ namespace ablastr::fields // allocate auto [it, success] = m_mf_register.emplace( - std::make_pair( - new_name, - MultiFabOwner{ - {mf_alias, amrex::make_alias, 0, mf_alias.nComp()}, - dir, - level, - alias.m_remake, - alias.m_redistribute_on_remake, - alias_name - } - ) + new_name, + MultiFabOwner{ + {mf_alias, amrex::make_alias, 0, mf_alias.nComp()}, + dir, + level, + alias.m_remake, + alias.m_redistribute_on_remake, + alias_name + } ); if (!success) { throw std::runtime_error("MultiFabRegister::alias_init failed for " + new_name); From 75187c7949e13f65222b516d19462df22036215d Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Tue, 17 Sep 2024 11:15:02 +0200 Subject: [PATCH 246/314] address clang-tidy issues --- Source/ablastr/fields/MultiFabRegister.H | 18 ++++----- Source/ablastr/fields/MultiFabRegister.cpp | 46 +++++++++++----------- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 9d1713849ca..7599aa68d07 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -251,7 +251,7 @@ namespace ablastr::fields */ void alloc_like ( - std::string other_name, + const std::string& other_name, int other_level ); @@ -361,12 +361,12 @@ namespace ablastr::fields */ MultiLevelScalarField get_mr_levels ( - std::string name, + const std::string& name, int finest_level ); [[nodiscard]] ConstMultiLevelScalarField get_mr_levels ( - std::string name, + const std::string& name, int finest_level ) const; @@ -380,12 +380,12 @@ namespace ablastr::fields */ VectorField get_alldirs ( - std::string name, + const std::string&, int level ); [[nodiscard]] ConstVectorField get_alldirs ( - std::string name, + const std::string& name, int level ) const; @@ -400,12 +400,12 @@ namespace ablastr::fields */ MultiLevelVectorField get_mr_levels_alldirs ( - std::string name, + const std::string& name, int finest_level ); [[nodiscard]] ConstMultiLevelVectorField get_mr_levels_alldirs ( - std::string name, + const std::string& name, int finest_level ) const; @@ -508,11 +508,11 @@ namespace ablastr::fields private: amrex::MultiFab * internal_get ( - std::string key + const std::string& key ); [[nodiscard]] amrex::MultiFab const * internal_get ( - std::string key + const std::string& key ) const; /** data storage: ownership and lifetime control */ diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index c043c044cd4..25b0505e197 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -119,7 +119,7 @@ namespace ablastr::fields void MultiFabRegister::alloc_like ( - std::string /* other_name */, + const std::string& /* other_name */, int /* other_level */ ) { @@ -267,7 +267,7 @@ namespace ablastr::fields // remake MultiFab with new distribution map if (mf_owner.m_level == level && !mf_owner.is_alias()) { - amrex::MultiFab & mf = mf_owner.m_mf; + const amrex::MultiFab & mf = mf_owner.m_mf; amrex::IntVect const & ng = mf.nGrowVect(); const auto tag = amrex::MFInfo().SetTag(mf.tags()[0]); amrex::MultiFab new_mf(mf.boxArray(), new_dm, mf.nComp(), ng, tag); @@ -294,7 +294,7 @@ namespace ablastr::fields } if (mf_owner.m_level == level && mf_owner.is_alias()) { - amrex::MultiFab & mf = m_mf_register[mf_owner.m_owner].m_mf; + const amrex::MultiFab & mf = m_mf_register[mf_owner.m_owner].m_mf; amrex::MultiFab new_mf(mf, amrex::make_alias, 0, mf.nComp()); // no copy via Redistribute: the owner was already redistributed @@ -330,7 +330,7 @@ namespace ablastr::fields amrex::MultiFab* MultiFabRegister::internal_get ( - std::string key + const std::string& key ) { if (m_mf_register.count(key) == 0) { @@ -345,7 +345,7 @@ namespace ablastr::fields amrex::MultiFab const * MultiFabRegister::internal_get ( - std::string key + const std::string& key ) const { if (m_mf_register.count(key) == 0) { @@ -402,7 +402,7 @@ namespace ablastr::fields MultiLevelScalarField MultiFabRegister::get_mr_levels ( - std::string name, + const std::string& name, int finest_level ) { @@ -417,7 +417,7 @@ namespace ablastr::fields ConstMultiLevelScalarField MultiFabRegister::get_mr_levels ( - std::string name, + const std::string& name, int finest_level ) const { @@ -432,18 +432,18 @@ namespace ablastr::fields VectorField MultiFabRegister::get_alldirs ( - std::string name, + const std::string& name, int level ) { // TODO: Technically, we should search field_on_level via std::unique_copy - std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; + const std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; // insert a new level VectorField vectorField; // insert components - for (Direction dir : all_dirs) + for (const Direction& dir : all_dirs) { vectorField[dir] = get(name, dir, level); } @@ -452,18 +452,18 @@ namespace ablastr::fields ConstVectorField MultiFabRegister::get_alldirs ( - std::string name, + const std::string& name, int level ) const { // TODO: Technically, we should search field_on_level via std::unique_copy - std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; + const std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; // insert a new level ConstVectorField vectorField; // insert components - for (Direction dir : all_dirs) + for (const Direction& dir: all_dirs) { vectorField[dir] = get(name, dir, level); } @@ -472,7 +472,7 @@ namespace ablastr::fields MultiLevelVectorField MultiFabRegister::get_mr_levels_alldirs ( - std::string name, + const std::string& name, int finest_level ) { @@ -480,7 +480,7 @@ namespace ablastr::fields field_on_level.reserve(finest_level+1); // TODO: Technically, we should search field_on_level via std::unique_copy - std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; + const std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; for (int lvl = 0; lvl <= finest_level; lvl++) { @@ -488,7 +488,7 @@ namespace ablastr::fields field_on_level.push_back(VectorField{}); // insert components - for (Direction dir : all_dirs) + for (const Direction& dir : all_dirs) { field_on_level[lvl][dir] = get(name, dir, lvl); } @@ -498,7 +498,7 @@ namespace ablastr::fields ConstMultiLevelVectorField MultiFabRegister::get_mr_levels_alldirs ( - std::string name, + const std::string& name, int finest_level ) const { @@ -506,7 +506,7 @@ namespace ablastr::fields field_on_level.reserve(finest_level+1); // TODO: Technically, we should search field_on_level via std::unique_copy - std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; + const std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; for (int lvl = 0; lvl <= finest_level; lvl++) { @@ -514,7 +514,7 @@ namespace ablastr::fields field_on_level.push_back(ConstVectorField{}); // insert components - for (Direction dir : all_dirs) + for (const Direction& dir : all_dirs) { field_on_level[lvl][dir] = get(name, dir, lvl); } @@ -610,12 +610,12 @@ namespace ablastr::fields const std::array< std::unique_ptr, 3 > & old_vectorfield ) { - std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; + const std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; VectorField field_on_level; // insert components - for (auto dir : {0, 1, 2}) + for (const auto dir : {0, 1, 2}) { field_on_level[Direction{dir}] = old_vectorfield[dir].get(); } @@ -632,7 +632,7 @@ namespace ablastr::fields MultiLevelVectorField field_on_level; field_on_level.reserve(finest_level+1); - std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; + const std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; for (int lvl = 0; lvl <= finest_level; lvl++) { @@ -640,7 +640,7 @@ namespace ablastr::fields field_on_level.push_back(VectorField{}); // insert components - for (auto dir : {0, 1, 2}) + for (const auto dir : {0, 1, 2}) { field_on_level[lvl][Direction{dir}] = old_vector_on_levels[lvl][dir].get(); } From 7f1a0472b489c7ec01f946d45a8de7362147b2c0 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Tue, 17 Sep 2024 14:27:37 +0200 Subject: [PATCH 247/314] address branch-clone issue --- .../FieldSolver/FiniteDifferenceSolver/EvolveB.cpp | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp index 0a69db6076d..2fcc18a33d0 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp @@ -76,23 +76,19 @@ void FiniteDifferenceSolver::EvolveB ( } ablastr::fields::VectorField face_areas; if (fields.has("face_areas", Direction{0}, lev)) { - face_areas = patch_type == PatchType::fine ? - fields.get_alldirs("face_areas", lev) : fields.get_alldirs("face_areas", lev); + face_areas = fields.get_alldirs("face_areas", lev); } ablastr::fields::VectorField area_mod; - if (fields.has("face_areas", Direction{0}, lev)) { - area_mod = patch_type == PatchType::fine ? - fields.get_alldirs("area_mod", lev) : fields.get_alldirs("area_mod", lev); + if (fields.has("area_mod", Direction{0}, lev)) { + area_mod = fields.get_alldirs("area_mod", lev); } ablastr::fields::VectorField ECTRhofield; if (fields.has("ECTRhofield", Direction{0}, lev)) { - ECTRhofield = patch_type == PatchType::fine ? - fields.get_alldirs("ECTRhofield", lev) : fields.get_alldirs("ECTRhofield", lev); + ECTRhofield = fields.get_alldirs("ECTRhofield", lev); } ablastr::fields::VectorField Venl; if (fields.has("Venl", Direction{0}, lev)) { - Venl = patch_type == PatchType::fine ? - fields.get_alldirs("Venl", lev) : fields.get_alldirs("Venl", lev); + Venl = fields.get_alldirs("Venl", lev); } if (m_grid_type == GridType::Collocated) { From 01e6f6319d29c8b4fac869f30f7253bf9b88e8b7 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 17 Sep 2024 06:02:28 -0700 Subject: [PATCH 248/314] Fix typo in moving window --- Source/Utils/WarpXMovingWindow.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index ec1d2bffc5f..1e4eb6bb6a9 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -241,7 +241,7 @@ WarpX::MoveWindow (const int step, bool move_j) } shiftMF(*m_fields.get("Bfield_fp", Direction{dim}, lev), geom[lev], num_shift, dir, lev, do_update_cost, m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); - shiftMF(*m_fields.get("Efield_fp",Direction{0},lev), geom[lev], num_shift, dir, lev, do_update_cost, + shiftMF(*m_fields.get("Efield_fp",Direction{dim},lev), geom[lev], num_shift, dir, lev, do_update_cost, m_p_ext_field_params->E_external_grid[dim], use_Eparser, Efield_parser); if (fft_do_time_averaging) { ablastr::fields::MultiLevelVectorField Efield_avg_fp = m_fields.get_mr_levels_alldirs("Efield_avg_fp", finest_level); @@ -270,9 +270,9 @@ WarpX::MoveWindow (const int step, bool move_j) #endif if (lev > 0) { // coarse grid - shiftMF(*m_fields.get("Bfield_cp",Direction{0},lev), geom[lev-1], num_shift_crse, dir, lev, do_update_cost, + shiftMF(*m_fields.get("Bfield_cp",Direction{dim},lev), geom[lev-1], num_shift_crse, dir, lev, do_update_cost, m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); - shiftMF(*m_fields.get("Efield_cp",Direction{0},lev), geom[lev-1], num_shift, dir, lev, do_update_cost, + shiftMF(*m_fields.get("Efield_cp",Direction{dim},lev), geom[lev-1], num_shift, dir, lev, do_update_cost, m_p_ext_field_params->E_external_grid[dim], use_Eparser, Efield_parser); shiftMF(*Bfield_aux[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost); shiftMF(*Efield_aux[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost); From 5c9f9ba68affdc6c0ab771429f0e8a36d7e8c01f Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 17 Sep 2024 06:12:26 -0700 Subject: [PATCH 249/314] Use more uniform style in the moving window code --- Source/Utils/WarpXMovingWindow.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 1e4eb6bb6a9..c6e8af6f2a1 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -149,9 +149,6 @@ WarpX::MoveWindow (const int step, bool move_j) } if (!moving_window_active(step)) { return 0; } - ablastr::fields::MultiLevelVectorField Efield_aux = m_fields.get_mr_levels_alldirs("Efield_aux", finest_level); - ablastr::fields::MultiLevelVectorField Bfield_aux = m_fields.get_mr_levels_alldirs("Bfield_aux", finest_level); - // Update the continuous position of the moving window, // and of the plasma injection moving_window_x += (moving_window_v - WarpX::beta_boost * PhysConst::c)/(1 - moving_window_v * WarpX::beta_boost / PhysConst::c) * dt[0]; @@ -274,8 +271,8 @@ WarpX::MoveWindow (const int step, bool move_j) m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); shiftMF(*m_fields.get("Efield_cp",Direction{dim},lev), geom[lev-1], num_shift, dir, lev, do_update_cost, m_p_ext_field_params->E_external_grid[dim], use_Eparser, Efield_parser); - shiftMF(*Bfield_aux[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost); - shiftMF(*Efield_aux[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost); + shiftMF(*m_fields.get("Efield_aux",Direction{dim},lev), geom[lev], num_shift, dir, lev, do_update_cost); + shiftMF(*m_fields.get("Efield_aux",Direction{dim},lev), geom[lev], num_shift, dir, lev, do_update_cost); if (fft_do_time_averaging) { ablastr::fields::MultiLevelVectorField Efield_avg_cp = m_fields.get_mr_levels_alldirs("Efield_avg_cp", finest_level); ablastr::fields::MultiLevelVectorField Bfield_avg_cp = m_fields.get_mr_levels_alldirs("Bfield_avg_cp", finest_level); From b43adaa1d1a56166387eb593f5460890da087ffe Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 17 Sep 2024 06:15:16 -0700 Subject: [PATCH 250/314] Correct typo --- Source/Utils/WarpXMovingWindow.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index c6e8af6f2a1..ca6d3bfb611 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -271,7 +271,7 @@ WarpX::MoveWindow (const int step, bool move_j) m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); shiftMF(*m_fields.get("Efield_cp",Direction{dim},lev), geom[lev-1], num_shift, dir, lev, do_update_cost, m_p_ext_field_params->E_external_grid[dim], use_Eparser, Efield_parser); - shiftMF(*m_fields.get("Efield_aux",Direction{dim},lev), geom[lev], num_shift, dir, lev, do_update_cost); + shiftMF(*m_fields.get("Bfield_aux",Direction{dim},lev), geom[lev], num_shift, dir, lev, do_update_cost); shiftMF(*m_fields.get("Efield_aux",Direction{dim},lev), geom[lev], num_shift, dir, lev, do_update_cost); if (fft_do_time_averaging) { ablastr::fields::MultiLevelVectorField Efield_avg_cp = m_fields.get_mr_levels_alldirs("Efield_avg_cp", finest_level); From fe87f9344ae7c7cc4e221db35e53657a12547500 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Tue, 17 Sep 2024 16:43:55 +0200 Subject: [PATCH 251/314] fix clang-tidy issues --- Source/BoundaryConditions/PML_RZ.cpp | 4 ++-- Source/Evolve/WarpXEvolve.cpp | 8 ++++---- .../LabFrameExplicitES.cpp | 8 ++++---- .../FiniteDifferenceSolver/EvolveB.cpp | 4 ++-- .../FiniteDifferenceSolver/EvolveBPML.cpp | 4 ++-- .../FiniteDifferenceSolver/EvolveE.cpp | 20 ++++++++----------- .../FiniteDifferenceSolver/EvolveEPML.cpp | 6 +++--- .../FiniteDifferenceSolver.H | 4 ++-- .../HybridPICModel/HybridPICModel.cpp | 4 ++-- .../ImplicitSolvers/WarpXSolverVec.H | 4 ++-- .../ImplicitSolvers/WarpXSolverVec.cpp | 6 +++--- Source/FieldSolver/WarpXPushFieldsEM.cpp | 10 +++++----- Source/WarpX.H | 2 +- Source/WarpX.cpp | 18 ++++++++--------- Source/ablastr/fields/MultiFabRegister.cpp | 10 ++++++---- 15 files changed, 55 insertions(+), 57 deletions(-) diff --git a/Source/BoundaryConditions/PML_RZ.cpp b/Source/BoundaryConditions/PML_RZ.cpp index cc05c126a55..0fc24bf8817 100644 --- a/Source/BoundaryConditions/PML_RZ.cpp +++ b/Source/BoundaryConditions/PML_RZ.cpp @@ -138,7 +138,7 @@ PML_RZ::FillBoundaryE (ablastr::fields::MultiFabRegister& fields, PatchType patc if (patch_type == PatchType::fine && pml_Er->nGrowVect().max() > 0) { amrex::Periodicity const& period = m_geom->periodicity(); - amrex::Vector mf = {pml_Er, pml_Et}; + const amrex::Vector mf = {pml_Er, pml_Et}; ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period, nodal_sync); } } @@ -152,7 +152,7 @@ PML_RZ::FillBoundaryB (ablastr::fields::MultiFabRegister& fields, PatchType patc amrex::MultiFab * pml_Bt = fields.get("pml_B_fp", Direction{1}, 0); amrex::Periodicity const& period = m_geom->periodicity(); - amrex::Vector mf = {pml_Br, pml_Bt}; + const amrex::Vector mf = {pml_Br, pml_Bt}; ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period, nodal_sync); } } diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index a4df5ea73bd..bd01aeaf359 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -663,8 +663,8 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // (after checking that pointer to rho_fp on MR level 0 is not null) if (m_fields.has("rho_fp", 0) && rho_in_time == RhoInTime::Linear) { - ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); - ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_fp", finest_level); + const ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); + const ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_fp", finest_level); // Deposit rho at relative time -dt // (dt[0] denotes the time step on mesh refinement level 0) @@ -732,8 +732,8 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // (after checking that pointer to rho_fp on MR level 0 is not null) if (m_fields.has("rho_fp", 0)) { - ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); - ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_cp", finest_level); + const ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); + const ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_cp", finest_level); // Move rho from new to old if rho is linear in time if (rho_in_time == RhoInTime::Linear) { PSATDMoveRhoNewToRhoOld(); } diff --git a/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp b/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp index de9f20a54aa..a2d45bcdb8d 100644 --- a/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp +++ b/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp @@ -29,10 +29,10 @@ void LabFrameExplicitES::ComputeSpaceChargeField ( using ablastr::fields::MultiLevelScalarField; using ablastr::fields::MultiLevelVectorField; - MultiLevelScalarField rho_fp = fields.get_mr_levels("rho_fp", max_level); - MultiLevelScalarField rho_cp = fields.get_mr_levels("rho_cp", max_level); + const MultiLevelScalarField rho_fp = fields.get_mr_levels("rho_fp", max_level); + const MultiLevelScalarField rho_cp = fields.get_mr_levels("rho_cp", max_level); MultiLevelScalarField phi_fp = fields.get_mr_levels("phi_fp", max_level); - MultiLevelVectorField Efield_fp = fields.get_mr_levels_alldirs("Efield_fp", max_level); + const MultiLevelVectorField Efield_fp = fields.get_mr_levels_alldirs("Efield_fp", max_level); mpc.DepositCharge(rho_fp, 0.0_rt); if (mfl) { @@ -41,7 +41,7 @@ void LabFrameExplicitES::ComputeSpaceChargeField ( } // Apply filter, perform MPI exchange, interpolate across levels - Vector > rho_buf(num_levels); + const Vector > rho_buf(num_levels); auto & warpx = WarpX::GetInstance(); warpx.SyncRho( rho_fp, rho_cp, amrex::GetVecOfPtrs(rho_buf) ); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp index 2fcc18a33d0..a5b85f7fbd7 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp @@ -56,9 +56,9 @@ void FiniteDifferenceSolver::EvolveB ( [[maybe_unused]] amrex::Real const dt ) { using ablastr::fields::Direction; - ablastr::fields::VectorField Bfield = patch_type == PatchType::fine ? + const ablastr::fields::VectorField Bfield = patch_type == PatchType::fine ? fields.get_alldirs("Bfield_fp", lev) : fields.get_alldirs("Bfield_cp", lev); - ablastr::fields::VectorField Efield = patch_type == PatchType::fine ? + const ablastr::fields::VectorField Efield = patch_type == PatchType::fine ? fields.get_alldirs("Efield_fp", lev) : fields.get_alldirs("Efield_cp", lev); // Select algorithm (The choice of algorithm is a runtime option, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp index 9a83b4bc072..5d46d18ff4e 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp @@ -56,9 +56,9 @@ void FiniteDifferenceSolver::EvolveBPML ( WARPX_ABORT_WITH_MESSAGE( "PML are not implemented in cylindrical geometry."); #else - ablastr::fields::VectorField Bfield = (patch_type == PatchType::fine) ? + const ablastr::fields::VectorField Bfield = (patch_type == PatchType::fine) ? fields.get_alldirs("pml_B_fp", level) : fields.get_alldirs("pml_B_cp", level); - ablastr::fields::VectorField Efield = (patch_type == PatchType::fine) ? + const ablastr::fields::VectorField Efield = (patch_type == PatchType::fine) ? fields.get_alldirs("pml_E_fp", level) : fields.get_alldirs("pml_E_cp", level); if (m_grid_type == ablastr::utils::enums::GridType::Collocated) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp index ac18b2f7fae..7e9f62c589b 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp @@ -58,9 +58,9 @@ void FiniteDifferenceSolver::EvolveE ( ) { using ablastr::fields::Direction; - ablastr::fields::VectorField Bfield = patch_type == PatchType::fine ? + const ablastr::fields::VectorField Bfield = patch_type == PatchType::fine ? fields.get_alldirs("Bfield_fp", lev) : fields.get_alldirs("Bfield_cp", lev); - ablastr::fields::VectorField Jfield = patch_type == PatchType::fine ? + const ablastr::fields::VectorField Jfield = patch_type == PatchType::fine ? fields.get_alldirs("current_fp", lev) : fields.get_alldirs("current_cp", lev); amrex::MultiFab* Ffield = nullptr; @@ -71,23 +71,19 @@ void FiniteDifferenceSolver::EvolveE ( ablastr::fields::VectorField edge_lengths; if (fields.has("edge_lengths", Direction{0}, lev)) { - edge_lengths = patch_type == PatchType::fine ? - fields.get_alldirs("edge_lengths", lev) : fields.get_alldirs("edge_lengths", lev); + edge_lengths = fields.get_alldirs("edge_lengths", lev); } ablastr::fields::VectorField face_areas; if (fields.has("face_areas", Direction{0}, lev)) { - face_areas = patch_type == PatchType::fine ? - fields.get_alldirs("face_areas", lev) : fields.get_alldirs("face_areas", lev); + face_areas = fields.get_alldirs("face_areas", lev); } ablastr::fields::VectorField area_mod; - if (fields.has("face_areas", Direction{0}, lev)) { - area_mod = patch_type == PatchType::fine ? - fields.get_alldirs("area_mod", lev) : fields.get_alldirs("area_mod", lev); + if (fields.has("area_mod", Direction{0}, lev)) { + area_mod = fields.get_alldirs("area_mod", lev); } ablastr::fields::VectorField ECTRhofield; if (fields.has("ECTRhofield", Direction{0}, lev)) { - ECTRhofield = patch_type == PatchType::fine ? - fields.get_alldirs("ECTRhofield", lev) : fields.get_alldirs("ECTRhofield", lev); + ECTRhofield = fields.get_alldirs("ECTRhofield", lev); } // Select algorithm (The choice of algorithm is a runtime option, @@ -221,7 +217,7 @@ void FiniteDifferenceSolver::EvolveECartesian ( if (Ffield) { // Extract field data for this grid/tile - Array4 F = Ffield->array(mfi); + const Array4 F = Ffield->array(mfi); // Loop over the cells and update the fields amrex::ParallelFor(tex, tey, tez, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp index 4da403156b0..d678bed3b01 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp @@ -59,11 +59,11 @@ void FiniteDifferenceSolver::EvolveEPML ( "PML are not implemented in cylindrical geometry."); #else using ablastr::fields::Direction; - ablastr::fields::VectorField Efield = (patch_type == PatchType::fine) ? + const ablastr::fields::VectorField Efield = (patch_type == PatchType::fine) ? fields.get_alldirs("pml_E_fp", level) : fields.get_alldirs("pml_E_cp", level); - ablastr::fields::VectorField Bfield = (patch_type == PatchType::fine) ? + const ablastr::fields::VectorField Bfield = (patch_type == PatchType::fine) ? fields.get_alldirs("pml_B_fp", level) : fields.get_alldirs("pml_B_cp", level); - ablastr::fields::VectorField Jfield = (patch_type == PatchType::fine) ? + const ablastr::fields::VectorField Jfield = (patch_type == PatchType::fine) ? fields.get_alldirs("pml_j_fp", level) : fields.get_alldirs("pml_j_cp", level); ablastr::fields::VectorField edge_lengths; if (fields.has("pml_edge_lengths", Direction{0}, level)) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 49d187431ce..03f51f7ba62 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -226,7 +226,7 @@ class FiniteDifferenceSolver void EvolveFCylindrical ( amrex::MultiFab* Ffield, ablastr::fields::VectorField const & Efield, - amrex::MultiFab* const rhofield, + amrex::MultiFab* rhofield, int rhocomp, amrex::Real dt ); @@ -322,7 +322,7 @@ class FiniteDifferenceSolver template< typename T_Algo > void EvolveBPMLCartesian ( std::array< amrex::MultiFab*, 3 > Bfield, - ablastr::fields::VectorField const Efield, + ablastr::fields::VectorField Efield, amrex::Real dt, bool dive_cleaning); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index b05b9ae1562..ac5bf34a1ff 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -468,8 +468,8 @@ void HybridPICModel::HybridPICSolveE ( auto& warpx = WarpX::GetInstance(); ablastr::fields::VectorField current_fp_ampere = warpx.m_fields.get_alldirs("hybrid_current_fp_ampere", lev); - ablastr::fields::VectorField current_fp_external = warpx.m_fields.get_alldirs("hybrid_current_fp_external", lev); - ablastr::fields::ScalarField electron_pressure_fp = warpx.m_fields.get("hybrid_electron_pressure_fp", lev); + const ablastr::fields::VectorField current_fp_external = warpx.m_fields.get_alldirs("hybrid_current_fp_external", lev); + const ablastr::fields::ScalarField electron_pressure_fp = warpx.m_fields.get("hybrid_electron_pressure_fp", lev); // Solve E field in regular cells warpx.get_pointer_fdtd_solver_fp(lev)->HybridPICSolveE( diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H index 74e45ba631c..b109229328e 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H +++ b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H @@ -68,8 +68,8 @@ public: [[nodiscard]] inline bool IsDefined () const { return m_is_defined; } void Define ( WarpX* a_WarpX, - std::string a_vector_type_name, - std::string a_scalar_type_name = "none" ); + const std::string& a_vector_type_name, + const std::string& a_scalar_type_name = "none" ); inline void Define ( const WarpXSolverVec& a_solver_vec ) diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp index da359f35248..10673704759 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp @@ -21,8 +21,8 @@ WarpXSolverVec::~WarpXSolverVec () } void WarpXSolverVec::Define ( WarpX* a_WarpX, - std::string a_vector_type_name, - std::string a_scalar_type_name ) + const std::string& a_vector_type_name, + const std::string& a_scalar_type_name ) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( !IsDefined(), @@ -41,7 +41,7 @@ void WarpXSolverVec::Define ( WarpX* a_WarpX, m_array_type = FieldType::Efield_fp; } else if (m_vector_type_name=="Bfield_fp") { - m_array_type = FieldType::Efield_fp; + m_array_type = FieldType::Bfield_fp; } else if (m_vector_type_name=="vector_potential_fp_nodal") { m_array_type = FieldType::vector_potential_fp; diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index ede04b39632..43a964c1806 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -669,11 +669,11 @@ WarpX::PushPSATD () const int rho_old = spectral_solver_fp[0]->m_spectral_index.rho_old; const int rho_new = spectral_solver_fp[0]->m_spectral_index.rho_new; - ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); - ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_fp", finest_level); - ablastr::fields::MultiLevelVectorField current_fp = m_fields.get_mr_levels_alldirs("current_fp", finest_level); - ablastr::fields::MultiLevelVectorField current_cp = m_fields.get_mr_levels_alldirs("current_cp", finest_level); - ablastr::fields::MultiLevelVectorField current_buf = m_fields.get_mr_levels_alldirs("current_buf", finest_level); + const ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); + const ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_fp", finest_level); + const ablastr::fields::MultiLevelVectorField current_fp = m_fields.get_mr_levels_alldirs("current_fp", finest_level); + const ablastr::fields::MultiLevelVectorField current_cp = m_fields.get_mr_levels_alldirs("current_cp", finest_level); + const ablastr::fields::MultiLevelVectorField current_buf = m_fields.get_mr_levels_alldirs("current_buf", finest_level); if (fft_periodic_single_box) { diff --git a/Source/WarpX.H b/Source/WarpX.H index 40ad6f031c3..338a88b28b1 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -485,7 +485,7 @@ public: * Set the dotMask container */ void SetDotMask( std::unique_ptr& field_dotMask, - std::string field_name, int lev, int dir ) const; + const std::string& field_name, int lev, int dir ) const; [[nodiscard]] bool DoPML () const {return do_pml;} [[nodiscard]] bool DoFluidSpecies () const {return do_fluid_species;} diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 4efa58b8fee..d89404f17bf 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2530,9 +2530,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm } if (mypc->m_B_ext_particle_s == "read_from_file") { // These fields will be added to the fields that the particles see, and need to match the index type - auto Bfield_aux_levl_0 = m_fields.get("Bfield_aux", Direction{0}, lev); - auto Bfield_aux_levl_1 = m_fields.get("Bfield_aux", Direction{1}, lev); - auto Bfield_aux_levl_2 = m_fields.get("Bfield_aux", Direction{2}, lev); + auto* Bfield_aux_levl_0 = m_fields.get("Bfield_aux", Direction{0}, lev); + auto* Bfield_aux_levl_1 = m_fields.get("Bfield_aux", Direction{1}, lev); + auto* Bfield_aux_levl_2 = m_fields.get("Bfield_aux", Direction{2}, lev); // Same as Bfield_fp for reading external field data m_fields.alloc_init( "B_external_particle_field", Direction{0}, lev, amrex::convert(ba, Bfield_aux_levl_0->ixType()), @@ -2553,9 +2553,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm } if (mypc->m_E_ext_particle_s == "read_from_file") { // These fields will be added to the fields that the particles see, and need to match the index type - auto Efield_aux_levl_0 = m_fields.get("Efield_aux", Direction{0}, lev); - auto Efield_aux_levl_1 = m_fields.get("Efield_aux", Direction{1}, lev); - auto Efield_aux_levl_2 = m_fields.get("Efield_aux", Direction{2}, lev); + auto* Efield_aux_levl_0 = m_fields.get("Efield_aux", Direction{0}, lev); + auto* Efield_aux_levl_1 = m_fields.get("Efield_aux", Direction{1}, lev); + auto* Efield_aux_levl_2 = m_fields.get("Efield_aux", Direction{2}, lev); // Same as Efield_fp for reading external field data m_fields.alloc_init( "E_external_particle_field", Direction{0}, lev, amrex::convert(ba, Efield_aux_levl_0->ixType()), @@ -2966,14 +2966,14 @@ WarpX::ComputeDivE(amrex::MultiFab& divE, const int lev) { if ( WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD ) { #ifdef WARPX_USE_FFT - ablastr::fields::VectorField Efield_aux_lev = m_fields.get_alldirs("Efield_aux", lev); + const ablastr::fields::VectorField Efield_aux_lev = m_fields.get_alldirs("Efield_aux", lev); spectral_solver_fp[lev]->ComputeSpectralDivE(lev, Efield_aux_lev, divE); #else WARPX_ABORT_WITH_MESSAGE( "ComputeDivE: PSATD requested but not compiled"); #endif } else { - ablastr::fields::VectorField Efield_aux_lev = m_fields.get_alldirs("Efield_aux", lev); + const ablastr::fields::VectorField Efield_aux_lev = m_fields.get_alldirs("Efield_aux", lev); m_fdtd_solver_fp[lev]->ComputeDivE(Efield_aux_lev, divE); } } @@ -3410,7 +3410,7 @@ WarpX::getFieldDotMaskPointer ( FieldType field_type, int lev, int dir ) const } void WarpX::SetDotMask( std::unique_ptr& field_dotMask, - std::string field_name, int lev, int dir ) const + const std::string& field_name, int lev, int dir ) const { // Define the dot mask for this field_type needed to properly compute dotProduct() // for field values that have shared locations on different MPI ranks diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 25b0505e197..4e91180c7eb 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -569,10 +569,12 @@ namespace ablastr::fields // C++20: Replace with std::erase_if for (auto first = m_mf_register.begin(), last = m_mf_register.end(); first != last;) { - if (first->second.m_level == level) + if (first->second.m_level == level){ first = m_mf_register.erase(first); - else + } + else{ ++first; + } } } @@ -627,7 +629,7 @@ namespace ablastr::fields const amrex::Vector, 3 > >& old_vector_on_levels ) { - int const finest_level = old_vector_on_levels.size() - 1u; + auto const finest_level = static_cast(old_vector_on_levels.size() - 1u); MultiLevelVectorField field_on_level; field_on_level.reserve(finest_level+1); @@ -653,7 +655,7 @@ namespace ablastr::fields const amrex::Vector >& old_scalar_on_levels ) { - int const finest_level = old_scalar_on_levels.size() - 1u; + auto const finest_level = static_cast(old_scalar_on_levels.size() - 1u); MultiLevelScalarField field_on_level; field_on_level.reserve(finest_level+1); From 35fa3960ed031fc3229481596acedadf9663ae3a Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Tue, 17 Sep 2024 09:12:22 -0700 Subject: [PATCH 252/314] Fix typo bugs in `WarpX::InitFromCheckpoint` --- Source/Diagnostics/WarpXIO.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp index a572eaa2ed3..26e7a025ff2 100644 --- a/Source/Diagnostics/WarpXIO.cpp +++ b/Source/Diagnostics/WarpXIO.cpp @@ -339,18 +339,18 @@ WarpX::InitFromCheckpoint () if (lev > 0) { - VisMF::Read(*m_fields.get("Efield_fp", Direction{0}, lev), + VisMF::Read(*m_fields.get("Efield_cp", Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ex_cp")); - VisMF::Read(*m_fields.get("Efield_fp", Direction{1}, lev), + VisMF::Read(*m_fields.get("Efield_cp", Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ey_cp")); - VisMF::Read(*m_fields.get("Efield_fp", Direction{2}, lev), + VisMF::Read(*m_fields.get("Efield_cp", Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ez_cp")); - VisMF::Read(*m_fields.get("Bfield_fp", Direction{0}, lev), + VisMF::Read(*m_fields.get("Bfield_cp", Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Bx_cp")); - VisMF::Read(*m_fields.get("Bfield_fp", Direction{1}, lev), + VisMF::Read(*m_fields.get("Bfield_cp", Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "By_cp")); - VisMF::Read(*m_fields.get("Bfield_fp", Direction{2}, lev), + VisMF::Read(*m_fields.get("Bfield_cp", Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Bz_cp")); if (WarpX::fft_do_time_averaging) From 2687d316788cb1c5b14d5ff59ac8d88171801709 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Tue, 17 Sep 2024 13:20:45 -0700 Subject: [PATCH 253/314] Fix typo bug in `MultiParticleContainer::Evolve` --- Source/Particles/MultiParticleContainer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index a44808195e0..472461d3317 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -472,7 +472,7 @@ MultiParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, fields.get(current_fp_string, Direction{2}, lev)->setVal(0.0); if (fields.has("current_buf", Direction{0}, lev)) { fields.get("current_buf", Direction{0}, lev)->setVal(0.0); } if (fields.has("current_buf", Direction{1}, lev)) { fields.get("current_buf", Direction{1}, lev)->setVal(0.0); } - if (fields.has("current_buf", Direction{1}, lev)) { fields.get("current_buf", Direction{2}, lev)->setVal(0.0); } + if (fields.has("current_buf", Direction{2}, lev)) { fields.get("current_buf", Direction{2}, lev)->setVal(0.0); } if (fields.has("rho_fp", lev)) { fields.get("rho_fp", lev)->setVal(0.0); } if (fields.has("rho_buf", lev)) { fields.get("rho_buf", lev)->setVal(0.0); } } From a5a9895d2e3d3ec70e52d358ea665c14b7f51227 Mon Sep 17 00:00:00 2001 From: David Grote Date: Tue, 17 Sep 2024 14:20:00 -0700 Subject: [PATCH 254/314] Fix EvolvE for F_fp --- Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp index ac18b2f7fae..60fa68ff918 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp @@ -64,7 +64,7 @@ void FiniteDifferenceSolver::EvolveE ( fields.get_alldirs("current_fp", lev) : fields.get_alldirs("current_cp", lev); amrex::MultiFab* Ffield = nullptr; - if (fields.has("F_fp", Direction{0}, lev)) { + if (fields.has("F_fp", lev)) { Ffield = patch_type == PatchType::fine ? fields.get("F_fp", lev) : fields.get("F_cp", lev); } From 006db81b6ea952eceffb5a20f0380d3088ba446f Mon Sep 17 00:00:00 2001 From: David Grote Date: Tue, 17 Sep 2024 14:20:43 -0700 Subject: [PATCH 255/314] Fix typos in HybridPICModel --- .../FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp | 2 +- Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index b05b9ae1562..07a4935a83b 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -539,7 +539,7 @@ void HybridPICModel::FillElectronPressureMF ( void HybridPICModel::BfieldEvolveRK ( ablastr::fields::MultiLevelVectorField const& Bfield, - ablastr::fields::MultiLevelVectorField const&Efield, + ablastr::fields::MultiLevelVectorField const& Efield, ablastr::fields::MultiLevelVectorField const& Jfield, ablastr::fields::MultiLevelScalarField const& rhofield, ablastr::fields::MultiLevelVectorField const& edge_lengths, diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index ea5517f31aa..a8d70d8d792 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -107,7 +107,7 @@ void WarpX::HybridPICEvolveFields () m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level), m_fields.get_mr_levels_alldirs("Efield_fp", finest_level), current_fp_temp, rho_fp_temp, - m_fields.get_mr_levels_alldirs("edge_lenghts", finest_level), + m_fields.get_mr_levels_alldirs("edge_lengths", finest_level), 0.5_rt/sub_steps*dt[0], DtType::FirstHalf, guard_cells.ng_FieldSolver, WarpX::sync_nodal_points @@ -134,7 +134,7 @@ void WarpX::HybridPICEvolveFields () m_fields.get_mr_levels_alldirs("Efield_fp", finest_level), m_fields.get_mr_levels_alldirs("current_fp", finest_level), rho_fp_temp, - m_fields.get_mr_levels_alldirs("edge_lenghts", finest_level), + m_fields.get_mr_levels_alldirs("edge_lengths", finest_level), 0.5_rt/sub_steps*dt[0], DtType::SecondHalf, guard_cells.ng_FieldSolver, WarpX::sync_nodal_points From cf86694826327e75f7ec0ea82b7d77d14ed57c9f Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Tue, 17 Sep 2024 14:39:18 -0700 Subject: [PATCH 256/314] Fix typo bugs for averaged fields --- Source/WarpX.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 4efa58b8fee..29caaced9c4 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2259,12 +2259,12 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (fft_do_time_averaging) { m_fields.alloc_init( "Bfield_avg_fp", Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Bfield_avg_fp", Direction{1}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Bfield_avg_fp", Direction{2}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Bfield_avg_fp", Direction{1}, lev, amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Bfield_avg_fp", Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); m_fields.alloc_init( "Efield_avg_fp", Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Efield_avg_fp", Direction{1}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Efield_avg_fp", Direction{2}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Efield_avg_fp", Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init( "Efield_avg_fp", Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); } if (EB::enabled()) { From 0615220cbbdd5dfd5bb7f746cb78f4a203e30498 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Tue, 17 Sep 2024 14:45:18 -0700 Subject: [PATCH 257/314] Fix typo bugs for averaged fields (coarse patch) --- Source/WarpX.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 29caaced9c4..55a8e3f2033 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2588,12 +2588,12 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (fft_do_time_averaging) { m_fields.alloc_init("Bfield_avg_cp", Direction{0}, lev, amrex::convert(cba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Bfield_avg_cp", Direction{1}, lev, amrex::convert(cba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Bfield_avg_cp", Direction{2}, lev, amrex::convert(cba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Bfield_avg_cp", Direction{1}, lev, amrex::convert(cba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Bfield_avg_cp", Direction{2}, lev, amrex::convert(cba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); m_fields.alloc_init("Efield_avg_cp", Direction{0}, lev, amrex::convert(cba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Efield_avg_cp", Direction{1}, lev, amrex::convert(cba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Efield_avg_cp", Direction{2}, lev, amrex::convert(cba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Efield_avg_cp", Direction{1}, lev, amrex::convert(cba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init("Efield_avg_cp", Direction{2}, lev, amrex::convert(cba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); } // Create the MultiFabs for the current From 020088c2b307653f92f92cd2cacb4f600249710b Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Tue, 17 Sep 2024 17:27:32 -0700 Subject: [PATCH 258/314] Remove redundant using-directive --- Source/Particles/PhysicalParticleContainer.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 1ff712538ef..4cf6345c520 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1762,7 +1762,6 @@ PhysicalParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, const iMultiFab* current_masks = WarpX::CurrentBufferMasks(lev); const iMultiFab* gather_masks = WarpX::GatherBufferMasks(lev); - using ablastr::fields::Direction; const bool has_rho = fields.has("rho_fp", lev); const bool has_cjx = fields.has("current_buf", Direction{0}, lev); const bool has_cEx = fields.has("Efield_cax", Direction{0}, lev); From e0eb5bcdce2b711b80f74f88652c305b85766f21 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Tue, 17 Sep 2024 17:32:20 -0700 Subject: [PATCH 259/314] Fix clang-tidy error --- Source/ablastr/fields/MultiFabRegister.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 25b0505e197..c290266176e 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -569,10 +569,11 @@ namespace ablastr::fields // C++20: Replace with std::erase_if for (auto first = m_mf_register.begin(), last = m_mf_register.end(); first != last;) { - if (first->second.m_level == level) + if (first->second.m_level == level) { first = m_mf_register.erase(first); - else + } else { ++first; + } } } From c5571bd9d4491eb812a58bd98607b0d3d97cca59 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Tue, 17 Sep 2024 17:35:00 -0700 Subject: [PATCH 260/314] Fix clang-tidy error --- Source/WarpX.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 55a8e3f2033..cabe71cc54f 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2530,9 +2530,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm } if (mypc->m_B_ext_particle_s == "read_from_file") { // These fields will be added to the fields that the particles see, and need to match the index type - auto Bfield_aux_levl_0 = m_fields.get("Bfield_aux", Direction{0}, lev); - auto Bfield_aux_levl_1 = m_fields.get("Bfield_aux", Direction{1}, lev); - auto Bfield_aux_levl_2 = m_fields.get("Bfield_aux", Direction{2}, lev); + auto *Bfield_aux_levl_0 = m_fields.get("Bfield_aux", Direction{0}, lev); + auto *Bfield_aux_levl_1 = m_fields.get("Bfield_aux", Direction{1}, lev); + auto *Bfield_aux_levl_2 = m_fields.get("Bfield_aux", Direction{2}, lev); // Same as Bfield_fp for reading external field data m_fields.alloc_init( "B_external_particle_field", Direction{0}, lev, amrex::convert(ba, Bfield_aux_levl_0->ixType()), @@ -2553,9 +2553,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm } if (mypc->m_E_ext_particle_s == "read_from_file") { // These fields will be added to the fields that the particles see, and need to match the index type - auto Efield_aux_levl_0 = m_fields.get("Efield_aux", Direction{0}, lev); - auto Efield_aux_levl_1 = m_fields.get("Efield_aux", Direction{1}, lev); - auto Efield_aux_levl_2 = m_fields.get("Efield_aux", Direction{2}, lev); + auto *Efield_aux_levl_0 = m_fields.get("Efield_aux", Direction{0}, lev); + auto *Efield_aux_levl_1 = m_fields.get("Efield_aux", Direction{1}, lev); + auto *Efield_aux_levl_2 = m_fields.get("Efield_aux", Direction{2}, lev); // Same as Efield_fp for reading external field data m_fields.alloc_init( "E_external_particle_field", Direction{0}, lev, amrex::convert(ba, Efield_aux_levl_0->ixType()), From 4f499d00e29b139c85ac40b332e47b4e0c06d1c8 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Tue, 17 Sep 2024 17:50:37 -0700 Subject: [PATCH 261/314] Fix clang-tidy error --- Source/WarpX.H | 2 +- Source/WarpX.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/WarpX.H b/Source/WarpX.H index 40ad6f031c3..45ffa150c3d 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -485,7 +485,7 @@ public: * Set the dotMask container */ void SetDotMask( std::unique_ptr& field_dotMask, - std::string field_name, int lev, int dir ) const; + std::string const & field_name, int lev, int dir ) const; [[nodiscard]] bool DoPML () const {return do_pml;} [[nodiscard]] bool DoFluidSpecies () const {return do_fluid_species;} diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index cabe71cc54f..84fabec29c5 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -3410,7 +3410,7 @@ WarpX::getFieldDotMaskPointer ( FieldType field_type, int lev, int dir ) const } void WarpX::SetDotMask( std::unique_ptr& field_dotMask, - std::string field_name, int lev, int dir ) const + std::string const & field_name, int lev, int dir ) const { // Define the dot mask for this field_type needed to properly compute dotProduct() // for field values that have shared locations on different MPI ranks From 1400de536c03b7289aaf81c1a13afe23c9659769 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Tue, 17 Sep 2024 17:54:10 -0700 Subject: [PATCH 262/314] Fix clang-tidy error --- Source/WarpX.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 84fabec29c5..5457b633504 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2966,14 +2966,14 @@ WarpX::ComputeDivE(amrex::MultiFab& divE, const int lev) { if ( WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD ) { #ifdef WARPX_USE_FFT - ablastr::fields::VectorField Efield_aux_lev = m_fields.get_alldirs("Efield_aux", lev); + const ablastr::fields::VectorField Efield_aux_lev = m_fields.get_alldirs("Efield_aux", lev); spectral_solver_fp[lev]->ComputeSpectralDivE(lev, Efield_aux_lev, divE); #else WARPX_ABORT_WITH_MESSAGE( "ComputeDivE: PSATD requested but not compiled"); #endif } else { - ablastr::fields::VectorField Efield_aux_lev = m_fields.get_alldirs("Efield_aux", lev); + const ablastr::fields::VectorField Efield_aux_lev = m_fields.get_alldirs("Efield_aux", lev); m_fdtd_solver_fp[lev]->ComputeDivE(Efield_aux_lev, divE); } } From 1dfeb9f0ff8b104502d31bfd8aa90827de58ea7a Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Tue, 17 Sep 2024 17:57:33 -0700 Subject: [PATCH 263/314] Fix clang-tidy error --- .../FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 49d187431ce..a3656142384 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -322,7 +322,7 @@ class FiniteDifferenceSolver template< typename T_Algo > void EvolveBPMLCartesian ( std::array< amrex::MultiFab*, 3 > Bfield, - ablastr::fields::VectorField const Efield, + ablastr::fields::VectorField Efield, amrex::Real dt, bool dive_cleaning); From a20f7ddb6025b637a0c221c8b76322db046acfaa Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 18 Sep 2024 10:31:32 +0200 Subject: [PATCH 264/314] Revert "fix merge conflict" This reverts commit 655834087bc64ce7fb7cb09e030faed8b5f93490, reversing changes made to 1dfeb9f0ff8b104502d31bfd8aa90827de58ea7a. --- Source/BoundaryConditions/PML_RZ.cpp | 4 ++-- Source/Evolve/WarpXEvolve.cpp | 8 ++++---- .../LabFrameExplicitES.cpp | 8 ++++---- .../FiniteDifferenceSolver/EvolveB.cpp | 4 ++-- .../FiniteDifferenceSolver/EvolveBPML.cpp | 4 ++-- .../FiniteDifferenceSolver/EvolveE.cpp | 20 +++++++++++-------- .../FiniteDifferenceSolver/EvolveEPML.cpp | 6 +++--- .../FiniteDifferenceSolver.H | 2 +- .../HybridPICModel/HybridPICModel.cpp | 4 ++-- .../ImplicitSolvers/WarpXSolverVec.H | 4 ++-- .../ImplicitSolvers/WarpXSolverVec.cpp | 6 +++--- Source/FieldSolver/WarpXPushFieldsEM.cpp | 10 +++++----- Source/WarpX.cpp | 12 ----------- Source/ablastr/fields/MultiFabRegister.cpp | 4 ++-- 14 files changed, 44 insertions(+), 52 deletions(-) diff --git a/Source/BoundaryConditions/PML_RZ.cpp b/Source/BoundaryConditions/PML_RZ.cpp index 0fc24bf8817..cc05c126a55 100644 --- a/Source/BoundaryConditions/PML_RZ.cpp +++ b/Source/BoundaryConditions/PML_RZ.cpp @@ -138,7 +138,7 @@ PML_RZ::FillBoundaryE (ablastr::fields::MultiFabRegister& fields, PatchType patc if (patch_type == PatchType::fine && pml_Er->nGrowVect().max() > 0) { amrex::Periodicity const& period = m_geom->periodicity(); - const amrex::Vector mf = {pml_Er, pml_Et}; + amrex::Vector mf = {pml_Er, pml_Et}; ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period, nodal_sync); } } @@ -152,7 +152,7 @@ PML_RZ::FillBoundaryB (ablastr::fields::MultiFabRegister& fields, PatchType patc amrex::MultiFab * pml_Bt = fields.get("pml_B_fp", Direction{1}, 0); amrex::Periodicity const& period = m_geom->periodicity(); - const amrex::Vector mf = {pml_Br, pml_Bt}; + amrex::Vector mf = {pml_Br, pml_Bt}; ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period, nodal_sync); } } diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index bd01aeaf359..a4df5ea73bd 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -663,8 +663,8 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // (after checking that pointer to rho_fp on MR level 0 is not null) if (m_fields.has("rho_fp", 0) && rho_in_time == RhoInTime::Linear) { - const ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); - const ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_fp", finest_level); + ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); + ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_fp", finest_level); // Deposit rho at relative time -dt // (dt[0] denotes the time step on mesh refinement level 0) @@ -732,8 +732,8 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // (after checking that pointer to rho_fp on MR level 0 is not null) if (m_fields.has("rho_fp", 0)) { - const ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); - const ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_cp", finest_level); + ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); + ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_cp", finest_level); // Move rho from new to old if rho is linear in time if (rho_in_time == RhoInTime::Linear) { PSATDMoveRhoNewToRhoOld(); } diff --git a/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp b/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp index a2d45bcdb8d..de9f20a54aa 100644 --- a/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp +++ b/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp @@ -29,10 +29,10 @@ void LabFrameExplicitES::ComputeSpaceChargeField ( using ablastr::fields::MultiLevelScalarField; using ablastr::fields::MultiLevelVectorField; - const MultiLevelScalarField rho_fp = fields.get_mr_levels("rho_fp", max_level); - const MultiLevelScalarField rho_cp = fields.get_mr_levels("rho_cp", max_level); + MultiLevelScalarField rho_fp = fields.get_mr_levels("rho_fp", max_level); + MultiLevelScalarField rho_cp = fields.get_mr_levels("rho_cp", max_level); MultiLevelScalarField phi_fp = fields.get_mr_levels("phi_fp", max_level); - const MultiLevelVectorField Efield_fp = fields.get_mr_levels_alldirs("Efield_fp", max_level); + MultiLevelVectorField Efield_fp = fields.get_mr_levels_alldirs("Efield_fp", max_level); mpc.DepositCharge(rho_fp, 0.0_rt); if (mfl) { @@ -41,7 +41,7 @@ void LabFrameExplicitES::ComputeSpaceChargeField ( } // Apply filter, perform MPI exchange, interpolate across levels - const Vector > rho_buf(num_levels); + Vector > rho_buf(num_levels); auto & warpx = WarpX::GetInstance(); warpx.SyncRho( rho_fp, rho_cp, amrex::GetVecOfPtrs(rho_buf) ); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp index a5b85f7fbd7..2fcc18a33d0 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp @@ -56,9 +56,9 @@ void FiniteDifferenceSolver::EvolveB ( [[maybe_unused]] amrex::Real const dt ) { using ablastr::fields::Direction; - const ablastr::fields::VectorField Bfield = patch_type == PatchType::fine ? + ablastr::fields::VectorField Bfield = patch_type == PatchType::fine ? fields.get_alldirs("Bfield_fp", lev) : fields.get_alldirs("Bfield_cp", lev); - const ablastr::fields::VectorField Efield = patch_type == PatchType::fine ? + ablastr::fields::VectorField Efield = patch_type == PatchType::fine ? fields.get_alldirs("Efield_fp", lev) : fields.get_alldirs("Efield_cp", lev); // Select algorithm (The choice of algorithm is a runtime option, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp index 5d46d18ff4e..9a83b4bc072 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp @@ -56,9 +56,9 @@ void FiniteDifferenceSolver::EvolveBPML ( WARPX_ABORT_WITH_MESSAGE( "PML are not implemented in cylindrical geometry."); #else - const ablastr::fields::VectorField Bfield = (patch_type == PatchType::fine) ? + ablastr::fields::VectorField Bfield = (patch_type == PatchType::fine) ? fields.get_alldirs("pml_B_fp", level) : fields.get_alldirs("pml_B_cp", level); - const ablastr::fields::VectorField Efield = (patch_type == PatchType::fine) ? + ablastr::fields::VectorField Efield = (patch_type == PatchType::fine) ? fields.get_alldirs("pml_E_fp", level) : fields.get_alldirs("pml_E_cp", level); if (m_grid_type == ablastr::utils::enums::GridType::Collocated) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp index d80ca9db15e..60fa68ff918 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp @@ -58,9 +58,9 @@ void FiniteDifferenceSolver::EvolveE ( ) { using ablastr::fields::Direction; - const ablastr::fields::VectorField Bfield = patch_type == PatchType::fine ? + ablastr::fields::VectorField Bfield = patch_type == PatchType::fine ? fields.get_alldirs("Bfield_fp", lev) : fields.get_alldirs("Bfield_cp", lev); - const ablastr::fields::VectorField Jfield = patch_type == PatchType::fine ? + ablastr::fields::VectorField Jfield = patch_type == PatchType::fine ? fields.get_alldirs("current_fp", lev) : fields.get_alldirs("current_cp", lev); amrex::MultiFab* Ffield = nullptr; @@ -71,19 +71,23 @@ void FiniteDifferenceSolver::EvolveE ( ablastr::fields::VectorField edge_lengths; if (fields.has("edge_lengths", Direction{0}, lev)) { - edge_lengths = fields.get_alldirs("edge_lengths", lev); + edge_lengths = patch_type == PatchType::fine ? + fields.get_alldirs("edge_lengths", lev) : fields.get_alldirs("edge_lengths", lev); } ablastr::fields::VectorField face_areas; if (fields.has("face_areas", Direction{0}, lev)) { - face_areas = fields.get_alldirs("face_areas", lev); + face_areas = patch_type == PatchType::fine ? + fields.get_alldirs("face_areas", lev) : fields.get_alldirs("face_areas", lev); } ablastr::fields::VectorField area_mod; - if (fields.has("area_mod", Direction{0}, lev)) { - area_mod = fields.get_alldirs("area_mod", lev); + if (fields.has("face_areas", Direction{0}, lev)) { + area_mod = patch_type == PatchType::fine ? + fields.get_alldirs("area_mod", lev) : fields.get_alldirs("area_mod", lev); } ablastr::fields::VectorField ECTRhofield; if (fields.has("ECTRhofield", Direction{0}, lev)) { - ECTRhofield = fields.get_alldirs("ECTRhofield", lev); + ECTRhofield = patch_type == PatchType::fine ? + fields.get_alldirs("ECTRhofield", lev) : fields.get_alldirs("ECTRhofield", lev); } // Select algorithm (The choice of algorithm is a runtime option, @@ -217,7 +221,7 @@ void FiniteDifferenceSolver::EvolveECartesian ( if (Ffield) { // Extract field data for this grid/tile - const Array4 F = Ffield->array(mfi); + Array4 F = Ffield->array(mfi); // Loop over the cells and update the fields amrex::ParallelFor(tex, tey, tez, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp index d678bed3b01..4da403156b0 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp @@ -59,11 +59,11 @@ void FiniteDifferenceSolver::EvolveEPML ( "PML are not implemented in cylindrical geometry."); #else using ablastr::fields::Direction; - const ablastr::fields::VectorField Efield = (patch_type == PatchType::fine) ? + ablastr::fields::VectorField Efield = (patch_type == PatchType::fine) ? fields.get_alldirs("pml_E_fp", level) : fields.get_alldirs("pml_E_cp", level); - const ablastr::fields::VectorField Bfield = (patch_type == PatchType::fine) ? + ablastr::fields::VectorField Bfield = (patch_type == PatchType::fine) ? fields.get_alldirs("pml_B_fp", level) : fields.get_alldirs("pml_B_cp", level); - const ablastr::fields::VectorField Jfield = (patch_type == PatchType::fine) ? + ablastr::fields::VectorField Jfield = (patch_type == PatchType::fine) ? fields.get_alldirs("pml_j_fp", level) : fields.get_alldirs("pml_j_cp", level); ablastr::fields::VectorField edge_lengths; if (fields.has("pml_edge_lengths", Direction{0}, level)) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 03f51f7ba62..a3656142384 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -226,7 +226,7 @@ class FiniteDifferenceSolver void EvolveFCylindrical ( amrex::MultiFab* Ffield, ablastr::fields::VectorField const & Efield, - amrex::MultiFab* rhofield, + amrex::MultiFab* const rhofield, int rhocomp, amrex::Real dt ); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index f8505d91d44..07a4935a83b 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -468,8 +468,8 @@ void HybridPICModel::HybridPICSolveE ( auto& warpx = WarpX::GetInstance(); ablastr::fields::VectorField current_fp_ampere = warpx.m_fields.get_alldirs("hybrid_current_fp_ampere", lev); - const ablastr::fields::VectorField current_fp_external = warpx.m_fields.get_alldirs("hybrid_current_fp_external", lev); - const ablastr::fields::ScalarField electron_pressure_fp = warpx.m_fields.get("hybrid_electron_pressure_fp", lev); + ablastr::fields::VectorField current_fp_external = warpx.m_fields.get_alldirs("hybrid_current_fp_external", lev); + ablastr::fields::ScalarField electron_pressure_fp = warpx.m_fields.get("hybrid_electron_pressure_fp", lev); // Solve E field in regular cells warpx.get_pointer_fdtd_solver_fp(lev)->HybridPICSolveE( diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H index b109229328e..74e45ba631c 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H +++ b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H @@ -68,8 +68,8 @@ public: [[nodiscard]] inline bool IsDefined () const { return m_is_defined; } void Define ( WarpX* a_WarpX, - const std::string& a_vector_type_name, - const std::string& a_scalar_type_name = "none" ); + std::string a_vector_type_name, + std::string a_scalar_type_name = "none" ); inline void Define ( const WarpXSolverVec& a_solver_vec ) diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp index 10673704759..da359f35248 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp @@ -21,8 +21,8 @@ WarpXSolverVec::~WarpXSolverVec () } void WarpXSolverVec::Define ( WarpX* a_WarpX, - const std::string& a_vector_type_name, - const std::string& a_scalar_type_name ) + std::string a_vector_type_name, + std::string a_scalar_type_name ) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( !IsDefined(), @@ -41,7 +41,7 @@ void WarpXSolverVec::Define ( WarpX* a_WarpX, m_array_type = FieldType::Efield_fp; } else if (m_vector_type_name=="Bfield_fp") { - m_array_type = FieldType::Bfield_fp; + m_array_type = FieldType::Efield_fp; } else if (m_vector_type_name=="vector_potential_fp_nodal") { m_array_type = FieldType::vector_potential_fp; diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 43a964c1806..ede04b39632 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -669,11 +669,11 @@ WarpX::PushPSATD () const int rho_old = spectral_solver_fp[0]->m_spectral_index.rho_old; const int rho_new = spectral_solver_fp[0]->m_spectral_index.rho_new; - const ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); - const ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_fp", finest_level); - const ablastr::fields::MultiLevelVectorField current_fp = m_fields.get_mr_levels_alldirs("current_fp", finest_level); - const ablastr::fields::MultiLevelVectorField current_cp = m_fields.get_mr_levels_alldirs("current_cp", finest_level); - const ablastr::fields::MultiLevelVectorField current_buf = m_fields.get_mr_levels_alldirs("current_buf", finest_level); + ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); + ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_fp", finest_level); + ablastr::fields::MultiLevelVectorField current_fp = m_fields.get_mr_levels_alldirs("current_fp", finest_level); + ablastr::fields::MultiLevelVectorField current_cp = m_fields.get_mr_levels_alldirs("current_cp", finest_level); + ablastr::fields::MultiLevelVectorField current_buf = m_fields.get_mr_levels_alldirs("current_buf", finest_level); if (fft_periodic_single_box) { diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 5ef8590d0ae..5457b633504 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2530,15 +2530,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm } if (mypc->m_B_ext_particle_s == "read_from_file") { // These fields will be added to the fields that the particles see, and need to match the index type -<<<<<<< HEAD - auto* Bfield_aux_levl_0 = m_fields.get("Bfield_aux", Direction{0}, lev); - auto* Bfield_aux_levl_1 = m_fields.get("Bfield_aux", Direction{1}, lev); - auto* Bfield_aux_levl_2 = m_fields.get("Bfield_aux", Direction{2}, lev); -======= auto *Bfield_aux_levl_0 = m_fields.get("Bfield_aux", Direction{0}, lev); auto *Bfield_aux_levl_1 = m_fields.get("Bfield_aux", Direction{1}, lev); auto *Bfield_aux_levl_2 = m_fields.get("Bfield_aux", Direction{2}, lev); ->>>>>>> 1dfeb9f0ff8b104502d31bfd8aa90827de58ea7a // Same as Bfield_fp for reading external field data m_fields.alloc_init( "B_external_particle_field", Direction{0}, lev, amrex::convert(ba, Bfield_aux_levl_0->ixType()), @@ -2559,15 +2553,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm } if (mypc->m_E_ext_particle_s == "read_from_file") { // These fields will be added to the fields that the particles see, and need to match the index type -<<<<<<< HEAD - auto* Efield_aux_levl_0 = m_fields.get("Efield_aux", Direction{0}, lev); - auto* Efield_aux_levl_1 = m_fields.get("Efield_aux", Direction{1}, lev); - auto* Efield_aux_levl_2 = m_fields.get("Efield_aux", Direction{2}, lev); -======= auto *Efield_aux_levl_0 = m_fields.get("Efield_aux", Direction{0}, lev); auto *Efield_aux_levl_1 = m_fields.get("Efield_aux", Direction{1}, lev); auto *Efield_aux_levl_2 = m_fields.get("Efield_aux", Direction{2}, lev); ->>>>>>> 1dfeb9f0ff8b104502d31bfd8aa90827de58ea7a // Same as Efield_fp for reading external field data m_fields.alloc_init( "E_external_particle_field", Direction{0}, lev, amrex::convert(ba, Efield_aux_levl_0->ixType()), diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index f5e19c19db1..c290266176e 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -628,7 +628,7 @@ namespace ablastr::fields const amrex::Vector, 3 > >& old_vector_on_levels ) { - auto const finest_level = static_cast(old_vector_on_levels.size() - 1u); + int const finest_level = old_vector_on_levels.size() - 1u; MultiLevelVectorField field_on_level; field_on_level.reserve(finest_level+1); @@ -654,7 +654,7 @@ namespace ablastr::fields const amrex::Vector >& old_scalar_on_levels ) { - auto const finest_level = static_cast(old_scalar_on_levels.size() - 1u); + int const finest_level = old_scalar_on_levels.size() - 1u; MultiLevelScalarField field_on_level; field_on_level.reserve(finest_level+1); From d4ea23be4a63627e00384c0695fdf37b187d5c4d Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 18 Sep 2024 11:08:44 +0200 Subject: [PATCH 265/314] fix clang-tidy issues --- Source/BoundaryConditions/PML_RZ.cpp | 4 ++-- Source/Evolve/WarpXEvolve.cpp | 8 ++++---- .../LabFrameExplicitES.cpp | 8 ++++---- .../FiniteDifferenceSolver/EvolveB.cpp | 4 ++-- .../FiniteDifferenceSolver/EvolveBPML.cpp | 4 ++-- .../FiniteDifferenceSolver/EvolveE.cpp | 20 ++++++++----------- .../FiniteDifferenceSolver/EvolveEPML.cpp | 6 +++--- .../HybridPICModel/HybridPICModel.cpp | 4 ++-- .../ImplicitSolvers/WarpXSolverVec.H | 4 ++-- .../ImplicitSolvers/WarpXSolverVec.cpp | 6 +++--- Source/FieldSolver/WarpXPushFieldsEM.cpp | 10 +++++----- Source/ablastr/fields/MultiFabRegister.cpp | 4 ++-- 12 files changed, 39 insertions(+), 43 deletions(-) diff --git a/Source/BoundaryConditions/PML_RZ.cpp b/Source/BoundaryConditions/PML_RZ.cpp index cc05c126a55..0fc24bf8817 100644 --- a/Source/BoundaryConditions/PML_RZ.cpp +++ b/Source/BoundaryConditions/PML_RZ.cpp @@ -138,7 +138,7 @@ PML_RZ::FillBoundaryE (ablastr::fields::MultiFabRegister& fields, PatchType patc if (patch_type == PatchType::fine && pml_Er->nGrowVect().max() > 0) { amrex::Periodicity const& period = m_geom->periodicity(); - amrex::Vector mf = {pml_Er, pml_Et}; + const amrex::Vector mf = {pml_Er, pml_Et}; ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period, nodal_sync); } } @@ -152,7 +152,7 @@ PML_RZ::FillBoundaryB (ablastr::fields::MultiFabRegister& fields, PatchType patc amrex::MultiFab * pml_Bt = fields.get("pml_B_fp", Direction{1}, 0); amrex::Periodicity const& period = m_geom->periodicity(); - amrex::Vector mf = {pml_Br, pml_Bt}; + const amrex::Vector mf = {pml_Br, pml_Bt}; ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period, nodal_sync); } } diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index a4df5ea73bd..bd01aeaf359 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -663,8 +663,8 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // (after checking that pointer to rho_fp on MR level 0 is not null) if (m_fields.has("rho_fp", 0) && rho_in_time == RhoInTime::Linear) { - ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); - ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_fp", finest_level); + const ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); + const ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_fp", finest_level); // Deposit rho at relative time -dt // (dt[0] denotes the time step on mesh refinement level 0) @@ -732,8 +732,8 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // (after checking that pointer to rho_fp on MR level 0 is not null) if (m_fields.has("rho_fp", 0)) { - ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); - ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_cp", finest_level); + const ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); + const ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_cp", finest_level); // Move rho from new to old if rho is linear in time if (rho_in_time == RhoInTime::Linear) { PSATDMoveRhoNewToRhoOld(); } diff --git a/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp b/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp index de9f20a54aa..a2d45bcdb8d 100644 --- a/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp +++ b/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp @@ -29,10 +29,10 @@ void LabFrameExplicitES::ComputeSpaceChargeField ( using ablastr::fields::MultiLevelScalarField; using ablastr::fields::MultiLevelVectorField; - MultiLevelScalarField rho_fp = fields.get_mr_levels("rho_fp", max_level); - MultiLevelScalarField rho_cp = fields.get_mr_levels("rho_cp", max_level); + const MultiLevelScalarField rho_fp = fields.get_mr_levels("rho_fp", max_level); + const MultiLevelScalarField rho_cp = fields.get_mr_levels("rho_cp", max_level); MultiLevelScalarField phi_fp = fields.get_mr_levels("phi_fp", max_level); - MultiLevelVectorField Efield_fp = fields.get_mr_levels_alldirs("Efield_fp", max_level); + const MultiLevelVectorField Efield_fp = fields.get_mr_levels_alldirs("Efield_fp", max_level); mpc.DepositCharge(rho_fp, 0.0_rt); if (mfl) { @@ -41,7 +41,7 @@ void LabFrameExplicitES::ComputeSpaceChargeField ( } // Apply filter, perform MPI exchange, interpolate across levels - Vector > rho_buf(num_levels); + const Vector > rho_buf(num_levels); auto & warpx = WarpX::GetInstance(); warpx.SyncRho( rho_fp, rho_cp, amrex::GetVecOfPtrs(rho_buf) ); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp index 2fcc18a33d0..a5b85f7fbd7 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp @@ -56,9 +56,9 @@ void FiniteDifferenceSolver::EvolveB ( [[maybe_unused]] amrex::Real const dt ) { using ablastr::fields::Direction; - ablastr::fields::VectorField Bfield = patch_type == PatchType::fine ? + const ablastr::fields::VectorField Bfield = patch_type == PatchType::fine ? fields.get_alldirs("Bfield_fp", lev) : fields.get_alldirs("Bfield_cp", lev); - ablastr::fields::VectorField Efield = patch_type == PatchType::fine ? + const ablastr::fields::VectorField Efield = patch_type == PatchType::fine ? fields.get_alldirs("Efield_fp", lev) : fields.get_alldirs("Efield_cp", lev); // Select algorithm (The choice of algorithm is a runtime option, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp index 9a83b4bc072..5d46d18ff4e 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp @@ -56,9 +56,9 @@ void FiniteDifferenceSolver::EvolveBPML ( WARPX_ABORT_WITH_MESSAGE( "PML are not implemented in cylindrical geometry."); #else - ablastr::fields::VectorField Bfield = (patch_type == PatchType::fine) ? + const ablastr::fields::VectorField Bfield = (patch_type == PatchType::fine) ? fields.get_alldirs("pml_B_fp", level) : fields.get_alldirs("pml_B_cp", level); - ablastr::fields::VectorField Efield = (patch_type == PatchType::fine) ? + const ablastr::fields::VectorField Efield = (patch_type == PatchType::fine) ? fields.get_alldirs("pml_E_fp", level) : fields.get_alldirs("pml_E_cp", level); if (m_grid_type == ablastr::utils::enums::GridType::Collocated) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp index 60fa68ff918..d80ca9db15e 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp @@ -58,9 +58,9 @@ void FiniteDifferenceSolver::EvolveE ( ) { using ablastr::fields::Direction; - ablastr::fields::VectorField Bfield = patch_type == PatchType::fine ? + const ablastr::fields::VectorField Bfield = patch_type == PatchType::fine ? fields.get_alldirs("Bfield_fp", lev) : fields.get_alldirs("Bfield_cp", lev); - ablastr::fields::VectorField Jfield = patch_type == PatchType::fine ? + const ablastr::fields::VectorField Jfield = patch_type == PatchType::fine ? fields.get_alldirs("current_fp", lev) : fields.get_alldirs("current_cp", lev); amrex::MultiFab* Ffield = nullptr; @@ -71,23 +71,19 @@ void FiniteDifferenceSolver::EvolveE ( ablastr::fields::VectorField edge_lengths; if (fields.has("edge_lengths", Direction{0}, lev)) { - edge_lengths = patch_type == PatchType::fine ? - fields.get_alldirs("edge_lengths", lev) : fields.get_alldirs("edge_lengths", lev); + edge_lengths = fields.get_alldirs("edge_lengths", lev); } ablastr::fields::VectorField face_areas; if (fields.has("face_areas", Direction{0}, lev)) { - face_areas = patch_type == PatchType::fine ? - fields.get_alldirs("face_areas", lev) : fields.get_alldirs("face_areas", lev); + face_areas = fields.get_alldirs("face_areas", lev); } ablastr::fields::VectorField area_mod; - if (fields.has("face_areas", Direction{0}, lev)) { - area_mod = patch_type == PatchType::fine ? - fields.get_alldirs("area_mod", lev) : fields.get_alldirs("area_mod", lev); + if (fields.has("area_mod", Direction{0}, lev)) { + area_mod = fields.get_alldirs("area_mod", lev); } ablastr::fields::VectorField ECTRhofield; if (fields.has("ECTRhofield", Direction{0}, lev)) { - ECTRhofield = patch_type == PatchType::fine ? - fields.get_alldirs("ECTRhofield", lev) : fields.get_alldirs("ECTRhofield", lev); + ECTRhofield = fields.get_alldirs("ECTRhofield", lev); } // Select algorithm (The choice of algorithm is a runtime option, @@ -221,7 +217,7 @@ void FiniteDifferenceSolver::EvolveECartesian ( if (Ffield) { // Extract field data for this grid/tile - Array4 F = Ffield->array(mfi); + const Array4 F = Ffield->array(mfi); // Loop over the cells and update the fields amrex::ParallelFor(tex, tey, tez, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp index 4da403156b0..d678bed3b01 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp @@ -59,11 +59,11 @@ void FiniteDifferenceSolver::EvolveEPML ( "PML are not implemented in cylindrical geometry."); #else using ablastr::fields::Direction; - ablastr::fields::VectorField Efield = (patch_type == PatchType::fine) ? + const ablastr::fields::VectorField Efield = (patch_type == PatchType::fine) ? fields.get_alldirs("pml_E_fp", level) : fields.get_alldirs("pml_E_cp", level); - ablastr::fields::VectorField Bfield = (patch_type == PatchType::fine) ? + const ablastr::fields::VectorField Bfield = (patch_type == PatchType::fine) ? fields.get_alldirs("pml_B_fp", level) : fields.get_alldirs("pml_B_cp", level); - ablastr::fields::VectorField Jfield = (patch_type == PatchType::fine) ? + const ablastr::fields::VectorField Jfield = (patch_type == PatchType::fine) ? fields.get_alldirs("pml_j_fp", level) : fields.get_alldirs("pml_j_cp", level); ablastr::fields::VectorField edge_lengths; if (fields.has("pml_edge_lengths", Direction{0}, level)) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 07a4935a83b..f8505d91d44 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -468,8 +468,8 @@ void HybridPICModel::HybridPICSolveE ( auto& warpx = WarpX::GetInstance(); ablastr::fields::VectorField current_fp_ampere = warpx.m_fields.get_alldirs("hybrid_current_fp_ampere", lev); - ablastr::fields::VectorField current_fp_external = warpx.m_fields.get_alldirs("hybrid_current_fp_external", lev); - ablastr::fields::ScalarField electron_pressure_fp = warpx.m_fields.get("hybrid_electron_pressure_fp", lev); + const ablastr::fields::VectorField current_fp_external = warpx.m_fields.get_alldirs("hybrid_current_fp_external", lev); + const ablastr::fields::ScalarField electron_pressure_fp = warpx.m_fields.get("hybrid_electron_pressure_fp", lev); // Solve E field in regular cells warpx.get_pointer_fdtd_solver_fp(lev)->HybridPICSolveE( diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H index 74e45ba631c..b109229328e 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H +++ b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H @@ -68,8 +68,8 @@ public: [[nodiscard]] inline bool IsDefined () const { return m_is_defined; } void Define ( WarpX* a_WarpX, - std::string a_vector_type_name, - std::string a_scalar_type_name = "none" ); + const std::string& a_vector_type_name, + const std::string& a_scalar_type_name = "none" ); inline void Define ( const WarpXSolverVec& a_solver_vec ) diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp index da359f35248..10673704759 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp @@ -21,8 +21,8 @@ WarpXSolverVec::~WarpXSolverVec () } void WarpXSolverVec::Define ( WarpX* a_WarpX, - std::string a_vector_type_name, - std::string a_scalar_type_name ) + const std::string& a_vector_type_name, + const std::string& a_scalar_type_name ) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( !IsDefined(), @@ -41,7 +41,7 @@ void WarpXSolverVec::Define ( WarpX* a_WarpX, m_array_type = FieldType::Efield_fp; } else if (m_vector_type_name=="Bfield_fp") { - m_array_type = FieldType::Efield_fp; + m_array_type = FieldType::Bfield_fp; } else if (m_vector_type_name=="vector_potential_fp_nodal") { m_array_type = FieldType::vector_potential_fp; diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index ede04b39632..43a964c1806 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -669,11 +669,11 @@ WarpX::PushPSATD () const int rho_old = spectral_solver_fp[0]->m_spectral_index.rho_old; const int rho_new = spectral_solver_fp[0]->m_spectral_index.rho_new; - ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); - ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_fp", finest_level); - ablastr::fields::MultiLevelVectorField current_fp = m_fields.get_mr_levels_alldirs("current_fp", finest_level); - ablastr::fields::MultiLevelVectorField current_cp = m_fields.get_mr_levels_alldirs("current_cp", finest_level); - ablastr::fields::MultiLevelVectorField current_buf = m_fields.get_mr_levels_alldirs("current_buf", finest_level); + const ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); + const ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_fp", finest_level); + const ablastr::fields::MultiLevelVectorField current_fp = m_fields.get_mr_levels_alldirs("current_fp", finest_level); + const ablastr::fields::MultiLevelVectorField current_cp = m_fields.get_mr_levels_alldirs("current_cp", finest_level); + const ablastr::fields::MultiLevelVectorField current_buf = m_fields.get_mr_levels_alldirs("current_buf", finest_level); if (fft_periodic_single_box) { diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index c290266176e..f5e19c19db1 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -628,7 +628,7 @@ namespace ablastr::fields const amrex::Vector, 3 > >& old_vector_on_levels ) { - int const finest_level = old_vector_on_levels.size() - 1u; + auto const finest_level = static_cast(old_vector_on_levels.size() - 1u); MultiLevelVectorField field_on_level; field_on_level.reserve(finest_level+1); @@ -654,7 +654,7 @@ namespace ablastr::fields const amrex::Vector >& old_scalar_on_levels ) { - int const finest_level = old_scalar_on_levels.size() - 1u; + auto const finest_level = static_cast(old_scalar_on_levels.size() - 1u); MultiLevelScalarField field_on_level; field_on_level.reserve(finest_level+1); From 5976c620e10433f53922be8cc28518e7468f1d69 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 18 Sep 2024 12:58:05 +0200 Subject: [PATCH 266/314] fix-clang-tidy issues --- .../ElectrostaticSolvers/RelativisticExplicitES.cpp | 2 +- .../HybridPICModel/HybridPICModel.H | 10 +++++----- .../HybridPICModel/HybridPICModel.cpp | 10 +++++----- Source/Fluids/WarpXFluidContainer.H | 4 ++-- Source/Fluids/WarpXFluidContainer.cpp | 4 ++-- Source/Initialization/WarpXInitData.cpp | 2 +- Source/Parallelization/WarpXComm.cpp | 8 ++++---- Source/Particles/LaserParticleContainer.H | 2 +- Source/Particles/LaserParticleContainer.cpp | 2 +- Source/Particles/PhotonParticleContainer.H | 2 +- Source/Particles/PhotonParticleContainer.cpp | 2 +- Source/Particles/PhysicalParticleContainer.H | 2 +- Source/Particles/PhysicalParticleContainer.cpp | 2 +- Source/Particles/RigidInjectedParticleContainer.H | 2 +- Source/Particles/RigidInjectedParticleContainer.cpp | 2 +- Source/Particles/WarpXParticleContainer.H | 2 +- Source/WarpX.H | 2 +- 17 files changed, 30 insertions(+), 30 deletions(-) diff --git a/Source/FieldSolver/ElectrostaticSolvers/RelativisticExplicitES.cpp b/Source/FieldSolver/ElectrostaticSolvers/RelativisticExplicitES.cpp index 3c9a15c931b..ae4006c2782 100644 --- a/Source/FieldSolver/ElectrostaticSolvers/RelativisticExplicitES.cpp +++ b/Source/FieldSolver/ElectrostaticSolvers/RelativisticExplicitES.cpp @@ -110,7 +110,7 @@ void RelativisticExplicitES::AddSpaceChargeField ( } // Apply filter, perform MPI exchange, interpolate across levels - Vector> rho_buf(num_levels); + const Vector> rho_buf(num_levels); warpx.SyncRho( amrex::GetVecOfPtrs(rho), amrex::GetVecOfPtrs(rho_coarse), diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index f59210315f6..b0f63dd8018 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -90,7 +90,7 @@ public: ablastr::fields::MultiLevelVectorField const& Bfield, ablastr::fields::MultiLevelScalarField const& rhofield, ablastr::fields::MultiLevelVectorField const& edge_lengths, - bool solve_for_Faraday); + bool solve_for_Faraday) const; void HybridPICSolveE ( ablastr::fields::VectorField const& Efield, @@ -98,7 +98,7 @@ public: ablastr::fields::VectorField const& Bfield, amrex::MultiFab const& rhofield, ablastr::fields::VectorField const& edge_lengths, - int lev, bool solve_for_Faraday); + int lev, bool solve_for_Faraday) const; void HybridPICSolveE ( ablastr::fields::VectorField const& Efield, @@ -106,7 +106,7 @@ public: ablastr::fields::VectorField const& Bfield, amrex::MultiFab const& rhofield, ablastr::fields::VectorField const& edge_lengths, - int lev, PatchType patch_type, bool solve_for_Faraday); + int lev, PatchType patch_type, bool solve_for_Faraday) const; void BfieldEvolveRK ( ablastr::fields::MultiLevelVectorField const& Bfield, @@ -140,8 +140,8 @@ public: * Function to calculate the electron pressure using the simulation charge * density. Used in the Ohm's law solver (kinetic-fluid hybrid model). */ - void CalculateElectronPressure (); - void CalculateElectronPressure (int lev); + void CalculateElectronPressure () const; + void CalculateElectronPressure (int lev) const; /** * \brief Fill the electron pressure multifab given the kinetic particle diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index f8505d91d44..f2219febef0 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -421,7 +421,7 @@ void HybridPICModel::HybridPICSolveE ( ablastr::fields::MultiLevelVectorField const& Bfield, ablastr::fields::MultiLevelScalarField const& rhofield, ablastr::fields::MultiLevelVectorField const& edge_lengths, - const bool solve_for_Faraday) + const bool solve_for_Faraday) const { auto& warpx = WarpX::GetInstance(); for (int lev = 0; lev <= warpx.finestLevel(); ++lev) @@ -439,7 +439,7 @@ void HybridPICModel::HybridPICSolveE ( ablastr::fields::VectorField const& Bfield, amrex::MultiFab const& rhofield, ablastr::fields::VectorField const& edge_lengths, - const int lev, const bool solve_for_Faraday) + const int lev, const bool solve_for_Faraday) const { WARPX_PROFILE("WarpX::HybridPICSolveE()"); @@ -461,7 +461,7 @@ void HybridPICModel::HybridPICSolveE ( amrex::MultiFab const& rhofield, ablastr::fields::VectorField const& edge_lengths, const int lev, PatchType patch_type, - const bool solve_for_Faraday) + const bool solve_for_Faraday) const { using ablastr::fields::va2vm; @@ -481,7 +481,7 @@ void HybridPICModel::HybridPICSolveE ( warpx.ApplyEfieldBoundary(lev, patch_type); } -void HybridPICModel::CalculateElectronPressure() +void HybridPICModel::CalculateElectronPressure() const { auto& warpx = WarpX::GetInstance(); for (int lev = 0; lev <= warpx.finestLevel(); ++lev) @@ -490,7 +490,7 @@ void HybridPICModel::CalculateElectronPressure() } } -void HybridPICModel::CalculateElectronPressure(const int lev) +void HybridPICModel::CalculateElectronPressure(const int lev) const { WARPX_PROFILE("WarpX::CalculateElectronPressure()"); diff --git a/Source/Fluids/WarpXFluidContainer.H b/Source/Fluids/WarpXFluidContainer.H index dbbe8267542..f3ea2d9a498 100644 --- a/Source/Fluids/WarpXFluidContainer.H +++ b/Source/Fluids/WarpXFluidContainer.H @@ -38,7 +38,7 @@ public: WarpXFluidContainer(WarpXFluidContainer&& ) = default; WarpXFluidContainer& operator=(WarpXFluidContainer&& ) = default; - void AllocateLevelMFs (ablastr::fields::MultiFabRegister& m_fields, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm, int lev); + void AllocateLevelMFs (ablastr::fields::MultiFabRegister& m_fields, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm, int lev) const; void InitData (ablastr::fields::MultiFabRegister& m_fields, amrex::Box init_box, amrex::Real cur_time, int lev); @@ -49,7 +49,7 @@ public: */ void Evolve (ablastr::fields::MultiFabRegister& fields, int lev, - std::string current_fp_string, + const std::string& current_fp_string, amrex::Real cur_time, bool skip_deposition=false); diff --git a/Source/Fluids/WarpXFluidContainer.cpp b/Source/Fluids/WarpXFluidContainer.cpp index 7d7d56ef62b..2bed7e059e6 100644 --- a/Source/Fluids/WarpXFluidContainer.cpp +++ b/Source/Fluids/WarpXFluidContainer.cpp @@ -136,7 +136,7 @@ void WarpXFluidContainer::ReadParameters() } } -void WarpXFluidContainer::AllocateLevelMFs(ablastr::fields::MultiFabRegister& fields, const BoxArray &ba, const DistributionMapping &dm, int lev) +void WarpXFluidContainer::AllocateLevelMFs(ablastr::fields::MultiFabRegister& fields, const BoxArray &ba, const DistributionMapping &dm, int lev) const { using ablastr::fields::Direction; const int ncomps = 1; @@ -254,7 +254,7 @@ void WarpXFluidContainer::InitData(ablastr::fields::MultiFabRegister& fields, am void WarpXFluidContainer::Evolve( ablastr::fields::MultiFabRegister& fields, int lev, - std::string current_fp_string, + const std::string& current_fp_string, amrex::Real cur_time, bool skip_deposition) { diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 56f4de1a561..0c54022505e 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -96,7 +96,7 @@ namespace */ void CheckGuardCells ( ablastr::fields::MultiFabRegister& fields, - std::string mf_name, + const std::string& mf_name, int lev ) { diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 7069982d0d7..fd3df0bd080 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -1067,7 +1067,7 @@ WarpX::FillBoundaryAux (int lev, IntVect ng) } void -WarpX::SyncCurrent (std::string current_fp_string) +WarpX::SyncCurrent (const std::string& current_fp_string) { using ablastr::fields::Direction; @@ -1237,13 +1237,13 @@ WarpX::SyncCurrent (std::string current_fp_string) void WarpX::SyncRho () { - ablastr::fields::MultiLevelScalarField rho_fp = m_fields.has("rho_fp", 0) ? + const ablastr::fields::MultiLevelScalarField rho_fp = m_fields.has("rho_fp", 0) ? m_fields.get_mr_levels("rho_fp", finest_level) : ablastr::fields::MultiLevelScalarField{static_cast(finest_level+1)}; - ablastr::fields::MultiLevelScalarField rho_cp = m_fields.has("rho_cp", 1) ? + const ablastr::fields::MultiLevelScalarField rho_cp = m_fields.has("rho_cp", 1) ? m_fields.get_mr_levels("rho_cp", finest_level) : ablastr::fields::MultiLevelScalarField{static_cast(finest_level+1)}; - ablastr::fields::MultiLevelScalarField rho_buf = m_fields.has("rho_buf", 1) ? + const ablastr::fields::MultiLevelScalarField rho_buf = m_fields.has("rho_buf", 1) ? m_fields.get_mr_levels("rho_buf", finest_level) : ablastr::fields::MultiLevelScalarField{static_cast(finest_level+1)}; diff --git a/Source/Particles/LaserParticleContainer.H b/Source/Particles/LaserParticleContainer.H index bb969747927..ea884a5f93f 100644 --- a/Source/Particles/LaserParticleContainer.H +++ b/Source/Particles/LaserParticleContainer.H @@ -66,7 +66,7 @@ public: void Evolve (ablastr::fields::MultiFabRegister& fields, int lev, - std::string current_fp_string, + const std::string& current_fp_string, amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, bool skip_deposition=false, PushType push_type=PushType::Explicit) final; diff --git a/Source/Particles/LaserParticleContainer.cpp b/Source/Particles/LaserParticleContainer.cpp index 234b32d5359..c91e358a1ff 100644 --- a/Source/Particles/LaserParticleContainer.cpp +++ b/Source/Particles/LaserParticleContainer.cpp @@ -558,7 +558,7 @@ LaserParticleContainer::InitData (int lev) void LaserParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, int lev, - std::string current_fp_string, + const std::string& current_fp_string, Real t, Real dt, DtType /*a_dt_type*/, bool skip_deposition, PushType push_type) { using ablastr::fields::Direction; diff --git a/Source/Particles/PhotonParticleContainer.H b/Source/Particles/PhotonParticleContainer.H index 18796519918..485f56dba43 100644 --- a/Source/Particles/PhotonParticleContainer.H +++ b/Source/Particles/PhotonParticleContainer.H @@ -48,7 +48,7 @@ public: void Evolve (ablastr::fields::MultiFabRegister& fields, int lev, - std::string current_fp_string, + const std::string& current_fp_string, amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, diff --git a/Source/Particles/PhotonParticleContainer.cpp b/Source/Particles/PhotonParticleContainer.cpp index ba33836fce1..47c426cd6ff 100644 --- a/Source/Particles/PhotonParticleContainer.cpp +++ b/Source/Particles/PhotonParticleContainer.cpp @@ -231,7 +231,7 @@ PhotonParticleContainer::PushPX (WarpXParIter& pti, void PhotonParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, int lev, - std::string current_fp_string, + const std::string& current_fp_string, Real t, Real dt, DtType a_dt_type, bool skip_deposition, PushType push_type) { diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index 5f5192ba262..18880239183 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -96,7 +96,7 @@ public: */ void Evolve (ablastr::fields::MultiFabRegister& fields, int lev, - std::string current_fp_string, + const std::string& current_fp_string, amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 4cf6345c520..3b40be5e470 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1746,7 +1746,7 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, void PhysicalParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, int lev, - std::string current_fp_string, + const std::string& current_fp_string, Real /*t*/, Real dt, DtType a_dt_type, bool skip_deposition, PushType push_type) { diff --git a/Source/Particles/RigidInjectedParticleContainer.H b/Source/Particles/RigidInjectedParticleContainer.H index 0f71161e4b1..d3565dd2df6 100644 --- a/Source/Particles/RigidInjectedParticleContainer.H +++ b/Source/Particles/RigidInjectedParticleContainer.H @@ -63,7 +63,7 @@ public: void Evolve (ablastr::fields::MultiFabRegister& fields, int lev, - std::string current_fp_string, + const std::string& current_fp_string, amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, diff --git a/Source/Particles/RigidInjectedParticleContainer.cpp b/Source/Particles/RigidInjectedParticleContainer.cpp index 5228eb3c606..d1e1f48ab38 100644 --- a/Source/Particles/RigidInjectedParticleContainer.cpp +++ b/Source/Particles/RigidInjectedParticleContainer.cpp @@ -293,7 +293,7 @@ RigidInjectedParticleContainer::PushPX (WarpXParIter& pti, void RigidInjectedParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, int lev, - std::string current_fp_string, + const std::string& current_fp_string, Real t, Real dt, DtType a_dt_type, bool skip_deposition, PushType push_type) { diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 4af9bee3734..9c316b110ee 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -149,7 +149,7 @@ public: */ virtual void Evolve (ablastr::fields::MultiFabRegister& fields, int lev, - std::string current_fp_string, + const std::string& current_fp_string, amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, bool skip_deposition=false, PushType push_type=PushType::Explicit) = 0; diff --git a/Source/WarpX.H b/Source/WarpX.H index 45ffa150c3d..9a8c59dff9c 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -805,7 +805,7 @@ public: * * \param[in] current_fp_string the coarse of fine patch to use for current */ - void SyncCurrent (std::string current_fp_string); + void SyncCurrent (const std::string& current_fp_string); void SyncRho (); From ffc94544c6fef7d7780fb72c8c8ef6e65923ccee Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 18 Sep 2024 06:38:02 -0700 Subject: [PATCH 267/314] Fix particles in PML --- Source/BoundaryConditions/PML.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index 655d3b59b07..6634256b87a 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -1060,15 +1060,15 @@ PML::CopyJtoPMLs ( { using ablastr::fields::Direction; - bool const has_j_fp = fields.has("j_fp", Direction{0}, lev); + bool const has_j_fp = fields.has("current_fp", Direction{0}, lev); bool const has_pml_j_fp = fields.has("pml_j_fp", Direction{0}, lev); - bool const has_j_cp = fields.has("j_cp", Direction{0}, lev); + bool const has_j_cp = fields.has("current_cp", Direction{0}, lev); bool const has_pml_j_cp = fields.has("pml_j_cp", Direction{0}, lev); if (patch_type == PatchType::fine && has_pml_j_fp && has_j_fp) { ablastr::fields::VectorField pml_j_fp = fields.get_alldirs("pml_j_fp", lev); - ablastr::fields::VectorField jp = fields.get_alldirs("j_fp", lev); + ablastr::fields::VectorField jp = fields.get_alldirs("current_fp", lev); CopyToPML(*pml_j_fp[0], *jp[0], *m_geom); CopyToPML(*pml_j_fp[1], *jp[1], *m_geom); CopyToPML(*pml_j_fp[2], *jp[2], *m_geom); @@ -1076,7 +1076,7 @@ PML::CopyJtoPMLs ( else if (patch_type == PatchType::coarse && has_j_cp && has_pml_j_cp) { ablastr::fields::VectorField pml_j_cp = fields.get_alldirs("pml_j_cp", lev); - ablastr::fields::VectorField jp = fields.get_alldirs("j_cp", lev); + ablastr::fields::VectorField jp = fields.get_alldirs("current_cp", lev); CopyToPML(*pml_j_cp[0], *jp[0], *m_cgeom); CopyToPML(*pml_j_cp[1], *jp[1], *m_cgeom); CopyToPML(*pml_j_cp[2], *jp[2], *m_cgeom); From 796c58339562c54bb60b800cc9a1b977d6cebb5a Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 18 Sep 2024 15:42:42 +0200 Subject: [PATCH 268/314] fix clang-tidy issue --- Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp b/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp index a2d45bcdb8d..f2b19b7e344 100644 --- a/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp +++ b/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp @@ -31,7 +31,7 @@ void LabFrameExplicitES::ComputeSpaceChargeField ( const MultiLevelScalarField rho_fp = fields.get_mr_levels("rho_fp", max_level); const MultiLevelScalarField rho_cp = fields.get_mr_levels("rho_cp", max_level); - MultiLevelScalarField phi_fp = fields.get_mr_levels("phi_fp", max_level); + const MultiLevelScalarField phi_fp = fields.get_mr_levels("phi_fp", max_level); const MultiLevelVectorField Efield_fp = fields.get_mr_levels_alldirs("Efield_fp", max_level); mpc.DepositCharge(rho_fp, 0.0_rt); From 4f9993828e7919d341240e71177c5262b2e9d210 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 18 Sep 2024 15:44:09 +0200 Subject: [PATCH 269/314] fix clang-tidy issue --- Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp index e7d443f9ebc..a8266f54135 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp @@ -176,7 +176,7 @@ WarpX::computeVectorPotential (ablastr::fields::MultiLevelVectorField const& cur } #if defined(AMREX_USE_EB) - ablastr::fields::MultiLevelVectorField Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); + const ablastr::fields::MultiLevelVectorField Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); const std::optional post_A_calculation( { Bfield_fp, From dd78d7c046c0a4753278a44b1b49de05d0c1a1b6 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 18 Sep 2024 16:02:07 +0200 Subject: [PATCH 270/314] fix bug introduced fixing clang-tidy issues --- Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.H | 2 +- Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.H b/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.H index c919b138c00..5606ebef2f1 100644 --- a/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.H +++ b/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.H @@ -29,7 +29,7 @@ public: void computePhiTriDiagonal ( const ablastr::fields::MultiLevelScalarField& rho, - ablastr::fields::MultiLevelScalarField& phi + const ablastr::fields::MultiLevelScalarField& phi ); }; diff --git a/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp b/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp index f2b19b7e344..eec5dce8f0e 100644 --- a/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp +++ b/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp @@ -94,7 +94,7 @@ void LabFrameExplicitES::ComputeSpaceChargeField ( */ void LabFrameExplicitES::computePhiTriDiagonal ( const ablastr::fields::MultiLevelScalarField& rho, - ablastr::fields::MultiLevelScalarField& phi) + const ablastr::fields::MultiLevelScalarField& phi) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE(num_levels == 1, "The tridiagonal solver cannot be used with mesh refinement"); From 873b50636e68d0fd9688b9335b1036ffc1e19cbe Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Wed, 18 Sep 2024 11:23:34 -0700 Subject: [PATCH 271/314] Apply bug fix by @RemiLehe --- Source/Parallelization/WarpXComm.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index fd3df0bd080..8a0d74f5904 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -432,6 +432,9 @@ WarpX::UpdateAuxilaryDataSameType () m_fields.get("Bfield_cp",Direction{1},lev)->nComp(), ng); MultiFab dBz(m_fields.get("Bfield_cp",Direction{2},lev)->boxArray(), dm, m_fields.get("Bfield_cp",Direction{2},lev)->nComp(), ng); + dBx.setVal(0.0); + dBy.setVal(0.0); + dBz.setVal(0.0); // Copy Bfield_aux to the dB MultiFabs, using up to ng_src (=ng_FieldGather) guard // cells from Bfield_aux and filling up to ng (=nGrow) guard cells in the dB MultiFabs From 9a7d86441a3103c2e452b5e9ab58b81d813602aa Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Wed, 18 Sep 2024 11:56:11 -0700 Subject: [PATCH 272/314] Fix merge bug --- Source/Evolve/WarpXEvolve.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 218f14d7d8d..c9e986381ec 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -62,6 +62,7 @@ using ablastr::utils::SignalHandling; void WarpX::Synchronize () { + using ablastr::fields::Direction; FillBoundaryE(guard_cells.ng_FieldGather); FillBoundaryB(guard_cells.ng_FieldGather); if (fft_do_time_averaging) From f146feed13f2f47b878bb737ae29747e66176ede Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 18 Sep 2024 12:22:26 -0700 Subject: [PATCH 273/314] Fix bug in mesh refinement with PML --- Source/Parallelization/WarpXComm.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index fd3df0bd080..8c64e49a424 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -730,8 +730,8 @@ WarpX::FillBoundaryE (const int lev, const PatchType patch_type, const amrex::In { const std::array mf_pml = (patch_type == PatchType::fine) ? - m_fields.get_alldirs("pml_E_fp", finest_level) : - m_fields.get_alldirs("pml_E_cp", finest_level); + m_fields.get_alldirs("pml_E_fp", lev) : + m_fields.get_alldirs("pml_E_cp", lev); pml[lev]->Exchange(mf_pml, mf, patch_type, do_pml_in_domain); pml[lev]->FillBoundary(mf_pml, patch_type, nodal_sync); @@ -795,8 +795,8 @@ WarpX::FillBoundaryB (const int lev, const PatchType patch_type, const amrex::In { const std::array mf_pml = (patch_type == PatchType::fine) ? - m_fields.get_alldirs("pml_B_fp", finest_level) : - m_fields.get_alldirs("pml_B_cp", finest_level); + m_fields.get_alldirs("pml_B_fp", lev) : + m_fields.get_alldirs("pml_B_cp", lev); pml[lev]->Exchange(mf_pml, mf, patch_type, do_pml_in_domain); pml[lev]->FillBoundary(mf_pml, patch_type, nodal_sync); From 2a5534edf31a174237ec885edfe2554f3ebec676 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 18 Sep 2024 13:23:11 -0700 Subject: [PATCH 274/314] Apply suggestions from code review --- .../FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index f2219febef0..24f9e07d39c 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -403,7 +403,7 @@ void HybridPICModel::CalculateCurrentAmpere ( WARPX_PROFILE("WarpX::CalculateCurrentAmpere()"); auto& warpx = WarpX::GetInstance(); - ablastr::fields::VectorField current_fp_ampere = warpx.m_fields.get_alldirs("hybrid_current_fp_ampere", warpx.finestLevel()); + ablastr::fields::VectorField current_fp_ampere = warpx.m_fields.get_alldirs("hybrid_current_fp_ampere", lev); warpx.get_pointer_fdtd_solver_fp(lev)->CalculateCurrentAmpere( current_fp_ampere, Bfield, edge_lengths, lev ); @@ -463,7 +463,6 @@ void HybridPICModel::HybridPICSolveE ( const int lev, PatchType patch_type, const bool solve_for_Faraday) const { - using ablastr::fields::va2vm; auto& warpx = WarpX::GetInstance(); From ede7a794aa1f0cc77d2b8b85717cb645980b6ad8 Mon Sep 17 00:00:00 2001 From: David Grote Date: Wed, 18 Sep 2024 13:37:35 -0700 Subject: [PATCH 275/314] Fix for HybridPIC, fixing name passed into SyncCurrent --- Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index a8d70d8d792..69ca47d9ad7 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -199,7 +199,7 @@ void WarpX::HybridPICDepositInitialRhoAndJ () mypc->DepositCharge(rho_fp_temp, 0._rt); mypc->DepositCurrent(current_fp_temp, dt[0], 0._rt); SyncRho(rho_fp_temp, m_fields.get_mr_levels("rho_cp", finest_level), m_fields.get_mr_levels("rho_buf", finest_level)); - SyncCurrent("current_fp"); + SyncCurrent("hybrid_current_fp_temp"); for (int lev=0; lev <= finest_level; ++lev) { // SyncCurrent does not include a call to FillBoundary, but it is needed // for the hybrid-PIC solver since current values are interpolated to From cf76e6a09a51a0d7baeac2feec8150cc3a70ae4d Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 18 Sep 2024 14:08:49 -0700 Subject: [PATCH 276/314] `WarpX::UpdateAuxilaryData`: Fix External E field Co-authored-by: Edoardo Zoni --- Source/Parallelization/WarpXComm.cpp | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index ed97a593772..5fa213cd8c1 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -69,21 +69,19 @@ WarpX::UpdateAuxilaryData () // When loading particle fields from file: add the external fields: for (int lev = 0; lev <= finest_level; ++lev) { - amrex::MultiFab *Bfield_aux_lvl_0 = m_fields.get("Bfield_aux", Direction{0}, lev); - amrex::MultiFab *Bfield_aux_lvl_1 = m_fields.get("Bfield_aux", Direction{1}, lev); - amrex::MultiFab *Bfield_aux_lvl_2 = m_fields.get("Bfield_aux", Direction{2}, lev); - if (mypc->m_E_ext_particle_s == "read_from_file") { + ablastr::fields::VectorField Efield_aux = m_fields.get_alldirs("Efield_aux", lev); const auto& E_ext_lev = m_fields.get_alldirs("E_external_particle_field", lev); - amrex::MultiFab::Add(*Bfield_aux_lvl_0, *E_ext_lev[0], 0, 0, E_ext_lev[0]->nComp(), guard_cells.ng_FieldGather); - amrex::MultiFab::Add(*Bfield_aux_lvl_1, *E_ext_lev[1], 0, 0, E_ext_lev[1]->nComp(), guard_cells.ng_FieldGather); - amrex::MultiFab::Add(*Bfield_aux_lvl_2, *E_ext_lev[2], 0, 0, E_ext_lev[2]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Efield_aux[0], *E_ext_lev[0], 0, 0, E_ext_lev[0]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Efield_aux[1], *E_ext_lev[1], 0, 0, E_ext_lev[1]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Efield_aux[2], *E_ext_lev[2], 0, 0, E_ext_lev[2]->nComp(), guard_cells.ng_FieldGather); } if (mypc->m_B_ext_particle_s == "read_from_file") { + ablastr::fields::VectorField Bfield_aux = m_fields.get_alldirs("Bfield_aux", lev); const auto& B_ext_lev = m_fields.get_alldirs("B_external_particle_field", lev); - amrex::MultiFab::Add(*Bfield_aux_lvl_0, *B_ext_lev[0], 0, 0, B_ext_lev[0]->nComp(), guard_cells.ng_FieldGather); - amrex::MultiFab::Add(*Bfield_aux_lvl_1, *B_ext_lev[1], 0, 0, B_ext_lev[1]->nComp(), guard_cells.ng_FieldGather); - amrex::MultiFab::Add(*Bfield_aux_lvl_2, *B_ext_lev[2], 0, 0, B_ext_lev[2]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Bfield_aux[0], *B_ext_lev[0], 0, 0, B_ext_lev[0]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Bfield_aux[1], *B_ext_lev[1], 0, 0, B_ext_lev[1]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Bfield_aux[2], *B_ext_lev[2], 0, 0, B_ext_lev[2]->nComp(), guard_cells.ng_FieldGather); } } From de0a3ea86c4e9c12328e9e4af0955489ec76c35a Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 18 Sep 2024 15:58:13 -0700 Subject: [PATCH 277/314] Fix subcycling typo --- Source/WarpX.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index af44afbcbd1..fdff1bbc2b2 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -3258,7 +3258,7 @@ WarpX::StoreCurrent (int lev) { using ablastr::fields::Direction; for (int idim = 0; idim < 3; ++idim) { - if (m_fields.get("current_store", Direction{idim},lev)) { + if (m_fields.has("current_store", Direction{idim},lev)) { MultiFab::Copy(*m_fields.get("current_store", Direction{idim}, lev), *m_fields.get("current_fp", Direction{idim}, lev), 0, 0, 1, m_fields.get("current_store", Direction{idim}, lev)->nGrowVect()); @@ -3272,7 +3272,7 @@ WarpX::RestoreCurrent (int lev) using ablastr::fields::Direction; for (int idim = 0; idim < 3; ++idim) { - if (m_fields.get("current_store", Direction{idim}, lev)) { + if (m_fields.has("current_store", Direction{idim}, lev)) { std::swap( *m_fields.get("current_fp", Direction{idim}, lev), *m_fields.get("current_store", Direction{idim}, lev) From f6ab6de085ca141ab91408c0894fec1b4105ee3f Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 18 Sep 2024 16:02:36 -0700 Subject: [PATCH 278/314] Fix another typo --- Source/Fluids/WarpXFluidContainer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/Fluids/WarpXFluidContainer.cpp b/Source/Fluids/WarpXFluidContainer.cpp index 2bed7e059e6..cc4c8777b18 100644 --- a/Source/Fluids/WarpXFluidContainer.cpp +++ b/Source/Fluids/WarpXFluidContainer.cpp @@ -261,7 +261,7 @@ void WarpXFluidContainer::Evolve( using ablastr::fields::Direction; WARPX_PROFILE("WarpXFluidContainer::Evolve"); - if (fields.get("rho_fp",lev) && ! skip_deposition && ! do_not_deposit) { + if (fields.has("rho_fp",lev) && ! skip_deposition && ! do_not_deposit) { // Deposit charge before particle push, in component 0 of MultiFab rho. DepositCharge(fields, *fields.get("rho_fp",lev), lev, 0); } @@ -294,7 +294,7 @@ void WarpXFluidContainer::Evolve( // Deposit rho to the simulation mesh // Deposit charge (end of the step) - if (fields.get("rho_fp",lev) && ! skip_deposition && ! do_not_deposit) { + if (fields.has("rho_fp",lev) && ! skip_deposition && ! do_not_deposit) { DepositCharge(fields, *fields.get("rho_fp",lev), lev, 1); } From 722520141631749bf12b4b06adec79a36d318c87 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 18 Sep 2024 16:10:16 -0700 Subject: [PATCH 279/314] Some minor refactoring --- Source/Initialization/WarpXInitData.cpp | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 0c54022505e..8701b1a4023 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -907,9 +907,6 @@ WarpX::InitLevelData (int lev, Real /*time*/) { using ablastr::fields::Direction; - ablastr::fields::MultiLevelVectorField Efield_aux = m_fields.get_mr_levels_alldirs("Efield_aux", max_level); - ablastr::fields::MultiLevelVectorField Bfield_aux = m_fields.get_mr_levels_alldirs("Bfield_aux", max_level); - // initialize the averaged fields only if the averaged algorithm // is activated ('psatd.do_time_averaging=1') const ParmParse pp_psatd("psatd"); @@ -929,7 +926,7 @@ WarpX::InitLevelData (int lev, Real /*time*/) } if (lev > 0) { - Bfield_aux[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); + m_fields.get("Bfield_aux", Direction{i}, lev)->setVal(m_p_ext_field_params->B_external_grid[i]); m_fields.get("Bfield_cp", Direction{i}, lev)->setVal(m_p_ext_field_params->B_external_grid[i]); if (fft_do_time_averaging) { m_fields.get("Bfield_avg_cp", Direction{i}, lev)->setVal(m_p_ext_field_params->B_external_grid[i]); @@ -948,7 +945,7 @@ WarpX::InitLevelData (int lev, Real /*time*/) m_fields.get("Efield_avg_fp", Direction{i}, lev)->setVal(m_p_ext_field_params->E_external_grid[i]); } if (lev > 0) { - Efield_aux[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); + m_fields.get("Efield_aux", Direction{i}, lev)->setVal(m_p_ext_field_params->E_external_grid[i]); m_fields.get("Efield_cp", Direction{i}, lev)->setVal(m_p_ext_field_params->E_external_grid[i]); if (fft_do_time_averaging) { m_fields.get("Efield_avg_cp", Direction{i}, lev)->setVal(m_p_ext_field_params->E_external_grid[i]); @@ -971,9 +968,9 @@ WarpX::InitLevelData (int lev, Real /*time*/) && (lev > 0) && (lev <= maxlevel_extEMfield_init)) { InitializeExternalFieldsOnGridUsingParser( - Bfield_aux[lev][0], - Bfield_aux[lev][1], - Bfield_aux[lev][2], + m_fields.get("Bfield_aux", Direction{0}, lev), + m_fields.get("Bfield_aux", Direction{1}, lev), + m_fields.get("Bfield_aux", Direction{2}, lev), m_p_ext_field_params->Bxfield_parser->compile<3>(), m_p_ext_field_params->Byfield_parser->compile<3>(), m_p_ext_field_params->Bzfield_parser->compile<3>(), @@ -1019,9 +1016,9 @@ WarpX::InitLevelData (int lev, Real /*time*/) if (lev > 0) { InitializeExternalFieldsOnGridUsingParser( - Efield_aux[lev][0], - Efield_aux[lev][1], - Efield_aux[lev][2], + m_fields.get("Efield_aux", Direction{0}, lev), + m_fields.get("Efield_aux", Direction{1}, lev), + m_fields.get("Efield_aux", Direction{2}, lev), m_p_ext_field_params->Exfield_parser->compile<3>(), m_p_ext_field_params->Eyfield_parser->compile<3>(), m_p_ext_field_params->Ezfield_parser->compile<3>(), From 240c8957407d80cfb16a037ad590db30d80b1f5e Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 18 Sep 2024 22:15:58 -0700 Subject: [PATCH 280/314] Clang-Tidy 3D --- Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp b/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp index 5b9aa940d6a..660a21aecde 100644 --- a/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp +++ b/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp @@ -193,12 +193,12 @@ computePhiIGF ( amrex::MultiFab const & rho, // Prepare to perform global FFT // Since there is 1 MPI rank per box, here each MPI rank obtains its local box and the associated boxid - int local_boxid = amrex::ParallelDescriptor::MyProc(); // because of how we made the DistributionMapping + int const local_boxid = amrex::ParallelDescriptor::MyProc(); // because of how we made the DistributionMapping if (local_boxid < realspace_ba.size()) { // When not using heFFTe, there is only one box (the global box) // It is taken care of my MPI rank 0 ; other ranks have no work (hence the if condition) - amrex::Box local_nodal_box = realspace_ba[local_boxid]; + amrex::Box const local_nodal_box = realspace_ba[local_boxid]; amrex::Box local_box(local_nodal_box.smallEnd(), local_nodal_box.bigEnd()); local_box.shift(-realspace_box.smallEnd()); // This simplifies the setup because the global lo is zero now // Since we the domain decompostion is in the z-direction, setting up c_local_box is simple. From a167d08de0be0b7db39c429a4b0bf5d48adfd729 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Thu, 19 Sep 2024 09:57:04 +0200 Subject: [PATCH 281/314] fix clang-tidy issues --- Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp | 2 +- .../FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp index d80ca9db15e..e9877c63f87 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp @@ -417,7 +417,7 @@ void FiniteDifferenceSolver::EvolveECylindrical ( if (Ffield) { // Extract field data for this grid/tile - Array4 F = Ffield->array(mfi); + const Array4 F = Ffield->array(mfi); // Loop over the cells and update the fields amrex::ParallelFor(ter, tet, tez, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index a3656142384..03f51f7ba62 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -226,7 +226,7 @@ class FiniteDifferenceSolver void EvolveFCylindrical ( amrex::MultiFab* Ffield, ablastr::fields::VectorField const & Efield, - amrex::MultiFab* const rhofield, + amrex::MultiFab* rhofield, int rhocomp, amrex::Real dt ); From 9f2f4e7db8e67d098e169555a8deb96af4506cd6 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 19 Sep 2024 10:55:26 -0700 Subject: [PATCH 282/314] Fix moving window bug --- Source/Utils/WarpXMovingWindow.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index ca6d3bfb611..f4676613fd3 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -269,10 +269,10 @@ WarpX::MoveWindow (const int step, bool move_j) // coarse grid shiftMF(*m_fields.get("Bfield_cp",Direction{dim},lev), geom[lev-1], num_shift_crse, dir, lev, do_update_cost, m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); - shiftMF(*m_fields.get("Efield_cp",Direction{dim},lev), geom[lev-1], num_shift, dir, lev, do_update_cost, + shiftMF(*m_fields.get("Efield_cp",Direction{dim},lev), geom[lev-1], num_shift_crse, dir, lev, do_update_cost, m_p_ext_field_params->E_external_grid[dim], use_Eparser, Efield_parser); - shiftMF(*m_fields.get("Bfield_aux",Direction{dim},lev), geom[lev], num_shift, dir, lev, do_update_cost); - shiftMF(*m_fields.get("Efield_aux",Direction{dim},lev), geom[lev], num_shift, dir, lev, do_update_cost); + shiftMF(*m_fields.get("Bfield_aux",Direction{dim},lev), geom[lev], num_shift_crse, dir, lev, do_update_cost); + shiftMF(*m_fields.get("Efield_aux",Direction{dim},lev), geom[lev], num_shift_crse, dir, lev, do_update_cost); if (fft_do_time_averaging) { ablastr::fields::MultiLevelVectorField Efield_avg_cp = m_fields.get_mr_levels_alldirs("Efield_avg_cp", finest_level); ablastr::fields::MultiLevelVectorField Bfield_avg_cp = m_fields.get_mr_levels_alldirs("Bfield_avg_cp", finest_level); From 7a64cc2686811b5cabb6d3f11720383afb6ba706 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 19 Sep 2024 11:00:41 -0700 Subject: [PATCH 283/314] Revert some of the changes --- Source/Utils/WarpXMovingWindow.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index f4676613fd3..e6cabd4aa65 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -271,8 +271,8 @@ WarpX::MoveWindow (const int step, bool move_j) m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); shiftMF(*m_fields.get("Efield_cp",Direction{dim},lev), geom[lev-1], num_shift_crse, dir, lev, do_update_cost, m_p_ext_field_params->E_external_grid[dim], use_Eparser, Efield_parser); - shiftMF(*m_fields.get("Bfield_aux",Direction{dim},lev), geom[lev], num_shift_crse, dir, lev, do_update_cost); - shiftMF(*m_fields.get("Efield_aux",Direction{dim},lev), geom[lev], num_shift_crse, dir, lev, do_update_cost); + shiftMF(*m_fields.get("Bfield_aux",Direction{dim},lev), geom[lev], num_shift, dir, lev, do_update_cost); + shiftMF(*m_fields.get("Efield_aux",Direction{dim},lev), geom[lev], num_shift, dir, lev, do_update_cost); if (fft_do_time_averaging) { ablastr::fields::MultiLevelVectorField Efield_avg_cp = m_fields.get_mr_levels_alldirs("Efield_avg_cp", finest_level); ablastr::fields::MultiLevelVectorField Bfield_avg_cp = m_fields.get_mr_levels_alldirs("Bfield_avg_cp", finest_level); From 4801c9992195633e6aa76f7031b9180aec216f21 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 19 Sep 2024 12:30:37 -0700 Subject: [PATCH 284/314] CI: MSVC Backtrace to Hang --- .github/workflows/windows.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index d6030743524..eade3909476 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -54,7 +54,11 @@ jobs: run: | $env:PATH += ';C:/Program Files (x86)/WarpX/bin/' - python3 Examples/Tests/gaussian_beam/inputs_test_3d_gaussian_beam_picmi.py + Start-ThreadJob -Name "test" -ScriptBlock { python3 Examples/Tests/gaussian_beam/inputs_test_3d_gaussian_beam_picmi.py } + Start-Sleep -Seconds 20 + Stop-Job -Name "test" + + cat Backtrace.0 # JSON writes are currently very slow (50min) with MSVC # --diagformat=openpmd From c7a29c424b90a9cfec8390f52404dba56d2d0561 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 19 Sep 2024 14:16:20 -0700 Subject: [PATCH 285/314] MSVC: Debug Further --- .github/workflows/windows.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index eade3909476..856648191f2 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -54,9 +54,9 @@ jobs: run: | $env:PATH += ';C:/Program Files (x86)/WarpX/bin/' - Start-ThreadJob -Name "test" -ScriptBlock { python3 Examples/Tests/gaussian_beam/inputs_test_3d_gaussian_beam_picmi.py } - Start-Sleep -Seconds 20 - Stop-Job -Name "test" + $job = Start-ThreadJob -Name "test" -ScriptBlock { & python3 Examples/Tests/gaussian_beam/inputs_test_3d_gaussian_beam_picmi.py } + Start-Sleep -Seconds 30 + Stop-Job $job.Id cat Backtrace.0 # JSON writes are currently very slow (50min) with MSVC From 95d9c721c3be780e0809778f9bf2629191ff531d Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 19 Sep 2024 14:16:47 -0700 Subject: [PATCH 286/314] More Docstrings --- Source/Parallelization/WarpXComm.cpp | 1 + Source/Python/MultiFabRegister.cpp | 3 +- Source/ablastr/fields/MultiFabRegister.H | 98 ++++++++++------------ Source/ablastr/fields/MultiFabRegister.cpp | 5 +- 4 files changed, 51 insertions(+), 56 deletions(-) diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 5fa213cd8c1..34a2fb18095 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -20,6 +20,7 @@ #include "WarpXSumGuardCells.H" #include "Particles/MultiParticleContainer.H" +#include #include #include diff --git a/Source/Python/MultiFabRegister.cpp b/Source/Python/MultiFabRegister.cpp index da0712a8909..f345f667743 100644 --- a/Source/Python/MultiFabRegister.cpp +++ b/Source/Python/MultiFabRegister.cpp @@ -99,7 +99,8 @@ void init_MultiFabRegister (py::module & m) .def("alloc_like", &MultiFabRegister::alloc_like, py::arg("other_name"), - py::arg("other_level") + py::arg("other_level"), + py::arg("level") ) .def("has", diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 7599aa68d07..79b7d0fa04f 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -241,27 +241,26 @@ namespace ablastr::fields std::optional initial_value = std::nullopt ); - /** title + /** Allocate a new MultiFab (field) with the same size and distribution as another. * - * body body - * body + * @todo this is reserved for future use and not yet implemented. + * @todo we will also add an overload with dir. * - * @param other_name ... - * @param other_level ... + * @param new_name new name + * @param other_name the other multifab + * @param level the MR level to represent */ void alloc_like ( - const std::string& other_name, - int other_level + std::string new_name, + std::string other_name, + int level ); - /** title - * - * body body - * body + /** Check if a scalar MultiFab (field) is registered. * - * @param name ... - * @param level ... + * @param name the name to check if registered + * @param level the MR level to check * @return true if contained, otherwise false */ [[nodiscard]] bool @@ -270,14 +269,11 @@ namespace ablastr::fields int level ) const; - /** title - * - * body body - * body + /** Check if a MultiFab that is part of a vector/tensor field is registered. * - * @param name ... - * @param dir ... - * @param level ... + * @param name the name to check if registered + * @param dir the field component for vector fields ("direction" of the unit vector) + * @param level the MR level to check * @return true if contained, otherwise false */ [[nodiscard]] bool @@ -287,46 +283,43 @@ namespace ablastr::fields int level ) const; - /** title + /** Return a scalar MultiFab (field). * - * body body - * body + * This throws a runtime error if the requested field is not present. * - * @param name ... - * @param level ... - * @return ... + * @param name the name to check if registered + * @param level the MR level to check + * @return a non-owning pointer to the MultiFab (field) */ - amrex::MultiFab* + [[nodiscard]] amrex::MultiFab* get ( std::string name, int level ); - /** title + /** Return a MultiFab that is part of a vector/tensor field. * - * body body - * body + * This throws a runtime error if the requested field is not present. * - * @param name ... - * @param dir ... - * @param level ... - * @return ... + * @param name the name to check if registered + * @param dir the field component for vector fields ("direction" of the unit vector) + * @param level the MR level to check + * @return a non-owning pointer to the MultiFab (field) */ - amrex::MultiFab* + [[nodiscard]] amrex::MultiFab* get ( std::string name, Direction dir, int level ); - /** title + /** Return a scalar MultiFab (field). * - * body body - * body + * This throws a runtime error if the requested field is not present. * - * @param name ... - * @param level ... - * @return ... + * @param name the name to check if registered + * @param level the MR level to check + * @return a non-owning pointer to the MultiFab (field) */ [[nodiscard]] amrex::MultiFab const * get ( @@ -334,15 +327,14 @@ namespace ablastr::fields int level ) const; - /** title + /** Return a MultiFab that is part of a vector/tensor field. * - * body body - * body + * This throws a runtime error if the requested field is not present. * - * @param name ... - * @param dir ... - * @param level ... - * @return ... + * @param name the name to check if registered + * @param dir the field component for vector fields ("direction" of the unit vector) + * @param level the MR level to check + * @return a non-owning pointer to the MultiFab (field) */ [[nodiscard]] amrex::MultiFab const * get ( @@ -359,7 +351,7 @@ namespace ablastr::fields * @param finest_level ... * @return ... */ - MultiLevelScalarField + [[nodiscard]] MultiLevelScalarField get_mr_levels ( const std::string& name, int finest_level @@ -378,7 +370,7 @@ namespace ablastr::fields * @param level ... * @return ... */ - VectorField + [[nodiscard]] VectorField get_alldirs ( const std::string&, int level @@ -398,7 +390,7 @@ namespace ablastr::fields * @param finest_level ... * @return ... */ - MultiLevelVectorField + [[nodiscard]] MultiLevelVectorField get_mr_levels_alldirs ( const std::string& name, int finest_level @@ -508,11 +500,11 @@ namespace ablastr::fields private: amrex::MultiFab * internal_get ( - const std::string& key + const std::string& key ); [[nodiscard]] amrex::MultiFab const * internal_get ( - const std::string& key + const std::string& key ) const; /** data storage: ownership and lifetime control */ diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index f5e19c19db1..3df75219dcf 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -119,8 +119,9 @@ namespace ablastr::fields void MultiFabRegister::alloc_like ( - const std::string& /* other_name */, - int /* other_level */ + std::string /* new_name */, + std::string /* other_name*/, + int /* level */ ) { // other_name = mf_name(other_name, other_level); From b35949b72ea653f86642958d6bf8a392244bf1d5 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 19 Sep 2024 15:18:03 -0700 Subject: [PATCH 287/314] CI: Windows... --- .github/workflows/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 856648191f2..5410454db3c 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -54,7 +54,7 @@ jobs: run: | $env:PATH += ';C:/Program Files (x86)/WarpX/bin/' - $job = Start-ThreadJob -Name "test" -ScriptBlock { & python3 Examples/Tests/gaussian_beam/inputs_test_3d_gaussian_beam_picmi.py } + $job = Start-ThreadJob -Name "test" -ScriptBlock { & "python3" "Examples/Tests/gaussian_beam/inputs_test_3d_gaussian_beam_picmi.py" } Start-Sleep -Seconds 30 Stop-Job $job.Id From 9f7b1c91b512ab3f297f8bdc9a7b1ea25f8baee4 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 19 Sep 2024 16:14:09 -0700 Subject: [PATCH 288/314] MSVC CI: Output? --- .github/workflows/windows.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 5410454db3c..2ab620c0e4a 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -58,6 +58,9 @@ jobs: Start-Sleep -Seconds 30 Stop-Job $job.Id + $jobOutput = $job | Receive-Job + $jobOutput + cat Backtrace.0 # JSON writes are currently very slow (50min) with MSVC # --diagformat=openpmd From 1a3146a7e3f02a28fa60dbeb87303e7dbb296b47 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Thu, 19 Sep 2024 18:25:25 -0700 Subject: [PATCH 289/314] Fix more typo bugs --- Source/Evolve/WarpXEvolve.cpp | 2 +- Source/FieldSolver/WarpXPushFieldsEM.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index c9e986381ec..caa87219cb7 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -682,7 +682,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) if (m_fields.has("rho_fp", 0) && rho_in_time == RhoInTime::Linear) { const ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); - const ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_fp", finest_level); + const ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_cp", finest_level); // Deposit rho at relative time -dt // (dt[0] denotes the time step on mesh refinement level 0) diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 43a964c1806..6492f5ce591 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -670,7 +670,7 @@ WarpX::PushPSATD () const int rho_old = spectral_solver_fp[0]->m_spectral_index.rho_old; const int rho_new = spectral_solver_fp[0]->m_spectral_index.rho_new; const ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); - const ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_fp", finest_level); + const ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_cp", finest_level); const ablastr::fields::MultiLevelVectorField current_fp = m_fields.get_mr_levels_alldirs("current_fp", finest_level); const ablastr::fields::MultiLevelVectorField current_cp = m_fields.get_mr_levels_alldirs("current_cp", finest_level); const ablastr::fields::MultiLevelVectorField current_buf = m_fields.get_mr_levels_alldirs("current_buf", finest_level); From 03798816e860c821e5fa3f6804e9ccd6043a88bf Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 20 Sep 2024 00:41:49 -0700 Subject: [PATCH 290/314] CI: Wait Longer for MSVC Run to hang --- .github/workflows/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 2ab620c0e4a..c850970bca6 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -55,7 +55,7 @@ jobs: $env:PATH += ';C:/Program Files (x86)/WarpX/bin/' $job = Start-ThreadJob -Name "test" -ScriptBlock { & "python3" "Examples/Tests/gaussian_beam/inputs_test_3d_gaussian_beam_picmi.py" } - Start-Sleep -Seconds 30 + Start-Sleep -Seconds 120 Stop-Job $job.Id $jobOutput = $job | Receive-Job From 7a7f462a3630d66c0d8631b19b9a6f97a9dc5834 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 20 Sep 2024 00:44:03 -0700 Subject: [PATCH 291/314] Clang-Tidy --- Source/ablastr/fields/MultiFabRegister.H | 4 ++-- Source/ablastr/fields/MultiFabRegister.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 79b7d0fa04f..7349ad5d91b 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -252,8 +252,8 @@ namespace ablastr::fields */ void alloc_like ( - std::string new_name, - std::string other_name, + std::string const & new_name, + std::string const & other_name, int level ); diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 3df75219dcf..14a24a57dec 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -119,8 +119,8 @@ namespace ablastr::fields void MultiFabRegister::alloc_like ( - std::string /* new_name */, - std::string /* other_name*/, + std::string const & /* new_name */, + std::string const & /* other_name */, int /* level */ ) { From 593430eff59ec695e216a9d5a0904fe4e9935d0f Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Fri, 20 Sep 2024 10:38:22 -0700 Subject: [PATCH 292/314] Clean up FFT functions for E, B --- Source/Evolve/WarpXEvolve.cpp | 11 +-- Source/FieldSolver/WarpXPushFieldsEM.cpp | 91 +++++++++++++++--------- Source/WarpX.H | 34 ++------- 3 files changed, 65 insertions(+), 71 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index caa87219cb7..ffbbe471edb 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -666,11 +666,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Initialize multi-J loop: // 1) Prepare E,B,F,G fields in spectral space - PSATDForwardTransformEB( - m_fields.get_mr_levels_alldirs("Efield_fp", finest_level), - m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level), - m_fields.get_mr_levels_alldirs("Efield_cp", finest_level), - m_fields.get_mr_levels_alldirs("Bfield_cp", finest_level) ); + PSATDForwardTransformEB(); if (WarpX::do_dive_cleaning) { PSATDForwardTransformF(); } if (WarpX::do_divb_cleaning) { PSATDForwardTransformG(); } @@ -778,10 +774,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // (the relative time reached here coincides with an integer full time step) if (i_deposit == n_deposit-1) { - PSATDBackwardTransformEB(m_fields.get_mr_levels_alldirs("Efield_fp", max_level), - m_fields.get_mr_levels_alldirs("Bfield_fp", max_level), - m_fields.get_mr_levels_alldirs("Efield_cp", max_level), - m_fields.get_mr_levels_alldirs("Bfield_cp", max_level)); + PSATDBackwardTransformEB(); if (WarpX::do_dive_cleaning) { PSATDBackwardTransformF(); } if (WarpX::do_divb_cleaning) { PSATDBackwardTransformG(); } } diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 6492f5ce591..25449061c45 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -100,55 +100,85 @@ namespace { } } -void WarpX::PSATDForwardTransformEB ( - const ablastr::fields::MultiLevelVectorField& E_fp, - const ablastr::fields::MultiLevelVectorField& B_fp, - const ablastr::fields::MultiLevelVectorField& E_cp, - const ablastr::fields::MultiLevelVectorField& B_cp) +void WarpX::PSATDForwardTransformEB () { const SpectralFieldIndex& Idx = spectral_solver_fp[0]->m_spectral_index; + const std::string Efield_fp_string = "Efield_fp"; + const std::string Efield_cp_string = "Efield_cp"; + const std::string Bfield_fp_string = "Bfield_fp"; + const std::string Bfield_cp_string = "Bfield_cp"; + for (int lev = 0; lev <= finest_level; ++lev) { - ForwardTransformVect(lev, *spectral_solver_fp[lev], E_fp[lev], Idx.Ex, Idx.Ey, Idx.Ez); - ForwardTransformVect(lev, *spectral_solver_fp[lev], B_fp[lev], Idx.Bx, Idx.By, Idx.Bz); + if (m_fields.has(Efield_fp_string, lev)) { + ablastr::fields::VectorField E_fp = m_fields.get_alldirs(Efield_fp_string, lev); + ForwardTransformVect(lev, *spectral_solver_fp[lev], E_fp, Idx.Ex, Idx.Ey, Idx.Ez); + } + if (m_fields.has(Bfield_fp_string, lev)) { + ablastr::fields::VectorField B_fp = m_fields.get_alldirs(Bfield_fp_string, lev); + ForwardTransformVect(lev, *spectral_solver_fp[lev], B_fp, Idx.Bx, Idx.By, Idx.Bz); + } if (spectral_solver_cp[lev]) { - ForwardTransformVect(lev, *spectral_solver_cp[lev], E_cp[lev], Idx.Ex, Idx.Ey, Idx.Ez); - ForwardTransformVect(lev, *spectral_solver_cp[lev], B_cp[lev], Idx.Bx, Idx.By, Idx.Bz); + if (m_fields.has(Efield_cp_string, lev)) { + ablastr::fields::VectorField E_cp = m_fields.get_alldirs(Efield_cp_string, lev); + ForwardTransformVect(lev, *spectral_solver_cp[lev], E_cp, Idx.Ex, Idx.Ey, Idx.Ez); + } + if (m_fields.has(Bfield_cp_string, lev)) { + ablastr::fields::VectorField B_cp = m_fields.get_alldirs(Bfield_cp_string, lev); + ForwardTransformVect(lev, *spectral_solver_cp[lev], B_cp, Idx.Bx, Idx.By, Idx.Bz); + } } } } -void WarpX::PSATDBackwardTransformEB ( - const ablastr::fields::MultiLevelVectorField& E_fp, - const ablastr::fields::MultiLevelVectorField& B_fp, - const ablastr::fields::MultiLevelVectorField& E_cp, - const ablastr::fields::MultiLevelVectorField& B_cp) +void WarpX::PSATDBackwardTransformEB () { const SpectralFieldIndex& Idx = spectral_solver_fp[0]->m_spectral_index; + const std::string Efield_fp_string = "Efield_fp"; + const std::string Efield_cp_string = "Efield_cp"; + const std::string Bfield_fp_string = "Bfield_fp"; + const std::string Bfield_cp_string = "Bfield_cp"; + for (int lev = 0; lev <= finest_level; ++lev) { - BackwardTransformVect(lev, *spectral_solver_fp[lev], E_fp[lev], - Idx.Ex, Idx.Ey, Idx.Ez, m_fill_guards_fields); - BackwardTransformVect(lev, *spectral_solver_fp[lev], B_fp[lev], - Idx.Bx, Idx.By, Idx.Bz, m_fill_guards_fields); + if (m_fields.has(Efield_fp_string, lev)) { + ablastr::fields::VectorField E_fp = m_fields.get_alldirs(Efield_fp_string, lev); + BackwardTransformVect(lev, *spectral_solver_fp[lev], E_fp, + Idx.Ex, Idx.Ey, Idx.Ez, m_fill_guards_fields); + } + if (m_fields.has(Bfield_fp_string, lev)) { + ablastr::fields::VectorField B_fp = m_fields.get_alldirs(Bfield_fp_string, lev); + BackwardTransformVect(lev, *spectral_solver_fp[lev], B_fp, + Idx.Bx, Idx.By, Idx.Bz, m_fill_guards_fields); + } if (spectral_solver_cp[lev]) { - BackwardTransformVect(lev, *spectral_solver_cp[lev], E_cp[lev], - Idx.Ex, Idx.Ey, Idx.Ez, m_fill_guards_fields); - BackwardTransformVect(lev, *spectral_solver_cp[lev], B_cp[lev], - Idx.Bx, Idx.By, Idx.Bz, m_fill_guards_fields); + if (m_fields.has(Efield_cp_string, lev)) { + ablastr::fields::VectorField E_cp = m_fields.get_alldirs(Efield_cp_string, lev); + BackwardTransformVect(lev, *spectral_solver_cp[lev], E_cp, + Idx.Ex, Idx.Ey, Idx.Ez, m_fill_guards_fields); + } + if (m_fields.has(Bfield_cp_string, lev)) { + ablastr::fields::VectorField B_cp = m_fields.get_alldirs(Bfield_cp_string, lev); + BackwardTransformVect(lev, *spectral_solver_cp[lev], B_cp, + Idx.Bx, Idx.By, Idx.Bz, m_fill_guards_fields); + } } } // Damp the fields in the guard cells for (int lev = 0; lev <= finest_level; ++lev) { - DampFieldsInGuards(lev, E_fp[lev], B_fp[lev]); + if (m_fields.has(Efield_fp_string, lev) && m_fields.has(Bfield_fp_string, lev)) { + ablastr::fields::VectorField E_fp = m_fields.get_alldirs(Efield_fp_string, lev); + ablastr::fields::VectorField B_fp = m_fields.get_alldirs(Bfield_fp_string, lev); + DampFieldsInGuards(lev, E_fp, B_fp); + } } } @@ -775,13 +805,13 @@ WarpX::PushPSATD () PSATDForwardTransformRho(rho_fp, rho_cp, 1, rho_new); } - auto Efield_fp = m_fields.get_mr_levels_alldirs("Efield_fp", finest_level); - auto Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); - auto Efield_cp = m_fields.get_mr_levels_alldirs("Efield_cp", finest_level); - auto Bfield_cp = m_fields.get_mr_levels_alldirs("Bfield_cp", finest_level); + //auto Efield_fp = m_fields.get_mr_levels_alldirs("Efield_fp", finest_level); + //auto Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); + //auto Efield_cp = m_fields.get_mr_levels_alldirs("Efield_cp", finest_level); + //auto Bfield_cp = m_fields.get_mr_levels_alldirs("Bfield_cp", finest_level); // FFT of E and B - PSATDForwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp ); + PSATDForwardTransformEB(); #ifdef WARPX_DIM_RZ if (pml_rz[0]) { pml_rz[0]->PushPSATD(0); } @@ -795,10 +825,7 @@ WarpX::PushPSATD () PSATDPushSpectralFields(); // Inverse FFT of E, B, F, and G - PSATDBackwardTransformEB( m_fields.get_mr_levels_alldirs("Efield_fp",finest_level), - Bfield_fp, - m_fields.get_mr_levels_alldirs("Efield_cp",finest_level), - Bfield_cp); + PSATDBackwardTransformEB(); if (WarpX::fft_do_time_averaging) { auto Efield_avg_fp = m_fields.get_mr_levels_alldirs("Efield_avg_fp", finest_level); auto Bfield_avg_fp = m_fields.get_mr_levels_alldirs("Bfield_avg_fp", finest_level); diff --git a/Source/WarpX.H b/Source/WarpX.H index bcce4caa3fc..aaac571f703 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1646,40 +1646,14 @@ private: /** * \brief Forward FFT of E,B on all mesh refinement levels - * - * \param E_fp Vector of three-dimensional arrays (for each level) - * storing the fine patch electric field to be transformed - * \param B_fp Vector of three-dimensional arrays (for each level) - * storing the fine patch magnetic field to be transformed - * \param E_cp Vector of three-dimensional arrays (for each level) - * storing the coarse patch electric field to be transformed - * \param B_cp Vector of three-dimensional arrays (for each level) - * storing the coarse patch magnetic field to be transformed - */ - void PSATDForwardTransformEB ( - const ablastr::fields::MultiLevelVectorField& E_fp, - const ablastr::fields::MultiLevelVectorField& B_fp, - const ablastr::fields::MultiLevelVectorField& E_cp, - const ablastr::fields::MultiLevelVectorField& B_cp); + */ + void PSATDForwardTransformEB (); /** * \brief Backward FFT of E,B on all mesh refinement levels, * with field damping in the guard cells (if needed) - * - * \param E_fp Vector of three-dimensional arrays (for each level) - * storing the fine patch electric field to be transformed - * \param B_fp Vector of three-dimensional arrays (for each level) - * storing the fine patch magnetic field to be transformed - * \param E_cp Vector of three-dimensional arrays (for each level) - * storing the coarse patch electric field to be transformed - * \param B_cp Vector of three-dimensional arrays (for each level) - * storing the coarse patch magnetic field to be transformed - */ - void PSATDBackwardTransformEB ( - const ablastr::fields::MultiLevelVectorField& E_fp, - const ablastr::fields::MultiLevelVectorField& B_fp, - const ablastr::fields::MultiLevelVectorField& E_cp, - const ablastr::fields::MultiLevelVectorField& B_cp); + */ + void PSATDBackwardTransformEB (); /** * \brief Backward FFT of averaged E,B on all mesh refinement levels From 83e2e253b0e5b440d0a898497e72c645b63f2cea Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 20 Sep 2024 10:42:16 -0700 Subject: [PATCH 293/314] Cleaning: Remove Unused Helper `va2vm` --- .../FieldSolver/WarpXPushFieldsHybridPIC.cpp | 3 -- Source/ablastr/fields/MultiFabRegister.H | 20 --------- Source/ablastr/fields/MultiFabRegister.cpp | 45 ------------------- 3 files changed, 68 deletions(-) diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index 69ca47d9ad7..13752458b9e 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -22,7 +22,6 @@ using namespace amrex; void WarpX::HybridPICEvolveFields () { using ablastr::fields::Direction; - using ablastr::fields::va2vm; WARPX_PROFILE("WarpX::HybridPICEvolveFields()"); @@ -192,8 +191,6 @@ void WarpX::HybridPICEvolveFields () void WarpX::HybridPICDepositInitialRhoAndJ () { - using ablastr::fields::va2vm; - ablastr::fields::MultiLevelScalarField rho_fp_temp = m_fields.get_mr_levels("hybrid_rho_fp_temp", finest_level); ablastr::fields::MultiLevelVectorField current_fp_temp = m_fields.get_mr_levels_alldirs("hybrid_current_fp_temp", finest_level); mypc->DepositCharge(rho_fp_temp, 0._rt); diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 7349ad5d91b..76ad4f30a27 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -523,26 +523,6 @@ namespace ablastr::fields const std::array< std::unique_ptr, 3 > & old_vectorfield ); - /** TODO: temporary, remove me - * - * Convert get_mr_levels_alldirs type to legacy amrex::Vector, 3 > >. - * - * @return - */ - MultiLevelVectorField - va2vm ( - const amrex::Vector, 3 > >& old_vector_on_levels - ); - - /** TODO: temporary, remove me - * - * @return - */ - MultiLevelScalarField - va2vm ( - const amrex::Vector >& old_scalar_on_levels - ); - } // namespace ablastr::fields #endif // ABLASTR_FIELDS_MF_REGISTER_H diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 14a24a57dec..2aa0ecba668 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -623,49 +623,4 @@ namespace ablastr::fields } return field_on_level; } - - MultiLevelVectorField - va2vm ( - const amrex::Vector, 3 > >& old_vector_on_levels - ) - { - auto const finest_level = static_cast(old_vector_on_levels.size() - 1u); - - MultiLevelVectorField field_on_level; - field_on_level.reserve(finest_level+1); - - const std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; - - for (int lvl = 0; lvl <= finest_level; lvl++) - { - // insert a new level - field_on_level.push_back(VectorField{}); - - // insert components - for (const auto dir : {0, 1, 2}) - { - field_on_level[lvl][Direction{dir}] = old_vector_on_levels[lvl][dir].get(); - } - } - return field_on_level; - } - - MultiLevelScalarField - va2vm ( - const amrex::Vector >& old_scalar_on_levels - ) - { - auto const finest_level = static_cast(old_scalar_on_levels.size() - 1u); - - MultiLevelScalarField field_on_level; - field_on_level.reserve(finest_level+1); - - for (int lvl = 0; lvl <= finest_level; lvl++) - { - // insert a scalar field on a level - field_on_level.push_back(ScalarField{old_scalar_on_levels[lvl].get()}); - - } - return field_on_level; - } } // namespace ablastr::fields From c2b481aa7ebbd7102f4d153297093003cdcfbd4a Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Fri, 20 Sep 2024 12:17:15 -0700 Subject: [PATCH 294/314] Add `has_vector`, fix bugs --- Source/FieldSolver/WarpXPushFieldsEM.cpp | 18 +++++++++--------- Source/ablastr/fields/MultiFabRegister.H | 12 ++++++++++++ Source/ablastr/fields/MultiFabRegister.cpp | 17 +++++++++++++++++ 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 90b552168bc..133043ebf96 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -111,22 +111,22 @@ void WarpX::PSATDForwardTransformEB () for (int lev = 0; lev <= finest_level; ++lev) { - if (m_fields.has(Efield_fp_string, lev)) { + if (m_fields.has_vector(Efield_fp_string, lev)) { ablastr::fields::VectorField E_fp = m_fields.get_alldirs(Efield_fp_string, lev); ForwardTransformVect(lev, *spectral_solver_fp[lev], E_fp, Idx.Ex, Idx.Ey, Idx.Ez); } - if (m_fields.has(Bfield_fp_string, lev)) { + if (m_fields.has_vector(Bfield_fp_string, lev)) { ablastr::fields::VectorField B_fp = m_fields.get_alldirs(Bfield_fp_string, lev); ForwardTransformVect(lev, *spectral_solver_fp[lev], B_fp, Idx.Bx, Idx.By, Idx.Bz); } if (spectral_solver_cp[lev]) { - if (m_fields.has(Efield_cp_string, lev)) { + if (m_fields.has_vector(Efield_cp_string, lev)) { ablastr::fields::VectorField E_cp = m_fields.get_alldirs(Efield_cp_string, lev); ForwardTransformVect(lev, *spectral_solver_cp[lev], E_cp, Idx.Ex, Idx.Ey, Idx.Ez); } - if (m_fields.has(Bfield_cp_string, lev)) { + if (m_fields.has_vector(Bfield_cp_string, lev)) { ablastr::fields::VectorField B_cp = m_fields.get_alldirs(Bfield_cp_string, lev); ForwardTransformVect(lev, *spectral_solver_cp[lev], B_cp, Idx.Bx, Idx.By, Idx.Bz); } @@ -145,12 +145,12 @@ void WarpX::PSATDBackwardTransformEB () for (int lev = 0; lev <= finest_level; ++lev) { - if (m_fields.has(Efield_fp_string, lev)) { + if (m_fields.has_vector(Efield_fp_string, lev)) { ablastr::fields::VectorField E_fp = m_fields.get_alldirs(Efield_fp_string, lev); BackwardTransformVect(lev, *spectral_solver_fp[lev], E_fp, Idx.Ex, Idx.Ey, Idx.Ez, m_fill_guards_fields); } - if (m_fields.has(Bfield_fp_string, lev)) { + if (m_fields.has_vector(Bfield_fp_string, lev)) { ablastr::fields::VectorField B_fp = m_fields.get_alldirs(Bfield_fp_string, lev); BackwardTransformVect(lev, *spectral_solver_fp[lev], B_fp, Idx.Bx, Idx.By, Idx.Bz, m_fill_guards_fields); @@ -158,12 +158,12 @@ void WarpX::PSATDBackwardTransformEB () if (spectral_solver_cp[lev]) { - if (m_fields.has(Efield_cp_string, lev)) { + if (m_fields.has_vector(Efield_cp_string, lev)) { ablastr::fields::VectorField E_cp = m_fields.get_alldirs(Efield_cp_string, lev); BackwardTransformVect(lev, *spectral_solver_cp[lev], E_cp, Idx.Ex, Idx.Ey, Idx.Ez, m_fill_guards_fields); } - if (m_fields.has(Bfield_cp_string, lev)) { + if (m_fields.has_vector(Bfield_cp_string, lev)) { ablastr::fields::VectorField B_cp = m_fields.get_alldirs(Bfield_cp_string, lev); BackwardTransformVect(lev, *spectral_solver_cp[lev], B_cp, Idx.Bx, Idx.By, Idx.Bz, m_fill_guards_fields); @@ -174,7 +174,7 @@ void WarpX::PSATDBackwardTransformEB () // Damp the fields in the guard cells for (int lev = 0; lev <= finest_level; ++lev) { - if (m_fields.has(Efield_fp_string, lev) && m_fields.has(Bfield_fp_string, lev)) { + if (m_fields.has_vector(Efield_fp_string, lev) && m_fields.has_vector(Bfield_fp_string, lev)) { ablastr::fields::VectorField E_fp = m_fields.get_alldirs(Efield_fp_string, lev); ablastr::fields::VectorField B_fp = m_fields.get_alldirs(Bfield_fp_string, lev); DampFieldsInGuards(lev, E_fp, B_fp); diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 76ad4f30a27..980598600c6 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -283,6 +283,18 @@ namespace ablastr::fields int level ) const; + /** Check if a MultiFab vector field is registered. + * + * @param name the name to check if registered + * @param level the MR level to check + * @return true if contained, otherwise false + */ + [[nodiscard]] bool + has_vector ( + std::string name, + int level + ) const; + /** Return a scalar MultiFab (field). * * This throws a runtime error if the requested field is not present. diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 2aa0ecba668..9360e065f61 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -329,6 +329,23 @@ namespace ablastr::fields return m_mf_register.count(name) > 0; } + bool + MultiFabRegister::has_vector ( + std::string name, + int level + ) const + { + const std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; + + int count = 0; + for (const Direction& dir : all_dirs) { + std::string internal_name = mf_name(name, dir, level); + count += m_mf_register.count(internal_name); + } + + return count == 3; + } + amrex::MultiFab* MultiFabRegister::internal_get ( const std::string& key From 98f96702f149d1a29ab1e269c8a75148f29441c4 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Fri, 20 Sep 2024 14:30:13 -0700 Subject: [PATCH 295/314] Clean up FFT functions for J, rho --- Source/Evolve/WarpXEvolve.cpp | 24 ++--- Source/FieldSolver/WarpXPushFieldsEM.cpp | 116 ++++++++++++++--------- Source/WarpX.H | 12 +-- 3 files changed, 89 insertions(+), 63 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index ffbbe471edb..dc684fc39ed 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -677,8 +677,10 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // (after checking that pointer to rho_fp on MR level 0 is not null) if (m_fields.has("rho_fp", 0) && rho_in_time == RhoInTime::Linear) { - const ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); - const ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_cp", finest_level); + ablastr::fields::MultiLevelScalarField const rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); + + std::string const rho_fp_string = "rho_fp"; + std::string const rho_cp_string = "rho_cp"; // Deposit rho at relative time -dt // (dt[0] denotes the time step on mesh refinement level 0) @@ -686,7 +688,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Filter, exchange boundary, and interpolate across levels SyncRho(); // Forward FFT of rho - PSATDForwardTransformRho(rho_fp, rho_cp, 0, rho_new); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 0, rho_new); } // 4) Deposit J at relative time -dt with time step dt @@ -702,9 +704,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // of guard cells. SyncCurrent("current_fp"); // Forward FFT of J - PSATDForwardTransformJ( - m_fields.get_mr_levels_alldirs( "current_fp", finest_level), - m_fields.get_mr_levels_alldirs( "current_cp", finest_level) ); + PSATDForwardTransformJ("current_fp", "current_cp"); } // Number of depositions for multi-J scheme @@ -738,16 +738,16 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // of guard cells. SyncCurrent("current_fp"); // Forward FFT of J - PSATDForwardTransformJ( - m_fields.get_mr_levels_alldirs( "current_fp", finest_level), - m_fields.get_mr_levels_alldirs( "current_cp", finest_level) ); + PSATDForwardTransformJ("current_fp", "current_cp"); // Deposit new rho // (after checking that pointer to rho_fp on MR level 0 is not null) if (m_fields.has("rho_fp", 0)) { - const ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); - const ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_cp", finest_level); + ablastr::fields::MultiLevelScalarField const rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); + + std::string const rho_fp_string = "rho_fp"; + std::string const rho_cp_string = "rho_cp"; // Move rho from new to old if rho is linear in time if (rho_in_time == RhoInTime::Linear) { PSATDMoveRhoNewToRhoOld(); } @@ -758,7 +758,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) SyncRho(); // Forward FFT of rho const int rho_idx = (rho_in_time == RhoInTime::Linear) ? rho_new : rho_mid; - PSATDForwardTransformRho(rho_fp, rho_cp, 0, rho_idx); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 0, rho_idx); } if (WarpX::current_correction) diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 133043ebf96..49e0bd2acb6 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -310,10 +310,12 @@ WarpX::PSATDBackwardTransformG () } void WarpX::PSATDForwardTransformJ ( - const ablastr::fields::MultiLevelVectorField& J_fp, - const ablastr::fields::MultiLevelVectorField& J_cp, + std::string const & J_fp_string, + std::string const & J_cp_string, const bool apply_kspace_filter) { + if (!m_fields.has_vector(J_fp_string, 0)) { return; } + SpectralFieldIndex Idx; int idx_jx, idx_jy, idx_jz; @@ -325,7 +327,10 @@ void WarpX::PSATDForwardTransformJ ( idx_jy = (J_in_time == JInTime::Linear) ? static_cast(Idx.Jy_new) : static_cast(Idx.Jy_mid); idx_jz = (J_in_time == JInTime::Linear) ? static_cast(Idx.Jz_new) : static_cast(Idx.Jz_mid); - ForwardTransformVect(lev, *spectral_solver_fp[lev], J_fp[lev], idx_jx, idx_jy, idx_jz); + if (m_fields.has_vector(J_fp_string, lev)) { + ablastr::fields::VectorField J_fp = m_fields.get_alldirs(J_fp_string, lev); + ForwardTransformVect(lev, *spectral_solver_fp[lev], J_fp, idx_jx, idx_jy, idx_jz); + } if (spectral_solver_cp[lev]) { @@ -335,7 +340,10 @@ void WarpX::PSATDForwardTransformJ ( idx_jy = (J_in_time == JInTime::Linear) ? static_cast(Idx.Jy_new) : static_cast(Idx.Jy_mid); idx_jz = (J_in_time == JInTime::Linear) ? static_cast(Idx.Jz_new) : static_cast(Idx.Jz_mid); - ForwardTransformVect(lev, *spectral_solver_cp[lev], J_cp[lev], idx_jx, idx_jy, idx_jz); + if (m_fields.has_vector(J_cp_string, lev)) { + ablastr::fields::VectorField J_cp = m_fields.get_alldirs(J_cp_string, lev); + ForwardTransformVect(lev, *spectral_solver_cp[lev], J_cp, idx_jx, idx_jy, idx_jz); + } } } @@ -371,9 +379,11 @@ void WarpX::PSATDForwardTransformJ ( } void WarpX::PSATDBackwardTransformJ ( - ablastr::fields::MultiLevelVectorField const & J_fp, - ablastr::fields::MultiLevelVectorField const & J_cp) + std::string const & J_fp_string, + std::string const & J_cp_string) { + if (!m_fields.has_vector(J_fp_string, 0)) { return; } + SpectralFieldIndex Idx; int idx_jx, idx_jy, idx_jz; @@ -387,8 +397,11 @@ void WarpX::PSATDBackwardTransformJ ( idx_jy = static_cast(Idx.Jy_mid); idx_jz = static_cast(Idx.Jz_mid); - BackwardTransformVect(lev, *spectral_solver_fp[lev], J_fp[lev], - idx_jx, idx_jy, idx_jz, m_fill_guards_current); + if (m_fields.has_vector(J_fp_string, lev)) { + ablastr::fields::VectorField J_fp = m_fields.get_alldirs(J_fp_string, lev); + BackwardTransformVect(lev, *spectral_solver_fp[lev], J_fp, + idx_jx, idx_jy, idx_jz, m_fill_guards_current); + } if (spectral_solver_cp[lev]) { @@ -400,26 +413,35 @@ void WarpX::PSATDBackwardTransformJ ( idx_jy = static_cast(Idx.Jy_mid); idx_jz = static_cast(Idx.Jz_mid); - BackwardTransformVect(lev, *spectral_solver_cp[lev], J_cp[lev], - idx_jx, idx_jy, idx_jz, m_fill_guards_current); + if (m_fields.has_vector(J_cp_string, lev)) { + ablastr::fields::VectorField J_cp = m_fields.get_alldirs(J_cp_string, lev); + BackwardTransformVect(lev, *spectral_solver_cp[lev], J_cp, + idx_jx, idx_jy, idx_jz, m_fill_guards_current); + } } } } void WarpX::PSATDForwardTransformRho ( - ablastr::fields::MultiLevelScalarField const & charge_fp, - ablastr::fields::MultiLevelScalarField const & charge_cp, + std::string const & charge_fp_string, + std::string const & charge_cp_string, const int icomp, const int dcomp, const bool apply_kspace_filter) { - if (charge_fp[0] == nullptr) { return; } + if (!m_fields.has(charge_fp_string, 0)) { return; } for (int lev = 0; lev <= finest_level; ++lev) { - if (charge_fp[lev]) { spectral_solver_fp[lev]->ForwardTransform(lev, *charge_fp[lev], dcomp, icomp); } + if (m_fields.has(charge_fp_string, lev)) { + amrex::MultiFab const & charge_fp = *m_fields.get(charge_fp_string, lev); + spectral_solver_fp[lev]->ForwardTransform(lev, charge_fp, dcomp, icomp); + } if (spectral_solver_cp[lev]) { - if (charge_cp[lev]) { spectral_solver_cp[lev]->ForwardTransform(lev, *charge_cp[lev], dcomp, icomp); } + if (m_fields.has(charge_cp_string, lev)) { + amrex::MultiFab const & charge_cp = *m_fields.get(charge_cp_string, lev); + spectral_solver_cp[lev]->ForwardTransform(lev, charge_cp, dcomp, icomp); + } } } @@ -699,53 +721,56 @@ WarpX::PushPSATD () const int rho_old = spectral_solver_fp[0]->m_spectral_index.rho_old; const int rho_new = spectral_solver_fp[0]->m_spectral_index.rho_new; - const ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); - const ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_cp", finest_level); + + std::string const rho_fp_string = "rho_fp"; + std::string const rho_cp_string = "rho_cp"; + const ablastr::fields::MultiLevelVectorField current_fp = m_fields.get_mr_levels_alldirs("current_fp", finest_level); - const ablastr::fields::MultiLevelVectorField current_cp = m_fields.get_mr_levels_alldirs("current_cp", finest_level); - const ablastr::fields::MultiLevelVectorField current_buf = m_fields.get_mr_levels_alldirs("current_buf", finest_level); + std::string current_fp_string = "current_fp"; + std::string const current_cp_string = "current_cp"; if (fft_periodic_single_box) { if (current_correction) { // FFT of J and rho - PSATDForwardTransformJ(current_fp, current_cp); - PSATDForwardTransformRho(rho_fp, rho_cp, 0, rho_old); - PSATDForwardTransformRho(rho_fp, rho_cp, 1, rho_new); + PSATDForwardTransformJ(current_fp_string, current_cp_string); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 0, rho_old); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 1, rho_new); // Correct J in k-space PSATDCurrentCorrection(); // Inverse FFT of J - PSATDBackwardTransformJ(current_fp, current_cp); + PSATDBackwardTransformJ(current_fp_string, current_cp_string); } else if (current_deposition_algo == CurrentDepositionAlgo::Vay) { // FFT of D and rho (if used) // TODO Replace current_cp with current_cp_vay once Vay deposition is implemented with MR - PSATDForwardTransformJ( - m_fields.get_mr_levels_alldirs("current_fp_vay", finest_level), current_cp); - PSATDForwardTransformRho(rho_fp, rho_cp, 0, rho_old); - PSATDForwardTransformRho(rho_fp, rho_cp, 1, rho_new); + current_fp_string = "current_fp_vay"; + PSATDForwardTransformJ(current_fp_string, current_cp_string); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 0, rho_old); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 1, rho_new); // Compute J from D in k-space PSATDVayDeposition(); // Inverse FFT of J, subtract cumulative sums of D - PSATDBackwardTransformJ(current_fp, current_cp); + current_fp_string = "current_fp"; + PSATDBackwardTransformJ(current_fp_string, current_cp_string); // TODO Cumulative sums need to be fixed with periodic single box PSATDSubtractCurrentPartialSumsAvg(); // FFT of J after subtraction of cumulative sums - PSATDForwardTransformJ(current_fp, current_cp); + PSATDForwardTransformJ(current_fp_string, current_cp_string); } else // no current correction, no Vay deposition { // FFT of J and rho (if used) - PSATDForwardTransformJ(current_fp, current_cp); - PSATDForwardTransformRho(rho_fp, rho_cp, 0, rho_old); - PSATDForwardTransformRho(rho_fp, rho_cp, 1, rho_new); + PSATDForwardTransformJ(current_fp_string, current_cp_string); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 0, rho_old); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 1, rho_new); } } else // no periodic single box @@ -757,20 +782,20 @@ WarpX::PushPSATD () // In RZ geometry, do not apply filtering here, since it is // applied in the subsequent calls to these functions (below) const bool apply_kspace_filter = false; - PSATDForwardTransformJ(current_fp, current_cp, apply_kspace_filter); - PSATDForwardTransformRho(rho_fp, rho_cp, 0, rho_old, apply_kspace_filter); - PSATDForwardTransformRho(rho_fp, rho_cp, 1, rho_new, apply_kspace_filter); + PSATDForwardTransformJ(current_fp_string, current_cp_string, apply_kspace_filter); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 0, rho_old, apply_kspace_filter); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 1, rho_new, apply_kspace_filter); #else - PSATDForwardTransformJ(current_fp, current_cp); - PSATDForwardTransformRho(rho_fp, rho_cp, 0, rho_old); - PSATDForwardTransformRho(rho_fp, rho_cp, 1, rho_new); + PSATDForwardTransformJ(current_fp_string, current_cp_string); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 0, rho_old); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 1, rho_new); #endif // Correct J in k-space PSATDCurrentCorrection(); // Inverse FFT of J - PSATDBackwardTransformJ(current_fp, current_cp); + PSATDBackwardTransformJ(current_fp_string, current_cp_string); // Synchronize J and rho SyncCurrent("current_fp"); @@ -779,14 +804,15 @@ WarpX::PushPSATD () else if (current_deposition_algo == CurrentDepositionAlgo::Vay) { // FFT of D - PSATDForwardTransformJ( - m_fields.get_mr_levels_alldirs("current_fp_vay", finest_level), current_cp); + current_fp_string = "current_fp_vay"; + PSATDForwardTransformJ(current_fp_string, current_cp_string); // Compute J from D in k-space PSATDVayDeposition(); // Inverse FFT of J, subtract cumulative sums of D - PSATDBackwardTransformJ(current_fp, current_cp); + current_fp_string = "current_fp"; + PSATDBackwardTransformJ(current_fp_string, current_cp_string); PSATDSubtractCurrentPartialSumsAvg(); // Synchronize J and rho (if used). @@ -800,9 +826,9 @@ WarpX::PushPSATD () } // FFT of J and rho (if used) - PSATDForwardTransformJ(current_fp, current_cp); - PSATDForwardTransformRho(rho_fp, rho_cp, 0, rho_old); - PSATDForwardTransformRho(rho_fp, rho_cp, 1, rho_new); + PSATDForwardTransformJ(current_fp_string, current_cp_string); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 0, rho_old); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 1, rho_new); } // FFT of E and B diff --git a/Source/WarpX.H b/Source/WarpX.H index aaac571f703..7b9b836aefc 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1685,8 +1685,8 @@ private: * (only used in RZ geometry to avoid double filtering) */ void PSATDForwardTransformJ ( - ablastr::fields::MultiLevelVectorField const& J_fp, - ablastr::fields::MultiLevelVectorField const& J_cp, + std::string const & J_fp_string, + std::string const & J_cp_string, bool apply_kspace_filter=true); /** @@ -1698,8 +1698,8 @@ private: * storing the coarse patch current to be transformed */ void PSATDBackwardTransformJ ( - ablastr::fields::MultiLevelVectorField const & J_fp, - ablastr::fields::MultiLevelVectorField const & J_cp); + std::string const & J_fp_string, + std::string const & J_cp_string); /** * \brief Forward FFT of rho on all mesh refinement levels, @@ -1713,8 +1713,8 @@ private: * (only used in RZ geometry to avoid double filtering) */ void PSATDForwardTransformRho ( - ablastr::fields::MultiLevelScalarField const & charge_fp, - ablastr::fields::MultiLevelScalarField const & charge_cp, + std::string const & charge_fp_string, + std::string const & charge_cp_string, int icomp, int dcomp, bool apply_kspace_filter=true); /** From 54bdb713131b57ff0009ddf0fe295b63ef189e1f Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Fri, 20 Sep 2024 14:57:37 -0700 Subject: [PATCH 296/314] Fix clang-tidy errors --- Source/FieldSolver/WarpXPushFieldsEM.cpp | 20 ++++++++++---------- Source/ablastr/fields/MultiFabRegister.H | 2 +- Source/ablastr/fields/MultiFabRegister.cpp | 6 +++--- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 49e0bd2acb6..fd1f55142f2 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -112,22 +112,22 @@ void WarpX::PSATDForwardTransformEB () for (int lev = 0; lev <= finest_level; ++lev) { if (m_fields.has_vector(Efield_fp_string, lev)) { - ablastr::fields::VectorField E_fp = m_fields.get_alldirs(Efield_fp_string, lev); + ablastr::fields::VectorField const E_fp = m_fields.get_alldirs(Efield_fp_string, lev); ForwardTransformVect(lev, *spectral_solver_fp[lev], E_fp, Idx.Ex, Idx.Ey, Idx.Ez); } if (m_fields.has_vector(Bfield_fp_string, lev)) { - ablastr::fields::VectorField B_fp = m_fields.get_alldirs(Bfield_fp_string, lev); + ablastr::fields::VectorField const B_fp = m_fields.get_alldirs(Bfield_fp_string, lev); ForwardTransformVect(lev, *spectral_solver_fp[lev], B_fp, Idx.Bx, Idx.By, Idx.Bz); } if (spectral_solver_cp[lev]) { if (m_fields.has_vector(Efield_cp_string, lev)) { - ablastr::fields::VectorField E_cp = m_fields.get_alldirs(Efield_cp_string, lev); + ablastr::fields::VectorField const E_cp = m_fields.get_alldirs(Efield_cp_string, lev); ForwardTransformVect(lev, *spectral_solver_cp[lev], E_cp, Idx.Ex, Idx.Ey, Idx.Ez); } if (m_fields.has_vector(Bfield_cp_string, lev)) { - ablastr::fields::VectorField B_cp = m_fields.get_alldirs(Bfield_cp_string, lev); + ablastr::fields::VectorField const B_cp = m_fields.get_alldirs(Bfield_cp_string, lev); ForwardTransformVect(lev, *spectral_solver_cp[lev], B_cp, Idx.Bx, Idx.By, Idx.Bz); } } @@ -146,12 +146,12 @@ void WarpX::PSATDBackwardTransformEB () for (int lev = 0; lev <= finest_level; ++lev) { if (m_fields.has_vector(Efield_fp_string, lev)) { - ablastr::fields::VectorField E_fp = m_fields.get_alldirs(Efield_fp_string, lev); + ablastr::fields::VectorField const E_fp = m_fields.get_alldirs(Efield_fp_string, lev); BackwardTransformVect(lev, *spectral_solver_fp[lev], E_fp, Idx.Ex, Idx.Ey, Idx.Ez, m_fill_guards_fields); } if (m_fields.has_vector(Bfield_fp_string, lev)) { - ablastr::fields::VectorField B_fp = m_fields.get_alldirs(Bfield_fp_string, lev); + ablastr::fields::VectorField const B_fp = m_fields.get_alldirs(Bfield_fp_string, lev); BackwardTransformVect(lev, *spectral_solver_fp[lev], B_fp, Idx.Bx, Idx.By, Idx.Bz, m_fill_guards_fields); } @@ -159,12 +159,12 @@ void WarpX::PSATDBackwardTransformEB () if (spectral_solver_cp[lev]) { if (m_fields.has_vector(Efield_cp_string, lev)) { - ablastr::fields::VectorField E_cp = m_fields.get_alldirs(Efield_cp_string, lev); + ablastr::fields::VectorField const E_cp = m_fields.get_alldirs(Efield_cp_string, lev); BackwardTransformVect(lev, *spectral_solver_cp[lev], E_cp, Idx.Ex, Idx.Ey, Idx.Ez, m_fill_guards_fields); } if (m_fields.has_vector(Bfield_cp_string, lev)) { - ablastr::fields::VectorField B_cp = m_fields.get_alldirs(Bfield_cp_string, lev); + ablastr::fields::VectorField const B_cp = m_fields.get_alldirs(Bfield_cp_string, lev); BackwardTransformVect(lev, *spectral_solver_cp[lev], B_cp, Idx.Bx, Idx.By, Idx.Bz, m_fill_guards_fields); } @@ -175,8 +175,8 @@ void WarpX::PSATDBackwardTransformEB () for (int lev = 0; lev <= finest_level; ++lev) { if (m_fields.has_vector(Efield_fp_string, lev) && m_fields.has_vector(Bfield_fp_string, lev)) { - ablastr::fields::VectorField E_fp = m_fields.get_alldirs(Efield_fp_string, lev); - ablastr::fields::VectorField B_fp = m_fields.get_alldirs(Bfield_fp_string, lev); + ablastr::fields::VectorField const E_fp = m_fields.get_alldirs(Efield_fp_string, lev); + ablastr::fields::VectorField const B_fp = m_fields.get_alldirs(Bfield_fp_string, lev); DampFieldsInGuards(lev, E_fp, B_fp); } } diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 980598600c6..7237fd6a575 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -291,7 +291,7 @@ namespace ablastr::fields */ [[nodiscard]] bool has_vector ( - std::string name, + std::string const & name, int level ) const; diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 9360e065f61..4abed6b372d 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -331,15 +331,15 @@ namespace ablastr::fields bool MultiFabRegister::has_vector ( - std::string name, + std::string const & name, int level ) const { const std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; - int count = 0; + unsigned long count = 0; for (const Direction& dir : all_dirs) { - std::string internal_name = mf_name(name, dir, level); + std::string const internal_name = mf_name(name, dir, level); count += m_mf_register.count(internal_name); } From bce62bb396b28e8dc4456060bb2784af3db7fcf9 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 20 Sep 2024 16:10:57 -0700 Subject: [PATCH 297/314] `MultiFabRegister`: `AMREX_ENUM` Key Support --- Source/Python/MultiFabRegister.cpp | 20 +- Source/ablastr/fields/MultiFabRegister.H | 265 +++++++++++++++--- Source/ablastr/fields/MultiFabRegister.cpp | 71 +++-- .../ablastr/fields/MultiFabRegister_string.H | 176 ++++++++++++ 4 files changed, 456 insertions(+), 76 deletions(-) create mode 100644 Source/ablastr/fields/MultiFabRegister_string.H diff --git a/Source/Python/MultiFabRegister.cpp b/Source/Python/MultiFabRegister.cpp index f345f667743..6ae86ebd61b 100644 --- a/Source/Python/MultiFabRegister.cpp +++ b/Source/Python/MultiFabRegister.cpp @@ -31,7 +31,7 @@ void init_MultiFabRegister (py::module & m) std::optional, bool, bool - >(&MultiFabRegister::alloc_init), + >(&MultiFabRegister::alloc_init), py::arg("name"), py::arg("level"), py::arg("ba"), @@ -55,7 +55,7 @@ void init_MultiFabRegister (py::module & m) std::optional, bool, bool - >(&MultiFabRegister::alloc_init), + >(&MultiFabRegister::alloc_init), py::arg("name"), py::arg("dir"), py::arg("level"), @@ -74,7 +74,7 @@ void init_MultiFabRegister (py::module & m) std::string, int, std::optional - >(&MultiFabRegister::alias_init), + >(&MultiFabRegister::alias_init), py::arg("new_name"), py::arg("alias_name"), py::arg("level"), @@ -88,7 +88,7 @@ void init_MultiFabRegister (py::module & m) ablastr::fields::Direction, int, std::optional - >(&MultiFabRegister::alias_init), + >(&MultiFabRegister::alias_init), py::arg("new_name"), py::arg("alias_name"), py::arg("dir"), @@ -107,7 +107,7 @@ void init_MultiFabRegister (py::module & m) py::overload_cast< std::string, int - >(&MultiFabRegister::has, py::const_), + >(&MultiFabRegister::has, py::const_), py::arg("name"), py::arg("level") ) @@ -117,7 +117,7 @@ void init_MultiFabRegister (py::module & m) std::string, ablastr::fields::Direction, int - >(&MultiFabRegister::has, py::const_), + >(&MultiFabRegister::has, py::const_), py::arg("name"), py::arg("dir"), py::arg("level") @@ -127,7 +127,7 @@ void init_MultiFabRegister (py::module & m) py::overload_cast< std::string, int - >(&MultiFabRegister::get), + >(&MultiFabRegister::get), py::arg("name"), py::arg("level") ) @@ -137,7 +137,7 @@ void init_MultiFabRegister (py::module & m) std::string, ablastr::fields::Direction, int - >(&MultiFabRegister::get), + >(&MultiFabRegister::get), py::arg("name"), py::arg("dir"), py::arg("level") @@ -152,7 +152,7 @@ void init_MultiFabRegister (py::module & m) py::overload_cast< std::string, int - >(&MultiFabRegister::erase), + >(&MultiFabRegister::erase), py::arg("name"), py::arg("level") ) @@ -162,7 +162,7 @@ void init_MultiFabRegister (py::module & m) std::string, ablastr::fields::Direction, int - >(&MultiFabRegister::erase), + >(&MultiFabRegister::erase), py::arg("name"), py::arg("dir"), py::arg("level") diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 7237fd6a575..ec088a74863 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -21,9 +22,35 @@ #include #include #include +#include #include +namespace +{ + // type trait helpers in lieu of an amrex::is_amrex_enum + template > + struct is_castable_to_string : std::false_type {}; + + template + struct is_castable_to_string(std::declval()))>> : std::true_type {}; + + /** helper to either cast a string/char array to string to query an AMREX_ENUM */ + template + std::string getExtractedName (T name) + { + if constexpr(is_castable_to_string()) + { + // already a unique string key + return std::string(name); + } else + { + // user-defined AMREX_ENUM or compile error + return amrex::getEnumNameString(name); + } + } +} + namespace ablastr::fields { /** Components (base vector directions) of vector/tensor fields. @@ -157,9 +184,10 @@ namespace ablastr::fields * @param redistribute_on_remake redistribute on @see amrex::AmrCore::RemakeLevel * @return pointer to newly allocated MultiFab */ + template amrex::MultiFab* alloc_init ( - std::string name, + T name, int level, amrex::BoxArray const & ba, amrex::DistributionMapping const & dm, @@ -168,7 +196,20 @@ namespace ablastr::fields std::optional initial_value = std::nullopt, bool remake = true, bool redistribute_on_remake = true - ); + ) + { + return alloc_init( + getExtractedName(name), + level, + ba, + dm, + ncomp, + ngrow, + initial_value, + remake, + redistribute_on_remake + ); + } /** Allocate and optionally initialize a MultiFab (field) * @@ -187,9 +228,10 @@ namespace ablastr::fields * @param redistribute_on_remake redistribute on @see amrex::AmrCore::RemakeLevel * @return pointer to newly allocated MultiFab */ + template amrex::MultiFab* alloc_init ( - std::string name, + T name, Direction dir, int level, amrex::BoxArray const & ba, @@ -199,7 +241,21 @@ namespace ablastr::fields std::optional initial_value = std::nullopt, bool remake = true, bool redistribute_on_remake = true - ); + ) + { + return alloc_init( + getExtractedName(name), + dir, + level, + ba, + dm, + ncomp, + ngrow, + initial_value, + remake, + redistribute_on_remake + ); + } /** Create an alias of a MultiFab (field) * @@ -212,13 +268,22 @@ namespace ablastr::fields * @param initial_value the optional value to assign * @return the newly aliased MultiFab */ + template amrex::MultiFab* alias_init ( - std::string new_name, - std::string alias_name, + N new_name, + A alias_name, int level, std::optional initial_value = std::nullopt - ); + ) + { + return alias_init( + getExtractedName(new_name), + getExtractedName(alias_name), + level, + initial_value + ); + } /** Create an alias of a MultiFab (field) * @@ -232,14 +297,24 @@ namespace ablastr::fields * @param initial_value the optional value to assign * @return the newly aliased MultiFab */ + template amrex::MultiFab* alias_init ( - std::string new_name, - std::string alias_name, + N new_name, + A alias_name, Direction dir, int level, std::optional initial_value = std::nullopt - ); + ) + { + return alias_init( + getExtractedName(new_name), + getExtractedName(alias_name), + dir, + level, + initial_value + ); + } /** Allocate a new MultiFab (field) with the same size and distribution as another. * @@ -263,11 +338,18 @@ namespace ablastr::fields * @param level the MR level to check * @return true if contained, otherwise false */ + template [[nodiscard]] bool has ( - std::string name, + T name, int level - ) const; + ) const + { + return has( + getExtractedName(name), + level + ); + } /** Check if a MultiFab that is part of a vector/tensor field is registered. * @@ -276,12 +358,20 @@ namespace ablastr::fields * @param level the MR level to check * @return true if contained, otherwise false */ + template [[nodiscard]] bool has ( - std::string name, + T name, Direction dir, int level - ) const; + ) const + { + return has( + getExtractedName(name), + dir, + level + ); + } /** Check if a MultiFab vector field is registered. * @@ -289,11 +379,18 @@ namespace ablastr::fields * @param level the MR level to check * @return true if contained, otherwise false */ + template [[nodiscard]] bool has_vector ( - std::string const & name, + T name, int level - ) const; + ) const + { + return has_vector( + getExtractedName(name), + level + ); + } /** Return a scalar MultiFab (field). * @@ -303,11 +400,18 @@ namespace ablastr::fields * @param level the MR level to check * @return a non-owning pointer to the MultiFab (field) */ + template [[nodiscard]] amrex::MultiFab* get ( - std::string name, + T name, int level - ); + ) + { + return get( + getExtractedName(name), + level + ); + } /** Return a MultiFab that is part of a vector/tensor field. * @@ -318,12 +422,20 @@ namespace ablastr::fields * @param level the MR level to check * @return a non-owning pointer to the MultiFab (field) */ + template [[nodiscard]] amrex::MultiFab* get ( - std::string name, + T name, Direction dir, int level - ); + ) + { + return get( + getExtractedName(name), + dir, + level + ); + } /** Return a scalar MultiFab (field). * @@ -333,11 +445,18 @@ namespace ablastr::fields * @param level the MR level to check * @return a non-owning pointer to the MultiFab (field) */ + template [[nodiscard]] amrex::MultiFab const * get ( - std::string name, + T name, int level - ) const; + ) const + { + return get( + getExtractedName(name), + level + ); + } /** Return a MultiFab that is part of a vector/tensor field. * @@ -348,12 +467,20 @@ namespace ablastr::fields * @param level the MR level to check * @return a non-owning pointer to the MultiFab (field) */ + template [[nodiscard]] amrex::MultiFab const * get ( - std::string name, + T name, Direction dir, int level - ) const; + ) const + { + return get( + getExtractedName(name), + dir, + level + ); + } /** title * @@ -363,16 +490,32 @@ namespace ablastr::fields * @param finest_level ... * @return ... */ + //@{ + template [[nodiscard]] MultiLevelScalarField get_mr_levels ( - const std::string& name, + T name, int finest_level - ); + ) + { + return get_mr_levels( + getExtractedName(name), + finest_level + ); + } + template [[nodiscard]] ConstMultiLevelScalarField get_mr_levels ( - const std::string& name, + T name, int finest_level - ) const; + ) const + { + return get_mr_levels( + getExtractedName(name), + finest_level + ); + } + //@} /** title * @@ -382,16 +525,32 @@ namespace ablastr::fields * @param level ... * @return ... */ + //@{ + template [[nodiscard]] VectorField get_alldirs ( - const std::string&, + T name, int level - ); + ) + { + return get_alldirs( + getExtractedName(name), + level + ); + } + template [[nodiscard]] ConstVectorField get_alldirs ( - const std::string& name, + T name, int level - ) const; + ) const + { + return get_alldirs( + getExtractedName(name), + level + ); + } + //@} /** Return a vector field on all MR levels. * @@ -402,16 +561,32 @@ namespace ablastr::fields * @param finest_level ... * @return ... */ + //@{ + template [[nodiscard]] MultiLevelVectorField get_mr_levels_alldirs ( - const std::string& name, + T name, int finest_level - ); + ) + { + return get_mr_levels_alldirs( + getExtractedName(name), + finest_level + ); + } + template [[nodiscard]] ConstMultiLevelVectorField get_mr_levels_alldirs ( - const std::string& name, + T name, int finest_level - ) const; + ) const + { + return get_mr_levels_alldirs( + getExtractedName(name), + finest_level + ); + } + //@} /** title * @@ -431,11 +606,15 @@ namespace ablastr::fields * @param name ... * @param level ... */ + template void erase ( - std::string name, + T name, int level - ); + ) + { + erase(getExtractedName(name), level); + } /** title * @@ -446,12 +625,16 @@ namespace ablastr::fields * @param dir ... * @param level ... */ + template void erase ( - std::string name, + T name, Direction dir, int level - ); + ) + { + erase(getExtractedName(name), dir, level); + } /** Erase all MultiFabs on a specific MR level. * @@ -537,4 +720,6 @@ namespace ablastr::fields } // namespace ablastr::fields +#include "MultiFabRegister_string.H" + #endif // ABLASTR_FIELDS_MF_REGISTER_H diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 4abed6b372d..27407ad70f2 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -14,8 +14,9 @@ namespace ablastr::fields { + template<> amrex::MultiFab* - MultiFabRegister::alloc_init ( + MultiFabRegister::alloc_init ( std::string name, int level, amrex::BoxArray const & ba, @@ -63,8 +64,9 @@ namespace ablastr::fields return &mf; } + template<> amrex::MultiFab* - MultiFabRegister::alloc_init ( + MultiFabRegister::alloc_init ( std::string name, Direction dir, int level, @@ -132,8 +134,9 @@ namespace ablastr::fields // TODO: does the other_name already exist? error } + template<> amrex::MultiFab* - MultiFabRegister::alias_init ( + MultiFabRegister::alias_init ( std::string new_name, std::string alias_name, int level, @@ -191,8 +194,9 @@ namespace ablastr::fields return &mf; } + template<> amrex::MultiFab* - MultiFabRegister::alias_init ( + MultiFabRegister::alias_init ( std::string new_name, std::string alias_name, Direction dir, @@ -306,8 +310,9 @@ namespace ablastr::fields } } + template<> bool - MultiFabRegister::has ( + MultiFabRegister::has ( std::string name, int level ) const @@ -317,8 +322,9 @@ namespace ablastr::fields return m_mf_register.count(name) > 0; } + template<> bool - MultiFabRegister::has ( + MultiFabRegister::has ( std::string name, Direction dir, int level @@ -329,9 +335,10 @@ namespace ablastr::fields return m_mf_register.count(name) > 0; } + template<> bool - MultiFabRegister::has_vector ( - std::string const & name, + MultiFabRegister::has_vector ( + std::string name, int level ) const { @@ -376,8 +383,9 @@ namespace ablastr::fields return &mf; } + template<> amrex::MultiFab* - MultiFabRegister::get ( + MultiFabRegister::get ( std::string name, int level ) @@ -386,8 +394,9 @@ namespace ablastr::fields return internal_get(name); } + template<> amrex::MultiFab* - MultiFabRegister::get ( + MultiFabRegister::get ( std::string name, Direction dir, int level @@ -397,8 +406,9 @@ namespace ablastr::fields return internal_get(name); } + template<> amrex::MultiFab const * - MultiFabRegister::get ( + MultiFabRegister::get ( std::string name, int level ) const @@ -407,8 +417,9 @@ namespace ablastr::fields return internal_get(name); } + template<> amrex::MultiFab const * - MultiFabRegister::get ( + MultiFabRegister::get ( std::string name, Direction dir, int level @@ -418,9 +429,10 @@ namespace ablastr::fields return internal_get(name); } + template<> MultiLevelScalarField - MultiFabRegister::get_mr_levels ( - const std::string& name, + MultiFabRegister::get_mr_levels ( + std::string name, int finest_level ) { @@ -433,9 +445,10 @@ namespace ablastr::fields return field_on_level; } + template<> ConstMultiLevelScalarField - MultiFabRegister::get_mr_levels ( - const std::string& name, + MultiFabRegister::get_mr_levels ( + std::string name, int finest_level ) const { @@ -448,9 +461,10 @@ namespace ablastr::fields return field_on_level; } + template<> VectorField - MultiFabRegister::get_alldirs ( - const std::string& name, + MultiFabRegister::get_alldirs ( + std::string name, int level ) { @@ -468,9 +482,10 @@ namespace ablastr::fields return vectorField; } + template<> ConstVectorField - MultiFabRegister::get_alldirs ( - const std::string& name, + MultiFabRegister::get_alldirs ( + std::string name, int level ) const { @@ -488,9 +503,10 @@ namespace ablastr::fields return vectorField; } + template<> MultiLevelVectorField - MultiFabRegister::get_mr_levels_alldirs ( - const std::string& name, + MultiFabRegister::get_mr_levels_alldirs ( + std::string name, int finest_level ) { @@ -514,9 +530,10 @@ namespace ablastr::fields return field_on_level; } + template<> ConstMultiLevelVectorField - MultiFabRegister::get_mr_levels_alldirs ( - const std::string& name, + MultiFabRegister::get_mr_levels_alldirs ( + std::string name, int finest_level ) const { @@ -550,8 +567,9 @@ namespace ablastr::fields return names; } + template<> void - MultiFabRegister::erase ( + MultiFabRegister::erase ( std::string name, int level ) @@ -564,8 +582,9 @@ namespace ablastr::fields m_mf_register.erase(name); } + template<> void - MultiFabRegister::erase ( + MultiFabRegister::erase ( std::string name, Direction dir, int level diff --git a/Source/ablastr/fields/MultiFabRegister_string.H b/Source/ablastr/fields/MultiFabRegister_string.H new file mode 100644 index 00000000000..f4139c7a540 --- /dev/null +++ b/Source/ablastr/fields/MultiFabRegister_string.H @@ -0,0 +1,176 @@ +/* Copyright 2024 The ABLAST Community + * + * This file is part of WarpX. + * It adds unique string key access to the MultiFabRegister. + * + * License: BSD-3-Clause-LBNL + * Authors: Axel Huebl + */ +#ifndef ABLASTR_FIELDS_MF_REGISTER_STRING_H +#define ABLASTR_FIELDS_MF_REGISTER_STRING_H + +#include "MultiFabRegister.H" + + +namespace ablastr::fields +{ + template<> + amrex::MultiFab* + MultiFabRegister::alloc_init ( + std::string name, + int level, + amrex::BoxArray const & ba, + amrex::DistributionMapping const & dm, + int ncomp, + amrex::IntVect const & ngrow, + std::optional initial_value, + bool remake, + bool redistribute_on_remake + ); + + template<> + amrex::MultiFab* + MultiFabRegister::alloc_init ( + std::string name, + Direction dir, + int level, + amrex::BoxArray const & ba, + amrex::DistributionMapping const & dm, + int ncomp, + amrex::IntVect const & ngrow, + std::optional initial_value, + bool remake, + bool redistribute_on_remake + ); + + template<> + amrex::MultiFab* + MultiFabRegister::alias_init ( + std::string new_name, + std::string alias_name, + int level, + std::optional initial_value + ); + + template<> + amrex::MultiFab* + MultiFabRegister::alias_init ( + std::string new_name, + std::string alias_name, + Direction dir, + int level, + std::optional initial_value + ); + + template<> + bool + MultiFabRegister::has ( + std::string name, + int level + ) const; + + template<> + bool + MultiFabRegister::has ( + std::string name, + Direction dir, + int level + ) const; + + template<> + bool + MultiFabRegister::has_vector ( + std::string name, + int level + ) const; + + template<> + amrex::MultiFab* + MultiFabRegister::get ( + std::string name, + int level + ); + + template<> + amrex::MultiFab* + MultiFabRegister::get ( + std::string name, + Direction dir, + int level + ); + + template<> + amrex::MultiFab const* + MultiFabRegister::get ( + std::string name, + int level + ) const; + + template<> + amrex::MultiFab const * + MultiFabRegister::get ( + std::string name, + Direction dir, + int level + ) const; + + template<> + MultiLevelScalarField + MultiFabRegister::get_mr_levels ( + std::string name, + int finest_level + ); + + template<> + ConstMultiLevelScalarField + MultiFabRegister::get_mr_levels ( + std::string name, + int finest_level + ) const; + + template<> + VectorField + MultiFabRegister::get_alldirs ( + std::string name, + int level + ); + + template<> + ConstVectorField + MultiFabRegister::get_alldirs ( + std::string name, + int level + ) const; + + template<> + MultiLevelVectorField + MultiFabRegister::get_mr_levels_alldirs ( + std::string name, + int finest_level + ); + + template<> + ConstMultiLevelVectorField + MultiFabRegister::get_mr_levels_alldirs ( + std::string name, + int finest_level + ) const; + + template<> + void + MultiFabRegister::erase ( + std::string name, + int level + ); + + template<> + void + MultiFabRegister::erase ( + std::string name, + Direction dir, + int level + ); + +} // namespace ablastr::fields + +#endif // ABLASTR_FIELDS_MF_REGISTER_STRING_H From cb64048c8b463d76a5f1b11f0be404a6fd6f7b4a Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 20 Sep 2024 16:40:39 -0700 Subject: [PATCH 298/314] `Fields.H`: Reintroduce & Move - more central - use `AMREX_ENUM` --- Source/BoundaryConditions/PML.cpp | 2 +- Source/BoundaryConditions/PML_RZ.cpp | 2 +- Source/Diagnostics/BTDiagnostics.cpp | 2 +- .../ComputeDiagFunctors/JFunctor.cpp | 2 +- .../ComputeDiagFunctors/JdispFunctor.cpp | 2 +- .../FlushFormats/FlushFormatCheckpoint.cpp | 2 +- .../FlushFormats/FlushFormatPlotfile.cpp | 2 +- Source/Diagnostics/FullDiagnostics.cpp | 2 +- Source/Diagnostics/ParticleIO.cpp | 2 +- .../Diagnostics/ReducedDiags/ChargeOnEB.cpp | 2 +- .../ReducedDiags/ColliderRelevant.cpp | 2 +- .../Diagnostics/ReducedDiags/FieldEnergy.cpp | 2 +- .../Diagnostics/ReducedDiags/FieldMaximum.cpp | 2 +- .../ReducedDiags/FieldMomentum.cpp | 2 +- .../Diagnostics/ReducedDiags/FieldProbe.cpp | 2 +- .../Diagnostics/ReducedDiags/FieldReduction.H | 2 +- .../ReducedDiags/LoadBalanceCosts.cpp | 2 +- .../ReducedDiags/ParticleExtrema.cpp | 2 +- Source/Diagnostics/SliceDiagnostic.cpp | 2 +- Source/FieldSolver/Fields.H | 38 ----- .../HybridPICModel/HybridPICModel.cpp | 2 +- .../MacroscopicProperties.cpp | 2 +- .../ImplicitSolvers/ThetaImplicitEM.cpp | 2 +- .../ImplicitSolvers/WarpXImplicitOps.cpp | 2 +- .../ImplicitSolvers/WarpXSolverVec.H | 2 +- Source/Fields.H | 131 ++++++++++++++++++ .../DivCleaner/ProjectionDivCleaner.H | 2 +- .../DivCleaner/ProjectionDivCleaner.cpp | 2 +- Source/Initialization/WarpXInitData.cpp | 2 +- Source/Particles/MultiParticleContainer.cpp | 2 +- Source/WarpX.H | 2 +- 31 files changed, 160 insertions(+), 67 deletions(-) delete mode 100644 Source/FieldSolver/Fields.H create mode 100644 Source/Fields.H diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index 6634256b87a..d905bf64d1f 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -10,7 +10,7 @@ #include "BoundaryConditions/PML.H" #include "BoundaryConditions/PMLComponent.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #ifdef WARPX_USE_FFT # include "FieldSolver/SpectralSolver/SpectralFieldData.H" #endif diff --git a/Source/BoundaryConditions/PML_RZ.cpp b/Source/BoundaryConditions/PML_RZ.cpp index 0fc24bf8817..b8875be734f 100644 --- a/Source/BoundaryConditions/PML_RZ.cpp +++ b/Source/BoundaryConditions/PML_RZ.cpp @@ -8,7 +8,7 @@ #include "PML_RZ.H" #include "BoundaryConditions/PML_RZ.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #ifdef WARPX_USE_FFT # include "FieldSolver/SpectralSolver/SpectralFieldDataRZ.H" #endif diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 3a7d27598c1..ab14dd3b86e 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -14,7 +14,7 @@ #include "Diagnostics/Diagnostics.H" #include "Diagnostics/FlushFormats/FlushFormat.H" #include "ComputeDiagFunctors/BackTransformParticleFunctor.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "Utils/Algorithms/IsIn.H" #include "Utils/Parser/ParserUtils.H" #include "Utils/TextMsg.H" diff --git a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp index 41bea408326..7f532c79e13 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp @@ -6,7 +6,7 @@ #include "JFunctor.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "Particles/MultiParticleContainer.H" #include "WarpX.H" diff --git a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp index 3c78bdf6d31..0e4a405c499 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp @@ -6,7 +6,7 @@ #include "JdispFunctor.H" #include "WarpX.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" #include "Particles/MultiParticleContainer.H" diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index 49d1e7e1ad0..2020be2d4b7 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -5,7 +5,7 @@ # include "BoundaryConditions/PML_RZ.H" #endif #include "Diagnostics/ParticleDiag/ParticleDiag.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "Particles/WarpXParticleContainer.H" #include "Utils/TextMsg.H" #include "Utils/WarpXProfilerWrapper.H" diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index d7cbbd98574..1731851dc3b 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -1,6 +1,6 @@ #include "FlushFormatPlotfile.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "Diagnostics/MultiDiagnostics.H" #include "Diagnostics/ParticleDiag/ParticleDiag.H" #include "Particles/Filter/FilterFunctors.H" diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index 95e1a04b40f..2b4c63880a8 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -12,7 +12,7 @@ #include "ComputeDiagFunctors/RhoFunctor.H" #include "Diagnostics/Diagnostics.H" #include "Diagnostics/ParticleDiag/ParticleDiag.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "FlushFormats/FlushFormat.H" #include "Particles/MultiParticleContainer.H" #include "Utils/Algorithms/IsIn.H" diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index 0ddab94d95a..51b28bc9a10 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -7,7 +7,7 @@ * License: BSD-3-Clause-LBNL */ -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "Particles/ParticleIO.H" #include "Particles/MultiParticleContainer.H" #include "Particles/PhysicalParticleContainer.H" diff --git a/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp b/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp index d0638605612..d5cf391a254 100644 --- a/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp +++ b/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp @@ -9,7 +9,7 @@ #include "Diagnostics/ReducedDiags/ReducedDiags.H" #include "EmbeddedBoundary/Enabled.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" #include "Utils/Parser/ParserUtils.H" diff --git a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp index addb7bdb873..463e6e160cd 100644 --- a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp @@ -8,7 +8,7 @@ #include "ColliderRelevant.H" #include "Diagnostics/ReducedDiags/ReducedDiags.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #if (defined WARPX_QED) # include "Particles/ElementaryProcess/QEDInternals/QedChiFunctions.H" #endif diff --git a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp index ecc7ee2466e..84b6df87af8 100644 --- a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp @@ -7,7 +7,7 @@ #include "FieldEnergy.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "Diagnostics/ReducedDiags/ReducedDiags.H" #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" diff --git a/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp b/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp index f875e6c0415..276c7928b43 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp @@ -7,7 +7,7 @@ #include "FieldMaximum.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "Utils/TextMsg.H" #include "WarpX.H" diff --git a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp index 016e222303d..f1857155457 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp @@ -7,7 +7,7 @@ #include "FieldMomentum.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" #include "WarpX.H" diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp index 535c38b2aad..bf5c68c9d83 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp @@ -7,7 +7,7 @@ #include "FieldProbe.H" #include "FieldProbeParticleContainer.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "Particles/Gather/FieldGather.H" #include "Particles/Pusher/GetAndSetPosition.H" #include "Particles/Pusher/UpdatePosition.H" diff --git a/Source/Diagnostics/ReducedDiags/FieldReduction.H b/Source/Diagnostics/ReducedDiags/FieldReduction.H index bfc2f9aef9e..65455039394 100644 --- a/Source/Diagnostics/ReducedDiags/FieldReduction.H +++ b/Source/Diagnostics/ReducedDiags/FieldReduction.H @@ -9,7 +9,7 @@ #define WARPX_DIAGNOSTICS_REDUCEDDIAGS_FIELDREDUCTION_H_ #include "ReducedDiags.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "WarpX.H" #include diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp index 7d5d26b5e0c..8ed3d21efc0 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp @@ -7,7 +7,7 @@ #include "LoadBalanceCosts.H" #include "Diagnostics/ReducedDiags/ReducedDiags.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "Particles/MultiParticleContainer.H" #include "Utils/TextMsg.H" #include "Utils/WarpXAlgorithmSelection.H" diff --git a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp index ca88241568e..5b4ff2908a9 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp @@ -11,7 +11,7 @@ #if (defined WARPX_QED) # include "Particles/ElementaryProcess/QEDInternals/QedChiFunctions.H" #endif -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "Particles/Gather/FieldGather.H" #include "Particles/Gather/GetExternalFields.H" #include "Particles/MultiParticleContainer.H" diff --git a/Source/Diagnostics/SliceDiagnostic.cpp b/Source/Diagnostics/SliceDiagnostic.cpp index 8e038e2bdf7..a9ebe4bf661 100644 --- a/Source/Diagnostics/SliceDiagnostic.cpp +++ b/Source/Diagnostics/SliceDiagnostic.cpp @@ -7,7 +7,7 @@ */ #include "SliceDiagnostic.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "Utils/TextMsg.H" #include "WarpX.H" diff --git a/Source/FieldSolver/Fields.H b/Source/FieldSolver/Fields.H deleted file mode 100644 index 8cc43306e6e..00000000000 --- a/Source/FieldSolver/Fields.H +++ /dev/null @@ -1,38 +0,0 @@ -/* Copyright 2024 Luca Fedeli - * - * This file is part of WarpX. - * - * License: BSD-3-Clause-LBNL - */ -#ifndef WARPX_FIELDS_H_ -#define WARPX_FIELDS_H_ - -#include -#include - -namespace warpx::fields -{ - enum struct FieldType : int - { - None, - Efield_fp, - Bfield_fp, - vector_potential_fp, - phi_fp - }; - - constexpr FieldType ArrayFieldTypes[] = { - FieldType::Efield_fp, - FieldType::Bfield_fp, - FieldType::vector_potential_fp }; - - inline bool - isFieldArray (const FieldType field_type) - { - return std::any_of( std::begin(ArrayFieldTypes), std::end(ArrayFieldTypes), - [field_type](const FieldType& f) { return f == field_type; }); - } - -} - -#endif //WARPX_FIELDS_H_ diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 24f9e07d39c..dba8a7de3f9 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -10,7 +10,7 @@ #include "HybridPICModel.H" #include "EmbeddedBoundary/Enabled.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "WarpX.H" using namespace amrex; diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp index a6a389fe056..ea1432dd1f9 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp @@ -1,6 +1,6 @@ #include "MacroscopicProperties.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "Utils/Parser/ParserUtils.H" #include "Utils/TextMsg.H" diff --git a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp index 262a33848d8..42d016bb34a 100644 --- a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp +++ b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp @@ -4,7 +4,7 @@ * * License: BSD-3-Clause-LBNL */ -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "ThetaImplicitEM.H" #include "WarpX.H" diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp index 06ec79779dc..a393bae0eaf 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp @@ -11,7 +11,7 @@ #include "Diagnostics/ReducedDiags/MultiReducedDiags.H" #include "Evolve/WarpXDtType.H" #include "Evolve/WarpXPushType.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H" #include "Parallelization/GuardCellManager.H" #include "Particles/MultiParticleContainer.H" diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H index b109229328e..29c808b48cd 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H +++ b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H @@ -8,7 +8,7 @@ #define WarpXSolverVec_H_ #include "Utils/TextMsg.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include #include diff --git a/Source/Fields.H b/Source/Fields.H new file mode 100644 index 00000000000..5e82e524fa2 --- /dev/null +++ b/Source/Fields.H @@ -0,0 +1,131 @@ +/* Copyright 2024 Luca Fedeli + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_FIELDS_H_ +#define WARPX_FIELDS_H_ + +#include + +#include + +#include +#include + + +namespace warpx::fields +{ + AMREX_ENUM(FieldType, + None, + Efield_aux, + Bfield_aux, + Efield_fp, + Bfield_fp, + Efield_fp_external, + Bfield_fp_external, + current_fp, + current_fp_nodal, + current_fp_vay, + current_buf, + current_store, + rho_buf, + rho_fp, + F_fp, + G_fp, + phi_fp, + vector_potential_fp, + vector_potential_fp_nodal, + vector_potential_grad_buf_e_stag, + vector_potential_grad_buf_b_stag, + hybrid_electron_pressure_fp, + hybrid_rho_fp_temp, + hybrid_current_fp_temp, + hybrid_current_fp_ampere, + hybrid_current_fp_external, + Efield_cp, + Bfield_cp, + current_cp, + rho_cp, + F_cp, + G_cp, + Efield_cax, + Bfield_cax, + E_external_particle_field, + B_external_particle_field, + distance_to_eb, + edge_lengths, + face_areas, + area_mod, + pml_E_fp, + pml_B_fp, + pml_j_fp, + pml_F_fp, + pml_G_fp, + pml_E_cp, + pml_B_cp, + pml_j_cp, + pml_F_cp, + pml_G_cp, + pml_edge_lengths, + Efield_avg_fp, + Bfield_avg_fp, + Efield_avg_cp, + Bfield_avg_cp, + Bold, + ECTRhofield, + Venl + ); + + /** these are vector fields */ + constexpr FieldType ArrayFieldTypes[] = { + FieldType::Efield_aux, + FieldType::Bfield_aux, + FieldType::Efield_fp, + FieldType::Bfield_fp, + FieldType::current_fp, + FieldType::current_fp_nodal, + FieldType::current_fp_vay, + FieldType::current_buf, + FieldType::current_store, + FieldType::vector_potential_fp, + FieldType::vector_potential_fp_nodal, + FieldType::vector_potential_grad_buf_e_stag, + FieldType::vector_potential_grad_buf_b_stag, + FieldType::hybrid_current_fp_temp, + FieldType::hybrid_current_fp_ampere, + FieldType::hybrid_current_fp_external, + FieldType::Efield_cp, + FieldType::Bfield_cp, + FieldType::current_cp, + FieldType::Efield_cax, + FieldType::Bfield_cax, + FieldType::E_external_particle_field, + FieldType::B_external_particle_field, + FieldType::pml_E_fp, + FieldType::pml_B_fp, + FieldType::pml_j_fp, + FieldType::pml_E_cp, + FieldType::pml_B_cp, + FieldType::pml_j_cp, + FieldType::Efield_avg_fp, + FieldType::Bfield_avg_fp, + FieldType::Efield_avg_cp, + FieldType::Bfield_avg_cp, + FieldType::Bold, + FieldType::ECTRhofield, + FieldType::Venl + }; + + /** Returns true if a FieldType represents a vector field */ + inline bool + isFieldArray (const FieldType field_type) + { + return std::any_of( std::begin(ArrayFieldTypes), std::end(ArrayFieldTypes), + [field_type](const FieldType& f) { return f == field_type; }); + } + +} + +#endif //WARPX_FIELDS_H_ diff --git a/Source/Initialization/DivCleaner/ProjectionDivCleaner.H b/Source/Initialization/DivCleaner/ProjectionDivCleaner.H index f2414323f25..2fedb83cd36 100644 --- a/Source/Initialization/DivCleaner/ProjectionDivCleaner.H +++ b/Source/Initialization/DivCleaner/ProjectionDivCleaner.H @@ -35,7 +35,7 @@ #include #include -#include +#include "Fields.H" #include "Utils/Parser/ParserUtils.H" namespace warpx::initialization { diff --git a/Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp b/Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp index 9fc6fab656a..ee8fcf40b9c 100644 --- a/Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp +++ b/Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp @@ -19,7 +19,7 @@ #else #include #endif -#include +#include "Fields.H" #include #include #include diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 8701b1a4023..0f950b3959d 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -17,7 +17,7 @@ #include "Diagnostics/MultiDiagnostics.H" #include "Diagnostics/ReducedDiags/MultiReducedDiags.H" #include "EmbeddedBoundary/Enabled.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "FieldSolver/ElectrostaticSolvers/ElectrostaticSolver.H" #include "FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H" #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 472461d3317..533ee574aba 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -11,7 +11,7 @@ */ #include "MultiParticleContainer.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "Particles/ElementaryProcess/Ionization.H" #ifdef WARPX_QED # include "Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H" diff --git a/Source/WarpX.H b/Source/WarpX.H index 7b9b836aefc..5575ea2036f 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -39,7 +39,7 @@ #include "AcceleratorLattice/AcceleratorLattice.H" #include "Evolve/WarpXDtType.H" #include "Evolve/WarpXPushType.H" -#include "FieldSolver/Fields.H" +#include "Fields.H" #include "FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H" #include "FieldSolver/ImplicitSolvers/ImplicitSolver.H" #include "FieldSolver/ImplicitSolvers/WarpXSolverVec.H" From dfa66533c2d053fa7422624c5d47dd317ab2b5f6 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sat, 21 Sep 2024 13:47:38 -0700 Subject: [PATCH 299/314] Fix Typo: `pml_edge_lengths` --- Source/BoundaryConditions/WarpXEvolvePML.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/BoundaryConditions/WarpXEvolvePML.cpp b/Source/BoundaryConditions/WarpXEvolvePML.cpp index b50010d83f6..7e8443415bb 100644 --- a/Source/BoundaryConditions/WarpXEvolvePML.cpp +++ b/Source/BoundaryConditions/WarpXEvolvePML.cpp @@ -284,7 +284,7 @@ WarpX::DampJPML (int lev, PatchType patch_type) // Skip the field update if this gridpoint is inside the embedded boundary amrex::Array4 eb_lxfab, eb_lyfab, eb_lzfab; if (EB::enabled()) { - const auto &pml_edge_lenghts = m_fields.get_alldirs("pmg_edge_lengths", lev); + const auto &pml_edge_lenghts = m_fields.get_alldirs("pml_edge_lengths", lev); eb_lxfab = pml_edge_lenghts[0]->array(mfi); eb_lyfab = pml_edge_lenghts[1]->array(mfi); From 4b03b41ac750e32a0899117cfbf6fff2cc3b976d Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sat, 21 Sep 2024 14:30:46 -0700 Subject: [PATCH 300/314] Use `FieldType` Instead of Strings --- Source/BoundaryConditions/PML.cpp | 150 ++++---- Source/BoundaryConditions/PML_RZ.cpp | 58 ++-- Source/BoundaryConditions/WarpXEvolvePML.cpp | 37 +- .../WarpXFieldBoundaries.cpp | 60 ++-- Source/Diagnostics/BTDiagnostics.cpp | 42 +-- .../ComputeDiagFunctors/JFunctor.cpp | 6 +- .../ComputeDiagFunctors/JdispFunctor.cpp | 8 +- .../FlushFormats/FlushFormatCheckpoint.cpp | 62 ++-- .../FlushFormats/FlushFormatPlotfile.cpp | 94 ++--- Source/Diagnostics/FullDiagnostics.cpp | 92 ++--- Source/Diagnostics/ParticleIO.cpp | 4 +- .../Diagnostics/ReducedDiags/ChargeOnEB.cpp | 8 +- .../ReducedDiags/ColliderRelevant.cpp | 14 +- .../Diagnostics/ReducedDiags/FieldEnergy.cpp | 14 +- .../Diagnostics/ReducedDiags/FieldMaximum.cpp | 14 +- .../ReducedDiags/FieldMomentum.cpp | 14 +- .../Diagnostics/ReducedDiags/FieldProbe.cpp | 14 +- .../Diagnostics/ReducedDiags/FieldReduction.H | 19 +- .../ReducedDiags/LoadBalanceCosts.cpp | 4 +- .../ReducedDiags/ParticleExtrema.cpp | 14 +- Source/Diagnostics/SliceDiagnostic.cpp | 14 +- Source/Diagnostics/WarpXIO.cpp | 78 ++--- .../EmbeddedBoundary/WarpXFaceExtensions.cpp | 44 +-- Source/EmbeddedBoundary/WarpXInitEB.cpp | 20 +- Source/Evolve/WarpXEvolve.cpp | 173 +++++----- .../ElectrostaticSolver.cpp | 18 +- .../LabFrameExplicitES.cpp | 10 +- .../RelativisticExplicitES.cpp | 14 +- .../FiniteDifferenceSolver/EvolveB.cpp | 30 +- .../FiniteDifferenceSolver/EvolveBPML.cpp | 6 +- .../FiniteDifferenceSolver/EvolveE.cpp | 27 +- .../FiniteDifferenceSolver/EvolveEPML.cpp | 17 +- .../HybridPICModel/HybridPICModel.cpp | 66 ++-- .../MacroscopicProperties.cpp | 2 +- .../ImplicitSolvers/SemiImplicitEM.cpp | 2 +- .../ImplicitSolvers/ThetaImplicitEM.cpp | 26 +- .../ImplicitSolvers/WarpXImplicitOps.cpp | 12 +- .../ImplicitSolvers/WarpXSolverVec.cpp | 6 +- .../MagnetostaticSolver.cpp | 25 +- Source/FieldSolver/WarpXPushFieldsEM.cpp | 114 ++++--- .../FieldSolver/WarpXPushFieldsHybridPIC.cpp | 67 ++-- Source/FieldSolver/WarpXSolveFieldsES.cpp | 8 +- .../FieldSolver/WarpX_QED_Field_Pushers.cpp | 38 ++- Source/Fluids/WarpXFluidContainer.cpp | 31 +- .../DivCleaner/ProjectionDivCleaner.cpp | 14 +- Source/Initialization/WarpXInitData.cpp | 181 +++++----- Source/Parallelization/WarpXComm.cpp | 320 +++++++++--------- Source/Parallelization/WarpXRegrid.cpp | 8 +- Source/Particles/LaserParticleContainer.cpp | 22 +- Source/Particles/MultiParticleContainer.cpp | 24 +- .../Particles/PhysicalParticleContainer.cpp | 54 +-- Source/Particles/WarpXParticleContainer.cpp | 4 +- Source/Utils/WarpXMovingWindow.cpp | 70 ++-- Source/WarpX.H | 5 +- Source/WarpX.cpp | 315 ++++++++--------- 55 files changed, 1346 insertions(+), 1247 deletions(-) diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index d905bf64d1f..4ac9a1588e5 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -57,7 +57,7 @@ #endif using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; namespace { @@ -703,33 +703,33 @@ PML::PML (const int lev, const BoxArray& grid_ba, auto& warpx = WarpX::GetInstance(); using ablastr::fields::Direction; - const amrex::BoxArray ba_Ex = amrex::convert(ba, warpx.m_fields.get("Efield_fp",Direction{0},0)->ixType().toIntVect()); - const amrex::BoxArray ba_Ey = amrex::convert(ba, warpx.m_fields.get("Efield_fp",Direction{1},0)->ixType().toIntVect()); - const amrex::BoxArray ba_Ez = amrex::convert(ba, warpx.m_fields.get("Efield_fp",Direction{2},0)->ixType().toIntVect()); - warpx.m_fields.alloc_init("pml_E_fp", Direction{0}, lev, ba_Ex, dm, ncompe, nge, 0.0_rt, false, false); - warpx.m_fields.alloc_init("pml_E_fp", Direction{1}, lev, ba_Ey, dm, ncompe, nge, 0.0_rt, false, false); - warpx.m_fields.alloc_init("pml_E_fp", Direction{2}, lev, ba_Ez, dm, ncompe, nge, 0.0_rt, false, false); - - const amrex::BoxArray ba_Bx = amrex::convert(ba, warpx.m_fields.get("Bfield_fp",Direction{0},0)->ixType().toIntVect()); - const amrex::BoxArray ba_By = amrex::convert(ba, warpx.m_fields.get("Bfield_fp",Direction{1},0)->ixType().toIntVect()); - const amrex::BoxArray ba_Bz = amrex::convert(ba, warpx.m_fields.get("Bfield_fp",Direction{2},0)->ixType().toIntVect()); - warpx.m_fields.alloc_init("pml_B_fp", Direction{0}, lev, ba_Bx, dm, ncompb, ngb, 0.0_rt, false, false); - warpx.m_fields.alloc_init("pml_B_fp", Direction{1}, lev, ba_By, dm, ncompb, ngb, 0.0_rt, false, false); - warpx.m_fields.alloc_init("pml_B_fp", Direction{2}, lev, ba_Bz, dm, ncompb, ngb, 0.0_rt, false, false); - - const amrex::BoxArray ba_jx = amrex::convert(ba, WarpX::GetInstance().m_fields.get("current_fp", Direction{0}, 0)->ixType().toIntVect()); - const amrex::BoxArray ba_jy = amrex::convert(ba, WarpX::GetInstance().m_fields.get("current_fp", Direction{1}, 0)->ixType().toIntVect()); - const amrex::BoxArray ba_jz = amrex::convert(ba, WarpX::GetInstance().m_fields.get("current_fp", Direction{2}, 0)->ixType().toIntVect()); - warpx.m_fields.alloc_init("pml_j_fp", Direction{0}, lev, ba_jx, dm, 1, ngb, 0.0_rt, false, false); - warpx.m_fields.alloc_init("pml_j_fp", Direction{1}, lev, ba_jy, dm, 1, ngb, 0.0_rt, false, false); - warpx.m_fields.alloc_init("pml_j_fp", Direction{2}, lev, ba_jz, dm, 1, ngb, 0.0_rt, false, false); + const amrex::BoxArray ba_Ex = amrex::convert(ba, warpx.m_fields.get(FieldType::Efield_fp, Direction{0}, 0)->ixType().toIntVect()); + const amrex::BoxArray ba_Ey = amrex::convert(ba, warpx.m_fields.get(FieldType::Efield_fp, Direction{1}, 0)->ixType().toIntVect()); + const amrex::BoxArray ba_Ez = amrex::convert(ba, warpx.m_fields.get(FieldType::Efield_fp, Direction{2}, 0)->ixType().toIntVect()); + warpx.m_fields.alloc_init(FieldType::pml_E_fp, Direction{0}, lev, ba_Ex, dm, ncompe, nge, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_E_fp, Direction{1}, lev, ba_Ey, dm, ncompe, nge, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_E_fp, Direction{2}, lev, ba_Ez, dm, ncompe, nge, 0.0_rt, false, false); + + const amrex::BoxArray ba_Bx = amrex::convert(ba, warpx.m_fields.get(FieldType::Bfield_fp, Direction{0}, 0)->ixType().toIntVect()); + const amrex::BoxArray ba_By = amrex::convert(ba, warpx.m_fields.get(FieldType::Bfield_fp, Direction{1}, 0)->ixType().toIntVect()); + const amrex::BoxArray ba_Bz = amrex::convert(ba, warpx.m_fields.get(FieldType::Bfield_fp, Direction{2}, 0)->ixType().toIntVect()); + warpx.m_fields.alloc_init(FieldType::pml_B_fp, Direction{0}, lev, ba_Bx, dm, ncompb, ngb, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_B_fp, Direction{1}, lev, ba_By, dm, ncompb, ngb, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_B_fp, Direction{2}, lev, ba_Bz, dm, ncompb, ngb, 0.0_rt, false, false); + + const amrex::BoxArray ba_jx = amrex::convert(ba, WarpX::GetInstance().m_fields.get(FieldType::current_fp, Direction{0}, 0)->ixType().toIntVect()); + const amrex::BoxArray ba_jy = amrex::convert(ba, WarpX::GetInstance().m_fields.get(FieldType::current_fp, Direction{1}, 0)->ixType().toIntVect()); + const amrex::BoxArray ba_jz = amrex::convert(ba, WarpX::GetInstance().m_fields.get(FieldType::current_fp, Direction{2}, 0)->ixType().toIntVect()); + warpx.m_fields.alloc_init(FieldType::pml_j_fp, Direction{0}, lev, ba_jx, dm, 1, ngb, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_j_fp, Direction{1}, lev, ba_jy, dm, 1, ngb, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_j_fp, Direction{2}, lev, ba_jz, dm, 1, ngb, 0.0_rt, false, false); #ifdef AMREX_USE_EB if (eb_enabled) { const amrex::IntVect max_guard_EB_vect = amrex::IntVect(max_guard_EB); - warpx.m_fields.alloc_init("pml_edge_lengths", Direction{0}, lev, ba_Ex, dm, WarpX::ncomps, max_guard_EB_vect, 0.0_rt, false, false); - warpx.m_fields.alloc_init("pml_edge_lengths", Direction{1}, lev, ba_Ey, dm, WarpX::ncomps, max_guard_EB_vect, 0.0_rt, false, false); - warpx.m_fields.alloc_init("pml_edge_lengths", Direction{2}, lev, ba_Ez, dm, WarpX::ncomps, max_guard_EB_vect, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_edge_lengths, Direction{0}, lev, ba_Ex, dm, WarpX::ncomps, max_guard_EB_vect, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_edge_lengths, Direction{1}, lev, ba_Ey, dm, WarpX::ncomps, max_guard_EB_vect, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_edge_lengths, Direction{2}, lev, ba_Ez, dm, WarpX::ncomps, max_guard_EB_vect, 0.0_rt, false, false); if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::Yee || WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC || @@ -737,7 +737,7 @@ PML::PML (const int lev, const BoxArray& grid_ba, auto const eb_fact = fieldEBFactory(); - ablastr::fields::VectorField t_pml_edge_lengths = warpx.m_fields.get_alldirs("pml_edge_lengths", lev); + ablastr::fields::VectorField t_pml_edge_lengths = warpx.m_fields.get_alldirs(FieldType::pml_edge_lengths, lev); WarpX::ComputeEdgeLengths(t_pml_edge_lengths, eb_fact); WarpX::ScaleEdges(t_pml_edge_lengths, WarpX::CellSize(lev)); @@ -749,7 +749,7 @@ PML::PML (const int lev, const BoxArray& grid_ba, if (m_dive_cleaning) { const amrex::BoxArray ba_F_nodal = amrex::convert(ba, amrex::IntVect::TheNodeVector()); - warpx.m_fields.alloc_init("pml_F_fp", lev, ba_F_nodal, dm, 3, ngf, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_F_fp, lev, ba_F_nodal, dm, 3, ngf, 0.0_rt, false, false); } if (m_divb_cleaning) @@ -759,7 +759,7 @@ PML::PML (const int lev, const BoxArray& grid_ba, (grid_type == GridType::Collocated) ? amrex::IntVect::TheNodeVector() : amrex::IntVect::TheCellVector(); const amrex::BoxArray ba_G_nodal = amrex::convert(ba, G_nodal_flag); - warpx.m_fields.alloc_init("pml_G_fp", lev, ba_G_nodal, dm, 3, ngf, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_G_fp, lev, ba_G_nodal, dm, 3, ngf, 0.0_rt, false, false); } Box single_domain_box = is_single_box_domain ? domain0 : Box(); @@ -841,24 +841,24 @@ PML::PML (const int lev, const BoxArray& grid_ba, cdm.define(cba); } - const amrex::BoxArray cba_Ex = amrex::convert(cba, WarpX::GetInstance().m_fields.get("Efield_cp", Direction{0}, 1)->ixType().toIntVect()); - const amrex::BoxArray cba_Ey = amrex::convert(cba, WarpX::GetInstance().m_fields.get("Efield_cp", Direction{1}, 1)->ixType().toIntVect()); - const amrex::BoxArray cba_Ez = amrex::convert(cba, WarpX::GetInstance().m_fields.get("Efield_cp", Direction{2}, 1)->ixType().toIntVect()); - warpx.m_fields.alloc_init("pml_E_cp", Direction{0}, lev, cba_Ex, cdm, ncompe, nge, 0.0_rt, false, false); - warpx.m_fields.alloc_init("pml_E_cp", Direction{1}, lev, cba_Ey, cdm, ncompe, nge, 0.0_rt, false, false); - warpx.m_fields.alloc_init("pml_E_cp", Direction{2}, lev, cba_Ez, cdm, ncompe, nge, 0.0_rt, false, false); + const amrex::BoxArray cba_Ex = amrex::convert(cba, WarpX::GetInstance().m_fields.get(FieldType::Efield_cp, Direction{0}, 1)->ixType().toIntVect()); + const amrex::BoxArray cba_Ey = amrex::convert(cba, WarpX::GetInstance().m_fields.get(FieldType::Efield_cp, Direction{1}, 1)->ixType().toIntVect()); + const amrex::BoxArray cba_Ez = amrex::convert(cba, WarpX::GetInstance().m_fields.get(FieldType::Efield_cp, Direction{2}, 1)->ixType().toIntVect()); + warpx.m_fields.alloc_init(FieldType::pml_E_cp, Direction{0}, lev, cba_Ex, cdm, ncompe, nge, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_E_cp, Direction{1}, lev, cba_Ey, cdm, ncompe, nge, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_E_cp, Direction{2}, lev, cba_Ez, cdm, ncompe, nge, 0.0_rt, false, false); - const amrex::BoxArray cba_Bx = amrex::convert(cba, WarpX::GetInstance().m_fields.get("Bfield_cp", Direction{0}, 1)->ixType().toIntVect()); - const amrex::BoxArray cba_By = amrex::convert(cba, WarpX::GetInstance().m_fields.get("Bfield_cp", Direction{1}, 1)->ixType().toIntVect()); - const amrex::BoxArray cba_Bz = amrex::convert(cba, WarpX::GetInstance().m_fields.get("Bfield_cp", Direction{2}, 1)->ixType().toIntVect()); - warpx.m_fields.alloc_init("pml_B_cp", Direction{0}, lev, cba_Bx, cdm, ncompb, ngb, 0.0_rt, false, false); - warpx.m_fields.alloc_init("pml_B_cp", Direction{1}, lev, cba_By, cdm, ncompb, ngb, 0.0_rt, false, false); - warpx.m_fields.alloc_init("pml_B_cp", Direction{2}, lev, cba_Bz, cdm, ncompb, ngb, 0.0_rt, false, false); + const amrex::BoxArray cba_Bx = amrex::convert(cba, WarpX::GetInstance().m_fields.get(FieldType::Bfield_cp, Direction{0}, 1)->ixType().toIntVect()); + const amrex::BoxArray cba_By = amrex::convert(cba, WarpX::GetInstance().m_fields.get(FieldType::Bfield_cp, Direction{1}, 1)->ixType().toIntVect()); + const amrex::BoxArray cba_Bz = amrex::convert(cba, WarpX::GetInstance().m_fields.get(FieldType::Bfield_cp, Direction{2}, 1)->ixType().toIntVect()); + warpx.m_fields.alloc_init(FieldType::pml_B_cp, Direction{0}, lev, cba_Bx, cdm, ncompb, ngb, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_B_cp, Direction{1}, lev, cba_By, cdm, ncompb, ngb, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_B_cp, Direction{2}, lev, cba_Bz, cdm, ncompb, ngb, 0.0_rt, false, false); if (m_dive_cleaning) { const amrex::BoxArray cba_F_nodal = amrex::convert(cba, amrex::IntVect::TheNodeVector()); - warpx.m_fields.alloc_init("pml_F_cp", lev, cba_F_nodal, cdm, 3, ngf, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_F_cp, lev, cba_F_nodal, cdm, 3, ngf, 0.0_rt, false, false); } if (m_divb_cleaning) @@ -868,15 +868,15 @@ PML::PML (const int lev, const BoxArray& grid_ba, (grid_type == GridType::Collocated) ? amrex::IntVect::TheNodeVector() : amrex::IntVect::TheCellVector(); const amrex::BoxArray cba_G_nodal = amrex::convert(cba, G_nodal_flag); - warpx.m_fields.alloc_init("pml_G_cp", lev, cba_G_nodal, cdm, 3, ngf, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_G_cp, lev, cba_G_nodal, cdm, 3, ngf, 0.0_rt, false, false); } - const amrex::BoxArray cba_jx = amrex::convert(cba, WarpX::GetInstance().m_fields.get("current_cp", Direction{0}, 1)->ixType().toIntVect()); - const amrex::BoxArray cba_jy = amrex::convert(cba, WarpX::GetInstance().m_fields.get("current_cp", Direction{1}, 1)->ixType().toIntVect()); - const amrex::BoxArray cba_jz = amrex::convert(cba, WarpX::GetInstance().m_fields.get("current_cp", Direction{2}, 1)->ixType().toIntVect()); - warpx.m_fields.alloc_init("pml_j_cp", Direction{0}, lev, cba_jx, cdm, 1, ngb, 0.0_rt, false, false); - warpx.m_fields.alloc_init("pml_j_cp", Direction{1}, lev, cba_jy, cdm, 1, ngb, 0.0_rt, false, false); - warpx.m_fields.alloc_init("pml_j_cp", Direction{2}, lev, cba_jz, cdm, 1, ngb, 0.0_rt, false, false); + const amrex::BoxArray cba_jx = amrex::convert(cba, WarpX::GetInstance().m_fields.get(FieldType::current_cp, Direction{0}, 1)->ixType().toIntVect()); + const amrex::BoxArray cba_jy = amrex::convert(cba, WarpX::GetInstance().m_fields.get(FieldType::current_cp, Direction{1}, 1)->ixType().toIntVect()); + const amrex::BoxArray cba_jz = amrex::convert(cba, WarpX::GetInstance().m_fields.get(FieldType::current_cp, Direction{2}, 1)->ixType().toIntVect()); + warpx.m_fields.alloc_init(FieldType::pml_j_cp, Direction{0}, lev, cba_jx, cdm, 1, ngb, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_j_cp, Direction{1}, lev, cba_jy, cdm, 1, ngb, 0.0_rt, false, false); + warpx.m_fields.alloc_init(FieldType::pml_j_cp, Direction{2}, lev, cba_jz, cdm, 1, ngb, 0.0_rt, false, false); single_domain_box = is_single_box_domain ? cdomain : Box(); sigba_cp = std::make_unique(cba, cdm, grid_cba_reduced, cgeom->CellSize(), @@ -1060,23 +1060,23 @@ PML::CopyJtoPMLs ( { using ablastr::fields::Direction; - bool const has_j_fp = fields.has("current_fp", Direction{0}, lev); - bool const has_pml_j_fp = fields.has("pml_j_fp", Direction{0}, lev); - bool const has_j_cp = fields.has("current_cp", Direction{0}, lev); - bool const has_pml_j_cp = fields.has("pml_j_cp", Direction{0}, lev); + bool const has_j_fp = fields.has(FieldType::current_fp, Direction{0}, lev); + bool const has_pml_j_fp = fields.has(FieldType::pml_j_fp, Direction{0}, lev); + bool const has_j_cp = fields.has(FieldType::current_cp, Direction{0}, lev); + bool const has_pml_j_cp = fields.has(FieldType::pml_j_cp, Direction{0}, lev); if (patch_type == PatchType::fine && has_pml_j_fp && has_j_fp) { - ablastr::fields::VectorField pml_j_fp = fields.get_alldirs("pml_j_fp", lev); - ablastr::fields::VectorField jp = fields.get_alldirs("current_fp", lev); + ablastr::fields::VectorField pml_j_fp = fields.get_alldirs(FieldType::pml_j_fp, lev); + ablastr::fields::VectorField jp = fields.get_alldirs(FieldType::current_fp, lev); CopyToPML(*pml_j_fp[0], *jp[0], *m_geom); CopyToPML(*pml_j_fp[1], *jp[1], *m_geom); CopyToPML(*pml_j_fp[2], *jp[2], *m_geom); } else if (patch_type == PatchType::coarse && has_j_cp && has_pml_j_cp) { - ablastr::fields::VectorField pml_j_cp = fields.get_alldirs("pml_j_cp", lev); - ablastr::fields::VectorField jp = fields.get_alldirs("current_cp", lev); + ablastr::fields::VectorField pml_j_cp = fields.get_alldirs(FieldType::pml_j_cp, lev); + ablastr::fields::VectorField jp = fields.get_alldirs(FieldType::current_cp, lev); CopyToPML(*pml_j_cp[0], *jp[0], *m_cgeom); CopyToPML(*pml_j_cp[1], *jp[1], *m_cgeom); CopyToPML(*pml_j_cp[2], *jp[2], *m_cgeom); @@ -1234,10 +1234,10 @@ PML::CheckPoint ( { using ablastr::fields::Direction; - if (fields.has("pml_E_fp", Direction{0}, 0)) + if (fields.has(FieldType::pml_E_fp, Direction{0}, 0)) { - ablastr::fields::VectorField pml_E_fp = fields.get_alldirs("pml_E_fp", 0); - ablastr::fields::VectorField pml_B_fp = fields.get_alldirs("pml_B_fp", 0); + ablastr::fields::VectorField pml_E_fp = fields.get_alldirs(FieldType::pml_E_fp, 0); + ablastr::fields::VectorField pml_B_fp = fields.get_alldirs(FieldType::pml_B_fp, 0); VisMF::AsyncWrite(*pml_E_fp[0], dir+"_Ex_fp"); VisMF::AsyncWrite(*pml_E_fp[1], dir+"_Ey_fp"); VisMF::AsyncWrite(*pml_E_fp[2], dir+"_Ez_fp"); @@ -1246,10 +1246,10 @@ PML::CheckPoint ( VisMF::AsyncWrite(*pml_B_fp[2], dir+"_Bz_fp"); } - if (fields.has("pml_E_cp", Direction{0}, 0)) + if (fields.has(FieldType::pml_E_cp, Direction{0}, 0)) { - ablastr::fields::VectorField pml_E_cp = fields.get_alldirs("pml_E_cp", 0); - ablastr::fields::VectorField pml_B_cp = fields.get_alldirs("pml_B_cp", 0); + ablastr::fields::VectorField pml_E_cp = fields.get_alldirs(FieldType::pml_E_cp, 0); + ablastr::fields::VectorField pml_B_cp = fields.get_alldirs(FieldType::pml_B_cp, 0); VisMF::AsyncWrite(*pml_E_cp[0], dir+"_Ex_cp"); VisMF::AsyncWrite(*pml_E_cp[1], dir+"_Ey_cp"); VisMF::AsyncWrite(*pml_E_cp[2], dir+"_Ez_cp"); @@ -1267,10 +1267,10 @@ PML::Restart ( { using ablastr::fields::Direction; - if (fields.has("pml_E_fp", Direction{0}, 0)) + if (fields.has(FieldType::pml_E_fp, Direction{0}, 0)) { - ablastr::fields::VectorField pml_E_fp = fields.get_alldirs("pml_E_fp", 0); - ablastr::fields::VectorField pml_B_fp = fields.get_alldirs("pml_B_fp", 0); + ablastr::fields::VectorField pml_E_fp = fields.get_alldirs(FieldType::pml_E_fp, 0); + ablastr::fields::VectorField pml_B_fp = fields.get_alldirs(FieldType::pml_B_fp, 0); VisMF::Read(*pml_E_fp[0], dir+"_Ex_fp"); VisMF::Read(*pml_E_fp[1], dir+"_Ey_fp"); VisMF::Read(*pml_E_fp[2], dir+"_Ez_fp"); @@ -1279,10 +1279,10 @@ PML::Restart ( VisMF::Read(*pml_B_fp[2], dir+"_Bz_fp"); } - if (fields.has("pml_E_cp", Direction{0}, 0)) + if (fields.has(FieldType::pml_E_cp, Direction{0}, 0)) { - ablastr::fields::VectorField pml_E_cp = fields.get_alldirs("pml_E_cp", 0); - ablastr::fields::VectorField pml_B_cp = fields.get_alldirs("pml_B_cp", 0); + ablastr::fields::VectorField pml_E_cp = fields.get_alldirs(FieldType::pml_E_cp, 0); + ablastr::fields::VectorField pml_B_cp = fields.get_alldirs(FieldType::pml_B_cp, 0); VisMF::Read(*pml_E_cp[0], dir+"_Ex_cp"); VisMF::Read(*pml_E_cp[1], dir+"_Ey_cp"); VisMF::Read(*pml_E_cp[2], dir+"_Ez_cp"); @@ -1296,18 +1296,18 @@ PML::Restart ( void PML::PushPSATD (ablastr::fields::MultiFabRegister& fields, const int lev) { - ablastr::fields::VectorField pml_E_fp = fields.get_alldirs("pml_E_fp", lev); - ablastr::fields::VectorField pml_B_fp = fields.get_alldirs("pml_B_fp", lev); - ablastr::fields::ScalarField pml_F_fp = fields.get("pml_F_fp", lev); - ablastr::fields::ScalarField pml_G_fp = fields.get("pml_G_fp", lev); + ablastr::fields::VectorField pml_E_fp = fields.get_alldirs(FieldType::pml_E_fp, lev); + ablastr::fields::VectorField pml_B_fp = fields.get_alldirs(FieldType::pml_B_fp, lev); + ablastr::fields::ScalarField pml_F_fp = fields.get(FieldType::pml_F_fp, lev); + ablastr::fields::ScalarField pml_G_fp = fields.get(FieldType::pml_G_fp, lev); // Update the fields on the fine and coarse patch PushPMLPSATDSinglePatch(lev, *spectral_solver_fp, pml_E_fp, pml_B_fp, pml_F_fp, pml_G_fp, m_fill_guards_fields); if (spectral_solver_cp) { - ablastr::fields::VectorField pml_E_cp = fields.get_alldirs("pml_E_cp", lev); - ablastr::fields::VectorField pml_B_cp = fields.get_alldirs("pml_B_cp", lev); - ablastr::fields::ScalarField pml_F_cp = fields.get("pml_F_cp", lev); - ablastr::fields::ScalarField pml_G_cp = fields.get("pml_G_cp", lev); + ablastr::fields::VectorField pml_E_cp = fields.get_alldirs(FieldType::pml_E_cp, lev); + ablastr::fields::VectorField pml_B_cp = fields.get_alldirs(FieldType::pml_B_cp, lev); + ablastr::fields::ScalarField pml_F_cp = fields.get(FieldType::pml_F_cp, lev); + ablastr::fields::ScalarField pml_G_cp = fields.get(FieldType::pml_G_cp, lev); PushPMLPSATDSinglePatch(lev, *spectral_solver_cp, pml_E_cp, pml_B_cp, pml_F_cp, pml_G_cp, m_fill_guards_fields); } } diff --git a/Source/BoundaryConditions/PML_RZ.cpp b/Source/BoundaryConditions/PML_RZ.cpp index b8875be734f..8fd6a1869ae 100644 --- a/Source/BoundaryConditions/PML_RZ.cpp +++ b/Source/BoundaryConditions/PML_RZ.cpp @@ -34,7 +34,7 @@ #include using namespace amrex::literals; -using namespace warpx::fields; +using warpx::fields::FieldType; using ablastr::fields::Direction; PML_RZ::PML_RZ (int lev, amrex::BoxArray const& grid_ba, amrex::DistributionMapping const& grid_dm, @@ -48,22 +48,22 @@ PML_RZ::PML_RZ (int lev, amrex::BoxArray const& grid_ba, amrex::DistributionMapp bool const remake = false; bool const redistribute_on_remake = false; - amrex::MultiFab const& Er_fp = *warpx.m_fields.get("Efield_fp",Direction{0},lev); - amrex::MultiFab const& Et_fp = *warpx.m_fields.get("Efield_fp",Direction{1},lev); + amrex::MultiFab const& Er_fp = *warpx.m_fields.get(FieldType::Efield_fp, Direction{0}, lev); + amrex::MultiFab const& Et_fp = *warpx.m_fields.get(FieldType::Efield_fp, Direction{1}, lev); amrex::BoxArray const ba_Er = amrex::convert(grid_ba, Er_fp.ixType().toIntVect()); amrex::BoxArray const ba_Et = amrex::convert(grid_ba, Et_fp.ixType().toIntVect()); - warpx.m_fields.alloc_init("pml_E_fp", Direction{0}, lev, ba_Er, grid_dm, Er_fp.nComp(), Er_fp.nGrowVect(), 0.0_rt, + warpx.m_fields.alloc_init(FieldType::pml_E_fp, Direction{0}, lev, ba_Er, grid_dm, Er_fp.nComp(), Er_fp.nGrowVect(), 0.0_rt, remake, redistribute_on_remake); - warpx.m_fields.alloc_init("pml_E_fp", Direction{1}, lev, ba_Et, grid_dm, Et_fp.nComp(), Et_fp.nGrowVect(), 0.0_rt, + warpx.m_fields.alloc_init(FieldType::pml_E_fp, Direction{1}, lev, ba_Et, grid_dm, Et_fp.nComp(), Et_fp.nGrowVect(), 0.0_rt, remake, redistribute_on_remake); - amrex::MultiFab const& Br_fp = *warpx.m_fields.get("Bfield_fp",Direction{0},lev); - amrex::MultiFab const& Bt_fp = *warpx.m_fields.get("Bfield_fp",Direction{1},lev); + amrex::MultiFab const& Br_fp = *warpx.m_fields.get(FieldType::Bfield_fp,Direction{0},lev); + amrex::MultiFab const& Bt_fp = *warpx.m_fields.get(FieldType::Bfield_fp,Direction{1},lev); amrex::BoxArray const ba_Br = amrex::convert(grid_ba, Br_fp.ixType().toIntVect()); amrex::BoxArray const ba_Bt = amrex::convert(grid_ba, Bt_fp.ixType().toIntVect()); - warpx.m_fields.alloc_init("pml_B_fp", Direction{0}, lev, ba_Br, grid_dm, Br_fp.nComp(), Br_fp.nGrowVect(), 0.0_rt, + warpx.m_fields.alloc_init(FieldType::pml_B_fp, Direction{0}, lev, ba_Br, grid_dm, Br_fp.nComp(), Br_fp.nGrowVect(), 0.0_rt, remake, redistribute_on_remake); - warpx.m_fields.alloc_init("pml_B_fp", Direction{1}, lev, ba_Bt, grid_dm, Bt_fp.nComp(), Bt_fp.nGrowVect(), 0.0_rt, + warpx.m_fields.alloc_init(FieldType::pml_B_fp, Direction{1}, lev, ba_Bt, grid_dm, Bt_fp.nComp(), Bt_fp.nGrowVect(), 0.0_rt, remake, redistribute_on_remake); } @@ -77,8 +77,8 @@ PML_RZ::ApplyDamping (amrex::MultiFab* Et_fp, amrex::MultiFab* Ez_fp, amrex::Real const dr = m_geom->CellSize(0); amrex::Real const cdt_over_dr = PhysConst::c*dt/dr; - amrex::MultiFab* pml_Et = fields.get("pml_E_fp", Direction{1}, 0); - amrex::MultiFab* pml_Bt = fields.get("pml_B_fp", Direction{1}, 0); + amrex::MultiFab* pml_Et = fields.get(FieldType::pml_E_fp, Direction{1}, 0); + amrex::MultiFab* pml_Bt = fields.get(FieldType::pml_B_fp, Direction{1}, 0); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) @@ -132,8 +132,8 @@ PML_RZ::ApplyDamping (amrex::MultiFab* Et_fp, amrex::MultiFab* Ez_fp, void PML_RZ::FillBoundaryE (ablastr::fields::MultiFabRegister& fields, PatchType patch_type, std::optional nodal_sync) { - amrex::MultiFab * pml_Er = fields.get("pml_E_fp", Direction{0}, 0); - amrex::MultiFab * pml_Et = fields.get("pml_E_fp", Direction{1}, 0); + amrex::MultiFab * pml_Er = fields.get(FieldType::pml_E_fp, Direction{0}, 0); + amrex::MultiFab * pml_Et = fields.get(FieldType::pml_E_fp, Direction{1}, 0); if (patch_type == PatchType::fine && pml_Er->nGrowVect().max() > 0) { @@ -148,8 +148,8 @@ PML_RZ::FillBoundaryB (ablastr::fields::MultiFabRegister& fields, PatchType patc { if (patch_type == PatchType::fine) { - amrex::MultiFab * pml_Br = fields.get("pml_B_fp", Direction{0}, 0); - amrex::MultiFab * pml_Bt = fields.get("pml_B_fp", Direction{1}, 0); + amrex::MultiFab * pml_Br = fields.get(FieldType::pml_B_fp, Direction{0}, 0); + amrex::MultiFab * pml_Bt = fields.get(FieldType::pml_B_fp, Direction{1}, 0); amrex::Periodicity const& period = m_geom->periodicity(); const amrex::Vector mf = {pml_Br, pml_Bt}; @@ -160,22 +160,22 @@ PML_RZ::FillBoundaryB (ablastr::fields::MultiFabRegister& fields, PatchType patc void PML_RZ::CheckPoint (ablastr::fields::MultiFabRegister& fields, std::string const& dir) const { - if (fields.has("pml_E_fp", Direction{0}, 0)) { - amrex::VisMF::AsyncWrite(*fields.get("pml_E_fp", Direction{0}, 0), dir+"_Er_fp"); - amrex::VisMF::AsyncWrite(*fields.get("pml_E_fp", Direction{1}, 0), dir+"_Et_fp"); - amrex::VisMF::AsyncWrite(*fields.get("pml_B_fp", Direction{0}, 0), dir+"_Br_fp"); - amrex::VisMF::AsyncWrite(*fields.get("pml_B_fp", Direction{1}, 0), dir+"_Bt_fp"); + if (fields.has(FieldType::pml_E_fp, Direction{0}, 0)) { + amrex::VisMF::AsyncWrite(*fields.get(FieldType::pml_E_fp, Direction{0}, 0), dir+"_Er_fp"); + amrex::VisMF::AsyncWrite(*fields.get(FieldType::pml_E_fp, Direction{1}, 0), dir+"_Et_fp"); + amrex::VisMF::AsyncWrite(*fields.get(FieldType::pml_B_fp, Direction{0}, 0), dir+"_Br_fp"); + amrex::VisMF::AsyncWrite(*fields.get(FieldType::pml_B_fp, Direction{1}, 0), dir+"_Bt_fp"); } } void PML_RZ::Restart (ablastr::fields::MultiFabRegister& fields, std::string const& dir) { - if (fields.has("pml_E_fp", Direction{0}, 0)) { - amrex::VisMF::Read(*fields.get("pml_E_fp", Direction{0}, 0), dir+"_Er_fp"); - amrex::VisMF::Read(*fields.get("pml_E_fp", Direction{1}, 0), dir+"_Et_fp"); - amrex::VisMF::Read(*fields.get("pml_B_fp", Direction{0}, 0), dir+"_Br_fp"); - amrex::VisMF::Read(*fields.get("pml_B_fp", Direction{1}, 0), dir+"_Bt_fp"); + if (fields.has(FieldType::pml_E_fp, Direction{0}, 0)) { + amrex::VisMF::Read(*fields.get(FieldType::pml_E_fp, Direction{0}, 0), dir+"_Er_fp"); + amrex::VisMF::Read(*fields.get(FieldType::pml_E_fp, Direction{1}, 0), dir+"_Et_fp"); + amrex::VisMF::Read(*fields.get(FieldType::pml_B_fp, Direction{0}, 0), dir+"_Br_fp"); + amrex::VisMF::Read(*fields.get(FieldType::pml_B_fp, Direction{1}, 0), dir+"_Bt_fp"); } } @@ -196,10 +196,10 @@ PML_RZ::PushPMLPSATDSinglePatchRZ ( ablastr::fields::MultiFabRegister& fields) { SpectralFieldIndex const& Idx = solver.m_spectral_index; - amrex::MultiFab * pml_Er = fields.get("pml_E_fp", Direction{0}, 0); - amrex::MultiFab * pml_Et = fields.get("pml_E_fp", Direction{1}, 0); - amrex::MultiFab * pml_Br = fields.get("pml_B_fp", Direction{0}, 0); - amrex::MultiFab * pml_Bt = fields.get("pml_B_fp", Direction{1}, 0); + amrex::MultiFab * pml_Er = fields.get(FieldType::pml_E_fp, Direction{0}, 0); + amrex::MultiFab * pml_Et = fields.get(FieldType::pml_E_fp, Direction{1}, 0); + amrex::MultiFab * pml_Br = fields.get(FieldType::pml_B_fp, Direction{0}, 0); + amrex::MultiFab * pml_Bt = fields.get(FieldType::pml_B_fp, Direction{1}, 0); // Perform forward Fourier transforms solver.ForwardTransform(lev, *pml_Er, Idx.Er_pml, *pml_Et, Idx.Et_pml); diff --git a/Source/BoundaryConditions/WarpXEvolvePML.cpp b/Source/BoundaryConditions/WarpXEvolvePML.cpp index 7e8443415bb..cfde83dcf5b 100644 --- a/Source/BoundaryConditions/WarpXEvolvePML.cpp +++ b/Source/BoundaryConditions/WarpXEvolvePML.cpp @@ -12,6 +12,7 @@ # include "BoundaryConditions/PML_RZ.H" #endif #include "EmbeddedBoundary/Enabled.H" +#include "Fields.H" #include "PML_current.H" #include "Utils/WarpXProfilerWrapper.H" #include "WarpX_PML_kernels.H" @@ -66,10 +67,11 @@ WarpX::DampPML (const int lev, PatchType patch_type) #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev]) { using ablastr::fields::Direction; - pml_rz[lev]->ApplyDamping( m_fields.get("Efield_fp",Direction{1},lev), - m_fields.get("Efield_fp",Direction{2},lev), - m_fields.get("Bfield_fp",Direction{1},lev), - m_fields.get("Bfield_fp",Direction{2},lev), + using warpx::fields::FieldType; + pml_rz[lev]->ApplyDamping( m_fields.get(FieldType::Efield_fp, Direction{1}, lev), + m_fields.get(FieldType::Efield_fp, Direction{2}, lev), + m_fields.get(FieldType::Bfield_fp, Direction{1}, lev), + m_fields.get(FieldType::Bfield_fp, Direction{2}, lev), dt[lev], m_fields); } #endif @@ -86,8 +88,10 @@ WarpX::DampPML_Cartesian (const int lev, PatchType patch_type) if (pml[lev]->ok()) { - const auto& pml_E = (patch_type == PatchType::fine) ? m_fields.get_alldirs("pml_E_fp", lev) : m_fields.get_alldirs("pml_E_cp", lev); - const auto& pml_B = (patch_type == PatchType::fine) ? m_fields.get_alldirs("pml_B_fp", lev) : m_fields.get_alldirs("pml_B_cp", lev); + using warpx::fields::FieldType; + + const auto& pml_E = (patch_type == PatchType::fine) ? m_fields.get_alldirs(FieldType::pml_E_fp, lev) : m_fields.get_alldirs(FieldType::pml_E_cp, lev); + const auto& pml_B = (patch_type == PatchType::fine) ? m_fields.get_alldirs(FieldType::pml_B_fp, lev) : m_fields.get_alldirs(FieldType::pml_B_cp, lev); const auto& sigba = (patch_type == PatchType::fine) ? pml[lev]->GetMultiSigmaBox_fp() : pml[lev]->GetMultiSigmaBox_cp(); const amrex::IntVect Ex_stag = pml_E[0]->ixType().toIntVect(); @@ -99,16 +103,16 @@ WarpX::DampPML_Cartesian (const int lev, PatchType patch_type) const amrex::IntVect Bz_stag = pml_B[2]->ixType().toIntVect(); amrex::IntVect F_stag; - if (m_fields.has("pml_F_fp", lev)) { + if (m_fields.has(FieldType::pml_F_fp, lev)) { amrex::MultiFab* pml_F = (patch_type == PatchType::fine) ? - m_fields.get("pml_F_fp", lev) : m_fields.get("pml_F_cp", lev); + m_fields.get(FieldType::pml_F_fp, lev) : m_fields.get(FieldType::pml_F_cp, lev); F_stag = pml_F->ixType().toIntVect(); } amrex::IntVect G_stag; - if (m_fields.has("pml_G_fp", lev)) { + if (m_fields.has(FieldType::pml_G_fp, lev)) { amrex::MultiFab* pml_G = (patch_type == PatchType::fine) ? - m_fields.get("pml_G_fp", lev) : m_fields.get("pml_G_cp", lev); + m_fields.get(FieldType::pml_G_fp, lev) : m_fields.get(FieldType::pml_G_cp, lev); G_stag = pml_G->ixType().toIntVect(); } @@ -199,9 +203,9 @@ WarpX::DampPML_Cartesian (const int lev, PatchType patch_type) // For warpx_damp_pml_F(), mfi.nodaltilebox is used in the ParallelFor loop and here we // use mfi.tilebox. However, it does not matter because in damp_pml, where nodaltilebox // is used, only a simple multiplication is performed. - if (m_fields.has("pml_F_fp", lev)) { + if (m_fields.has(FieldType::pml_F_fp, lev)) { amrex::MultiFab* pml_F = (patch_type == PatchType::fine) ? - m_fields.get("pml_F_fp", lev) : m_fields.get("pml_F_cp", lev); + m_fields.get(FieldType::pml_F_fp, lev) : m_fields.get(FieldType::pml_F_cp, lev); const Box& tnd = mfi.nodaltilebox(); auto const& pml_F_fab = pml_F->array(mfi); amrex::ParallelFor(tnd, [=] AMREX_GPU_DEVICE (int i, int j, int k) @@ -212,9 +216,9 @@ WarpX::DampPML_Cartesian (const int lev, PatchType patch_type) } // Damp G when WarpX::do_divb_cleaning = true - if (m_fields.has("pml_G_fp", lev)) { + if (m_fields.has(FieldType::pml_G_fp, lev)) { amrex::MultiFab* pml_G = (patch_type == PatchType::fine) ? - m_fields.get("pml_G_fp", lev) : m_fields.get("pml_G_cp", lev); + m_fields.get(FieldType::pml_G_fp, lev) : m_fields.get(FieldType::pml_G_cp, lev); const Box& tb = mfi.tilebox(G_stag); auto const& pml_G_fab = pml_G->array(mfi); @@ -254,8 +258,9 @@ WarpX::DampJPML (int lev, PatchType patch_type) if (pml[lev]->ok()) { + using warpx::fields::FieldType; - const auto& pml_j = (patch_type == PatchType::fine) ? m_fields.get_alldirs("pml_j_fp", lev) : m_fields.get_alldirs("pml_j_cp", lev); + const auto& pml_j = (patch_type == PatchType::fine) ? m_fields.get_alldirs(FieldType::pml_j_fp, lev) : m_fields.get_alldirs(FieldType::pml_j_cp, lev); const auto& sigba = (patch_type == PatchType::fine) ? pml[lev]->GetMultiSigmaBox_fp() : pml[lev]->GetMultiSigmaBox_cp(); @@ -284,7 +289,7 @@ WarpX::DampJPML (int lev, PatchType patch_type) // Skip the field update if this gridpoint is inside the embedded boundary amrex::Array4 eb_lxfab, eb_lyfab, eb_lzfab; if (EB::enabled()) { - const auto &pml_edge_lenghts = m_fields.get_alldirs("pml_edge_lengths", lev); + const auto &pml_edge_lenghts = m_fields.get_alldirs(FieldType::pml_edge_lengths, lev); eb_lxfab = pml_edge_lenghts[0]->array(mfi); eb_lyfab = pml_edge_lenghts[1]->array(mfi); diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index 6e4b1b451e2..dc41e95f40f 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -18,7 +18,7 @@ using namespace amrex; using namespace amrex::literals; -using namespace warpx::fields; +using warpx::fields::FieldType; namespace { @@ -55,9 +55,9 @@ void WarpX::ApplyEfieldBoundary(const int lev, PatchType patch_type) if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { if (patch_type == PatchType::fine) { PEC::ApplyPECtoEfield( - {m_fields.get("Efield_fp",Direction{0},lev), - m_fields.get("Efield_fp",Direction{1},lev), - m_fields.get("Efield_fp",Direction{2},lev)}, + {m_fields.get(FieldType::Efield_fp, Direction{0}, lev), + m_fields.get(FieldType::Efield_fp, Direction{1}, lev), + m_fields.get(FieldType::Efield_fp, Direction{2}, lev)}, field_boundary_lo, field_boundary_hi, get_ng_fieldgather(), Geom(lev), lev, patch_type, ref_ratio); @@ -65,7 +65,7 @@ void WarpX::ApplyEfieldBoundary(const int lev, PatchType patch_type) // apply pec on split E-fields in PML region const bool split_pml_field = true; PEC::ApplyPECtoEfield( - m_fields.get_alldirs("pml_E_fp",lev), + m_fields.get_alldirs(FieldType::pml_E_fp, lev), field_boundary_lo, field_boundary_hi, get_ng_fieldgather(), Geom(lev), lev, patch_type, ref_ratio, @@ -73,9 +73,9 @@ void WarpX::ApplyEfieldBoundary(const int lev, PatchType patch_type) } } else { PEC::ApplyPECtoEfield( - {m_fields.get("Efield_cp",Direction{0},lev), - m_fields.get("Efield_cp",Direction{1},lev), - m_fields.get("Efield_cp",Direction{2},lev)}, + {m_fields.get(FieldType::Efield_cp,Direction{0},lev), + m_fields.get(FieldType::Efield_cp,Direction{1},lev), + m_fields.get(FieldType::Efield_cp,Direction{2},lev)}, field_boundary_lo, field_boundary_hi, get_ng_fieldgather(), Geom(lev), lev, patch_type, ref_ratio); @@ -83,7 +83,7 @@ void WarpX::ApplyEfieldBoundary(const int lev, PatchType patch_type) // apply pec on split E-fields in PML region const bool split_pml_field = true; PEC::ApplyPECtoEfield( - m_fields.get_alldirs("pml_E_cp",lev), + m_fields.get_alldirs(FieldType::pml_E_cp, lev), field_boundary_lo, field_boundary_hi, get_ng_fieldgather(), Geom(lev), lev, patch_type, ref_ratio, @@ -94,13 +94,13 @@ void WarpX::ApplyEfieldBoundary(const int lev, PatchType patch_type) #ifdef WARPX_DIM_RZ if (patch_type == PatchType::fine) { - ApplyFieldBoundaryOnAxis(m_fields.get("Efield_fp",Direction{0},lev), - m_fields.get("Efield_fp",Direction{1},lev), - m_fields.get("Efield_fp",Direction{2},lev), lev); + ApplyFieldBoundaryOnAxis(m_fields.get(FieldType::Efield_fp, Direction{0}, lev), + m_fields.get(FieldType::Efield_fp, Direction{1}, lev), + m_fields.get(FieldType::Efield_fp, Direction{2}, lev), lev); } else { - ApplyFieldBoundaryOnAxis(m_fields.get("Efield_cp",Direction{0},lev), - m_fields.get("Efield_cp",Direction{1},lev), - m_fields.get("Efield_cp",Direction{2},lev), lev); + ApplyFieldBoundaryOnAxis(m_fields.get(FieldType::Efield_cp, Direction{0}, lev), + m_fields.get(FieldType::Efield_cp, Direction{1}, lev), + m_fields.get(FieldType::Efield_cp, Direction{2}, lev), lev); } #endif } @@ -112,17 +112,17 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { if (patch_type == PatchType::fine) { PEC::ApplyPECtoBfield( { - m_fields.get("Bfield_fp",Direction{0},lev), - m_fields.get("Bfield_fp",Direction{1},lev), - m_fields.get("Bfield_fp",Direction{2},lev) }, + m_fields.get(FieldType::Bfield_fp,Direction{0},lev), + m_fields.get(FieldType::Bfield_fp,Direction{1},lev), + m_fields.get(FieldType::Bfield_fp,Direction{2},lev) }, field_boundary_lo, field_boundary_hi, get_ng_fieldgather(), Geom(lev), lev, patch_type, ref_ratio); } else { PEC::ApplyPECtoBfield( { - m_fields.get("Bfield_cp",Direction{0},lev), - m_fields.get("Bfield_cp",Direction{1},lev), - m_fields.get("Bfield_cp",Direction{2},lev) }, + m_fields.get(FieldType::Bfield_cp,Direction{0},lev), + m_fields.get(FieldType::Bfield_cp,Direction{1},lev), + m_fields.get(FieldType::Bfield_cp,Direction{2},lev) }, field_boundary_lo, field_boundary_hi, get_ng_fieldgather(), Geom(lev), lev, patch_type, ref_ratio); @@ -135,8 +135,8 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d if (lev == 0) { if (a_dt_type == DtType::FirstHalf) { if(::isAnyBoundary(field_boundary_lo, field_boundary_hi)){ - auto Efield_fp = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); - auto Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp",max_level); + auto Efield_fp = m_fields.get_mr_levels_alldirs(FieldType::Efield_fp, max_level); + auto Bfield_fp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_fp, max_level); m_fdtd_solver_fp[0]->ApplySilverMuellerBoundary( Efield_fp[lev], Bfield_fp[lev], Geom(lev).Domain(), dt[lev], @@ -147,13 +147,13 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d #ifdef WARPX_DIM_RZ if (patch_type == PatchType::fine) { - ApplyFieldBoundaryOnAxis(m_fields.get("Bfield_fp",Direction{0},lev), - m_fields.get("Bfield_fp",Direction{1},lev), - m_fields.get("Bfield_fp",Direction{2},lev), lev); + ApplyFieldBoundaryOnAxis(m_fields.get(FieldType::Bfield_fp,Direction{0},lev), + m_fields.get(FieldType::Bfield_fp,Direction{1},lev), + m_fields.get(FieldType::Bfield_fp,Direction{2},lev), lev); } else { - ApplyFieldBoundaryOnAxis(m_fields.get("Bfield_cp",Direction{0},lev), - m_fields.get("Bfield_cp",Direction{1},lev), - m_fields.get("Bfield_cp",Direction{2},lev), lev); + ApplyFieldBoundaryOnAxis(m_fields.get(FieldType::Bfield_cp,Direction{0},lev), + m_fields.get(FieldType::Bfield_cp,Direction{1},lev), + m_fields.get(FieldType::Bfield_cp,Direction{2},lev), lev); } #endif } @@ -274,7 +274,7 @@ void WarpX::ApplyElectronPressureBoundary (const int lev, PatchType patch_type) { if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { if (patch_type == PatchType::fine) { - ablastr::fields::ScalarField electron_pressure_fp = m_fields.get("hybrid_electron_pressure_fp", lev); + ablastr::fields::ScalarField electron_pressure_fp = m_fields.get(FieldType::hybrid_electron_pressure_fp, lev); PEC::ApplyPECtoElectronPressure( electron_pressure_fp, field_boundary_lo, field_boundary_hi, diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index ab14dd3b86e..e00c30aa78e 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -48,7 +48,7 @@ #include using namespace amrex::literals; -using namespace warpx::fields; +using warpx::fields::FieldType; namespace { @@ -570,23 +570,23 @@ BTDiagnostics::InitializeFieldFunctors (int lev) m_cell_center_functors.at(lev).size()); for (int comp=0; comp(warpx.m_fields.get("Efield_aux", Direction{0}, lev), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "Ey" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Efield_aux", Direction{1}, lev), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "Ez" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Efield_aux", Direction{2}, lev), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{2}, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "Bx" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{0}, lev), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "By" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{1}, lev), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "Bz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{2}, lev), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{2}, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "jx" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp",Direction{0}, lev), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::current_fp,Direction{0}, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "jy" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp",Direction{1}, lev), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::current_fp,Direction{1}, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "jz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp",Direction{2}, lev), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::current_fp,Direction{2}, lev), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "rho" ){ m_cell_center_functors[lev][comp] = std::make_unique(lev, m_crse_ratio); } @@ -602,7 +602,7 @@ BTDiagnostics::UpdateVarnamesForRZopenPMD () #ifdef WARPX_DIM_RZ auto & warpx = WarpX::GetInstance(); using ablastr::fields::Direction; - const int ncomp_multimodefab = warpx.m_fields.get("Efield_aux", Direction{0}, 0)->nComp(); + const int ncomp_multimodefab = warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, 0)->nComp(); const int ncomp = ncomp_multimodefab; @@ -663,7 +663,7 @@ BTDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) using ablastr::fields::Direction; auto & warpx = WarpX::GetInstance(); - const int ncomp_multimodefab = warpx.m_fields.get("Efield_aux", Direction{0}, 0)->nComp(); + const int ncomp_multimodefab = warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, 0)->nComp(); const int ncomp = ncomp_multimodefab; // Clear any pre-existing vector to release stored data // This ensures that when domain is load-balanced, the functors point @@ -689,23 +689,23 @@ BTDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) const auto m_cell_center_functors_at_lev_size = static_cast(m_cell_center_functors.at(lev).size()); for (int comp=0; comp(warpx.m_fields.get("Efield_aux", Direction{0}, lev), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Et" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Efield_aux", Direction{1}, lev), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Ez" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Efield_aux", Direction{2}, lev), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{2}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Br" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{0}, lev), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Bt" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{1}, lev), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Bz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{2}, lev), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{2}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "jr" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", Direction{0}, lev), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::current_fp, Direction{0}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "jt" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", Direction{1}, lev), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::current_fp, Direction{1}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "jz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get("current_fp", Direction{2}, lev), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::current_fp, Direction{2}, lev), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "rho" ){ m_cell_center_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, false, -1, false, ncomp); } diff --git a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp index 7f532c79e13..df25bf7ff03 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp @@ -17,7 +17,7 @@ #include #include -using namespace warpx::fields; +using warpx::fields::FieldType; JFunctor::JFunctor (const int dir, int lev, amrex::IntVect crse_ratio, @@ -35,14 +35,14 @@ JFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/ auto& warpx = WarpX::GetInstance(); /** pointer to source multifab (can be multi-component) */ - amrex::MultiFab* m_mf_src = warpx.m_fields.get("current_fp",Direction{m_dir},m_lev); + amrex::MultiFab* m_mf_src = warpx.m_fields.get(FieldType::current_fp,Direction{m_dir},m_lev); // Deposit current if no solver or the electrostatic solver is being used if (m_deposit_current) { // allocate temporary multifab to deposit current density into ablastr::fields::MultiLevelVectorField current_fp_temp { - warpx.m_fields.get_alldirs("current_fp", m_lev) + warpx.m_fields.get_alldirs(FieldType::current_fp, m_lev) }; auto& mypc = warpx.GetPartContainer(); diff --git a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp index 0e4a405c499..b4f286506a8 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp @@ -16,7 +16,7 @@ #include using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; JdispFunctor::JdispFunctor (int dir, int lev, amrex::IntVect crse_ratio, bool convertRZmodes2cartesian, int ncomp) @@ -33,14 +33,14 @@ JdispFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buff auto* hybrid_pic_model = warpx.get_pointer_HybridPICModel(); /** pointer to total simulation current (J) multifab */ - amrex::MultiFab* mf_j = warpx.m_fields.get("current_fp", Direction{m_dir}, m_lev); + amrex::MultiFab* mf_j = warpx.m_fields.get(FieldType::current_fp, Direction{m_dir}, m_lev); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(hybrid_pic_model, "Displacement current diagnostic is only implemented for the HybridPICModel."); AMREX_ASSUME(hybrid_pic_model != nullptr); /** pointer to current calculated from Ampere's Law (Jamp) multifab */ - amrex::MultiFab* mf_curlB = warpx.m_fields.get("hybrid_current_fp_ampere", Direction{m_dir}, m_lev); + amrex::MultiFab* mf_curlB = warpx.m_fields.get(FieldType::hybrid_current_fp_ampere, Direction{m_dir}, m_lev); //if (!hybrid_pic_model) { // To finish this implementation, we need to implement a method to @@ -66,7 +66,7 @@ JdispFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buff if (hybrid_pic_model) { // Subtract the interpolated j_external value from j_displacement. /** pointer to external currents (Jext) multifab */ - amrex::MultiFab* mf_j_external = warpx.m_fields.get("hybrid_current_fp_external", Direction{m_dir}, m_lev); + amrex::MultiFab* mf_j_external = warpx.m_fields.get(FieldType::hybrid_current_fp_external, Direction{m_dir}, m_lev); // Index type required for interpolating Jext from their respective // staggering (nodal) to the Jx_displacement, Jy_displacement, Jz_displacement diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index 2020be2d4b7..4d721dd6abe 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -22,7 +22,7 @@ #include using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; namespace { @@ -68,85 +68,85 @@ FlushFormatCheckpoint::WriteToFile ( for (int lev = 0; lev < nlev; ++lev) { - VisMF::Write(*warpx.m_fields.get("Efield_fp", Direction{0}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::Efield_fp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ex_fp")); - VisMF::Write(*warpx.m_fields.get("Efield_fp", Direction{1}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::Efield_fp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ey_fp")); - VisMF::Write(*warpx.m_fields.get("Efield_fp", Direction{2}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::Efield_fp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ez_fp")); - VisMF::Write(*warpx.m_fields.get("Bfield_fp", Direction{0}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::Bfield_fp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bx_fp")); - VisMF::Write(*warpx.m_fields.get("Bfield_fp", Direction{1}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::Bfield_fp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "By_fp")); - VisMF::Write(*warpx.m_fields.get("Bfield_fp", Direction{2}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::Bfield_fp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bz_fp")); if (WarpX::fft_do_time_averaging) { - VisMF::Write(*warpx.m_fields.get("Efield_avg_fp", Direction{0}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::Efield_avg_fp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ex_avg_fp")); - VisMF::Write(*warpx.m_fields.get("Efield_avg_fp", Direction{1}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::Efield_avg_fp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ey_avg_fp")); - VisMF::Write(*warpx.m_fields.get("Efield_avg_fp", Direction{2}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::Efield_avg_fp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ez_avg_fp")); - VisMF::Write(*warpx.m_fields.get("Bfield_avg_fp", Direction{0}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::Bfield_avg_fp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bx_avg_fp")); - VisMF::Write(*warpx.m_fields.get("Bfield_avg_fp", Direction{1}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::Bfield_avg_fp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "By_avg_fp")); - VisMF::Write(*warpx.m_fields.get("Bfield_avg_fp", Direction{2}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::Bfield_avg_fp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bz_avg_fp")); } if (warpx.getis_synchronized()) { // Need to save j if synchronized because after restart we need j to evolve E by dt/2. - VisMF::Write(*warpx.m_fields.get("current_fp", Direction{0}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::current_fp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jx_fp")); - VisMF::Write(*warpx.m_fields.get("current_fp", Direction{1}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::current_fp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jy_fp")); - VisMF::Write(*warpx.m_fields.get("current_fp", Direction{2}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::current_fp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jz_fp")); } if (lev > 0) { - VisMF::Write(*warpx.m_fields.get("Efield_cp", Direction{0}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::Efield_cp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ex_cp")); - VisMF::Write(*warpx.m_fields.get("Efield_cp", Direction{1}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::Efield_cp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ey_cp")); - VisMF::Write(*warpx.m_fields.get("Efield_cp", Direction{2}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::Efield_cp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ez_cp")); - VisMF::Write(*warpx.m_fields.get("Bfield_cp", Direction{0}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::Bfield_cp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bx_cp")); - VisMF::Write(*warpx.m_fields.get("Bfield_cp", Direction{1}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::Bfield_cp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "By_cp")); - VisMF::Write(*warpx.m_fields.get("Bfield_cp", Direction{2}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::Bfield_cp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bz_cp")); if (WarpX::fft_do_time_averaging) { - VisMF::Write(*warpx.m_fields.get("Efield_avg_cp", Direction{0}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::Efield_avg_cp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ex_avg_cp")); - VisMF::Write(*warpx.m_fields.get("Efield_avg_cp", Direction{1}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::Efield_avg_cp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ey_avg_cp")); - VisMF::Write(*warpx.m_fields.get("Efield_avg_cp", Direction{2}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::Efield_avg_cp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ez_avg_cp")); - VisMF::Write(*warpx.m_fields.get("Bfield_avg_cp", Direction{0}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::Bfield_avg_cp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bx_avg_cp")); - VisMF::Write(*warpx.m_fields.get("Bfield_avg_cp", Direction{1}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::Bfield_avg_cp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "By_avg_cp")); - VisMF::Write(*warpx.m_fields.get("Bfield_avg_cp", Direction{2}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::Bfield_avg_cp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bz_avg_cp")); } if (warpx.getis_synchronized()) { // Need to save j if synchronized because after restart we need j to evolve E by dt/2. - VisMF::Write(*warpx.m_fields.get("current_cp", Direction{0}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::current_cp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jx_cp")); - VisMF::Write(*warpx.m_fields.get("current_cp", Direction{1}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::current_cp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jy_cp")); - VisMF::Write(*warpx.m_fields.get("current_cp", Direction{2}, lev), + VisMF::Write(*warpx.m_fields.get(FieldType::current_cp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jz_cp")); } } diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index 1731851dc3b..0f05496e4c0 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -50,7 +50,7 @@ #include using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; namespace { @@ -568,103 +568,103 @@ FlushFormatPlotfile::WriteAllRawFields( // Auxiliary patch - WriteRawMF( *warpx.m_fields.get("Efield_aux", Direction{0}, lev), dm, raw_pltname, default_level_prefix, "Ex_aux", lev, plot_raw_fields_guards); - WriteRawMF( *warpx.m_fields.get("Efield_aux", Direction{1}, lev), dm, raw_pltname, default_level_prefix, "Ey_aux", lev, plot_raw_fields_guards); - WriteRawMF( *warpx.m_fields.get("Efield_aux", Direction{2}, lev), dm, raw_pltname, default_level_prefix, "Ez_aux", lev, plot_raw_fields_guards); - WriteRawMF( *warpx.m_fields.get("Bfield_aux", Direction{0}, lev), dm, raw_pltname, default_level_prefix, "Bx_aux", lev, plot_raw_fields_guards); - WriteRawMF( *warpx.m_fields.get("Bfield_aux", Direction{1}, lev), dm, raw_pltname, default_level_prefix, "By_aux", lev, plot_raw_fields_guards); - WriteRawMF( *warpx.m_fields.get("Bfield_aux", Direction{2}, lev), dm, raw_pltname, default_level_prefix, "Bz_aux", lev, plot_raw_fields_guards); + WriteRawMF( *warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev), dm, raw_pltname, default_level_prefix, "Ex_aux", lev, plot_raw_fields_guards); + WriteRawMF( *warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, lev), dm, raw_pltname, default_level_prefix, "Ey_aux", lev, plot_raw_fields_guards); + WriteRawMF( *warpx.m_fields.get(FieldType::Efield_aux, Direction{2}, lev), dm, raw_pltname, default_level_prefix, "Ez_aux", lev, plot_raw_fields_guards); + WriteRawMF( *warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, lev), dm, raw_pltname, default_level_prefix, "Bx_aux", lev, plot_raw_fields_guards); + WriteRawMF( *warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, lev), dm, raw_pltname, default_level_prefix, "By_aux", lev, plot_raw_fields_guards); + WriteRawMF( *warpx.m_fields.get(FieldType::Bfield_aux, Direction{2}, lev), dm, raw_pltname, default_level_prefix, "Bz_aux", lev, plot_raw_fields_guards); // fine patch - WriteRawMF( *warpx.m_fields.get("Efield_fp", Direction{0}, lev), dm, raw_pltname, + WriteRawMF( *warpx.m_fields.get(FieldType::Efield_fp, Direction{0}, lev), dm, raw_pltname, default_level_prefix, "Ex_fp", lev, plot_raw_fields_guards ); - WriteRawMF( *warpx.m_fields.get("Efield_fp", Direction{1}, lev), dm, raw_pltname, + WriteRawMF( *warpx.m_fields.get(FieldType::Efield_fp, Direction{1}, lev), dm, raw_pltname, default_level_prefix, "Ey_fp", lev, plot_raw_fields_guards ); - WriteRawMF( *warpx.m_fields.get("Efield_fp", Direction{2}, lev), dm, raw_pltname, + WriteRawMF( *warpx.m_fields.get(FieldType::Efield_fp, Direction{2}, lev), dm, raw_pltname, default_level_prefix, "Ez_fp", lev, plot_raw_fields_guards ); - WriteRawMF( *warpx.m_fields.get("current_fp",Direction{0}, lev), dm, raw_pltname, + WriteRawMF( *warpx.m_fields.get(FieldType::current_fp,Direction{0}, lev), dm, raw_pltname, default_level_prefix, "jx_fp", lev,plot_raw_fields_guards ); - WriteRawMF( *warpx.m_fields.get("current_fp",Direction{1}, lev), dm, raw_pltname, + WriteRawMF( *warpx.m_fields.get(FieldType::current_fp,Direction{1}, lev), dm, raw_pltname, default_level_prefix, "jy_fp", lev,plot_raw_fields_guards ); - WriteRawMF( *warpx.m_fields.get("current_fp",Direction{2}, lev), dm, raw_pltname, + WriteRawMF( *warpx.m_fields.get(FieldType::current_fp,Direction{2}, lev), dm, raw_pltname, default_level_prefix, "jz_fp", lev,plot_raw_fields_guards ); - WriteRawMF( *warpx.m_fields.get("Bfield_fp", Direction{0}, lev), dm, raw_pltname, + WriteRawMF( *warpx.m_fields.get(FieldType::Bfield_fp, Direction{0}, lev), dm, raw_pltname, default_level_prefix, "Bx_fp", lev, plot_raw_fields_guards ); - WriteRawMF( *warpx.m_fields.get("Bfield_fp", Direction{1}, lev), dm, raw_pltname, + WriteRawMF( *warpx.m_fields.get(FieldType::Bfield_fp, Direction{1}, lev), dm, raw_pltname, default_level_prefix, "By_fp", lev, plot_raw_fields_guards ); - WriteRawMF( *warpx.m_fields.get("Bfield_fp", Direction{2}, lev), dm, raw_pltname, + WriteRawMF( *warpx.m_fields.get(FieldType::Bfield_fp, Direction{2}, lev), dm, raw_pltname, default_level_prefix, "Bz_fp", lev, plot_raw_fields_guards ); - if (warpx.m_fields.has("F_fp", lev)) + if (warpx.m_fields.has(FieldType::F_fp, lev)) { - WriteRawMF( *warpx.m_fields.get("F_fp", lev), dm, raw_pltname, + WriteRawMF( *warpx.m_fields.get(FieldType::F_fp, lev), dm, raw_pltname, default_level_prefix, "F_fp", lev, plot_raw_fields_guards ); } - if (warpx.m_fields.has("rho_fp", lev)) + if (warpx.m_fields.has(FieldType::rho_fp, lev)) { // rho_fp will have either ncomps or 2*ncomps (2 being the old and new). When 2, return the new so // there is time synchronization. - const int nstart = warpx.m_fields.get("rho_fp", lev)->nComp() - WarpX::ncomps; - const MultiFab rho_new(*warpx.m_fields.get("rho_fp", lev), amrex::make_alias, nstart, WarpX::ncomps); + const int nstart = warpx.m_fields.get(FieldType::rho_fp, lev)->nComp() - WarpX::ncomps; + const MultiFab rho_new(*warpx.m_fields.get(FieldType::rho_fp, lev), amrex::make_alias, nstart, WarpX::ncomps); WriteRawMF(rho_new, dm, raw_pltname, default_level_prefix, "rho_fp", lev, plot_raw_fields_guards); } - if (warpx.m_fields.has("phi_fp", lev)) { - WriteRawMF( *warpx.m_fields.get("phi_fp", lev), dm, raw_pltname, + if (warpx.m_fields.has(FieldType::phi_fp, lev)) { + WriteRawMF( *warpx.m_fields.get(FieldType::phi_fp, lev), dm, raw_pltname, default_level_prefix, "phi_fp", lev, plot_raw_fields_guards ); } // Averaged fields on fine patch if (WarpX::fft_do_time_averaging) { - WriteRawMF(*warpx.m_fields.get("Efield_avg_fp", Direction{0}, lev) , dm, raw_pltname, default_level_prefix, + WriteRawMF(*warpx.m_fields.get(FieldType::Efield_avg_fp, Direction{0}, lev) , dm, raw_pltname, default_level_prefix, "Ex_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(*warpx.m_fields.get("Efield_avg_fp", Direction{1}, lev) , dm, raw_pltname, default_level_prefix, + WriteRawMF(*warpx.m_fields.get(FieldType::Efield_avg_fp, Direction{1}, lev) , dm, raw_pltname, default_level_prefix, "Ey_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(*warpx.m_fields.get("Efield_avg_fp", Direction{2}, lev) , dm, raw_pltname, default_level_prefix, + WriteRawMF(*warpx.m_fields.get(FieldType::Efield_avg_fp, Direction{2}, lev) , dm, raw_pltname, default_level_prefix, "Ez_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(*warpx.m_fields.get("Bfield_avg_fp", Direction{0}, lev) , dm, raw_pltname, default_level_prefix, + WriteRawMF(*warpx.m_fields.get(FieldType::Bfield_avg_fp, Direction{0}, lev) , dm, raw_pltname, default_level_prefix, "Bx_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(*warpx.m_fields.get("Bfield_avg_fp", Direction{1}, lev) , dm, raw_pltname, default_level_prefix, + WriteRawMF(*warpx.m_fields.get(FieldType::Bfield_avg_fp, Direction{1}, lev) , dm, raw_pltname, default_level_prefix, "By_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(*warpx.m_fields.get("Bfield_avg_fp", Direction{2}, lev) , dm, raw_pltname, default_level_prefix, + WriteRawMF(*warpx.m_fields.get(FieldType::Bfield_avg_fp, Direction{2}, lev) , dm, raw_pltname, default_level_prefix, "Bz_avg_fp", lev, plot_raw_fields_guards); } // Coarse path if (lev > 0) { WriteCoarseVector( "E", - warpx.m_fields.get("Efield_cp", Direction{0}, lev), - warpx.m_fields.get("Efield_cp", Direction{1}, lev), - warpx.m_fields.get("Efield_cp", Direction{2}, lev), - warpx.m_fields.get("Efield_fp", Direction{0}, lev), - warpx.m_fields.get("Efield_fp", Direction{1}, lev), - warpx.m_fields.get("Efield_fp", Direction{2}, lev), + warpx.m_fields.get(FieldType::Efield_cp, Direction{0}, lev), + warpx.m_fields.get(FieldType::Efield_cp, Direction{1}, lev), + warpx.m_fields.get(FieldType::Efield_cp, Direction{2}, lev), + warpx.m_fields.get(FieldType::Efield_fp, Direction{0}, lev), + warpx.m_fields.get(FieldType::Efield_fp, Direction{1}, lev), + warpx.m_fields.get(FieldType::Efield_fp, Direction{2}, lev), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards); WriteCoarseVector( "B", - warpx.m_fields.get("Bfield_cp", Direction{0}, lev), - warpx.m_fields.get("Bfield_cp", Direction{1}, lev), - warpx.m_fields.get("Bfield_cp", Direction{2}, lev), - warpx.m_fields.get("Bfield_fp", Direction{0}, lev), - warpx.m_fields.get("Bfield_fp", Direction{1}, lev), - warpx.m_fields.get("Bfield_fp", Direction{2}, lev), + warpx.m_fields.get(FieldType::Bfield_cp, Direction{0}, lev), + warpx.m_fields.get(FieldType::Bfield_cp, Direction{1}, lev), + warpx.m_fields.get(FieldType::Bfield_cp, Direction{2}, lev), + warpx.m_fields.get(FieldType::Bfield_fp, Direction{0}, lev), + warpx.m_fields.get(FieldType::Bfield_fp, Direction{1}, lev), + warpx.m_fields.get(FieldType::Bfield_fp, Direction{2}, lev), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards); WriteCoarseVector( "j", - warpx.m_fields.get("current_cp", Direction{0}, lev), warpx.m_fields.get("current_cp", Direction{1}, lev), warpx.m_fields.get("current_cp", Direction{2}, lev), - warpx.m_fields.get("current_fp", Direction{0}, lev), warpx.m_fields.get("current_fp", Direction{1}, lev), warpx.m_fields.get("current_fp", Direction{2}, lev), + warpx.m_fields.get(FieldType::current_cp, Direction{0}, lev), warpx.m_fields.get(FieldType::current_cp, Direction{1}, lev), warpx.m_fields.get(FieldType::current_cp, Direction{2}, lev), + warpx.m_fields.get(FieldType::current_fp, Direction{0}, lev), warpx.m_fields.get(FieldType::current_fp, Direction{1}, lev), warpx.m_fields.get(FieldType::current_fp, Direction{2}, lev), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards); - if (warpx.m_fields.has("F_fp", lev) && warpx.m_fields.has("F_cp", lev)) + if (warpx.m_fields.has(FieldType::F_fp, lev) && warpx.m_fields.has(FieldType::F_cp, lev)) { - WriteCoarseScalar("F", warpx.m_fields.get("F_cp", lev), warpx.m_fields.get("F_fp", lev), + WriteCoarseScalar("F", warpx.m_fields.get(FieldType::F_cp, lev), warpx.m_fields.get(FieldType::F_fp, lev), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards, 0); } - if (warpx.m_fields.has("rho_fp", lev) && warpx.m_fields.has("rho_cp", lev)) + if (warpx.m_fields.has(FieldType::rho_fp, lev) && warpx.m_fields.has(FieldType::rho_cp, lev)) { // Use the component 1 of `rho_cp`, i.e. rho_new for time synchronization - WriteCoarseScalar("rho", warpx.m_fields.get("rho_cp", lev), warpx.m_fields.get("rho_fp", lev), + WriteCoarseScalar("rho", warpx.m_fields.get(FieldType::rho_cp, lev), warpx.m_fields.get(FieldType::rho_fp, lev), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards, 1); } } diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index 2b4c63880a8..e5eefc82de5 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -45,7 +45,7 @@ #include using namespace amrex::literals; -using namespace warpx::fields; +using warpx::fields::FieldType; FullDiagnostics::FullDiagnostics (int i, const std::string& name): Diagnostics{i, name}, @@ -177,15 +177,15 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) using ablastr::fields::Direction; auto & warpx = WarpX::GetInstance(); - const int ncomp_multimodefab = warpx.m_fields.get("Efield_aux", Direction{0}, 0)->nComp(); + const int ncomp_multimodefab = warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, 0)->nComp(); // Make sure all multifabs have the same number of components for (int dim=0; dim<3; dim++){ AMREX_ALWAYS_ASSERT( - warpx.m_fields.get("Efield_aux", Direction{dim}, lev)->nComp() == ncomp_multimodefab ); + warpx.m_fields.get(FieldType::Efield_aux, Direction{dim}, lev)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( - warpx.m_fields.get("Bfield_aux", Direction{dim}, lev)->nComp() == ncomp_multimodefab ); + warpx.m_fields.get(FieldType::Bfield_aux, Direction{dim}, lev)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( - warpx.m_fields.get("current_fp", Direction{dim}, lev)->nComp() == ncomp_multimodefab ); + warpx.m_fields.get(FieldType::current_fp, Direction{dim}, lev)->nComp() == ncomp_multimodefab ); } // Species index to loop over species that dump rho per species @@ -220,37 +220,37 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) const auto m_varname_fields_size = static_cast(m_varnames_fields.size()); for (int comp=0; comp(warpx.m_fields.get("Efield_aux", Direction{0}, lev), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Er"), ncomp); } } else if ( m_varnames_fields[comp] == "Et" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Efield_aux", Direction{1}, lev), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Et"), ncomp); } } else if ( m_varnames_fields[comp] == "Ez" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Efield_aux", Direction{2}, lev), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{2}, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Ez"), ncomp); } } else if ( m_varnames_fields[comp] == "Br" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{0}, lev), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Br"), ncomp); } } else if ( m_varnames_fields[comp] == "Bt" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{1}, lev), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Bt"), ncomp); } } else if ( m_varnames_fields[comp] == "Bz" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{2}, lev), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{2}, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Bz"), ncomp); @@ -317,19 +317,19 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) } i_T_species++; } else if ( m_varnames_fields[comp] == "F" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("F_fp", lev), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::F_fp, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("F"), ncomp); } } else if ( m_varnames_fields[comp] == "G" ){ - m_all_field_functors[lev][comp] = std::make_unique( warpx.m_fields.get("G_fp", lev), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique( warpx.m_fields.get(FieldType::G_fp, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("G"), ncomp); } } else if ( m_varnames_fields[comp] == "phi" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("phi_fp", lev), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::phi_fp, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("phi"), ncomp); @@ -346,14 +346,14 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) } } else if ( m_varnames_fields[comp] == "divB" ){ m_all_field_functors[lev][comp] = std::make_unique( - warpx.m_fields.get_alldirs("Bfield_aux", lev), + warpx.m_fields.get_alldirs(FieldType::Bfield_aux, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("divB"), ncomp); } } else if ( m_varnames_fields[comp] == "divE" ){ m_all_field_functors[lev][comp] = std::make_unique( - warpx.m_fields.get_alldirs("Efield_aux", lev), + warpx.m_fields.get_alldirs(FieldType::Efield_aux, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("divE"), ncomp); @@ -401,15 +401,15 @@ FullDiagnostics::AddRZModesToDiags (int lev) if (!m_dump_rz_modes) { return; } auto & warpx = WarpX::GetInstance(); - const int ncomp_multimodefab = warpx.m_fields.get("Efield_aux", Direction{0}, 0)->nComp(); + const int ncomp_multimodefab = warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, 0)->nComp(); // Make sure all multifabs have the same number of components for (int dim=0; dim<3; dim++){ AMREX_ALWAYS_ASSERT( - warpx.m_fields.get("Efield_aux", Direction{dim}, lev)->nComp() == ncomp_multimodefab ); + warpx.m_fields.get(FieldType::Efield_aux, Direction{dim}, lev)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( - warpx.m_fields.get("Bfield_aux", Direction{dim}, lev)->nComp() == ncomp_multimodefab ); + warpx.m_fields.get(FieldType::Bfield_aux, Direction{dim}, lev)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( - warpx.m_fields.get("current_fp", Direction{dim}, lev)->nComp() == ncomp_multimodefab ); + warpx.m_fields.get(FieldType::current_fp, Direction{dim}, lev)->nComp() == ncomp_multimodefab ); } // Check if divE is requested @@ -444,19 +444,19 @@ FullDiagnostics::AddRZModesToDiags (int lev) for (int dim=0; dim<3; dim++){ // 3 components, r theta z m_all_field_functors[lev].push_back(std::make_unique( - warpx.m_fields.get("Efield_aux", Direction{dim}, lev), lev, + warpx.m_fields.get(FieldType::Efield_aux, Direction{dim}, lev), lev, m_crse_ratio, false, ncomp_multimodefab)); AddRZModesToOutputNames(std::string("E") + coord[dim], - warpx.m_fields.get("Efield_aux", Direction{0}, 0)->nComp()); + warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, 0)->nComp()); } // B for (int dim=0; dim<3; dim++){ // 3 components, r theta z m_all_field_functors[lev].push_back(std::make_unique( - warpx.m_fields.get("Bfield_aux", Direction{dim}, lev), lev, + warpx.m_fields.get(FieldType::Bfield_aux, Direction{dim}, lev), lev, m_crse_ratio, false, ncomp_multimodefab)); AddRZModesToOutputNames(std::string("B") + coord[dim], - warpx.m_fields.get("Bfield_aux", Direction{0}, 0)->nComp()); + warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, 0)->nComp()); } // j for (int dim=0; dim<3; dim++){ @@ -465,12 +465,12 @@ FullDiagnostics::AddRZModesToDiags (int lev) dim, lev, m_crse_ratio, false, deposit_current, ncomp_multimodefab)); deposit_current = false; AddRZModesToOutputNames(std::string("J") + coord[dim], - warpx.m_fields.get("current_fp",Direction{0},0)->nComp()); + warpx.m_fields.get(FieldType::current_fp,Direction{0},0)->nComp()); } // divE if (divE_requested) { m_all_field_functors[lev].push_back(std::make_unique( - warpx.m_fields.get_alldirs("Efield_aux", lev), + warpx.m_fields.get_alldirs(FieldType::Efield_aux, lev), lev, m_crse_ratio, false, ncomp_multimodefab)); AddRZModesToOutputNames(std::string("divE"), ncomp_multimodefab); } @@ -668,16 +668,16 @@ FullDiagnostics::InitializeFieldFunctors (int lev) // Fill vector of functors for all components except individual cylindrical modes. for (int comp=0; comp(warpx.m_fields.get("Efield_aux", Direction{2}, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{2}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Bz" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{2}, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{2}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "jz" ){ m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, true, deposit_current); deposit_current = false; } else if ( m_varnames[comp] == "jz_displacement" ) { m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, true); } else if ( m_varnames[comp] == "Az" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("vector_potential_fp_nodal", Direction{2}, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::vector_potential_fp_nodal, Direction{2}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "rho" ){ // Initialize rho functor to dump total rho m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true); @@ -690,31 +690,31 @@ FullDiagnostics::InitializeFieldFunctors (int lev) m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, m_T_per_species_index[i_T_species]); i_T_species++; } else if ( m_varnames[comp] == "F" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("F_fp", lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::F_fp, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "G" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("G_fp", lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::G_fp, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "phi" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("phi_fp", lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::phi_fp, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "part_per_cell" ){ m_all_field_functors[lev][comp] = std::make_unique(nullptr, lev, m_crse_ratio); } else if ( m_varnames[comp] == "part_per_grid" ){ m_all_field_functors[lev][comp] = std::make_unique(nullptr, lev, m_crse_ratio); } else if ( m_varnames[comp] == "divB" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get_alldirs("Bfield_aux", lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get_alldirs(FieldType::Bfield_aux, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "divE" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get_alldirs("Efield_aux", lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get_alldirs(FieldType::Efield_aux, lev), lev, m_crse_ratio); } else { #ifdef WARPX_DIM_RZ if ( m_varnames[comp] == "Er" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Efield_aux", Direction{0}, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Et" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Efield_aux", Direction{1}, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Br" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{0}, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Bt" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{1}, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "jr" ){ m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, true, deposit_current); deposit_current = false; @@ -726,22 +726,22 @@ FullDiagnostics::InitializeFieldFunctors (int lev) } else if (m_varnames[comp] == "jt_displacement" ){ m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, true); } else if ( m_varnames[comp] == "Ar" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("vector_potential_fp_nodal", Direction{0}, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::vector_potential_fp_nodal, Direction{0}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "At" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("vector_potential_fp_nodal", Direction{1}, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::vector_potential_fp_nodal, Direction{1}, lev), lev, m_crse_ratio); } else { WARPX_ABORT_WITH_MESSAGE(m_varnames[comp] + " is not a known field output type for RZ geometry"); } #else // Valid transverse fields in Cartesian coordinates if ( m_varnames[comp] == "Ex" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Efield_aux", Direction{0}, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Ey" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Efield_aux", Direction{1}, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Bx" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{0}, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "By" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("Bfield_aux", Direction{1}, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "jx" ){ m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, true, deposit_current); deposit_current = false; @@ -753,9 +753,9 @@ FullDiagnostics::InitializeFieldFunctors (int lev) } else if ( m_varnames[comp] == "jy_displacement" ){ m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio); } else if ( m_varnames[comp] == "Ax" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("vector_potential_fp_nodal", Direction{0}, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::vector_potential_fp_nodal, Direction{0}, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Ay" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get("vector_potential_fp_nodal", Direction{1}, lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::vector_potential_fp_nodal, Direction{1}, lev), lev, m_crse_ratio); } else { std::cout << "Error on component " << m_varnames[comp] << std::endl; WARPX_ABORT_WITH_MESSAGE(m_varnames[comp] + " is not a known field output type for this geometry"); diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index 51b28bc9a10..e94039ec079 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -43,7 +43,7 @@ #include using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; void LaserParticleContainer::ReadHeader (std::istream& is) @@ -268,7 +268,7 @@ storePhiOnParticles ( PinnedMemoryParticleContainer& tmp, const amrex::Geometry& geom = warpx.Geom(lev); auto plo = geom.ProbLoArray(); auto dxi = geom.InvCellSizeArray(); - amrex::MultiFab const& phi = *warpx.m_fields.get("phi_fp", lev); + amrex::MultiFab const& phi = *warpx.m_fields.get(FieldType::phi_fp, lev); for (PinnedParIter pti(tmp, lev); pti.isValid(); ++pti) { diff --git a/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp b/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp index d5cf391a254..d3592c6c6f2 100644 --- a/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp +++ b/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp @@ -29,7 +29,7 @@ #include using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; // constructor @@ -106,9 +106,9 @@ void ChargeOnEB::ComputeDiags (const int step) int const lev = 0; // get MultiFab data at lev - const amrex::MultiFab & Ex = *warpx.m_fields.get("Efield_fp",Direction{0},lev); - const amrex::MultiFab & Ey = *warpx.m_fields.get("Efield_fp",Direction{1},lev); - const amrex::MultiFab & Ez = *warpx.m_fields.get("Efield_fp",Direction{2},lev); + const amrex::MultiFab & Ex = *warpx.m_fields.get(FieldType::Efield_fp, Direction{0}, lev); + const amrex::MultiFab & Ey = *warpx.m_fields.get(FieldType::Efield_fp, Direction{1}, lev); + const amrex::MultiFab & Ez = *warpx.m_fields.get(FieldType::Efield_fp, Direction{2}, lev); // get EB structures amrex::EBFArrayBoxFactory const& eb_box_factory = warpx.fieldEBFactory(lev); diff --git a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp index 463e6e160cd..c0532163232 100644 --- a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp @@ -59,7 +59,7 @@ #include using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; ColliderRelevant::ColliderRelevant (const std::string& rd_name) : ReducedDiags{rd_name} @@ -444,12 +444,12 @@ void ColliderRelevant::ComputeDiags (int step) // define variables in preparation for field gathering const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(lev, 0)); - const amrex::MultiFab & Ex = *warpx.m_fields.get("Efield_aux", Direction{0}, lev); - const amrex::MultiFab & Ey = *warpx.m_fields.get("Efield_aux", Direction{1}, lev); - const amrex::MultiFab & Ez = *warpx.m_fields.get("Efield_aux", Direction{2}, lev); - const amrex::MultiFab & Bx = *warpx.m_fields.get("Bfield_aux", Direction{0}, lev); - const amrex::MultiFab & By = *warpx.m_fields.get("Bfield_aux", Direction{1}, lev); - const amrex::MultiFab & Bz = *warpx.m_fields.get("Bfield_aux", Direction{2}, lev); + const amrex::MultiFab & Ex = *warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev); + const amrex::MultiFab & Ey = *warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, lev); + const amrex::MultiFab & Ez = *warpx.m_fields.get(FieldType::Efield_aux, Direction{2}, lev); + const amrex::MultiFab & Bx = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, lev); + const amrex::MultiFab & By = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, lev); + const amrex::MultiFab & Bz = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{2}, lev); // declare reduce_op ReduceOps reduce_op; diff --git a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp index 84b6df87af8..1a984368b4e 100644 --- a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp @@ -31,7 +31,7 @@ #include using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; // constructor FieldEnergy::FieldEnergy (const std::string& rd_name) @@ -95,12 +95,12 @@ void FieldEnergy::ComputeDiags (int step) for (int lev = 0; lev < nLevel; ++lev) { // get MultiFab data at lev - const MultiFab & Ex = *warpx.m_fields.get("Efield_aux", Direction{0}, lev); - const MultiFab & Ey = *warpx.m_fields.get("Efield_aux", Direction{1}, lev); - const MultiFab & Ez = *warpx.m_fields.get("Efield_aux", Direction{2}, lev); - const MultiFab & Bx = *warpx.m_fields.get("Bfield_aux", Direction{0}, lev); - const MultiFab & By = *warpx.m_fields.get("Bfield_aux", Direction{1}, lev); - const MultiFab & Bz = *warpx.m_fields.get("Bfield_aux", Direction{2}, lev); + const MultiFab & Ex = *warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev); + const MultiFab & Ey = *warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, lev); + const MultiFab & Ez = *warpx.m_fields.get(FieldType::Efield_aux, Direction{2}, lev); + const MultiFab & Bx = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, lev); + const MultiFab & By = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, lev); + const MultiFab & Bz = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{2}, lev); // get cell volume const std::array &dx = WarpX::CellSize(lev); diff --git a/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp b/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp index 276c7928b43..8c7eb6b4dec 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp @@ -40,7 +40,7 @@ #include using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; // constructor FieldMaximum::FieldMaximum (const std::string& rd_name) @@ -119,12 +119,12 @@ void FieldMaximum::ComputeDiags (int step) for (int lev = 0; lev < nLevel; ++lev) { // get MultiFab data at lev - const MultiFab & Ex = *warpx.m_fields.get("Efield_aux", Direction{0}, lev); - const MultiFab & Ey = *warpx.m_fields.get("Efield_aux", Direction{1}, lev); - const MultiFab & Ez = *warpx.m_fields.get("Efield_aux", Direction{2}, lev); - const MultiFab & Bx = *warpx.m_fields.get("Bfield_aux", Direction{0}, lev); - const MultiFab & By = *warpx.m_fields.get("Bfield_aux", Direction{1}, lev); - const MultiFab & Bz = *warpx.m_fields.get("Bfield_aux", Direction{2}, lev); + const MultiFab & Ex = *warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev); + const MultiFab & Ey = *warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, lev); + const MultiFab & Ez = *warpx.m_fields.get(FieldType::Efield_aux, Direction{2}, lev); + const MultiFab & Bx = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, lev); + const MultiFab & By = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, lev); + const MultiFab & Bz = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{2}, lev); constexpr int noutputs = 8; // max of Ex,Ey,Ez,|E|,Bx,By,Bz and |B| constexpr int index_Ex = 0; diff --git a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp index f1857155457..764e9874c39 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp @@ -39,7 +39,7 @@ #include using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; FieldMomentum::FieldMomentum (const std::string& rd_name) : ReducedDiags{rd_name} @@ -111,12 +111,12 @@ void FieldMomentum::ComputeDiags (int step) for (int lev = 0; lev < nLevel; ++lev) { // Get MultiFab data at given refinement level - const amrex::MultiFab & Ex = *warpx.m_fields.get("Efield_aux", Direction{0}, lev); - const amrex::MultiFab & Ey = *warpx.m_fields.get("Efield_aux", Direction{1}, lev); - const amrex::MultiFab & Ez = *warpx.m_fields.get("Efield_aux", Direction{2}, lev); - const amrex::MultiFab & Bx = *warpx.m_fields.get("Bfield_aux", Direction{0}, lev); - const amrex::MultiFab & By = *warpx.m_fields.get("Bfield_aux", Direction{1}, lev); - const amrex::MultiFab & Bz = *warpx.m_fields.get("Bfield_aux", Direction{2}, lev); + const amrex::MultiFab & Ex = *warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev); + const amrex::MultiFab & Ey = *warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, lev); + const amrex::MultiFab & Ez = *warpx.m_fields.get(FieldType::Efield_aux, Direction{2}, lev); + const amrex::MultiFab & Bx = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, lev); + const amrex::MultiFab & By = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, lev); + const amrex::MultiFab & Bz = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{2}, lev); // Cell-centered index type const amrex::GpuArray cc{0,0,0}; diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp index bf5c68c9d83..923ae727d08 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp @@ -46,7 +46,7 @@ #include using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; // constructor @@ -401,12 +401,12 @@ void FieldProbe::ComputeDiags (int step) } // get MultiFab data at lev - const amrex::MultiFab &Ex = *warpx.m_fields.get("Efield_aux", Direction{0}, lev); - const amrex::MultiFab &Ey = *warpx.m_fields.get("Efield_aux", Direction{1}, lev); - const amrex::MultiFab &Ez = *warpx.m_fields.get("Efield_aux", Direction{2}, lev); - const amrex::MultiFab &Bx = *warpx.m_fields.get("Bfield_aux", Direction{0}, lev); - const amrex::MultiFab &By = *warpx.m_fields.get("Bfield_aux", Direction{1}, lev); - const amrex::MultiFab &Bz = *warpx.m_fields.get("Bfield_aux", Direction{2}, lev); + const amrex::MultiFab &Ex = *warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev); + const amrex::MultiFab &Ey = *warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, lev); + const amrex::MultiFab &Ez = *warpx.m_fields.get(FieldType::Efield_aux, Direction{2}, lev); + const amrex::MultiFab &Bx = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, lev); + const amrex::MultiFab &By = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, lev); + const amrex::MultiFab &Bz = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{2}, lev); /* * Prepare interpolation of field components to probe_position diff --git a/Source/Diagnostics/ReducedDiags/FieldReduction.H b/Source/Diagnostics/ReducedDiags/FieldReduction.H index 65455039394..d2c6dc6f6da 100644 --- a/Source/Diagnostics/ReducedDiags/FieldReduction.H +++ b/Source/Diagnostics/ReducedDiags/FieldReduction.H @@ -89,6 +89,7 @@ public: { using ablastr::fields::Direction; using namespace amrex::literals; + using warpx::fields::FieldType; // get a reference to WarpX instance auto & warpx = WarpX::GetInstance(); @@ -100,15 +101,15 @@ public: const auto dx = geom.CellSizeArray(); // get MultiFab data - const amrex::MultiFab & Ex = *warpx.m_fields.get("Efield_aux", Direction{0}, lev); - const amrex::MultiFab & Ey = *warpx.m_fields.get("Efield_aux", Direction{1}, lev); - const amrex::MultiFab & Ez = *warpx.m_fields.get("Efield_aux", Direction{2}, lev); - const amrex::MultiFab & Bx = *warpx.m_fields.get("Bfield_aux", Direction{0}, lev); - const amrex::MultiFab & By = *warpx.m_fields.get("Bfield_aux", Direction{1}, lev); - const amrex::MultiFab & Bz = *warpx.m_fields.get("Bfield_aux", Direction{2}, lev); - const amrex::MultiFab & jx = *warpx.m_fields.get("current_fp",Direction{0},lev); - const amrex::MultiFab & jy = *warpx.m_fields.get("current_fp",Direction{1},lev); - const amrex::MultiFab & jz = *warpx.m_fields.get("current_fp",Direction{2},lev); + const amrex::MultiFab & Ex = *warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev); + const amrex::MultiFab & Ey = *warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, lev); + const amrex::MultiFab & Ez = *warpx.m_fields.get(FieldType::Efield_aux, Direction{2}, lev); + const amrex::MultiFab & Bx = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, lev); + const amrex::MultiFab & By = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, lev); + const amrex::MultiFab & Bz = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{2}, lev); + const amrex::MultiFab & jx = *warpx.m_fields.get(FieldType::current_fp, Direction{0},lev); + const amrex::MultiFab & jy = *warpx.m_fields.get(FieldType::current_fp, Direction{1},lev); + const amrex::MultiFab & jz = *warpx.m_fields.get(FieldType::current_fp, Direction{2},lev); // General preparation of interpolation and reduction operations diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp index 8ed3d21efc0..c496300c54e 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp @@ -38,7 +38,7 @@ #include using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; namespace { @@ -131,7 +131,7 @@ void LoadBalanceCosts::ComputeDiags (int step) for (int lev = 0; lev < nLevels; ++lev) { const amrex::DistributionMapping& dm = warpx.DistributionMap(lev); - const MultiFab & Ex = *warpx.m_fields.get("Efield_aux", Direction{0}, lev); + const MultiFab & Ex = *warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev); for (MFIter mfi(Ex, false); mfi.isValid(); ++mfi) { const Box& tbx = mfi.tilebox(); diff --git a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp index 5b4ff2908a9..c82b060b67c 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp @@ -54,7 +54,7 @@ #include using namespace amrex::literals; -using namespace warpx::fields; +using warpx::fields::FieldType; // constructor ParticleExtrema::ParticleExtrema (const std::string& rd_name) @@ -270,12 +270,12 @@ void ParticleExtrema::ComputeDiags (int step) // define variables in preparation for field gathering const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(lev, 0)); - const amrex::MultiFab & Ex = *warpx.m_fields.get("Efield_aux", Direction{0}, lev); - const amrex::MultiFab & Ey = *warpx.m_fields.get("Efield_aux", Direction{1}, lev); - const amrex::MultiFab & Ez = *warpx.m_fields.get("Efield_aux", Direction{2}, lev); - const amrex::MultiFab & Bx = *warpx.m_fields.get("Bfield_aux", Direction{0}, lev); - const amrex::MultiFab & By = *warpx.m_fields.get("Bfield_aux", Direction{1}, lev); - const amrex::MultiFab & Bz = *warpx.m_fields.get("Bfield_aux", Direction{2}, lev); + const amrex::MultiFab & Ex = *warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev); + const amrex::MultiFab & Ey = *warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, lev); + const amrex::MultiFab & Ez = *warpx.m_fields.get(FieldType::Efield_aux, Direction{2}, lev); + const amrex::MultiFab & Bx = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, lev); + const amrex::MultiFab & By = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, lev); + const amrex::MultiFab & Bz = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{2}, lev); // declare reduce_op amrex::ReduceOps reduce_op; diff --git a/Source/Diagnostics/SliceDiagnostic.cpp b/Source/Diagnostics/SliceDiagnostic.cpp index a9ebe4bf661..bcb6070abdf 100644 --- a/Source/Diagnostics/SliceDiagnostic.cpp +++ b/Source/Diagnostics/SliceDiagnostic.cpp @@ -42,7 +42,7 @@ #include using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; /* \brief * The functions creates the slice for diagnostics based on the user-input. @@ -201,27 +201,27 @@ CreateSlice( const MultiFab& mf, const Vector &dom_geom, amrex::amrex_avgdown_nodes(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio); } - if( SliceType == warpx.m_fields.get("Efield_aux", Direction{0}, 0)->ixType().toIntVect() ) { + if( SliceType == warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, 0)->ixType().toIntVect() ) { amrex::amrex_avgdown_edges(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 0); } - if( SliceType == warpx.m_fields.get("Efield_aux", Direction{1}, 0)->ixType().toIntVect() ) { + if( SliceType == warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, 0)->ixType().toIntVect() ) { amrex::amrex_avgdown_edges(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 1); } - if( SliceType == warpx.m_fields.get("Efield_aux", Direction{2}, 0)->ixType().toIntVect() ) { + if( SliceType == warpx.m_fields.get(FieldType::Efield_aux, Direction{2}, 0)->ixType().toIntVect() ) { amrex::amrex_avgdown_edges(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 2); } - if( SliceType == warpx.m_fields.get("Bfield_aux", Direction{0}, 0)->ixType().toIntVect() ) { + if( SliceType == warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, 0)->ixType().toIntVect() ) { amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 0); } - if( SliceType == warpx.m_fields.get("Bfield_aux", Direction{1}, 0)->ixType().toIntVect() ) { + if( SliceType == warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, 0)->ixType().toIntVect() ) { amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 1); } - if( SliceType == warpx.m_fields.get("Bfield_aux", Direction{2}, 0)->ixType().toIntVect() ) { + if( SliceType == warpx.m_fields.get(FieldType::Bfield_aux, Direction{2}, 0)->ixType().toIntVect() ) { amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 2); } diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp index 26e7a025ff2..43415daf151 100644 --- a/Source/Diagnostics/WarpXIO.cpp +++ b/Source/Diagnostics/WarpXIO.cpp @@ -12,6 +12,7 @@ # include "BoundaryConditions/PML_RZ.H" #endif #include "EmbeddedBoundary/Enabled.H" +#include "Fields.H" #include "FieldIO.H" #include "Particles/MultiParticleContainer.H" #include "Utils/TextMsg.H" @@ -89,6 +90,7 @@ void WarpX::InitFromCheckpoint () { using ablastr::fields::Direction; + using warpx::fields::FieldType; WARPX_PROFILE("WarpX::InitFromCheckpoint()"); @@ -281,101 +283,101 @@ WarpX::InitFromCheckpoint () for (int lev = 0; lev < nlevs; ++lev) { for (int i = 0; i < 3; ++i) { - m_fields.get("current_fp",Direction{i},lev)->setVal(0.0); - m_fields.get("Efield_fp",Direction{i},lev)->setVal(0.0); - m_fields.get("Bfield_fp",Direction{i},lev)->setVal(0.0); + m_fields.get(FieldType::current_fp, Direction{i}, lev)->setVal(0.0); + m_fields.get(FieldType::Efield_fp, Direction{i}, lev)->setVal(0.0); + m_fields.get(FieldType::Bfield_fp, Direction{i}, lev)->setVal(0.0); } if (lev > 0) { for (int i = 0; i < 3; ++i) { - m_fields.get("Efield_aux", Direction{i}, lev)->setVal(0.0); - m_fields.get("Bfield_aux", Direction{i}, lev)->setVal(0.0); + m_fields.get(FieldType::Efield_aux, Direction{i}, lev)->setVal(0.0); + m_fields.get(FieldType::Bfield_aux, Direction{i}, lev)->setVal(0.0); - m_fields.get("current_cp",Direction{i},lev)->setVal(0.0); - m_fields.get("Efield_cp",Direction{i},lev)->setVal(0.0); - m_fields.get("Bfield_cp",Direction{i},lev)->setVal(0.0); + m_fields.get(FieldType::current_cp, Direction{i}, lev)->setVal(0.0); + m_fields.get(FieldType::Efield_cp, Direction{i}, lev)->setVal(0.0); + m_fields.get(FieldType::Bfield_cp, Direction{i}, lev)->setVal(0.0); } } - VisMF::Read(*m_fields.get("Efield_fp", Direction{0}, lev), + VisMF::Read(*m_fields.get(FieldType::Efield_fp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ex_fp")); - VisMF::Read(*m_fields.get("Efield_fp", Direction{1}, lev), + VisMF::Read(*m_fields.get(FieldType::Efield_fp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ey_fp")); - VisMF::Read(*m_fields.get("Efield_fp", Direction{2}, lev), + VisMF::Read(*m_fields.get(FieldType::Efield_fp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ez_fp")); - VisMF::Read(*m_fields.get("Bfield_fp", Direction{0}, lev), + VisMF::Read(*m_fields.get(FieldType::Bfield_fp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Bx_fp")); - VisMF::Read(*m_fields.get("Bfield_fp", Direction{1}, lev), + VisMF::Read(*m_fields.get(FieldType::Bfield_fp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "By_fp")); - VisMF::Read(*m_fields.get("Bfield_fp", Direction{2}, lev), + VisMF::Read(*m_fields.get(FieldType::Bfield_fp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Bz_fp")); if (WarpX::fft_do_time_averaging) { - VisMF::Read(*m_fields.get("Efield_avg_fp", Direction{0}, lev), + VisMF::Read(*m_fields.get(FieldType::Efield_avg_fp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ex_avg_fp")); - VisMF::Read(*m_fields.get("Efield_avg_fp", Direction{1}, lev), + VisMF::Read(*m_fields.get(FieldType::Efield_avg_fp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ey_avg_fp")); - VisMF::Read(*m_fields.get("Efield_avg_fp", Direction{2}, lev), + VisMF::Read(*m_fields.get(FieldType::Efield_avg_fp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ez_avg_fp")); - VisMF::Read(*m_fields.get("Bfield_avg_fp", Direction{0}, lev), + VisMF::Read(*m_fields.get(FieldType::Bfield_avg_fp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Bx_avg_fp")); - VisMF::Read(*m_fields.get("Bfield_avg_fp", Direction{1}, lev), + VisMF::Read(*m_fields.get(FieldType::Bfield_avg_fp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "By_avg_fp")); - VisMF::Read(*m_fields.get("Bfield_avg_fp", Direction{2}, lev), + VisMF::Read(*m_fields.get(FieldType::Bfield_avg_fp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Bz_avg_fp")); } if (is_synchronized) { - VisMF::Read(*m_fields.get("current_fp", Direction{0}, lev), + VisMF::Read(*m_fields.get(FieldType::current_fp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "jx_fp")); - VisMF::Read(*m_fields.get("current_fp", Direction{1}, lev), + VisMF::Read(*m_fields.get(FieldType::current_fp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "jy_fp")); - VisMF::Read(*m_fields.get("current_fp", Direction{2}, lev), + VisMF::Read(*m_fields.get(FieldType::current_fp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "jz_fp")); } if (lev > 0) { - VisMF::Read(*m_fields.get("Efield_cp", Direction{0}, lev), + VisMF::Read(*m_fields.get(FieldType::Efield_cp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ex_cp")); - VisMF::Read(*m_fields.get("Efield_cp", Direction{1}, lev), + VisMF::Read(*m_fields.get(FieldType::Efield_cp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ey_cp")); - VisMF::Read(*m_fields.get("Efield_cp", Direction{2}, lev), + VisMF::Read(*m_fields.get(FieldType::Efield_cp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ez_cp")); - VisMF::Read(*m_fields.get("Bfield_cp", Direction{0}, lev), + VisMF::Read(*m_fields.get(FieldType::Bfield_cp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Bx_cp")); - VisMF::Read(*m_fields.get("Bfield_cp", Direction{1}, lev), + VisMF::Read(*m_fields.get(FieldType::Bfield_cp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "By_cp")); - VisMF::Read(*m_fields.get("Bfield_cp", Direction{2}, lev), + VisMF::Read(*m_fields.get(FieldType::Bfield_cp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Bz_cp")); if (WarpX::fft_do_time_averaging) { - VisMF::Read(*m_fields.get("Efield_avg_cp", Direction{0}, lev), + VisMF::Read(*m_fields.get(FieldType::Efield_avg_cp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ex_avg_cp")); - VisMF::Read(*m_fields.get("Efield_avg_cp", Direction{1}, lev), + VisMF::Read(*m_fields.get(FieldType::Efield_avg_cp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ey_avg_cp")); - VisMF::Read(*m_fields.get("Efield_avg_cp", Direction{2}, lev), + VisMF::Read(*m_fields.get(FieldType::Efield_avg_cp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Ez_avg_cp")); - VisMF::Read(*m_fields.get("Bfield_avg_cp", Direction{0}, lev), + VisMF::Read(*m_fields.get(FieldType::Bfield_avg_cp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Bx_avg_cp")); - VisMF::Read(*m_fields.get("Bfield_avg_cp", Direction{1}, lev), + VisMF::Read(*m_fields.get(FieldType::Bfield_avg_cp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "By_avg_cp")); - VisMF::Read(*m_fields.get("Bfield_avg_cp", Direction{2}, lev), + VisMF::Read(*m_fields.get(FieldType::Bfield_avg_cp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "Bz_avg_cp")); } if (is_synchronized) { - VisMF::Read(*m_fields.get("current_cp", Direction{0}, lev), + VisMF::Read(*m_fields.get(FieldType::current_cp, Direction{0}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "jx_cp")); - VisMF::Read(*m_fields.get("current_cp", Direction{1}, lev), + VisMF::Read(*m_fields.get(FieldType::current_cp, Direction{1}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "jy_cp")); - VisMF::Read(*m_fields.get("current_cp", Direction{2}, lev), + VisMF::Read(*m_fields.get(FieldType::current_cp, Direction{2}, lev), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "jz_cp")); } } diff --git a/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp b/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp index b52ac88d289..61009fb46e0 100644 --- a/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp +++ b/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp @@ -7,6 +7,7 @@ #include "WarpXFaceInfoBox.H" #include "EmbeddedBoundary/Enabled.H" +#include "Fields.H" #include "Utils/TextMsg.H" #include "WarpX.H" @@ -18,6 +19,7 @@ #include using namespace ablastr::fields; +using warpx::fields::FieldType; /** * \brief Get the value of arr in the neighbor (i_n, j_n) on the plane with normal 'dim'. @@ -286,7 +288,7 @@ WarpX::ComputeFaceExtensions () void WarpX::InitBorrowing() { int idim = 0; - for (amrex::MFIter mfi(*m_fields.get("Bfield_fp", Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get(FieldType::Bfield_fp, Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { amrex::Box const &box = mfi.validbox(); auto &borrowing_x = (*m_borrowing[maxLevel()][idim])[mfi]; borrowing_x.inds_pointer.resize(box); @@ -302,7 +304,7 @@ WarpX::InitBorrowing() { } idim = 1; - for (amrex::MFIter mfi(*m_fields.get("Bfield_fp", Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get(FieldType::Bfield_fp, Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { amrex::Box const &box = mfi.validbox(); auto &borrowing_y = (*m_borrowing[maxLevel()][idim])[mfi]; borrowing_y.inds_pointer.resize(box); @@ -315,7 +317,7 @@ WarpX::InitBorrowing() { } idim = 2; - for (amrex::MFIter mfi(*m_fields.get("Bfield_fp", Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get(FieldType::Bfield_fp, Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { amrex::Box const &box = mfi.validbox(); auto &borrowing_z = (*m_borrowing[maxLevel()][idim])[mfi]; borrowing_z.inds_pointer.resize(box); @@ -456,10 +458,10 @@ WarpX::ComputeOneWayExtensions () WARPX_ABORT_WITH_MESSAGE( "ComputeOneWayExtensions: Only implemented in 2D3V and 3D3V"); #endif - for (amrex::MFIter mfi(*m_fields.get("Bfield_fp", Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get(FieldType::Bfield_fp, Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { amrex::Box const &box = mfi.validbox(); - auto const &S = m_fields.get("face_areas", Direction{idim}, maxLevel())->array(mfi); + auto const &S = m_fields.get(FieldType::face_areas, Direction{idim}, maxLevel())->array(mfi); auto const &flag_ext_face = m_flag_ext_face[maxLevel()][idim]->array(mfi); auto const &flag_info_face = m_flag_info_face[maxLevel()][idim]->array(mfi); auto &borrowing = (*m_borrowing[maxLevel()][idim])[mfi]; @@ -471,11 +473,11 @@ WarpX::ComputeOneWayExtensions () amrex::Real* borrowing_area = borrowing.area.data(); int& vecs_size = borrowing.vecs_size; - auto const &S_mod = m_fields.get("area_mod", Direction{idim}, maxLevel())->array(mfi); + auto const &S_mod = m_fields.get(FieldType::area_mod, Direction{idim}, maxLevel())->array(mfi); - const auto &lx = m_fields.get("edge_lengths", Direction{0}, maxLevel())->array(mfi); - const auto &ly = m_fields.get("edge_lengths", Direction{1}, maxLevel())->array(mfi); - const auto &lz = m_fields.get("edge_lengths", Direction{2}, maxLevel())->array(mfi); + const auto &lx = m_fields.get(FieldType::edge_lengths, Direction{0}, maxLevel())->array(mfi); + const auto &ly = m_fields.get(FieldType::edge_lengths, Direction{1}, maxLevel())->array(mfi); + const auto &lz = m_fields.get(FieldType::edge_lengths, Direction{2}, maxLevel())->array(mfi); vecs_size = amrex::Scan::PrefixSum(ncells, [=] AMREX_GPU_DEVICE (int icell) { @@ -583,11 +585,11 @@ WarpX::ComputeEightWaysExtensions () WARPX_ABORT_WITH_MESSAGE( "ComputeEightWaysExtensions: Only implemented in 2D3V and 3D3V"); #endif - for (amrex::MFIter mfi(*m_fields.get("Bfield_fp", Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get(FieldType::Bfield_fp, Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { amrex::Box const &box = mfi.validbox(); - auto const &S = m_fields.get("face_areas", Direction{idim}, maxLevel())->array(mfi); + auto const &S = m_fields.get(FieldType::face_areas, Direction{idim}, maxLevel())->array(mfi); auto const &flag_ext_face = m_flag_ext_face[maxLevel()][idim]->array(mfi); auto const &flag_info_face = m_flag_info_face[maxLevel()][idim]->array(mfi); auto &borrowing = (*m_borrowing[maxLevel()][idim])[mfi]; @@ -599,11 +601,11 @@ WarpX::ComputeEightWaysExtensions () amrex::Real* borrowing_area = borrowing.area.data(); int& vecs_size = borrowing.vecs_size; - auto const &S_mod = m_fields.get("area_mod", Direction{idim}, maxLevel())->array(mfi); + auto const &S_mod = m_fields.get(FieldType::area_mod, Direction{idim}, maxLevel())->array(mfi); - const auto &lx = m_fields.get("edge_lengths", Direction{0}, maxLevel())->array(mfi); - const auto &ly = m_fields.get("edge_lengths", Direction{1}, maxLevel())->array(mfi); - const auto &lz = m_fields.get("edge_lengths", Direction{2}, maxLevel())->array(mfi); + const auto &lx = m_fields.get(FieldType::edge_lengths, Direction{0}, maxLevel())->array(mfi); + const auto &ly = m_fields.get(FieldType::edge_lengths, Direction{1}, maxLevel())->array(mfi); + const auto &lz = m_fields.get(FieldType::edge_lengths, Direction{2}, maxLevel())->array(mfi); vecs_size += amrex::Scan::PrefixSum(ncells, [=] AMREX_GPU_DEVICE (int icell){ @@ -735,15 +737,15 @@ WarpX::ApplyBCKCorrection (const int idim) const amrex::Real dy = cell_size[1]; const amrex::Real dz = cell_size[2]; - for (amrex::MFIter mfi(*m_fields.get("Bfield_fp", Direction{idim}, maxLevel()), amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get(FieldType::Bfield_fp, Direction{idim}, maxLevel()), amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { const amrex::Box &box = mfi.tilebox(); const amrex::Array4 &flag_ext_face = m_flag_ext_face[maxLevel()][idim]->array(mfi); const amrex::Array4 &flag_info_face = m_flag_info_face[maxLevel()][idim]->array(mfi); - const amrex::Array4 &S = m_fields.get("face_areas", Direction{idim}, maxLevel())->array(mfi); - const amrex::Array4 &lx = m_fields.get("face_areas", Direction{0}, maxLevel())->array(mfi);; - const amrex::Array4 &ly = m_fields.get("face_areas", Direction{1}, maxLevel())->array(mfi);; - const amrex::Array4 &lz = m_fields.get("face_areas", Direction{2}, maxLevel())->array(mfi);; + const amrex::Array4 &S = m_fields.get(FieldType::face_areas, Direction{idim}, maxLevel())->array(mfi); + const amrex::Array4 &lx = m_fields.get(FieldType::face_areas, Direction{0}, maxLevel())->array(mfi);; + const amrex::Array4 &ly = m_fields.get(FieldType::face_areas, Direction{1}, maxLevel())->array(mfi);; + const amrex::Array4 &lz = m_fields.get(FieldType::face_areas, Direction{2}, maxLevel())->array(mfi);; amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k) { if (flag_ext_face(i, j, k)) { @@ -763,7 +765,7 @@ void WarpX::ShrinkBorrowing () { for(int idim = 0; idim < AMREX_SPACEDIM; idim++) { - for (amrex::MFIter mfi(*m_fields.get("Bfield_fp", Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get(FieldType::Bfield_fp, Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { auto &borrowing = (*m_borrowing[maxLevel()][idim])[mfi]; borrowing.inds.resize(borrowing.vecs_size); borrowing.neigh_faces.resize(borrowing.vecs_size); diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index 0b98e13b5ab..edbc97a8efe 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -9,6 +9,7 @@ #include "EmbeddedBoundary/Enabled.H" #ifdef AMREX_USE_EB +# include "Fields.H" # include "Utils/Parser/ParserUtils.H" # include "Utils/TextMsg.H" @@ -292,8 +293,10 @@ WarpX::ScaleAreas (ablastr::fields::VectorField& face_areas, void -WarpX::MarkCells (){ +WarpX::MarkCells () +{ using ablastr::fields::Direction; + using warpx::fields::FieldType; #ifndef WARPX_DIM_RZ auto const &cell_size = CellSize(maxLevel()); @@ -310,9 +313,9 @@ WarpX::MarkCells (){ continue; } #endif - for (amrex::MFIter mfi(*m_fields.get("Bfield_fp",Direction{idim},maxLevel())); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*m_fields.get(FieldType::Bfield_fp, Direction{idim}, maxLevel())); mfi.isValid(); ++mfi) { auto* face_areas_idim_max_lev = - m_fields.get("face_areas", Direction{idim}, maxLevel()); + m_fields.get(FieldType::face_areas, Direction{idim}, maxLevel()); const amrex::Box& box = mfi.tilebox(face_areas_idim_max_lev->ixType().toIntVect(), face_areas_idim_max_lev->nGrowVect() ); @@ -320,10 +323,10 @@ WarpX::MarkCells (){ auto const &S = face_areas_idim_max_lev->array(mfi); auto const &flag_info_face = m_flag_info_face[maxLevel()][idim]->array(mfi); auto const &flag_ext_face = m_flag_ext_face[maxLevel()][idim]->array(mfi); - const auto &lx = m_fields.get("edge_lengths", Direction{0}, maxLevel())->array(mfi); - const auto &ly = m_fields.get("edge_lengths", Direction{1}, maxLevel())->array(mfi); - const auto &lz = m_fields.get("edge_lengths", Direction{2}, maxLevel())->array(mfi); - auto const &mod_areas_dim = m_fields.get("area_mod", Direction{idim}, maxLevel())->array(mfi); + const auto &lx = m_fields.get(FieldType::edge_lengths, Direction{0}, maxLevel())->array(mfi); + const auto &ly = m_fields.get(FieldType::edge_lengths, Direction{1}, maxLevel())->array(mfi); + const auto &lz = m_fields.get(FieldType::edge_lengths, Direction{2}, maxLevel())->array(mfi); + auto const &mod_areas_dim = m_fields.get(FieldType::area_mod, Direction{idim}, maxLevel())->array(mfi); const amrex::Real dx = cell_size[0]; const amrex::Real dy = cell_size[1]; @@ -387,11 +390,12 @@ WarpX::ComputeDistanceToEB () } #ifdef AMREX_USE_EB BL_PROFILE("ComputeDistanceToEB"); + using warpx::fields::FieldType; const amrex::EB2::IndexSpace& eb_is = amrex::EB2::IndexSpace::top(); for (int lev=0; lev<=maxLevel(); lev++) { const amrex::EB2::Level& eb_level = eb_is.getLevel(Geom(lev)); auto const eb_fact = fieldEBFactory(lev); - amrex::FillSignedDistance(*m_fields.get("distance_to_eb", lev), eb_level, eb_fact, 1); + amrex::FillSignedDistance(*m_fields.get(FieldType::distance_to_eb, lev), eb_level, eb_fact, 1); } #endif } diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index dc684fc39ed..93d265d598f 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -15,6 +15,7 @@ #include "Diagnostics/ReducedDiags/MultiReducedDiags.H" #include "EmbeddedBoundary/Enabled.H" #include "Evolve/WarpXDtType.H" +#include "Fields.H" #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" #ifdef WARPX_USE_FFT # ifdef WARPX_DIM_RZ @@ -63,6 +64,8 @@ using ablastr::utils::SignalHandling; void WarpX::Synchronize () { using ablastr::fields::Direction; + using warpx::fields::FieldType; + FillBoundaryE(guard_cells.ng_FieldGather); FillBoundaryB(guard_cells.ng_FieldGather); if (fft_do_time_averaging) @@ -76,12 +79,12 @@ WarpX::Synchronize () { mypc->PushP( lev, 0.5_rt*dt[lev], - *m_fields.get("Efield_aux", Direction{0}, lev), - *m_fields.get("Efield_aux", Direction{1}, lev), - *m_fields.get("Efield_aux", Direction{2}, lev), - *m_fields.get("Bfield_aux", Direction{0}, lev), - *m_fields.get("Bfield_aux", Direction{1}, lev), - *m_fields.get("Bfield_aux", Direction{2}, lev) + *m_fields.get(FieldType::Efield_aux, Direction{0}, lev), + *m_fields.get(FieldType::Efield_aux, Direction{1}, lev), + *m_fields.get(FieldType::Efield_aux, Direction{2}, lev), + *m_fields.get(FieldType::Bfield_aux, Direction{0}, lev), + *m_fields.get(FieldType::Bfield_aux, Direction{1}, lev), + *m_fields.get(FieldType::Bfield_aux, Direction{2}, lev) ); } is_synchronized = true; @@ -473,6 +476,7 @@ void WarpX::ExplicitFillBoundaryEBUpdateAux () "Cannot call WarpX::ExplicitFillBoundaryEBUpdateAux without Explicit evolve scheme set!"); using ablastr::fields::Direction; + using warpx::fields::FieldType; // At the beginning, we have B^{n} and E^{n}. // Particles have p^{n} and x^{n}. @@ -491,12 +495,12 @@ void WarpX::ExplicitFillBoundaryEBUpdateAux () mypc->PushP( lev, -0.5_rt*dt[lev], - *m_fields.get("Efield_aux", Direction{0}, lev), - *m_fields.get("Efield_aux", Direction{1}, lev), - *m_fields.get("Efield_aux", Direction{2}, lev), - *m_fields.get("Bfield_aux", Direction{0}, lev), - *m_fields.get("Bfield_aux", Direction{1}, lev), - *m_fields.get("Bfield_aux", Direction{2}, lev) + *m_fields.get(FieldType::Efield_aux, Direction{0}, lev), + *m_fields.get(FieldType::Efield_aux, Direction{1}, lev), + *m_fields.get(FieldType::Efield_aux, Direction{2}, lev), + *m_fields.get(FieldType::Bfield_aux, Direction{0}, lev), + *m_fields.get(FieldType::Bfield_aux, Direction{1}, lev), + *m_fields.get(FieldType::Bfield_aux, Direction{2}, lev) ); } is_synchronized = false; @@ -561,9 +565,10 @@ void WarpX::HandleParticlesAtBoundaries (int step, amrex::Real cur_time, int num // interact the particles with EB walls (if present) if (EB::enabled()) { - mypc->ScrapeParticlesAtEB(m_fields.get_mr_levels("distance_to_eb", finest_level)); + using warpx::fields::FieldType; + mypc->ScrapeParticlesAtEB(m_fields.get_mr_levels(FieldType::distance_to_eb, finest_level)); m_particle_boundary_buffer->gatherParticlesFromEmbeddedBoundaries( - *mypc, m_fields.get_mr_levels("distance_to_eb", finest_level)); + *mypc, m_fields.get_mr_levels(FieldType::distance_to_eb, finest_level)); mypc->deleteInvalidParticles(); } @@ -578,6 +583,7 @@ void WarpX::HandleParticlesAtBoundaries (int step, amrex::Real cur_time, int num void WarpX::SyncCurrentAndRho () { using ablastr::fields::Direction; + using warpx::fields::FieldType; if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { @@ -610,7 +616,7 @@ void WarpX::SyncCurrentAndRho () // TODO This works only without mesh refinement const int lev = 0; if (use_filter) { - ApplyFilterJ(m_fields.get_mr_levels_alldirs("current_fp_vay", finest_level), lev); + ApplyFilterJ(m_fields.get_mr_levels_alldirs(FieldType::current_fp_vay, finest_level), lev); } } } @@ -624,22 +630,22 @@ void WarpX::SyncCurrentAndRho () // Reflect charge and current density over PEC boundaries, if needed. for (int lev = 0; lev <= finest_level; ++lev) { - if (m_fields.has("rho_fp", lev)) { - ApplyRhofieldBoundary(lev, m_fields.get("rho_fp",lev), PatchType::fine); + if (m_fields.has(FieldType::rho_fp, lev)) { + ApplyRhofieldBoundary(lev, m_fields.get(FieldType::rho_fp,lev), PatchType::fine); } ApplyJfieldBoundary(lev, - m_fields.get("current_fp",Direction{0},lev), - m_fields.get("current_fp",Direction{1},lev), - m_fields.get("current_fp",Direction{2},lev), + m_fields.get(FieldType::current_fp, Direction{0}, lev), + m_fields.get(FieldType::current_fp, Direction{1}, lev), + m_fields.get(FieldType::current_fp, Direction{2}, lev), PatchType::fine); if (lev > 0) { - if (m_fields.has("rho_cp", lev)) { - ApplyRhofieldBoundary(lev, m_fields.get("rho_cp",lev), PatchType::coarse); + if (m_fields.has(FieldType::rho_cp, lev)) { + ApplyRhofieldBoundary(lev, m_fields.get(FieldType::rho_cp,lev), PatchType::coarse); } ApplyJfieldBoundary(lev, - m_fields.get("current_cp",Direction{0},lev), - m_fields.get("current_cp",Direction{1},lev), - m_fields.get("current_cp",Direction{2},lev), + m_fields.get(FieldType::current_cp, Direction{0}, lev), + m_fields.get(FieldType::current_cp, Direction{1}, lev), + m_fields.get(FieldType::current_cp, Direction{2}, lev), PatchType::coarse); } } @@ -655,6 +661,8 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) "multi-J algorithm not implemented for FDTD" ); + using warpx::fields::FieldType; + const int rho_mid = spectral_solver_fp[0]->m_spectral_index.rho_mid; const int rho_new = spectral_solver_fp[0]->m_spectral_index.rho_new; @@ -675,9 +683,9 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // 3) Deposit rho (in rho_new, since it will be moved during the loop) // (after checking that pointer to rho_fp on MR level 0 is not null) - if (m_fields.has("rho_fp", 0) && rho_in_time == RhoInTime::Linear) + if (m_fields.has(FieldType::rho_fp, 0) && rho_in_time == RhoInTime::Linear) { - ablastr::fields::MultiLevelScalarField const rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); + ablastr::fields::MultiLevelScalarField const rho_fp = m_fields.get_mr_levels(FieldType::rho_fp, finest_level); std::string const rho_fp_string = "rho_fp"; std::string const rho_cp_string = "rho_cp"; @@ -742,9 +750,9 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Deposit new rho // (after checking that pointer to rho_fp on MR level 0 is not null) - if (m_fields.has("rho_fp", 0)) + if (m_fields.has(FieldType::rho_fp, 0)) { - ablastr::fields::MultiLevelScalarField const rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); + ablastr::fields::MultiLevelScalarField const rho_fp = m_fields.get_mr_levels(FieldType::rho_fp, finest_level); std::string const rho_fp_string = "rho_fp"; std::string const rho_cp_string = "rho_cp"; @@ -786,10 +794,10 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // We summed the integral of the field over 2*dt PSATDScaleAverageFields(1._rt / (2._rt*dt[0])); PSATDBackwardTransformEBavg( - m_fields.get_mr_levels_alldirs("Efield_avg_fp", finest_level), - m_fields.get_mr_levels_alldirs("Bfield_avg_fp", finest_level), - m_fields.get_mr_levels_alldirs("Efield_avg_cp", finest_level), - m_fields.get_mr_levels_alldirs("Bfield_avg_cp", finest_level) + m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_fp, finest_level), + m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_fp, finest_level), + m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_cp, finest_level), + m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_cp, finest_level) ); } @@ -858,22 +866,24 @@ WarpX::OneStep_sub1 (Real cur_time) const int fine_lev = 1; const int coarse_lev = 0; + using warpx::fields::FieldType; + // i) Push particles and fields on the fine patch (first fine step) PushParticlesandDeposit(fine_lev, cur_time, DtType::FirstHalf); RestrictCurrentFromFineToCoarsePatch( - m_fields.get_mr_levels_alldirs("current_fp", finest_level), - m_fields.get_mr_levels_alldirs("current_cp", finest_level), fine_lev); + m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), + m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level), fine_lev); RestrictRhoFromFineToCoarsePatch(fine_lev); if (use_filter) { - ApplyFilterJ( m_fields.get_mr_levels_alldirs("current_fp", finest_level), fine_lev); + ApplyFilterJ( m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), fine_lev); } SumBoundaryJ( - m_fields.get_mr_levels_alldirs("current_fp", finest_level), + m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), fine_lev, Geom(fine_lev).periodicity()); ApplyFilterandSumBoundaryRho( - m_fields.get_mr_levels("rho_fp", finest_level), - m_fields.get_mr_levels("rho_cp", finest_level), + m_fields.get_mr_levels(FieldType::rho_fp, finest_level), + m_fields.get_mr_levels(FieldType::rho_cp, finest_level), fine_lev, PatchType::fine, 0, 2*ncomps); EvolveB(fine_lev, PatchType::fine, 0.5_rt*dt[fine_lev], DtType::FirstHalf); @@ -903,13 +913,13 @@ WarpX::OneStep_sub1 (Real cur_time) PushParticlesandDeposit(coarse_lev, cur_time, DtType::Full); StoreCurrent(coarse_lev); AddCurrentFromFineLevelandSumBoundary( - m_fields.get_mr_levels_alldirs("current_fp", finest_level), - m_fields.get_mr_levels_alldirs("current_cp", finest_level), - m_fields.get_mr_levels_alldirs("current_buf", finest_level), coarse_lev); + m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), + m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level), + m_fields.get_mr_levels_alldirs(FieldType::current_buf, finest_level), coarse_lev); AddRhoFromFineLevelandSumBoundary( - m_fields.get_mr_levels("rho_fp", finest_level), - m_fields.get_mr_levels("rho_cp", finest_level), - m_fields.get_mr_levels("rho_buf", finest_level), + m_fields.get_mr_levels(FieldType::rho_fp, finest_level), + m_fields.get_mr_levels(FieldType::rho_cp, finest_level), + m_fields.get_mr_levels(FieldType::rho_buf, finest_level), coarse_lev, 0, ncomps); EvolveB(fine_lev, PatchType::coarse, dt[fine_lev], DtType::FirstHalf); @@ -939,16 +949,16 @@ WarpX::OneStep_sub1 (Real cur_time) // iv) Push particles and fields on the fine patch (second fine step) PushParticlesandDeposit(fine_lev, cur_time + dt[fine_lev], DtType::SecondHalf); RestrictCurrentFromFineToCoarsePatch( - m_fields.get_mr_levels_alldirs("current_fp", finest_level), - m_fields.get_mr_levels_alldirs("current_cp", finest_level), fine_lev); + m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), + m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level), fine_lev); RestrictRhoFromFineToCoarsePatch(fine_lev); if (use_filter) { - ApplyFilterJ( m_fields.get_mr_levels_alldirs("current_fp", finest_level), fine_lev); + ApplyFilterJ( m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), fine_lev); } - SumBoundaryJ( m_fields.get_mr_levels_alldirs("current_fp", finest_level), fine_lev, Geom(fine_lev).periodicity()); + SumBoundaryJ( m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), fine_lev, Geom(fine_lev).periodicity()); ApplyFilterandSumBoundaryRho( - m_fields.get_mr_levels("rho_fp", finest_level), - m_fields.get_mr_levels("rho_cp", finest_level), + m_fields.get_mr_levels(FieldType::rho_fp, finest_level), + m_fields.get_mr_levels(FieldType::rho_cp, finest_level), fine_lev, PatchType::fine, 0, ncomps); EvolveB(fine_lev, PatchType::fine, 0.5_rt*dt[fine_lev], DtType::FirstHalf); @@ -977,14 +987,14 @@ WarpX::OneStep_sub1 (Real cur_time) // by only half a coarse step (second half) RestoreCurrent(coarse_lev); AddCurrentFromFineLevelandSumBoundary( - m_fields.get_mr_levels_alldirs("current_fp", finest_level), - m_fields.get_mr_levels_alldirs("current_cp", finest_level), - m_fields.get_mr_levels_alldirs("current_buf", finest_level), + m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), + m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level), + m_fields.get_mr_levels_alldirs(FieldType::current_buf, finest_level), coarse_lev); AddRhoFromFineLevelandSumBoundary( - m_fields.get_mr_levels("rho_fp", finest_level), - m_fields.get_mr_levels("rho_cp", finest_level), - m_fields.get_mr_levels("rho_buf", finest_level), + m_fields.get_mr_levels(FieldType::rho_fp, finest_level), + m_fields.get_mr_levels(FieldType::rho_cp, finest_level), + m_fields.get_mr_levels(FieldType::rho_buf, finest_level), coarse_lev, ncomps, ncomps); EvolveE(fine_lev, PatchType::coarse, dt[fine_lev]); @@ -1047,15 +1057,16 @@ void WarpX::doFieldIonization (int lev) { using ablastr::fields::Direction; + using warpx::fields::FieldType; mypc->doFieldIonization( lev, - *m_fields.get("Efield_aux", Direction{0}, lev), - *m_fields.get("Efield_aux", Direction{1}, lev), - *m_fields.get("Efield_aux", Direction{2}, lev), - *m_fields.get("Bfield_aux", Direction{0}, lev), - *m_fields.get("Bfield_aux", Direction{1}, lev), - *m_fields.get("Bfield_aux", Direction{2}, lev) + *m_fields.get(FieldType::Efield_aux, Direction{0}, lev), + *m_fields.get(FieldType::Efield_aux, Direction{1}, lev), + *m_fields.get(FieldType::Efield_aux, Direction{2}, lev), + *m_fields.get(FieldType::Bfield_aux, Direction{0}, lev), + *m_fields.get(FieldType::Bfield_aux, Direction{1}, lev), + *m_fields.get(FieldType::Bfield_aux, Direction{2}, lev) ); } @@ -1072,15 +1083,16 @@ void WarpX::doQEDEvents (int lev) { using ablastr::fields::Direction; + using warpx::fields::FieldType; mypc->doQedEvents( lev, - *m_fields.get("Efield_aux", Direction{0}, lev), - *m_fields.get("Efield_aux", Direction{1}, lev), - *m_fields.get("Efield_aux", Direction{2}, lev), - *m_fields.get("Bfield_aux", Direction{0}, lev), - *m_fields.get("Bfield_aux", Direction{1}, lev), - *m_fields.get("Bfield_aux", Direction{2}, lev) + *m_fields.get(FieldType::Efield_aux, Direction{0}, lev), + *m_fields.get(FieldType::Efield_aux, Direction{1}, lev), + *m_fields.get(FieldType::Efield_aux, Direction{2}, lev), + *m_fields.get(FieldType::Bfield_aux, Direction{0}, lev), + *m_fields.get(FieldType::Bfield_aux, Direction{1}, lev), + *m_fields.get(FieldType::Bfield_aux, Direction{2}, lev) ); } #endif @@ -1100,6 +1112,7 @@ WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, PushType push_type) { using ablastr::fields::Direction; + using warpx::fields::FieldType; std::string current_fp_string; @@ -1130,21 +1143,21 @@ WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, #ifdef WARPX_DIM_RZ // This is called after all particles have deposited their current and charge. ApplyInverseVolumeScalingToCurrentDensity( - m_fields.get("current_fp", Direction{0}, lev), - m_fields.get("current_fp", Direction{1}, lev), - m_fields.get("current_fp", Direction{2}, lev), + m_fields.get(FieldType::current_fp, Direction{0}, lev), + m_fields.get(FieldType::current_fp, Direction{1}, lev), + m_fields.get(FieldType::current_fp, Direction{2}, lev), lev); - if (m_fields.has("current_buf", Direction{0}, lev)) { + if (m_fields.has(FieldType::current_buf, Direction{0}, lev)) { ApplyInverseVolumeScalingToCurrentDensity( - m_fields.get("current_buf", Direction{0}, lev), - m_fields.get("current_buf", Direction{1}, lev), - m_fields.get("current_buf", Direction{2}, lev), + m_fields.get(FieldType::current_buf, Direction{0}, lev), + m_fields.get(FieldType::current_buf, Direction{1}, lev), + m_fields.get(FieldType::current_buf, Direction{2}, lev), lev-1); } - if (m_fields.has("rho_fp", lev)) { - ApplyInverseVolumeScalingToChargeDensity(m_fields.get("rho_fp", lev), lev); - if (m_fields.has("rho_buf", lev)) { - ApplyInverseVolumeScalingToChargeDensity(m_fields.get("rho_buf", lev), lev-1); + if (m_fields.has(FieldType::rho_fp, lev)) { + ApplyInverseVolumeScalingToChargeDensity(m_fields.get(FieldType::rho_fp, lev), lev); + if (m_fields.has(FieldType::rho_buf, lev)) { + ApplyInverseVolumeScalingToChargeDensity(m_fields.get(FieldType::rho_buf, lev), lev-1); } } // #else diff --git a/Source/FieldSolver/ElectrostaticSolvers/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolvers/ElectrostaticSolver.cpp index 50a9aab4980..1ced0a07152 100644 --- a/Source/FieldSolver/ElectrostaticSolvers/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolvers/ElectrostaticSolver.cpp @@ -8,10 +8,14 @@ */ #include "ElectrostaticSolver.H" -#include #include "EmbeddedBoundary/Enabled.H" +#include "Fields.H" + +#include + using namespace amrex; +using warpx::fields::FieldType; ElectrostaticSolver::ElectrostaticSolver (int nlevs_max) : num_levels{nlevs_max} { @@ -153,18 +157,18 @@ ElectrostaticSolver::computePhi ( e_field.push_back( #if defined(WARPX_DIM_1D_Z) amrex::Array{ - warpx.m_fields.get("Efield_fp", Direction{2}, lev) + warpx.m_fields.get(FieldType::Efield_fp, Direction{2}, lev) } #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) amrex::Array{ - warpx.m_fields.get("Efield_fp", Direction{0}, lev), - warpx.m_fields.get("Efield_fp", Direction{2}, lev) + warpx.m_fields.get(FieldType::Efield_fp, Direction{0}, lev), + warpx.m_fields.get(FieldType::Efield_fp, Direction{2}, lev) } #elif defined(WARPX_DIM_3D) amrex::Array{ - warpx.m_fields.get("Efield_fp", Direction{0}, lev), - warpx.m_fields.get("Efield_fp", Direction{1}, lev), - warpx.m_fields.get("Efield_fp", Direction{2}, lev) + warpx.m_fields.get(FieldType::Efield_fp, Direction{0}, lev), + warpx.m_fields.get(FieldType::Efield_fp, Direction{1}, lev), + warpx.m_fields.get(FieldType::Efield_fp, Direction{2}, lev) } #endif ); diff --git a/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp b/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp index eec5dce8f0e..e973ae66975 100644 --- a/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp +++ b/Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp @@ -9,6 +9,7 @@ #include "LabFrameExplicitES.H" #include "Fluids/MultiFluidContainer_fwd.H" #include "EmbeddedBoundary/Enabled.H" +#include "Fields.H" #include "Particles/MultiParticleContainer_fwd.H" #include "Python/callbacks.H" #include "WarpX.H" @@ -28,11 +29,12 @@ void LabFrameExplicitES::ComputeSpaceChargeField ( { using ablastr::fields::MultiLevelScalarField; using ablastr::fields::MultiLevelVectorField; + using warpx::fields::FieldType; - const MultiLevelScalarField rho_fp = fields.get_mr_levels("rho_fp", max_level); - const MultiLevelScalarField rho_cp = fields.get_mr_levels("rho_cp", max_level); - const MultiLevelScalarField phi_fp = fields.get_mr_levels("phi_fp", max_level); - const MultiLevelVectorField Efield_fp = fields.get_mr_levels_alldirs("Efield_fp", max_level); + const MultiLevelScalarField rho_fp = fields.get_mr_levels(FieldType::rho_fp, max_level); + const MultiLevelScalarField rho_cp = fields.get_mr_levels(FieldType::rho_cp, max_level); + const MultiLevelScalarField phi_fp = fields.get_mr_levels(FieldType::phi_fp, max_level); + const MultiLevelVectorField Efield_fp = fields.get_mr_levels_alldirs(FieldType::Efield_fp, max_level); mpc.DepositCharge(rho_fp, 0.0_rt); if (mfl) { diff --git a/Source/FieldSolver/ElectrostaticSolvers/RelativisticExplicitES.cpp b/Source/FieldSolver/ElectrostaticSolvers/RelativisticExplicitES.cpp index ae4006c2782..69647da1702 100644 --- a/Source/FieldSolver/ElectrostaticSolvers/RelativisticExplicitES.cpp +++ b/Source/FieldSolver/ElectrostaticSolvers/RelativisticExplicitES.cpp @@ -6,12 +6,13 @@ * * License: BSD-3-Clause-LBNL */ -#include "WarpX.H" - #include "RelativisticExplicitES.H" +#include "Fields.H" #include "Particles/MultiParticleContainer.H" #include "Particles/WarpXParticleContainer.H" +#include "WarpX.H" + using namespace amrex; @@ -35,14 +36,15 @@ void RelativisticExplicitES::ComputeSpaceChargeField ( [[maybe_unused]] MultiFluidContainer* mfl, int max_level) { - using ablastr::fields::MultiLevelVectorField; - WARPX_PROFILE("RelativisticExplicitES::ComputeSpaceChargeField"); + using ablastr::fields::MultiLevelVectorField; + using warpx::fields::FieldType; + const bool always_run_solve = (WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::Relativistic); - MultiLevelVectorField Efield_fp = fields.get_mr_levels_alldirs("Efield_fp", max_level); - MultiLevelVectorField Bfield_fp = fields.get_mr_levels_alldirs("Bfield_fp", max_level); + MultiLevelVectorField Efield_fp = fields.get_mr_levels_alldirs(FieldType::Efield_fp, max_level); + MultiLevelVectorField Bfield_fp = fields.get_mr_levels_alldirs(FieldType::Bfield_fp, max_level); // Loop over the species and add their space-charge contribution to E and B. // Note that the fields calculated here does not include the E field diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp index a5b85f7fbd7..63b51cb8416 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp @@ -7,6 +7,7 @@ #include "FiniteDifferenceSolver.H" #include "EmbeddedBoundary/WarpXFaceInfoBox.H" +#include "Fields.H" #ifndef WARPX_DIM_RZ # include "FiniteDifferenceAlgorithms/CartesianYeeAlgorithm.H" # include "FiniteDifferenceAlgorithms/CartesianCKCAlgorithm.H" @@ -53,13 +54,16 @@ void FiniteDifferenceSolver::EvolveB ( PatchType patch_type, [[maybe_unused]] std::array< std::unique_ptr, 3 >& flag_info_cell, [[maybe_unused]] std::array< std::unique_ptr >, 3 >& borrowing, - [[maybe_unused]] amrex::Real const dt ) { + [[maybe_unused]] amrex::Real const dt ) +{ using ablastr::fields::Direction; + using warpx::fields::FieldType; + const ablastr::fields::VectorField Bfield = patch_type == PatchType::fine ? - fields.get_alldirs("Bfield_fp", lev) : fields.get_alldirs("Bfield_cp", lev); + fields.get_alldirs(FieldType::Bfield_fp, lev) : fields.get_alldirs(FieldType::Bfield_cp, lev); const ablastr::fields::VectorField Efield = patch_type == PatchType::fine ? - fields.get_alldirs("Efield_fp", lev) : fields.get_alldirs("Efield_cp", lev); + fields.get_alldirs(FieldType::Efield_fp, lev) : fields.get_alldirs(FieldType::Efield_cp, lev); // Select algorithm (The choice of algorithm is a runtime option, // but we compile code for each algorithm, using templates) @@ -70,25 +74,25 @@ void FiniteDifferenceSolver::EvolveB ( #else amrex::MultiFab const * Gfield = nullptr; - if (fields.has("G_fp", lev)) { + if (fields.has(FieldType::G_fp, lev)) { Gfield = patch_type == PatchType::fine ? - fields.get("G_fp", lev) : fields.get("G_cp", lev); + fields.get(FieldType::G_fp, lev) : fields.get(FieldType::G_cp, lev); } ablastr::fields::VectorField face_areas; - if (fields.has("face_areas", Direction{0}, lev)) { - face_areas = fields.get_alldirs("face_areas", lev); + if (fields.has(FieldType::face_areas, Direction{0}, lev)) { + face_areas = fields.get_alldirs(FieldType::face_areas, lev); } ablastr::fields::VectorField area_mod; - if (fields.has("area_mod", Direction{0}, lev)) { - area_mod = fields.get_alldirs("area_mod", lev); + if (fields.has(FieldType::area_mod, Direction{0}, lev)) { + area_mod = fields.get_alldirs(FieldType::area_mod, lev); } ablastr::fields::VectorField ECTRhofield; - if (fields.has("ECTRhofield", Direction{0}, lev)) { - ECTRhofield = fields.get_alldirs("ECTRhofield", lev); + if (fields.has(FieldType::ECTRhofield, Direction{0}, lev)) { + ECTRhofield = fields.get_alldirs(FieldType::ECTRhofield, lev); } ablastr::fields::VectorField Venl; - if (fields.has("Venl", Direction{0}, lev)) { - Venl = fields.get_alldirs("Venl", lev); + if (fields.has(FieldType::Venl, Direction{0}, lev)) { + Venl = fields.get_alldirs(FieldType::Venl, lev); } if (m_grid_type == GridType::Collocated) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp index 5d46d18ff4e..e3289d52cfe 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp @@ -7,6 +7,7 @@ #include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H" #include "BoundaryConditions/PMLComponent.H" +#include "Fields.H" #ifndef WARPX_DIM_RZ # include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianYeeAlgorithm.H" @@ -48,6 +49,7 @@ void FiniteDifferenceSolver::EvolveBPML ( const bool dive_cleaning ) { + using warpx::fields::FieldType; // Select algorithm (The choice of algorithm is a runtime option, // but we compile code for each algorithm, using templates) @@ -57,9 +59,9 @@ void FiniteDifferenceSolver::EvolveBPML ( "PML are not implemented in cylindrical geometry."); #else const ablastr::fields::VectorField Bfield = (patch_type == PatchType::fine) ? - fields.get_alldirs("pml_B_fp", level) : fields.get_alldirs("pml_B_cp", level); + fields.get_alldirs(FieldType::pml_B_fp, level) : fields.get_alldirs(FieldType::pml_B_cp, level); const ablastr::fields::VectorField Efield = (patch_type == PatchType::fine) ? - fields.get_alldirs("pml_E_fp", level) : fields.get_alldirs("pml_E_cp", level); + fields.get_alldirs(FieldType::pml_E_fp, level) : fields.get_alldirs(FieldType::pml_E_cp, level); if (m_grid_type == ablastr::utils::enums::GridType::Collocated) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp index e9877c63f87..db8e80cc972 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp @@ -6,6 +6,7 @@ */ #include "FiniteDifferenceSolver.H" +#include "Fields.H" #ifndef WARPX_DIM_RZ # include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianYeeAlgorithm.H" # include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CartesianCKCAlgorithm.H" @@ -58,32 +59,34 @@ void FiniteDifferenceSolver::EvolveE ( ) { using ablastr::fields::Direction; + using warpx::fields::FieldType; + const ablastr::fields::VectorField Bfield = patch_type == PatchType::fine ? - fields.get_alldirs("Bfield_fp", lev) : fields.get_alldirs("Bfield_cp", lev); + fields.get_alldirs(FieldType::Bfield_fp, lev) : fields.get_alldirs(FieldType::Bfield_cp, lev); const ablastr::fields::VectorField Jfield = patch_type == PatchType::fine ? - fields.get_alldirs("current_fp", lev) : fields.get_alldirs("current_cp", lev); + fields.get_alldirs(FieldType::current_fp, lev) : fields.get_alldirs(FieldType::current_cp, lev); amrex::MultiFab* Ffield = nullptr; - if (fields.has("F_fp", lev)) { + if (fields.has(FieldType::F_fp, lev)) { Ffield = patch_type == PatchType::fine ? - fields.get("F_fp", lev) : fields.get("F_cp", lev); + fields.get(FieldType::F_fp, lev) : fields.get(FieldType::F_cp, lev); } ablastr::fields::VectorField edge_lengths; - if (fields.has("edge_lengths", Direction{0}, lev)) { - edge_lengths = fields.get_alldirs("edge_lengths", lev); + if (fields.has(FieldType::edge_lengths, Direction{0}, lev)) { + edge_lengths = fields.get_alldirs(FieldType::edge_lengths, lev); } ablastr::fields::VectorField face_areas; - if (fields.has("face_areas", Direction{0}, lev)) { - face_areas = fields.get_alldirs("face_areas", lev); + if (fields.has(FieldType::face_areas, Direction{0}, lev)) { + face_areas = fields.get_alldirs(FieldType::face_areas, lev); } ablastr::fields::VectorField area_mod; - if (fields.has("area_mod", Direction{0}, lev)) { - area_mod = fields.get_alldirs("area_mod", lev); + if (fields.has(FieldType::area_mod, Direction{0}, lev)) { + area_mod = fields.get_alldirs(FieldType::area_mod, lev); } ablastr::fields::VectorField ECTRhofield; - if (fields.has("ECTRhofield", Direction{0}, lev)) { - ECTRhofield = fields.get_alldirs("ECTRhofield", lev); + if (fields.has(FieldType::ECTRhofield, Direction{0}, lev)) { + ECTRhofield = fields.get_alldirs(FieldType::ECTRhofield, lev); } // Select algorithm (The choice of algorithm is a runtime option, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp index d678bed3b01..9ecae05516d 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp @@ -17,6 +17,7 @@ # include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H" #endif #include "EmbeddedBoundary/Enabled.H" +#include "Fields.H" #include "Utils/TextMsg.H" #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXConst.H" @@ -59,20 +60,22 @@ void FiniteDifferenceSolver::EvolveEPML ( "PML are not implemented in cylindrical geometry."); #else using ablastr::fields::Direction; + using warpx::fields::FieldType; + const ablastr::fields::VectorField Efield = (patch_type == PatchType::fine) ? - fields.get_alldirs("pml_E_fp", level) : fields.get_alldirs("pml_E_cp", level); + fields.get_alldirs(FieldType::pml_E_fp, level) : fields.get_alldirs(FieldType::pml_E_cp, level); const ablastr::fields::VectorField Bfield = (patch_type == PatchType::fine) ? - fields.get_alldirs("pml_B_fp", level) : fields.get_alldirs("pml_B_cp", level); + fields.get_alldirs(FieldType::pml_B_fp, level) : fields.get_alldirs(FieldType::pml_B_cp, level); const ablastr::fields::VectorField Jfield = (patch_type == PatchType::fine) ? - fields.get_alldirs("pml_j_fp", level) : fields.get_alldirs("pml_j_cp", level); + fields.get_alldirs(FieldType::pml_j_fp, level) : fields.get_alldirs(FieldType::pml_j_cp, level); ablastr::fields::VectorField edge_lengths; - if (fields.has("pml_edge_lengths", Direction{0}, level)) { - edge_lengths = fields.get_alldirs("pml_edge_lengths", level); + if (fields.has(FieldType::pml_edge_lengths, Direction{0}, level)) { + edge_lengths = fields.get_alldirs(FieldType::pml_edge_lengths, level); } amrex::MultiFab * Ffield = nullptr; - if (fields.has("pml_F_fp", level)) { + if (fields.has(FieldType::pml_F_fp, level)) { Ffield = (patch_type == PatchType::fine) ? - fields.get("pml_F_fp", level) : fields.get("pml_F_cp", level); + fields.get(FieldType::pml_F_fp, level) : fields.get(FieldType::pml_F_cp, level); } if (m_grid_type == GridType::Collocated) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index dba8a7de3f9..dbf56a0e899 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -14,7 +14,7 @@ #include "WarpX.H" using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; HybridPICModel::HybridPICModel () { @@ -73,42 +73,42 @@ void HybridPICModel::AllocateLevelMFs (ablastr::fields::MultiFabRegister & field // interpolated or extrapolated to appropriate timesteps. // The "hybrid_current_fp_ampere" multifab stores the total current calculated as // the curl of B. - fields.alloc_init("hybrid_electron_pressure_fp", + fields.alloc_init(FieldType::hybrid_electron_pressure_fp, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngRho, 0.0_rt); - fields.alloc_init("hybrid_rho_fp_temp", + fields.alloc_init(FieldType::hybrid_rho_fp_temp, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngRho, 0.0_rt); - fields.alloc_init("hybrid_current_fp_temp", Direction{0}, + fields.alloc_init(FieldType::hybrid_current_fp_temp, Direction{0}, lev, amrex::convert(ba, jx_nodal_flag), dm, ncomps, ngJ, 0.0_rt); - fields.alloc_init("hybrid_current_fp_temp", Direction{1}, + fields.alloc_init(FieldType::hybrid_current_fp_temp, Direction{1}, lev, amrex::convert(ba, jy_nodal_flag), dm, ncomps, ngJ, 0.0_rt); - fields.alloc_init("hybrid_current_fp_temp", Direction{2}, + fields.alloc_init(FieldType::hybrid_current_fp_temp, Direction{2}, lev, amrex::convert(ba, jz_nodal_flag), dm, ncomps, ngJ, 0.0_rt); - fields.alloc_init("hybrid_current_fp_ampere", Direction{0}, + fields.alloc_init(FieldType::hybrid_current_fp_ampere, Direction{0}, lev, amrex::convert(ba, jx_nodal_flag), dm, ncomps, ngJ, 0.0_rt); - fields.alloc_init("hybrid_current_fp_ampere", Direction{1}, + fields.alloc_init(FieldType::hybrid_current_fp_ampere, Direction{1}, lev, amrex::convert(ba, jy_nodal_flag), dm, ncomps, ngJ, 0.0_rt); - fields.alloc_init("hybrid_current_fp_ampere", Direction{2}, + fields.alloc_init(FieldType::hybrid_current_fp_ampere, Direction{2}, lev, amrex::convert(ba, jz_nodal_flag), dm, ncomps, ngJ, 0.0_rt); // the external current density multifab is made nodal to avoid needing to interpolate // to a nodal grid as has to be done for the ion and total current density multifabs // this also allows the external current multifab to not have any ghost cells - fields.alloc_init("hybrid_current_fp_external", Direction{0}, + fields.alloc_init(FieldType::hybrid_current_fp_external, Direction{0}, lev, amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), dm, ncomps, IntVect(AMREX_D_DECL(0,0,0)), 0.0_rt); - fields.alloc_init("hybrid_current_fp_external", Direction{1}, + fields.alloc_init(FieldType::hybrid_current_fp_external, Direction{1}, lev, amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), dm, ncomps, IntVect(AMREX_D_DECL(0,0,0)), 0.0_rt); - fields.alloc_init("hybrid_current_fp_external", Direction{2}, + fields.alloc_init(FieldType::hybrid_current_fp_external, Direction{2}, lev, amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), dm, ncomps, IntVect(AMREX_D_DECL(0,0,0)), 0.0_rt); @@ -147,15 +147,15 @@ void HybridPICModel::InitData () using ablastr::fields::Direction; // Get the grid staggering of the fields involved in calculating E - amrex::IntVect Jx_stag = warpx.m_fields.get("current_fp",Direction{0},0)->ixType().toIntVect(); - amrex::IntVect Jy_stag = warpx.m_fields.get("current_fp",Direction{1},0)->ixType().toIntVect(); - amrex::IntVect Jz_stag = warpx.m_fields.get("current_fp",Direction{2},0)->ixType().toIntVect(); - amrex::IntVect Bx_stag = warpx.m_fields.get("Bfield_fp",Direction{0},0)->ixType().toIntVect(); - amrex::IntVect By_stag = warpx.m_fields.get("Bfield_fp",Direction{1},0)->ixType().toIntVect(); - amrex::IntVect Bz_stag = warpx.m_fields.get("Bfield_fp",Direction{2},0)->ixType().toIntVect(); - amrex::IntVect Ex_stag = warpx.m_fields.get("Efield_fp",Direction{0},0)->ixType().toIntVect(); - amrex::IntVect Ey_stag = warpx.m_fields.get("Efield_fp",Direction{1},0)->ixType().toIntVect(); - amrex::IntVect Ez_stag = warpx.m_fields.get("Efield_fp",Direction{2},0)->ixType().toIntVect(); + amrex::IntVect Jx_stag = warpx.m_fields.get(FieldType::current_fp, Direction{0}, 0)->ixType().toIntVect(); + amrex::IntVect Jy_stag = warpx.m_fields.get(FieldType::current_fp, Direction{1}, 0)->ixType().toIntVect(); + amrex::IntVect Jz_stag = warpx.m_fields.get(FieldType::current_fp, Direction{2}, 0)->ixType().toIntVect(); + amrex::IntVect Bx_stag = warpx.m_fields.get(FieldType::Bfield_fp, Direction{0}, 0)->ixType().toIntVect(); + amrex::IntVect By_stag = warpx.m_fields.get(FieldType::Bfield_fp, Direction{1}, 0)->ixType().toIntVect(); + amrex::IntVect Bz_stag = warpx.m_fields.get(FieldType::Bfield_fp, Direction{2}, 0)->ixType().toIntVect(); + amrex::IntVect Ex_stag = warpx.m_fields.get(FieldType::Efield_fp, Direction{0}, 0)->ixType().toIntVect(); + amrex::IntVect Ey_stag = warpx.m_fields.get(FieldType::Efield_fp, Direction{1}, 0)->ixType().toIntVect(); + amrex::IntVect Ez_stag = warpx.m_fields.get(FieldType::Efield_fp, Direction{2}, 0)->ixType().toIntVect(); // Check that the grid types are appropriate const bool appropriate_grids = ( @@ -224,9 +224,9 @@ void HybridPICModel::InitData () #ifdef AMREX_USE_EB if (EB::enabled()) { using ablastr::fields::Direction; - auto const & edge_lengths_x = *warpx.m_fields.get("edge_lengths", Direction{0}, lev); - auto const & edge_lengths_y = *warpx.m_fields.get("edge_lengths", Direction{1}, lev); - auto const & edge_lengths_z = *warpx.m_fields.get("edge_lengths", Direction{2}, lev); + auto const & edge_lengths_x = *warpx.m_fields.get(FieldType::edge_lengths, Direction{0}, lev); + auto const & edge_lengths_y = *warpx.m_fields.get(FieldType::edge_lengths, Direction{1}, lev); + auto const & edge_lengths_z = *warpx.m_fields.get(FieldType::edge_lengths, Direction{2}, lev); edge_lengths = std::array< std::unique_ptr, 3 >{ std::make_unique( @@ -269,9 +269,9 @@ void HybridPICModel::GetCurrentExternal ( const RealBox& real_box = warpx.Geom(lev).ProbDomain(); using ablastr::fields::Direction; - amrex::MultiFab * mfx = warpx.m_fields.get("hybrid_current_fp_external", Direction{0}, lev); - amrex::MultiFab * mfy = warpx.m_fields.get("hybrid_current_fp_external", Direction{1}, lev); - amrex::MultiFab * mfz = warpx.m_fields.get("hybrid_current_fp_external", Direction{2}, lev); + amrex::MultiFab * mfx = warpx.m_fields.get(FieldType::hybrid_current_fp_external, Direction{0}, lev); + amrex::MultiFab * mfy = warpx.m_fields.get(FieldType::hybrid_current_fp_external, Direction{1}, lev); + amrex::MultiFab * mfz = warpx.m_fields.get(FieldType::hybrid_current_fp_external, Direction{2}, lev); const amrex::IntVect x_nodal_flag = mfx->ixType().toIntVect(); const amrex::IntVect y_nodal_flag = mfy->ixType().toIntVect(); @@ -403,7 +403,7 @@ void HybridPICModel::CalculateCurrentAmpere ( WARPX_PROFILE("WarpX::CalculateCurrentAmpere()"); auto& warpx = WarpX::GetInstance(); - ablastr::fields::VectorField current_fp_ampere = warpx.m_fields.get_alldirs("hybrid_current_fp_ampere", lev); + ablastr::fields::VectorField current_fp_ampere = warpx.m_fields.get_alldirs(FieldType::hybrid_current_fp_ampere, lev); warpx.get_pointer_fdtd_solver_fp(lev)->CalculateCurrentAmpere( current_fp_ampere, Bfield, edge_lengths, lev ); @@ -466,9 +466,9 @@ void HybridPICModel::HybridPICSolveE ( auto& warpx = WarpX::GetInstance(); - ablastr::fields::VectorField current_fp_ampere = warpx.m_fields.get_alldirs("hybrid_current_fp_ampere", lev); - const ablastr::fields::VectorField current_fp_external = warpx.m_fields.get_alldirs("hybrid_current_fp_external", lev); - const ablastr::fields::ScalarField electron_pressure_fp = warpx.m_fields.get("hybrid_electron_pressure_fp", lev); + ablastr::fields::VectorField current_fp_ampere = warpx.m_fields.get_alldirs(FieldType::hybrid_current_fp_ampere, lev); + const ablastr::fields::VectorField current_fp_external = warpx.m_fields.get_alldirs(FieldType::hybrid_current_fp_external, lev); + const ablastr::fields::ScalarField electron_pressure_fp = warpx.m_fields.get(FieldType::hybrid_electron_pressure_fp, lev); // Solve E field in regular cells warpx.get_pointer_fdtd_solver_fp(lev)->HybridPICSolveE( @@ -494,8 +494,8 @@ void HybridPICModel::CalculateElectronPressure(const int lev) const WARPX_PROFILE("WarpX::CalculateElectronPressure()"); auto& warpx = WarpX::GetInstance(); - ablastr::fields::ScalarField electron_pressure_fp = warpx.m_fields.get("hybrid_electron_pressure_fp", lev); - ablastr::fields::ScalarField rho_fp = warpx.m_fields.get("rho_fp", lev); + ablastr::fields::ScalarField electron_pressure_fp = warpx.m_fields.get(FieldType::hybrid_electron_pressure_fp, lev); + ablastr::fields::ScalarField rho_fp = warpx.m_fields.get(FieldType::rho_fp, lev); // Calculate the electron pressure using rho^{n+1}. FillElectronPressureMF( diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp index ea1432dd1f9..583e0a43903 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp @@ -23,7 +23,7 @@ #include using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; MacroscopicProperties::MacroscopicProperties () { diff --git a/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp index e43fe058c90..2236118a30c 100644 --- a/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp +++ b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp @@ -7,7 +7,7 @@ #include "SemiImplicitEM.H" #include "WarpX.H" -using namespace warpx::fields; +using warpx::fields::FieldType; using namespace amrex::literals; void SemiImplicitEM::Define ( WarpX* a_WarpX ) diff --git a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp index 42d016bb34a..e62ced29f6d 100644 --- a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp +++ b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp @@ -8,7 +8,7 @@ #include "ThetaImplicitEM.H" #include "WarpX.H" -using namespace warpx::fields; +using warpx::fields::FieldType; using namespace amrex::literals; void ThetaImplicitEM::Define ( WarpX* const a_WarpX ) @@ -28,14 +28,14 @@ void ThetaImplicitEM::Define ( WarpX* const a_WarpX ) using ablastr::fields::Direction; const int num_levels = 1; for (int lev = 0; lev < num_levels; ++lev) { - const auto& ba_Bx = m_WarpX->m_fields.get("Bfield_fp",Direction{0},lev)->boxArray(); - const auto& ba_By = m_WarpX->m_fields.get("Bfield_fp",Direction{1},lev)->boxArray(); - const auto& ba_Bz = m_WarpX->m_fields.get("Bfield_fp",Direction{2},lev)->boxArray(); - const auto& dm = m_WarpX->m_fields.get("Bfield_fp",Direction{0},lev)->DistributionMap(); - const amrex::IntVect ngb = m_WarpX->m_fields.get("Bfield_fp",Direction{0},lev)->nGrowVect(); - m_WarpX->m_fields.alloc_init("Bold", Direction{0}, lev, ba_Bx, dm, 1, ngb, 0.0_rt); - m_WarpX->m_fields.alloc_init("Bold", Direction{1}, lev, ba_By, dm, 1, ngb, 0.0_rt); - m_WarpX->m_fields.alloc_init("Bold", Direction{2}, lev, ba_Bz, dm, 1, ngb, 0.0_rt); + const auto& ba_Bx = m_WarpX->m_fields.get(FieldType::Bfield_fp, Direction{0}, lev)->boxArray(); + const auto& ba_By = m_WarpX->m_fields.get(FieldType::Bfield_fp, Direction{1}, lev)->boxArray(); + const auto& ba_Bz = m_WarpX->m_fields.get(FieldType::Bfield_fp, Direction{2}, lev)->boxArray(); + const auto& dm = m_WarpX->m_fields.get(FieldType::Bfield_fp, Direction{0}, lev)->DistributionMap(); + const amrex::IntVect ngb = m_WarpX->m_fields.get(FieldType::Bfield_fp, Direction{0}, lev)->nGrowVect(); + m_WarpX->m_fields.alloc_init(FieldType::Bold, Direction{0}, lev, ba_Bx, dm, 1, ngb, 0.0_rt); + m_WarpX->m_fields.alloc_init(FieldType::Bold, Direction{1}, lev, ba_By, dm, 1, ngb, 0.0_rt); + m_WarpX->m_fields.alloc_init(FieldType::Bold, Direction{2}, lev, ba_Bz, dm, 1, ngb, 0.0_rt); } // Parse theta-implicit solver specific parameters @@ -91,8 +91,8 @@ void ThetaImplicitEM::OneStep ( const amrex::Real a_time, const int num_levels = 1; for (int lev = 0; lev < num_levels; ++lev) { - const ablastr::fields::VectorField Bfp = m_WarpX->m_fields.get_alldirs("Bfield_fp",lev); - ablastr::fields::VectorField Bold = m_WarpX->m_fields.get_alldirs("Bold",lev); + const ablastr::fields::VectorField Bfp = m_WarpX->m_fields.get_alldirs(FieldType::Bfield_fp, lev); + ablastr::fields::VectorField Bold = m_WarpX->m_fields.get_alldirs(FieldType::Bold, lev); for (int n = 0; n < 3; ++n) { amrex::MultiFab::Copy( *Bold[n], *Bfp[n], 0, 0, Bold[n]->nComp(), Bold[n]->nGrowVect() ); @@ -147,7 +147,7 @@ void ThetaImplicitEM::UpdateWarpXFields ( const WarpXSolverVec& a_E, m_WarpX->SetElectricFieldAndApplyBCs( a_E ); // Update Bfield_fp owned by WarpX - ablastr::fields::MultiLevelVectorField const& Bold = m_WarpX->m_fields.get_mr_levels_alldirs("Bold",0); + ablastr::fields::MultiLevelVectorField const& Bold = m_WarpX->m_fields.get_mr_levels_alldirs(FieldType::Bold, 0); m_WarpX->UpdateMagneticFieldAndApplyBCs( Bold, m_theta*a_dt ); } @@ -163,7 +163,7 @@ void ThetaImplicitEM::FinishFieldUpdate ( amrex::Real a_new_time ) const amrex::Real c1 = 1._rt - c0; m_E.linComb( c0, m_E, c1, m_Eold ); m_WarpX->SetElectricFieldAndApplyBCs( m_E ); - ablastr::fields::MultiLevelVectorField const & Bold = m_WarpX->m_fields.get_mr_levels_alldirs("Bold",0); + ablastr::fields::MultiLevelVectorField const & Bold = m_WarpX->m_fields.get_mr_levels_alldirs(FieldType::Bold, 0); m_WarpX->FinishMagneticFieldAndApplyBCs( Bold, m_theta ); } diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp index a393bae0eaf..806c3412990 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp @@ -73,7 +73,9 @@ WarpX::SetElectricFieldAndApplyBCs ( const WarpXSolverVec& a_E ) a_E.getArrayVecType()==warpx::fields::FieldType::Efield_fp, "WarpX::SetElectricFieldAndApplyBCs() must be called with Efield_fp type"); - ablastr::fields::MultiLevelVectorField Efield_fp = m_fields.get_mr_levels_alldirs("Efield_fp",finest_level); + using warpx::fields::FieldType; + + ablastr::fields::MultiLevelVectorField Efield_fp = m_fields.get_mr_levels_alldirs(FieldType::Efield_fp, finest_level); const ablastr::fields::MultiLevelVectorField& Evec = a_E.getArrayVec(); amrex::MultiFab::Copy(*Efield_fp[0][0], *Evec[0][0], 0, 0, ncomps, Evec[0][0]->nGrowVect()); amrex::MultiFab::Copy(*Efield_fp[0][1], *Evec[0][1], 0, 0, ncomps, Evec[0][1]->nGrowVect()); @@ -87,8 +89,10 @@ WarpX::UpdateMagneticFieldAndApplyBCs( ablastr::fields::MultiLevelVectorField co amrex::Real a_thetadt ) { using ablastr::fields::Direction; + using warpx::fields::FieldType; + for (int lev = 0; lev <= finest_level; ++lev) { - ablastr::fields::VectorField Bfp = m_fields.get_alldirs("Bfield_fp",lev); + ablastr::fields::VectorField Bfp = m_fields.get_alldirs(FieldType::Bfield_fp, lev); amrex::MultiFab::Copy(*Bfp[0], *a_Bn[lev][0], 0, 0, ncomps, a_Bn[lev][0]->nGrowVect()); amrex::MultiFab::Copy(*Bfp[1], *a_Bn[lev][1], 0, 0, ncomps, a_Bn[lev][1]->nGrowVect()); amrex::MultiFab::Copy(*Bfp[2], *a_Bn[lev][2], 0, 0, ncomps, a_Bn[lev][2]->nGrowVect()); @@ -101,7 +105,9 @@ void WarpX::FinishMagneticFieldAndApplyBCs( ablastr::fields::MultiLevelVectorField const& a_Bn, amrex::Real a_theta ) { - FinishImplicitField(m_fields.get_mr_levels_alldirs("Bfield_fp", 0), a_Bn, a_theta); + using warpx::fields::FieldType; + + FinishImplicitField(m_fields.get_mr_levels_alldirs(FieldType::Bfield_fp, 0), a_Bn, a_theta); ApplyMagneticFieldBCs(); } diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp index 10673704759..6a0e6bb8a91 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp @@ -7,7 +7,7 @@ #include "FieldSolver/ImplicitSolvers/WarpXSolverVec.H" #include "WarpX.H" -using namespace warpx::fields; +using warpx::fields::FieldType; WarpXSolverVec::~WarpXSolverVec () { @@ -73,7 +73,7 @@ void WarpXSolverVec::Define ( WarpX* a_WarpX, "WarpXSolverVec::Define() called with array_type not an array field"); for (int lev = 0; lev < m_num_amr_levels; ++lev) { - const ablastr::fields::VectorField this_array = m_WarpX->m_fields.get_alldirs(m_vector_type_name,lev); + const ablastr::fields::VectorField this_array = m_WarpX->m_fields.get_alldirs(m_vector_type_name, lev); for (int n = 0; n < 3; n++) { m_array_vec[lev][n] = new amrex::MultiFab( this_array[n]->boxArray(), this_array[n]->DistributionMap(), @@ -122,7 +122,7 @@ void WarpXSolverVec::Copy ( FieldType a_array_type, for (int lev = 0; lev < m_num_amr_levels; ++lev) { if (m_array_type != FieldType::None) { - const ablastr::fields::VectorField this_array = m_WarpX->m_fields.get_alldirs(m_vector_type_name,lev); + const ablastr::fields::VectorField this_array = m_WarpX->m_fields.get_alldirs(m_vector_type_name, lev); for (int n = 0; n < 3; ++n) { amrex::MultiFab::Copy( *m_array_vec[lev][n], *this_array[n], 0, 0, m_ncomp, amrex::IntVect::TheZeroVector() ); diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp index a8266f54135..84efe8bf45a 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp @@ -6,6 +6,7 @@ */ #include "WarpX.H" +#include "Fields.H" #include "FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H" #include "EmbeddedBoundary/Enabled.H" #include "Parallelization/GuardCellManager.H" @@ -73,6 +74,7 @@ void WarpX::AddMagnetostaticFieldLabFrame() { using ablastr::fields::Direction; + using warpx::fields::FieldType; WARPX_PROFILE("WarpX::AddMagnetostaticFieldLabFrame"); @@ -90,7 +92,7 @@ WarpX::AddMagnetostaticFieldLabFrame() // reset current_fp before depositing current density for this step for (int lev = 0; lev <= max_level; lev++) { for (int dim=0; dim < 3; dim++) { - m_fields.get("current_fp", Direction{dim}, lev)->setVal(0.); + m_fields.get(FieldType::current_fp, Direction{dim}, lev)->setVal(0.); } } @@ -99,7 +101,7 @@ WarpX::AddMagnetostaticFieldLabFrame() WarpXParticleContainer& species = mypc->GetParticleContainer(ispecies); if (!species.do_not_deposit) { species.DepositCurrent( - m_fields.get_mr_levels_alldirs("current_fp", finest_level), + m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), dt[0], 0.); } } @@ -107,9 +109,9 @@ WarpX::AddMagnetostaticFieldLabFrame() #ifdef WARPX_DIM_RZ for (int lev = 0; lev <= max_level; lev++) { ApplyInverseVolumeScalingToCurrentDensity( - m_fields.get("current_fp", Direction{0}, lev), - m_fields.get("current_fp", Direction{1}, lev), - m_fields.get("current_fp", Direction{2}, lev), + m_fields.get(FieldType::current_fp, Direction{0}, lev), + m_fields.get(FieldType::current_fp, Direction{1}, lev), + m_fields.get(FieldType::current_fp, Direction{2}, lev), lev ); } #endif @@ -117,7 +119,7 @@ WarpX::AddMagnetostaticFieldLabFrame() SyncCurrent("current_fp"); // set the boundary and current density potentials - setVectorPotentialBC(m_fields.get_mr_levels_alldirs("vector_potential_fp_nodal", finest_level)); + setVectorPotentialBC(m_fields.get_mr_levels_alldirs(FieldType::vector_potential_fp_nodal, finest_level)); // Compute the vector potential A, by solving the Poisson equation WARPX_ALWAYS_ASSERT_WITH_MESSAGE( !IsPythonCallbackInstalled("poissonsolver"), @@ -131,8 +133,8 @@ WarpX::AddMagnetostaticFieldLabFrame() const int self_fields_verbosity = 2; computeVectorPotential( - m_fields.get_mr_levels_alldirs("current_fp", finest_level), - m_fields.get_mr_levels_alldirs("vector_potential_fp_nodal", finest_level), + m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), + m_fields.get_mr_levels_alldirs(FieldType::vector_potential_fp_nodal, finest_level), self_fields_required_precision, magnetostatic_absolute_tolerance, self_fields_max_iters, self_fields_verbosity); } @@ -162,6 +164,7 @@ WarpX::computeVectorPotential (ablastr::fields::MultiLevelVectorField const& cur int const verbosity) // const // This breaks non-const m_fields.get_mr_levels_alldirs { using ablastr::fields::Direction; + using warpx::fields::FieldType; // create a vector to our fields, sorted by level amrex::Vector> sorted_curr; @@ -176,12 +179,12 @@ WarpX::computeVectorPotential (ablastr::fields::MultiLevelVectorField const& cur } #if defined(AMREX_USE_EB) - const ablastr::fields::MultiLevelVectorField Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); + const ablastr::fields::MultiLevelVectorField Bfield_fp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_fp, finest_level); const std::optional post_A_calculation( { Bfield_fp, - m_fields.get_mr_levels_alldirs("vector_potential_grad_buf_e_stag", finest_level), - m_fields.get_mr_levels_alldirs("vector_potential_grad_buf_b_stag", finest_level) + m_fields.get_mr_levels_alldirs(FieldType::vector_potential_grad_buf_e_stag, finest_level), + m_fields.get_mr_levels_alldirs(FieldType::vector_potential_grad_buf_b_stag, finest_level) }); amrex::Vector factories; diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index fd1f55142f2..db2dcea65ac 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -10,6 +10,7 @@ #include "BoundaryConditions/PML.H" #include "Evolve/WarpXDtType.H" +#include "Fields.H" #include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H" #if defined(WARPX_USE_FFT) # include "FieldSolver/SpectralSolver/SpectralFieldData.H" @@ -53,6 +54,7 @@ #include using namespace amrex; +using warpx::fields::FieldType; #ifdef WARPX_USE_FFT namespace { @@ -214,14 +216,14 @@ WarpX::PSATDForwardTransformF () for (int lev = 0; lev <= finest_level; ++lev) { - if (m_fields.has("F_fp", lev)) { - spectral_solver_fp[lev]->ForwardTransform(lev, *m_fields.get("F_fp", lev), Idx.F); + if (m_fields.has(FieldType::F_fp, lev)) { + spectral_solver_fp[lev]->ForwardTransform(lev, *m_fields.get(FieldType::F_fp, lev), Idx.F); } if (spectral_solver_cp[lev]) { - if (m_fields.has("F_cp", lev)) { - spectral_solver_cp[lev]->ForwardTransform(lev, *m_fields.get("F_cp", lev), Idx.F); + if (m_fields.has(FieldType::F_cp, lev)) { + spectral_solver_cp[lev]->ForwardTransform(lev, *m_fields.get(FieldType::F_cp, lev), Idx.F); } } } @@ -235,17 +237,17 @@ WarpX::PSATDBackwardTransformF () for (int lev = 0; lev <= finest_level; ++lev) { #ifdef WARPX_DIM_RZ - if (m_fields.has("F_fp", lev)) { spectral_solver_fp[lev]->BackwardTransform(lev, *m_fields.get("F_fp", lev), Idx.F); } + if (m_fields.has(FieldType::F_fp, lev)) { spectral_solver_fp[lev]->BackwardTransform(lev, *m_fields.get(FieldType::F_fp, lev), Idx.F); } #else - if (m_fields.has("F_fp", lev)) { spectral_solver_fp[lev]->BackwardTransform(lev, *m_fields.get("F_fp", lev), Idx.F, m_fill_guards_fields); } + if (m_fields.has(FieldType::F_fp, lev)) { spectral_solver_fp[lev]->BackwardTransform(lev, *m_fields.get(FieldType::F_fp, lev), Idx.F, m_fill_guards_fields); } #endif if (spectral_solver_cp[lev]) { #ifdef WARPX_DIM_RZ - if (m_fields.has("F_cp", lev)) { spectral_solver_cp[lev]->BackwardTransform(lev, *m_fields.get("F_cp", lev), Idx.F); } + if (m_fields.has(FieldType::F_cp, lev)) { spectral_solver_cp[lev]->BackwardTransform(lev, *m_fields.get(FieldType::F_cp, lev), Idx.F); } #else - if (m_fields.has("F_cp", lev)) { spectral_solver_cp[lev]->BackwardTransform(lev, *m_fields.get("F_cp", lev), Idx.F, m_fill_guards_fields); } + if (m_fields.has(FieldType::F_cp, lev)) { spectral_solver_cp[lev]->BackwardTransform(lev, *m_fields.get(FieldType::F_cp, lev), Idx.F, m_fill_guards_fields); } #endif } } @@ -253,7 +255,7 @@ WarpX::PSATDBackwardTransformF () // Damp the field in the guard cells for (int lev = 0; lev <= finest_level; ++lev) { - DampFieldsInGuards(lev, m_fields.get("F_fp", lev)); + DampFieldsInGuards(lev, m_fields.get(FieldType::F_fp, lev)); } } @@ -264,14 +266,14 @@ WarpX::PSATDForwardTransformG () for (int lev = 0; lev <= finest_level; ++lev) { - if (m_fields.has("G_fp", lev)) { - spectral_solver_fp[lev]->ForwardTransform(lev, *m_fields.get("G_fp", lev), Idx.G); + if (m_fields.has(FieldType::G_fp, lev)) { + spectral_solver_fp[lev]->ForwardTransform(lev, *m_fields.get(FieldType::G_fp, lev), Idx.G); } if (spectral_solver_cp[lev]) { - if (m_fields.has("G_cp", lev)) { - spectral_solver_fp[lev]->ForwardTransform(lev, *m_fields.get("G_cp", lev), Idx.G); + if (m_fields.has(FieldType::G_cp, lev)) { + spectral_solver_fp[lev]->ForwardTransform(lev, *m_fields.get(FieldType::G_cp, lev), Idx.G); } } } @@ -284,8 +286,8 @@ WarpX::PSATDBackwardTransformG () for (int lev = 0; lev <= finest_level; ++lev) { - if (m_fields.has("G_fp", lev)) { - MultiFab* G_fp = m_fields.get("G_fp", lev); + if (m_fields.has(FieldType::G_fp, lev)) { + MultiFab* G_fp = m_fields.get(FieldType::G_fp, lev); #ifdef WARPX_DIM_RZ spectral_solver_fp[lev]->BackwardTransform(lev, *G_fp, Idx.G); #else @@ -297,8 +299,8 @@ WarpX::PSATDBackwardTransformG () if (spectral_solver_cp[lev]) { - if (m_fields.has("G_cp", lev)) { - MultiFab* G_cp = m_fields.get("G_cp", lev); + if (m_fields.has(FieldType::G_cp, lev)) { + MultiFab* G_cp = m_fields.get(FieldType::G_cp, lev); #ifdef WARPX_DIM_RZ spectral_solver_fp[lev]->BackwardTransform(lev, *G_cp, Idx.G); #else @@ -505,15 +507,15 @@ void WarpX::PSATDSubtractCurrentPartialSumsAvg () { const std::array& dx = WarpX::CellSize(lev); - amrex::MultiFab const& Dx = *m_fields.get("current_fp_vay", Direction{0}, lev); - amrex::MultiFab const& Dy = *m_fields.get("current_fp_vay", Direction{1}, lev); - amrex::MultiFab const& Dz = *m_fields.get("current_fp_vay", Direction{2}, lev); + amrex::MultiFab const& Dx = *m_fields.get(FieldType::current_fp_vay, Direction{0}, lev); + amrex::MultiFab const& Dy = *m_fields.get(FieldType::current_fp_vay, Direction{1}, lev); + amrex::MultiFab const& Dz = *m_fields.get(FieldType::current_fp_vay, Direction{2}, lev); #if defined (WARPX_DIM_XZ) amrex::ignore_unused(Dy); #endif - amrex::MultiFab& Jx = *m_fields.get("current_fp", Direction{0}, lev); + amrex::MultiFab& Jx = *m_fields.get(FieldType::current_fp, Direction{0}, lev); #ifdef AMREX_USE_OMP @@ -544,7 +546,7 @@ void WarpX::PSATDSubtractCurrentPartialSumsAvg () #if defined (WARPX_DIM_3D) // Subtract average of cumulative sum from Jy - amrex::MultiFab& Jy = *m_fields.get("current_fp", Direction{1}, lev);; + amrex::MultiFab& Jy = *m_fields.get(FieldType::current_fp, Direction{1}, lev);; for (amrex::MFIter mfi(Jy); mfi.isValid(); ++mfi) { const amrex::Box& bx = mfi.fabbox(); @@ -569,7 +571,7 @@ void WarpX::PSATDSubtractCurrentPartialSumsAvg () #endif // Subtract average of cumulative sum from Jz - amrex::MultiFab& Jz = *m_fields.get("current_fp", Direction{2}, lev); + amrex::MultiFab& Jz = *m_fields.get(FieldType::current_fp, Direction{2}, lev); for (amrex::MFIter mfi(Jz); mfi.isValid(); ++mfi) { const amrex::Box& bx = mfi.fabbox(); @@ -725,7 +727,7 @@ WarpX::PushPSATD () std::string const rho_fp_string = "rho_fp"; std::string const rho_cp_string = "rho_cp"; - const ablastr::fields::MultiLevelVectorField current_fp = m_fields.get_mr_levels_alldirs("current_fp", finest_level); + const ablastr::fields::MultiLevelVectorField current_fp = m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level); std::string current_fp_string = "current_fp"; std::string const current_cp_string = "current_cp"; @@ -848,10 +850,10 @@ WarpX::PushPSATD () // Inverse FFT of E, B, F, and G PSATDBackwardTransformEB(); if (WarpX::fft_do_time_averaging) { - auto Efield_avg_fp = m_fields.get_mr_levels_alldirs("Efield_avg_fp", finest_level); - auto Bfield_avg_fp = m_fields.get_mr_levels_alldirs("Bfield_avg_fp", finest_level); - auto Efield_avg_cp = m_fields.get_mr_levels_alldirs("Efield_avg_cp", finest_level); - auto Bfield_avg_cp = m_fields.get_mr_levels_alldirs("Bfield_avg_cp", finest_level); + auto Efield_avg_fp = m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_fp, finest_level); + auto Bfield_avg_fp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_fp, finest_level); + auto Efield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_cp, finest_level); + auto Bfield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_cp, finest_level); PSATDBackwardTransformEBavg(Efield_avg_fp, Bfield_avg_fp, Efield_avg_cp, Bfield_avg_cp); } if (WarpX::do_dive_cleaning) { PSATDBackwardTransformF(); } @@ -956,13 +958,13 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) m_fdtd_solver_fp[lev]->EvolveE( m_fields, lev, patch_type, - m_fields.get_alldirs("Efield_fp",lev), + m_fields.get_alldirs(FieldType::Efield_fp, lev), a_dt ); } else { m_fdtd_solver_cp[lev]->EvolveE( m_fields, lev, patch_type, - m_fields.get_alldirs("Efield_cp",lev), + m_fields.get_alldirs(FieldType::Efield_cp, lev), a_dt ); } @@ -992,16 +994,16 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) #ifdef AMREX_USE_EB if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { if (patch_type == PatchType::fine) { - m_fdtd_solver_fp[lev]->EvolveECTRho( m_fields.get_alldirs("Efield_fp",lev), - m_fields.get_alldirs("edge_lengths", lev), - m_fields.get_alldirs("face_areas", lev), - m_fields.get_alldirs("ECTRhofield", lev), + m_fdtd_solver_fp[lev]->EvolveECTRho( m_fields.get_alldirs(FieldType::Efield_fp, lev), + m_fields.get_alldirs(FieldType::edge_lengths, lev), + m_fields.get_alldirs(FieldType::face_areas, lev), + m_fields.get_alldirs(FieldType::ECTRhofield, lev), lev ); } else { - m_fdtd_solver_cp[lev]->EvolveECTRho( m_fields.get_alldirs("Efield_cp",lev), - m_fields.get_alldirs("edge_lengths", lev), - m_fields.get_alldirs("face_areas", lev), - m_fields.get_alldirs("ECTRhofield", lev), + m_fdtd_solver_cp[lev]->EvolveECTRho( m_fields.get_alldirs(FieldType::Efield_cp, lev), + m_fields.get_alldirs(FieldType::edge_lengths, lev), + m_fields.get_alldirs(FieldType::face_areas, lev), + m_fields.get_alldirs(FieldType::ECTRhofield, lev), lev); } } @@ -1040,26 +1042,26 @@ WarpX::EvolveF (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_typ // Evolve F field in regular cells if (patch_type == PatchType::fine) { - m_fdtd_solver_fp[lev]->EvolveF( m_fields.get("F_fp", lev), - m_fields.get_alldirs("Efield_fp", lev), - m_fields.get("rho_fp",lev), rhocomp, a_dt ); + m_fdtd_solver_fp[lev]->EvolveF( m_fields.get(FieldType::F_fp, lev), + m_fields.get_alldirs(FieldType::Efield_fp, lev), + m_fields.get(FieldType::rho_fp,lev), rhocomp, a_dt ); } else { - m_fdtd_solver_cp[lev]->EvolveF( m_fields.get("F_cp", lev), - m_fields.get_alldirs("Efield_cp", lev), - m_fields.get("rho_cp",lev), rhocomp, a_dt ); + m_fdtd_solver_cp[lev]->EvolveF( m_fields.get(FieldType::F_cp, lev), + m_fields.get_alldirs(FieldType::Efield_cp, lev), + m_fields.get(FieldType::rho_cp,lev), rhocomp, a_dt ); } // Evolve F field in PML cells if (do_pml && pml[lev]->ok()) { if (patch_type == PatchType::fine) { m_fdtd_solver_fp[lev]->EvolveFPML( - m_fields.get("pml_F_fp", lev), - m_fields.get_alldirs("pml_E_fp", lev), + m_fields.get(FieldType::pml_F_fp, lev), + m_fields.get_alldirs(FieldType::pml_E_fp, lev), a_dt ); } else { m_fdtd_solver_cp[lev]->EvolveFPML( - m_fields.get("pml_F_cp", lev), - m_fields.get_alldirs("pml_E_cp", lev), + m_fields.get(FieldType::pml_F_cp, lev), + m_fields.get_alldirs(FieldType::pml_E_cp, lev), a_dt ); } } @@ -1099,16 +1101,16 @@ WarpX::EvolveG (int lev, PatchType patch_type, amrex::Real a_dt, DtType /*a_dt_t // Evolve G field in regular cells if (patch_type == PatchType::fine) { - ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); + ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_fp, finest_level); m_fdtd_solver_fp[lev]->EvolveG( - m_fields.get("G_fp", lev), + m_fields.get(FieldType::G_fp, lev), Bfield_fp[lev], a_dt); } else // coarse patch { - ablastr::fields::MultiLevelVectorField const& Bfield_cp_new = m_fields.get_mr_levels_alldirs("Bfield_cp", finest_level); + ablastr::fields::MultiLevelVectorField const& Bfield_cp_new = m_fields.get_mr_levels_alldirs(FieldType::Bfield_cp, finest_level); m_fdtd_solver_cp[lev]->EvolveG( - m_fields.get("G_cp", lev), + m_fields.get(FieldType::G_cp, lev), Bfield_cp_new[lev], a_dt); } @@ -1145,10 +1147,10 @@ WarpX::MacroscopicEvolveE (int lev, PatchType patch_type, amrex::Real a_dt) { ); m_fdtd_solver_fp[lev]->MacroscopicEvolveE( - m_fields.get_alldirs("Efield_fp", lev), - m_fields.get_alldirs("Bfield_fp", lev), - m_fields.get_alldirs("current_fp", lev), - m_fields.get_alldirs("edge_lengths", lev), + m_fields.get_alldirs(FieldType::Efield_fp, lev), + m_fields.get_alldirs(FieldType::Bfield_fp, lev), + m_fields.get_alldirs(FieldType::current_fp, lev), + m_fields.get_alldirs(FieldType::edge_lengths, lev), a_dt, m_macroscopic_properties); if (do_pml && pml[lev]->ok()) { diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index 13752458b9e..556b8f8fca4 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -7,6 +7,7 @@ * License: BSD-3-Clause-LBNL */ #include "Evolve/WarpXDtType.H" +#include "Fields.H" #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" #include "Particles/MultiParticleContainer.H" #include "Utils/TextMsg.H" @@ -17,11 +18,13 @@ #include + using namespace amrex; void WarpX::HybridPICEvolveFields () { using ablastr::fields::Direction; + using warpx::fields::FieldType; WARPX_PROFILE("WarpX::HybridPICEvolveFields()"); @@ -32,18 +35,18 @@ void WarpX::HybridPICEvolveFields () // The particles have now been pushed to their t_{n+1} positions. // Perform charge deposition in component 0 of rho_fp at t_{n+1}. - mypc->DepositCharge(m_fields.get_mr_levels("rho_fp", finest_level), 0._rt); + mypc->DepositCharge(m_fields.get_mr_levels(FieldType::rho_fp, finest_level), 0._rt); // Perform current deposition at t_{n+1/2}. - mypc->DepositCurrent(m_fields.get_mr_levels_alldirs("current_fp", finest_level), dt[0], -0.5_rt * dt[0]); + mypc->DepositCurrent(m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), dt[0], -0.5_rt * dt[0]); // Deposit cold-relativistic fluid charge and current if (do_fluid_species) { int const lev = 0; - myfl->DepositCharge(m_fields, *m_fields.get("rho_fp", lev), lev); + myfl->DepositCharge(m_fields, *m_fields.get(FieldType::rho_fp, lev), lev); myfl->DepositCurrent(m_fields, - *m_fields.get("current_fp", Direction{0}, lev), - *m_fields.get("current_fp", Direction{1}, lev), - *m_fields.get("current_fp", Direction{2}, lev), + *m_fields.get(FieldType::current_fp, Direction{0}, lev), + *m_fields.get(FieldType::current_fp, Direction{1}, lev), + *m_fields.get(FieldType::current_fp, Direction{2}, lev), lev); } @@ -57,7 +60,7 @@ void WarpX::HybridPICEvolveFields () // a nodal grid for (int lev = 0; lev <= finest_level; ++lev) { for (int idim = 0; idim < 3; ++idim) { - m_fields.get("current_fp", Direction{idim}, lev)->FillBoundary(Geom(lev).periodicity()); + m_fields.get(FieldType::current_fp, Direction{idim}, lev)->FillBoundary(Geom(lev).periodicity()); } } @@ -66,11 +69,11 @@ void WarpX::HybridPICEvolveFields () // Get the external current m_hybrid_pic_model->GetCurrentExternal( - m_fields.get_mr_levels_alldirs("edge_lengths", finest_level)); + m_fields.get_mr_levels_alldirs(FieldType::edge_lengths, finest_level)); // Reference hybrid-PIC multifabs - ablastr::fields::MultiLevelScalarField rho_fp_temp = m_fields.get_mr_levels("hybrid_rho_fp_temp", finest_level); - ablastr::fields::MultiLevelVectorField current_fp_temp = m_fields.get_mr_levels_alldirs("hybrid_current_fp_temp", finest_level); + ablastr::fields::MultiLevelScalarField rho_fp_temp = m_fields.get_mr_levels(FieldType::hybrid_rho_fp_temp, finest_level); + ablastr::fields::MultiLevelVectorField current_fp_temp = m_fields.get_mr_levels_alldirs(FieldType::hybrid_current_fp_temp, finest_level); // During the above deposition the charge and current density were updated // so that, at this time, we have rho^{n} in rho_fp_temp, rho{n+1} in the @@ -91,7 +94,7 @@ void WarpX::HybridPICEvolveFields () MultiFab::LinComb( *current_fp_temp[lev][idim], 0.5_rt, *current_fp_temp[lev][idim], 0, - 0.5_rt, *m_fields.get("current_fp", Direction{idim}, lev), 0, + 0.5_rt, *m_fields.get(FieldType::current_fp, Direction{idim}, lev), 0, 0, 1, current_fp_temp[lev][idim]->nGrowVect() ); } @@ -103,10 +106,10 @@ void WarpX::HybridPICEvolveFields () for (int sub_step = 0; sub_step < sub_steps; sub_step++) { m_hybrid_pic_model->BfieldEvolveRK( - m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level), - m_fields.get_mr_levels_alldirs("Efield_fp", finest_level), + m_fields.get_mr_levels_alldirs(FieldType::Bfield_fp, finest_level), + m_fields.get_mr_levels_alldirs(FieldType::Efield_fp, finest_level), current_fp_temp, rho_fp_temp, - m_fields.get_mr_levels_alldirs("edge_lengths", finest_level), + m_fields.get_mr_levels_alldirs(FieldType::edge_lengths, finest_level), 0.5_rt/sub_steps*dt[0], DtType::FirstHalf, guard_cells.ng_FieldSolver, WarpX::sync_nodal_points @@ -121,7 +124,7 @@ void WarpX::HybridPICEvolveFields () // the result into the 0'th index of `rho_fp_temp[lev]` MultiFab::LinComb( *rho_fp_temp[lev], 0.5_rt, *rho_fp_temp[lev], 0, - 0.5_rt, *m_fields.get("rho_fp", lev), 0, 0, 1, rho_fp_temp[lev]->nGrowVect() + 0.5_rt, *m_fields.get(FieldType::rho_fp, lev), 0, 0, 1, rho_fp_temp[lev]->nGrowVect() ); } @@ -129,11 +132,11 @@ void WarpX::HybridPICEvolveFields () for (int sub_step = 0; sub_step < sub_steps; sub_step++) { m_hybrid_pic_model->BfieldEvolveRK( - m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level), - m_fields.get_mr_levels_alldirs("Efield_fp", finest_level), - m_fields.get_mr_levels_alldirs("current_fp", finest_level), + m_fields.get_mr_levels_alldirs(FieldType::Bfield_fp, finest_level), + m_fields.get_mr_levels_alldirs(FieldType::Efield_fp, finest_level), + m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), rho_fp_temp, - m_fields.get_mr_levels_alldirs("edge_lengths", finest_level), + m_fields.get_mr_levels_alldirs(FieldType::edge_lengths, finest_level), 0.5_rt/sub_steps*dt[0], DtType::SecondHalf, guard_cells.ng_FieldSolver, WarpX::sync_nodal_points @@ -152,7 +155,7 @@ void WarpX::HybridPICEvolveFields () MultiFab::LinComb( *current_fp_temp[lev][idim], -1._rt, *current_fp_temp[lev][idim], 0, - 2._rt, *m_fields.get("current_fp", Direction{idim}, lev), 0, + 2._rt, *m_fields.get(FieldType::current_fp, Direction{idim}, lev), 0, 0, 1, current_fp_temp[lev][idim]->nGrowVect() ); } @@ -163,14 +166,14 @@ void WarpX::HybridPICEvolveFields () // Update the E field to t=n+1 using the extrapolated J_i^n+1 value m_hybrid_pic_model->CalculateCurrentAmpere( - m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level), - m_fields.get_mr_levels_alldirs("edge_lengths", finest_level)); + m_fields.get_mr_levels_alldirs(FieldType::Bfield_fp, finest_level), + m_fields.get_mr_levels_alldirs(FieldType::edge_lengths, finest_level)); m_hybrid_pic_model->HybridPICSolveE( - m_fields.get_mr_levels_alldirs("Efield_fp", finest_level), + m_fields.get_mr_levels_alldirs(FieldType::Efield_fp, finest_level), current_fp_temp, - m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level), - m_fields.get_mr_levels("rho_fp", finest_level), - m_fields.get_mr_levels_alldirs("edge_lengths", finest_level), false + m_fields.get_mr_levels_alldirs(FieldType::Bfield_fp, finest_level), + m_fields.get_mr_levels(FieldType::rho_fp, finest_level), + m_fields.get_mr_levels_alldirs(FieldType::edge_lengths, finest_level), false ); FillBoundaryE(guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); @@ -180,10 +183,10 @@ void WarpX::HybridPICEvolveFields () for (int lev = 0; lev <= finest_level; ++lev) { // copy 1 component value starting at index 0 to index 0 - MultiFab::Copy(*rho_fp_temp[lev], *m_fields.get("rho_fp", lev), + MultiFab::Copy(*rho_fp_temp[lev], *m_fields.get(FieldType::rho_fp, lev), 0, 0, 1, rho_fp_temp[lev]->nGrowVect()); for (int idim = 0; idim < 3; ++idim) { - MultiFab::Copy(*current_fp_temp[lev][idim], *m_fields.get("current_fp", Direction{idim}, lev), + MultiFab::Copy(*current_fp_temp[lev][idim], *m_fields.get(FieldType::current_fp, Direction{idim}, lev), 0, 0, 1, current_fp_temp[lev][idim]->nGrowVect()); } } @@ -191,11 +194,13 @@ void WarpX::HybridPICEvolveFields () void WarpX::HybridPICDepositInitialRhoAndJ () { - ablastr::fields::MultiLevelScalarField rho_fp_temp = m_fields.get_mr_levels("hybrid_rho_fp_temp", finest_level); - ablastr::fields::MultiLevelVectorField current_fp_temp = m_fields.get_mr_levels_alldirs("hybrid_current_fp_temp", finest_level); + using warpx::fields::FieldType; + + ablastr::fields::MultiLevelScalarField rho_fp_temp = m_fields.get_mr_levels(FieldType::hybrid_rho_fp_temp, finest_level); + ablastr::fields::MultiLevelVectorField current_fp_temp = m_fields.get_mr_levels_alldirs(FieldType::hybrid_current_fp_temp, finest_level); mypc->DepositCharge(rho_fp_temp, 0._rt); mypc->DepositCurrent(current_fp_temp, dt[0], 0._rt); - SyncRho(rho_fp_temp, m_fields.get_mr_levels("rho_cp", finest_level), m_fields.get_mr_levels("rho_buf", finest_level)); + SyncRho(rho_fp_temp, m_fields.get_mr_levels(FieldType::rho_cp, finest_level), m_fields.get_mr_levels(FieldType::rho_buf, finest_level)); SyncCurrent("hybrid_current_fp_temp"); for (int lev=0; lev <= finest_level; ++lev) { // SyncCurrent does not include a call to FillBoundary, but it is needed diff --git a/Source/FieldSolver/WarpXSolveFieldsES.cpp b/Source/FieldSolver/WarpXSolveFieldsES.cpp index 32362204b4a..6194570cd2d 100644 --- a/Source/FieldSolver/WarpXSolveFieldsES.cpp +++ b/Source/FieldSolver/WarpXSolveFieldsES.cpp @@ -7,21 +7,25 @@ * License: BSD-3-Clause-LBNL */ #include "FieldSolver/ElectrostaticSolvers/ElectrostaticSolver.H" + +#include "Fields.H" #include "Utils/WarpXProfilerWrapper.H" #include "WarpX.H" + void WarpX::ComputeSpaceChargeField (bool const reset_fields) { WARPX_PROFILE("WarpX::ComputeSpaceChargeField"); using ablastr::fields::Direction; + using warpx::fields::FieldType; if (reset_fields) { // Reset all E and B fields to 0, before calculating space-charge fields WARPX_PROFILE("WarpX::ComputeSpaceChargeField::reset_fields"); for (int lev = 0; lev <= max_level; lev++) { for (int comp=0; comp<3; comp++) { - m_fields.get("Efield_fp",Direction{comp},lev)->setVal(0); - m_fields.get("Bfield_fp",Direction{comp},lev)->setVal(0); + m_fields.get(FieldType::Efield_fp, Direction{comp}, lev)->setVal(0); + m_fields.get(FieldType::Bfield_fp, Direction{comp}, lev)->setVal(0); } } } diff --git a/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp b/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp index b95fb938a39..1ff1d1f866d 100644 --- a/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp +++ b/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp @@ -6,6 +6,7 @@ */ #include "WarpX.H" +#include "Fields.H" #include "Utils/TextMsg.H" #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXProfilerWrapper.H" @@ -70,6 +71,7 @@ void WarpX::Hybrid_QED_Push (int lev, PatchType patch_type, amrex::Real a_dt) { using ablastr::fields::Direction; + using warpx::fields::FieldType; const int patch_level = (patch_type == PatchType::fine) ? lev : lev-1; const std::array& dx_vec= WarpX::CellSize(patch_level); @@ -82,27 +84,27 @@ WarpX::Hybrid_QED_Push (int lev, PatchType patch_type, amrex::Real a_dt) MultiFab *Ex, *Ey, *Ez, *Bx, *By, *Bz, *Jx, *Jy, *Jz; if (patch_type == PatchType::fine) { - Ex = m_fields.get("Efield_fp",Direction{0},lev); - Ey = m_fields.get("Efield_fp",Direction{1},lev); - Ez = m_fields.get("Efield_fp",Direction{2},lev); - Bx = m_fields.get("Bfield_fp",Direction{0},lev); - By = m_fields.get("Bfield_fp",Direction{1},lev); - Bz = m_fields.get("Bfield_fp",Direction{2},lev); - Jx = m_fields.get("current_fp", Direction{0}, lev); - Jy = m_fields.get("current_fp", Direction{1}, lev); - Jz = m_fields.get("current_fp", Direction{2}, lev); + Ex = m_fields.get(FieldType::Efield_fp, Direction{0}, lev); + Ey = m_fields.get(FieldType::Efield_fp, Direction{1}, lev); + Ez = m_fields.get(FieldType::Efield_fp, Direction{2}, lev); + Bx = m_fields.get(FieldType::Bfield_fp, Direction{0}, lev); + By = m_fields.get(FieldType::Bfield_fp, Direction{1}, lev); + Bz = m_fields.get(FieldType::Bfield_fp, Direction{2}, lev); + Jx = m_fields.get(FieldType::current_fp, Direction{0}, lev); + Jy = m_fields.get(FieldType::current_fp, Direction{1}, lev); + Jz = m_fields.get(FieldType::current_fp, Direction{2}, lev); } else { - Ex = m_fields.get("Efield_cp",Direction{0},lev); - Ey = m_fields.get("Efield_cp",Direction{1},lev); - Ez = m_fields.get("Efield_cp",Direction{2},lev); - Bx = m_fields.get("Bfield_cp",Direction{0},lev); - By = m_fields.get("Bfield_cp",Direction{1},lev); - Bz = m_fields.get("Bfield_cp",Direction{2},lev); - Jx = m_fields.get("current_cp", Direction{0}, lev); - Jy = m_fields.get("current_cp", Direction{1}, lev); - Jz = m_fields.get("current_cp", Direction{2}, lev); + Ex = m_fields.get(FieldType::Efield_cp, Direction{0}, lev); + Ey = m_fields.get(FieldType::Efield_cp, Direction{1}, lev); + Ez = m_fields.get(FieldType::Efield_cp, Direction{2}, lev); + Bx = m_fields.get(FieldType::Bfield_cp, Direction{0}, lev); + By = m_fields.get(FieldType::Bfield_cp, Direction{1}, lev); + Bz = m_fields.get(FieldType::Bfield_cp, Direction{2}, lev); + Jx = m_fields.get(FieldType::current_cp, Direction{0}, lev); + Jy = m_fields.get(FieldType::current_cp, Direction{1}, lev); + Jz = m_fields.get(FieldType::current_cp, Direction{2}, lev); } amrex::LayoutData* cost = WarpX::getCosts(lev); diff --git a/Source/Fluids/WarpXFluidContainer.cpp b/Source/Fluids/WarpXFluidContainer.cpp index cc4c8777b18..326ce30c844 100644 --- a/Source/Fluids/WarpXFluidContainer.cpp +++ b/Source/Fluids/WarpXFluidContainer.cpp @@ -4,21 +4,24 @@ * * License: BSD-3-Clause-LBNL */ -#include "ablastr/coarsen/sample.H" +#include "Fields.H" #include "Particles/Pusher/UpdateMomentumHigueraCary.H" #include "Utils/WarpXProfilerWrapper.H" #include "MusclHancockUtils.H" #include "Fluids/WarpXFluidContainer.H" -#include "WarpX.H" -#include #include "Utils/Parser/ParserUtils.H" #include "Utils/WarpXUtil.H" #include "Utils/SpeciesUtils.H" +#include "WarpX.H" + +#include +#include using namespace ablastr::utils::communication; using namespace amrex; + WarpXFluidContainer::WarpXFluidContainer(int ispecies, const std::string &name): species_id{ispecies}, species_name{name} @@ -259,22 +262,24 @@ void WarpXFluidContainer::Evolve( bool skip_deposition) { using ablastr::fields::Direction; + using warpx::fields::FieldType; + WARPX_PROFILE("WarpXFluidContainer::Evolve"); - if (fields.has("rho_fp",lev) && ! skip_deposition && ! do_not_deposit) { + if (fields.has(FieldType::rho_fp,lev) && ! skip_deposition && ! do_not_deposit) { // Deposit charge before particle push, in component 0 of MultiFab rho. - DepositCharge(fields, *fields.get("rho_fp",lev), lev, 0); + DepositCharge(fields, *fields.get(FieldType::rho_fp,lev), lev, 0); } // Step the Lorentz Term if(!do_not_gather){ GatherAndPush(fields, - *fields.get("Efield_aux", Direction{0}, lev), - *fields.get("Efield_aux", Direction{1}, lev), - *fields.get("Efield_aux", Direction{2}, lev), - *fields.get("Bfield_aux", Direction{0}, lev), - *fields.get("Bfield_aux", Direction{1}, lev), - *fields.get("Bfield_aux", Direction{2}, lev), + *fields.get(FieldType::Efield_aux, Direction{0}, lev), + *fields.get(FieldType::Efield_aux, Direction{1}, lev), + *fields.get(FieldType::Efield_aux, Direction{2}, lev), + *fields.get(FieldType::Bfield_aux, Direction{0}, lev), + *fields.get(FieldType::Bfield_aux, Direction{1}, lev), + *fields.get(FieldType::Bfield_aux, Direction{2}, lev), cur_time, lev); } @@ -294,8 +299,8 @@ void WarpXFluidContainer::Evolve( // Deposit rho to the simulation mesh // Deposit charge (end of the step) - if (fields.has("rho_fp",lev) && ! skip_deposition && ! do_not_deposit) { - DepositCharge(fields, *fields.get("rho_fp",lev), lev, 1); + if (fields.has(FieldType::rho_fp,lev) && ! skip_deposition && ! do_not_deposit) { + DepositCharge(fields, *fields.get(FieldType::rho_fp,lev), lev, 1); } // Deposit J to the simulation mesh diff --git a/Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp b/Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp index ee8fcf40b9c..670f962f7c3 100644 --- a/Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp +++ b/Source/Initialization/DivCleaner/ProjectionDivCleaner.cpp @@ -49,7 +49,7 @@ ProjectionDivCleaner::ProjectionDivCleaner(std::string const& a_field_name) : m_source.resize(m_levels); const int ncomps = WarpX::ncomps; - auto const& ng = warpx.m_fields.get(m_field_name,Direction{0},0)->nGrowVect(); + auto const& ng = warpx.m_fields.get(m_field_name, Direction{0}, 0)->nGrowVect(); for (int lev = 0; lev < m_levels; ++lev) { @@ -214,9 +214,9 @@ ProjectionDivCleaner::setSourceFromBfield () WarpX::ComputeDivB( *m_source[ilev], 0, - {warpx.m_fields.get(m_field_name,Direction{0},ilev), - warpx.m_fields.get(m_field_name,Direction{1},ilev), - warpx.m_fields.get(m_field_name,Direction{2},ilev)}, + {warpx.m_fields.get(m_field_name, Direction{0}, ilev), + warpx.m_fields.get(m_field_name, Direction{1}, ilev), + warpx.m_fields.get(m_field_name, Direction{2}, ilev)}, WarpX::CellSize(0) ); @@ -243,9 +243,9 @@ ProjectionDivCleaner::correctBfield () for (int ilev = 0; ilev < m_levels; ++ilev) { // Grab B-field multifabs at this level - amrex::MultiFab* Bx = warpx.m_fields.get(m_field_name,Direction{0},ilev); - amrex::MultiFab* By = warpx.m_fields.get(m_field_name,Direction{1},ilev); - amrex::MultiFab* Bz = warpx.m_fields.get(m_field_name,Direction{2},ilev); + amrex::MultiFab* Bx = warpx.m_fields.get(m_field_name, Direction{0}, ilev); + amrex::MultiFab* By = warpx.m_fields.get(m_field_name, Direction{1}, ilev); + amrex::MultiFab* Bz = warpx.m_fields.get(m_field_name, Direction{2}, ilev); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 0f950b3959d..70bf20d0905 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -503,8 +503,11 @@ void WarpX::InitData () { WARPX_PROFILE("WarpX::InitData()"); - ablastr::parallelization::check_mpi_thread_level(); + using ablastr::fields::Direction; + using warpx::fields::FieldType; + + ablastr::parallelization::check_mpi_thread_level(); #ifdef WARPX_QED Print() << "PICSAR (" << WarpX::PicsarVersion() << ")\n"; @@ -555,9 +558,9 @@ WarpX::InitData () const int lev_zero = 0; m_macroscopic_properties->InitData( Geom(lev_zero), - m_fields.get("Efield_fp",Direction{0},lev_zero)->ixType().toIntVect(), - m_fields.get("Efield_fp",Direction{1},lev_zero)->ixType().toIntVect(), - m_fields.get("Efield_fp",Direction{2},lev_zero)->ixType().toIntVect() + m_fields.get(FieldType::Efield_fp, Direction{0}, lev_zero)->ixType().toIntVect(), + m_fields.get(FieldType::Efield_fp, Direction{1}, lev_zero)->ixType().toIntVect(), + m_fields.get(FieldType::Efield_fp, Direction{2}, lev_zero)->ixType().toIntVect() ); } @@ -629,34 +632,36 @@ WarpX::InitData () } void -WarpX::AddExternalFields (int const lev) { +WarpX::AddExternalFields (int const lev) +{ using ablastr::fields::Direction; + using warpx::fields::FieldType; // FIXME: RZ multimode has more than one component for all these if (m_p_ext_field_params->E_ext_grid_type != ExternalFieldType::default_zero) { - ablastr::fields::MultiLevelVectorField Efield_fp = m_fields.get_mr_levels_alldirs("Efield_fp",max_level); + ablastr::fields::MultiLevelVectorField Efield_fp = m_fields.get_mr_levels_alldirs(FieldType::Efield_fp, max_level); if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::constant) { Efield_fp[lev][0]->plus(m_p_ext_field_params->E_external_grid[0], guard_cells.ng_alloc_EB.min()); Efield_fp[lev][1]->plus(m_p_ext_field_params->E_external_grid[1], guard_cells.ng_alloc_EB.min()); Efield_fp[lev][2]->plus(m_p_ext_field_params->E_external_grid[2], guard_cells.ng_alloc_EB.min()); } else { - amrex::MultiFab::Add(*Efield_fp[lev][0], *m_fields.get("Efield_fp_external",Direction{0},lev), 0, 0, 1, guard_cells.ng_alloc_EB); - amrex::MultiFab::Add(*Efield_fp[lev][1], *m_fields.get("Efield_fp_external",Direction{1},lev), 0, 0, 1, guard_cells.ng_alloc_EB); - amrex::MultiFab::Add(*Efield_fp[lev][2], *m_fields.get("Efield_fp_external",Direction{2},lev), 0, 0, 1, guard_cells.ng_alloc_EB); + amrex::MultiFab::Add(*Efield_fp[lev][0], *m_fields.get(FieldType::Efield_fp_external, Direction{0}, lev), 0, 0, 1, guard_cells.ng_alloc_EB); + amrex::MultiFab::Add(*Efield_fp[lev][1], *m_fields.get(FieldType::Efield_fp_external, Direction{1}, lev), 0, 0, 1, guard_cells.ng_alloc_EB); + amrex::MultiFab::Add(*Efield_fp[lev][2], *m_fields.get(FieldType::Efield_fp_external, Direction{2}, lev), 0, 0, 1, guard_cells.ng_alloc_EB); } } if (m_p_ext_field_params->B_ext_grid_type != ExternalFieldType::default_zero) { - ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", max_level); + ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_fp, max_level); if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::constant) { Bfield_fp[lev][0]->plus(m_p_ext_field_params->B_external_grid[0], guard_cells.ng_alloc_EB.min()); Bfield_fp[lev][1]->plus(m_p_ext_field_params->B_external_grid[1], guard_cells.ng_alloc_EB.min()); Bfield_fp[lev][2]->plus(m_p_ext_field_params->B_external_grid[2], guard_cells.ng_alloc_EB.min()); } else { - amrex::MultiFab::Add(*Bfield_fp[lev][0], *m_fields.get("Bfield_fp_external",Direction{0},lev), 0, 0, 1, guard_cells.ng_alloc_EB); - amrex::MultiFab::Add(*Bfield_fp[lev][1], *m_fields.get("Bfield_fp_external",Direction{1},lev), 0, 0, 1, guard_cells.ng_alloc_EB); - amrex::MultiFab::Add(*Bfield_fp[lev][2], *m_fields.get("Bfield_fp_external",Direction{2},lev), 0, 0, 1, guard_cells.ng_alloc_EB); + amrex::MultiFab::Add(*Bfield_fp[lev][0], *m_fields.get(FieldType::Bfield_fp_external, Direction{0}, lev), 0, 0, 1, guard_cells.ng_alloc_EB); + amrex::MultiFab::Add(*Bfield_fp[lev][1], *m_fields.get(FieldType::Bfield_fp_external, Direction{1}, lev), 0, 0, 1, guard_cells.ng_alloc_EB); + amrex::MultiFab::Add(*Bfield_fp[lev][2], *m_fields.get(FieldType::Bfield_fp_external, Direction{2}, lev), 0, 0, 1, guard_cells.ng_alloc_EB); } } } @@ -906,6 +911,7 @@ void WarpX::InitLevelData (int lev, Real /*time*/) { using ablastr::fields::Direction; + using warpx::fields::FieldType; // initialize the averaged fields only if the averaged algorithm // is activated ('psatd.do_time_averaging=1') @@ -922,14 +928,14 @@ WarpX::InitLevelData (int lev, Real /*time*/) if ( is_B_ext_const && (lev <= maxlevel_extEMfield_init) ) { if (fft_do_time_averaging) { - m_fields.get("Bfield_avg_fp", Direction{i}, lev)->setVal(m_p_ext_field_params->B_external_grid[i]); + m_fields.get(FieldType::Bfield_avg_fp, Direction{i}, lev)->setVal(m_p_ext_field_params->B_external_grid[i]); } if (lev > 0) { - m_fields.get("Bfield_aux", Direction{i}, lev)->setVal(m_p_ext_field_params->B_external_grid[i]); - m_fields.get("Bfield_cp", Direction{i}, lev)->setVal(m_p_ext_field_params->B_external_grid[i]); + m_fields.get(FieldType::Bfield_aux, Direction{i}, lev)->setVal(m_p_ext_field_params->B_external_grid[i]); + m_fields.get(FieldType::Bfield_cp, Direction{i}, lev)->setVal(m_p_ext_field_params->B_external_grid[i]); if (fft_do_time_averaging) { - m_fields.get("Bfield_avg_cp", Direction{i}, lev)->setVal(m_p_ext_field_params->B_external_grid[i]); + m_fields.get(FieldType::Bfield_avg_cp, Direction{i}, lev)->setVal(m_p_ext_field_params->B_external_grid[i]); } } } @@ -942,13 +948,13 @@ WarpX::InitLevelData (int lev, Real /*time*/) if ( is_E_ext_const && (lev <= maxlevel_extEMfield_init) ) { if (fft_do_time_averaging) { - m_fields.get("Efield_avg_fp", Direction{i}, lev)->setVal(m_p_ext_field_params->E_external_grid[i]); + m_fields.get(FieldType::Efield_avg_fp, Direction{i}, lev)->setVal(m_p_ext_field_params->E_external_grid[i]); } if (lev > 0) { - m_fields.get("Efield_aux", Direction{i}, lev)->setVal(m_p_ext_field_params->E_external_grid[i]); - m_fields.get("Efield_cp", Direction{i}, lev)->setVal(m_p_ext_field_params->E_external_grid[i]); + m_fields.get(FieldType::Efield_aux, Direction{i}, lev)->setVal(m_p_ext_field_params->E_external_grid[i]); + m_fields.get(FieldType::Efield_cp, Direction{i}, lev)->setVal(m_p_ext_field_params->E_external_grid[i]); if (fft_do_time_averaging) { - m_fields.get("Efield_avg_cp", Direction{i}, lev)->setVal(m_p_ext_field_params->E_external_grid[i]); + m_fields.get(FieldType::Efield_avg_cp, Direction{i}, lev)->setVal(m_p_ext_field_params->E_external_grid[i]); } } } @@ -968,26 +974,26 @@ WarpX::InitLevelData (int lev, Real /*time*/) && (lev > 0) && (lev <= maxlevel_extEMfield_init)) { InitializeExternalFieldsOnGridUsingParser( - m_fields.get("Bfield_aux", Direction{0}, lev), - m_fields.get("Bfield_aux", Direction{1}, lev), - m_fields.get("Bfield_aux", Direction{2}, lev), + m_fields.get(FieldType::Bfield_aux, Direction{0}, lev), + m_fields.get(FieldType::Bfield_aux, Direction{1}, lev), + m_fields.get(FieldType::Bfield_aux, Direction{2}, lev), m_p_ext_field_params->Bxfield_parser->compile<3>(), m_p_ext_field_params->Byfield_parser->compile<3>(), m_p_ext_field_params->Bzfield_parser->compile<3>(), - m_fields.get_alldirs("edge_lengths", lev), - m_fields.get_alldirs("face_areas", lev), + m_fields.get_alldirs(FieldType::edge_lengths, lev), + m_fields.get_alldirs(FieldType::face_areas, lev), 'B', lev, PatchType::fine); InitializeExternalFieldsOnGridUsingParser( - m_fields.get("Bfield_cp", Direction{0}, lev), - m_fields.get("Bfield_cp", Direction{1}, lev), - m_fields.get("Bfield_cp", Direction{2}, lev), + m_fields.get(FieldType::Bfield_cp, Direction{0}, lev), + m_fields.get(FieldType::Bfield_cp, Direction{1}, lev), + m_fields.get(FieldType::Bfield_cp, Direction{2}, lev), m_p_ext_field_params->Bxfield_parser->compile<3>(), m_p_ext_field_params->Byfield_parser->compile<3>(), m_p_ext_field_params->Bzfield_parser->compile<3>(), - m_fields.get_alldirs("edge_lengths", lev), - m_fields.get_mr_levels_alldirs("face_areas", max_level)[lev], + m_fields.get_alldirs(FieldType::edge_lengths, lev), + m_fields.get_mr_levels_alldirs(FieldType::face_areas, max_level)[lev], 'B', lev, PatchType::coarse); } @@ -1005,10 +1011,10 @@ WarpX::InitLevelData (int lev, Real /*time*/) // We initialize ECTRhofield consistently with the Efield if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { m_fdtd_solver_fp[lev]->EvolveECTRho( - m_fields.get_alldirs("Efield_fp",lev), - m_fields.get_alldirs("edge_lengths", lev), - m_fields.get_mr_levels_alldirs("face_areas", max_level)[lev], - m_fields.get_alldirs("ECTRhofield", lev), + m_fields.get_alldirs(FieldType::Efield_fp, lev), + m_fields.get_alldirs(FieldType::edge_lengths, lev), + m_fields.get_mr_levels_alldirs(FieldType::face_areas, max_level)[lev], + m_fields.get_alldirs(FieldType::ECTRhofield, lev), lev); } } @@ -1016,26 +1022,26 @@ WarpX::InitLevelData (int lev, Real /*time*/) if (lev > 0) { InitializeExternalFieldsOnGridUsingParser( - m_fields.get("Efield_aux", Direction{0}, lev), - m_fields.get("Efield_aux", Direction{1}, lev), - m_fields.get("Efield_aux", Direction{2}, lev), + m_fields.get(FieldType::Efield_aux, Direction{0}, lev), + m_fields.get(FieldType::Efield_aux, Direction{1}, lev), + m_fields.get(FieldType::Efield_aux, Direction{2}, lev), m_p_ext_field_params->Exfield_parser->compile<3>(), m_p_ext_field_params->Eyfield_parser->compile<3>(), m_p_ext_field_params->Ezfield_parser->compile<3>(), - m_fields.get_alldirs("edge_lengths", lev), - m_fields.get_alldirs("face_areas", lev), + m_fields.get_alldirs(FieldType::edge_lengths, lev), + m_fields.get_alldirs(FieldType::face_areas, lev), 'E', lev, PatchType::fine); InitializeExternalFieldsOnGridUsingParser( - m_fields.get("Efield_cp", Direction{0}, lev), - m_fields.get("Efield_cp", Direction{1}, lev), - m_fields.get("Efield_cp", Direction{2}, lev), + m_fields.get(FieldType::Efield_cp, Direction{0}, lev), + m_fields.get(FieldType::Efield_cp, Direction{1}, lev), + m_fields.get(FieldType::Efield_cp, Direction{2}, lev), m_p_ext_field_params->Exfield_parser->compile<3>(), m_p_ext_field_params->Eyfield_parser->compile<3>(), m_p_ext_field_params->Ezfield_parser->compile<3>(), - m_fields.get_alldirs("edge_lengths", lev), - m_fields.get_alldirs("face_areas", lev), + m_fields.get_alldirs(FieldType::edge_lengths, lev), + m_fields.get_alldirs(FieldType::face_areas, lev), 'E', lev, PatchType::coarse); #ifdef AMREX_USE_EB @@ -1043,10 +1049,10 @@ WarpX::InitLevelData (int lev, Real /*time*/) if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { // We initialize ECTRhofield consistently with the Efield m_fdtd_solver_cp[lev]->EvolveECTRho( - m_fields.get_alldirs("Efield_cp",lev), - m_fields.get_alldirs("edge_lengths", lev), - m_fields.get_mr_levels_alldirs("face_areas", max_level)[lev], - m_fields.get_alldirs("ECTRhofield", lev), + m_fields.get_alldirs(FieldType::Efield_cp, lev), + m_fields.get_alldirs(FieldType::edge_lengths, lev), + m_fields.get_mr_levels_alldirs(FieldType::face_areas, max_level)[lev], + m_fields.get_alldirs(FieldType::ECTRhofield, lev), lev); } } @@ -1284,15 +1290,17 @@ void WarpX::InitializeEBGridData (int lev) "particles are close to embedded boundaries"); } - if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD ) { + if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD ) + { + using warpx::fields::FieldType; auto const eb_fact = fieldEBFactory(lev); - auto edge_lengths_lev = m_fields.get_alldirs("edge_lengths", lev); + auto edge_lengths_lev = m_fields.get_alldirs(FieldType::edge_lengths, lev); ComputeEdgeLengths(edge_lengths_lev, eb_fact); ScaleEdges(edge_lengths_lev, CellSize(lev)); - auto face_areas_lev = m_fields.get_alldirs("face_areas", lev); + auto face_areas_lev = m_fields.get_alldirs(FieldType::face_areas, lev); ComputeFaceAreas(face_areas_lev, eb_fact); ScaleAreas(face_areas_lev, CellSize(lev)); @@ -1360,6 +1368,7 @@ void WarpX::LoadExternalFields (int const lev) { using ablastr::fields::Direction; + using warpx::fields::FieldType; // External fields from file are currently not compatible with the moving window // In order to support the moving window, the MultiFab containing the external @@ -1378,14 +1387,14 @@ WarpX::LoadExternalFields (int const lev) if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::parse_ext_grid_function) { // Initialize Bfield_fp_external with external function InitializeExternalFieldsOnGridUsingParser( - m_fields.get("Bfield_fp_external",Direction{0},lev), - m_fields.get("Bfield_fp_external",Direction{1},lev), - m_fields.get("Bfield_fp_external",Direction{2},lev), + m_fields.get(FieldType::Bfield_fp_external, Direction{0}, lev), + m_fields.get(FieldType::Bfield_fp_external, Direction{1}, lev), + m_fields.get(FieldType::Bfield_fp_external, Direction{2}, lev), m_p_ext_field_params->Bxfield_parser->compile<3>(), m_p_ext_field_params->Byfield_parser->compile<3>(), m_p_ext_field_params->Bzfield_parser->compile<3>(), - m_fields.get_alldirs("edge_lengths", lev), - m_fields.get_alldirs("face_areas", lev), + m_fields.get_alldirs(FieldType::edge_lengths, lev), + m_fields.get_alldirs(FieldType::face_areas, lev), 'B', lev, PatchType::fine); } @@ -1393,27 +1402,27 @@ WarpX::LoadExternalFields (int const lev) #if defined(WARPX_DIM_RZ) WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, "External field reading is not implemented for more than one RZ mode (see #3829)"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get("Bfield_fp_external",Direction{0},lev), "B", "r"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get("Bfield_fp_external",Direction{1},lev), "B", "t"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get("Bfield_fp_external",Direction{2},lev), "B", "z"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get(FieldType::Bfield_fp_external,Direction{0},lev), "B", "r"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get(FieldType::Bfield_fp_external,Direction{1},lev), "B", "t"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get(FieldType::Bfield_fp_external,Direction{2},lev), "B", "z"); #else - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get("Bfield_fp_external",Direction{0},lev), "B", "x"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get("Bfield_fp_external",Direction{1},lev), "B", "y"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get("Bfield_fp_external",Direction{2},lev), "B", "z"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get(FieldType::Bfield_fp_external, Direction{0}, lev), "B", "x"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get(FieldType::Bfield_fp_external, Direction{1}, lev), "B", "y"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get(FieldType::Bfield_fp_external, Direction{2}, lev), "B", "z"); #endif } if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::parse_ext_grid_function) { // Initialize Efield_fp_external with external function InitializeExternalFieldsOnGridUsingParser( - m_fields.get("Efield_fp_external",Direction{0},lev), - m_fields.get("Efield_fp_external",Direction{1},lev), - m_fields.get("Efield_fp_external",Direction{2},lev), + m_fields.get(FieldType::Efield_fp_external, Direction{0}, lev), + m_fields.get(FieldType::Efield_fp_external, Direction{1}, lev), + m_fields.get(FieldType::Efield_fp_external, Direction{2}, lev), m_p_ext_field_params->Exfield_parser->compile<3>(), m_p_ext_field_params->Eyfield_parser->compile<3>(), m_p_ext_field_params->Ezfield_parser->compile<3>(), - m_fields.get_alldirs("edge_lengths", lev), - m_fields.get_alldirs("face_areas", lev), + m_fields.get_alldirs(FieldType::edge_lengths, lev), + m_fields.get_alldirs(FieldType::face_areas, lev), 'E', lev, PatchType::fine); } @@ -1421,13 +1430,13 @@ WarpX::LoadExternalFields (int const lev) #if defined(WARPX_DIM_RZ) WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, "External field reading is not implemented for more than one RZ mode (see #3829)"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get("Efield_fp_external",Direction{0},lev), "E", "r"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get("Efield_fp_external",Direction{1},lev), "E", "t"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get("Efield_fp_external",Direction{2},lev), "E", "z"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get(FieldType::Efield_fp_external,Direction{0},lev), "E", "r"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get(FieldType::Efield_fp_external,Direction{1},lev), "E", "t"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get(FieldType::Efield_fp_external,Direction{2},lev), "E", "z"); #else - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get("Efield_fp_external",Direction{0},lev), "E", "x"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get("Efield_fp_external",Direction{1},lev), "E", "y"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get("Efield_fp_external",Direction{2},lev), "E", "z"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get(FieldType::Efield_fp_external, Direction{0}, lev), "E", "x"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get(FieldType::Efield_fp_external, Direction{1}, lev), "E", "y"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, m_fields.get(FieldType::Efield_fp_external, Direction{2}, lev), "E", "z"); #endif } @@ -1445,23 +1454,23 @@ WarpX::LoadExternalFields (int const lev) WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, "External field reading is not implemented for more than one RZ mode (see #3829)"); ReadExternalFieldFromFile(external_fields_path, - m_fields.get("B_external_particle_field", Direction{0}, lev), + m_fields.get(FieldType::B_external_particle_field, Direction{0}, lev), "B", "r"); ReadExternalFieldFromFile(external_fields_path, - m_fields.get("B_external_particle_field", Direction{1}, lev), + m_fields.get(FieldType::B_external_particle_field, Direction{1}, lev), "B", "t"); ReadExternalFieldFromFile(external_fields_path, - m_fields.get("B_external_particle_field", Direction{2}, lev), + m_fields.get(FieldType::B_external_particle_field, Direction{2}, lev), "B", "z"); #else ReadExternalFieldFromFile(external_fields_path, - m_fields.get("B_external_particle_field", Direction{0}, lev), + m_fields.get(FieldType::B_external_particle_field, Direction{0}, lev), "B", "x"); ReadExternalFieldFromFile(external_fields_path, - m_fields.get("B_external_particle_field", Direction{1}, lev), + m_fields.get(FieldType::B_external_particle_field, Direction{1}, lev), "B", "y"); ReadExternalFieldFromFile(external_fields_path, - m_fields.get("B_external_particle_field", Direction{2}, lev), + m_fields.get(FieldType::B_external_particle_field, Direction{2}, lev), "B", "z"); #endif } @@ -1473,23 +1482,23 @@ WarpX::LoadExternalFields (int const lev) WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, "External field reading is not implemented for more than one RZ mode (see #3829)"); ReadExternalFieldFromFile(external_fields_path, - m_fields.get("E_external_particle_field", Direction{0}, lev), + m_fields.get(FieldType::E_external_particle_field, Direction{0}, lev), "E", "r"); ReadExternalFieldFromFile(external_fields_path, - m_fields.get("E_external_particle_field", Direction{1}, lev), + m_fields.get(FieldType::E_external_particle_field, Direction{1}, lev), "E", "t"); ReadExternalFieldFromFile(external_fields_path, - m_fields.get("E_external_particle_field", Direction{2}, lev), + m_fields.get(FieldType::E_external_particle_field, Direction{2}, lev), "E", "z"); #else ReadExternalFieldFromFile(external_fields_path, - m_fields.get("E_external_particle_field", Direction{0}, lev), + m_fields.get(FieldType::E_external_particle_field, Direction{0}, lev), "E", "x"); ReadExternalFieldFromFile(external_fields_path, - m_fields.get("E_external_particle_field", Direction{1}, lev), + m_fields.get(FieldType::E_external_particle_field, Direction{1}, lev), "E", "y"); ReadExternalFieldFromFile(external_fields_path, - m_fields.get("E_external_particle_field", Direction{2}, lev), + m_fields.get(FieldType::E_external_particle_field, Direction{2}, lev), "E", "z"); #endif } diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 34a2fb18095..ac797d1e706 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -12,6 +12,7 @@ #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) # include "BoundaryConditions/PML_RZ.H" #endif +#include "Fields.H" #include "Filter/BilinearFilter.H" #include "Utils/TextMsg.H" #include "Utils/WarpXAlgorithmSelection.H" @@ -50,6 +51,7 @@ #include using namespace amrex; +using warpx::fields::FieldType; void WarpX::UpdateAuxilaryData () @@ -58,9 +60,9 @@ WarpX::UpdateAuxilaryData () using ablastr::fields::Direction; - amrex::MultiFab *Bfield_aux_lvl0_0 = m_fields.get("Bfield_aux", Direction{0}, 0); + amrex::MultiFab *Bfield_aux_lvl0_0 = m_fields.get(FieldType::Bfield_aux, Direction{0}, 0); - ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); + ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_fp, finest_level); if (Bfield_aux_lvl0_0->ixType() == Bfield_fp[0][0]->ixType()) { UpdateAuxilaryDataSameType(); @@ -71,15 +73,15 @@ WarpX::UpdateAuxilaryData () // When loading particle fields from file: add the external fields: for (int lev = 0; lev <= finest_level; ++lev) { if (mypc->m_E_ext_particle_s == "read_from_file") { - ablastr::fields::VectorField Efield_aux = m_fields.get_alldirs("Efield_aux", lev); - const auto& E_ext_lev = m_fields.get_alldirs("E_external_particle_field", lev); + ablastr::fields::VectorField Efield_aux = m_fields.get_alldirs(FieldType::Efield_aux, lev); + const auto& E_ext_lev = m_fields.get_alldirs(FieldType::E_external_particle_field, lev); amrex::MultiFab::Add(*Efield_aux[0], *E_ext_lev[0], 0, 0, E_ext_lev[0]->nComp(), guard_cells.ng_FieldGather); amrex::MultiFab::Add(*Efield_aux[1], *E_ext_lev[1], 0, 0, E_ext_lev[1]->nComp(), guard_cells.ng_FieldGather); amrex::MultiFab::Add(*Efield_aux[2], *E_ext_lev[2], 0, 0, E_ext_lev[2]->nComp(), guard_cells.ng_FieldGather); } if (mypc->m_B_ext_particle_s == "read_from_file") { - ablastr::fields::VectorField Bfield_aux = m_fields.get_alldirs("Bfield_aux", lev); - const auto& B_ext_lev = m_fields.get_alldirs("B_external_particle_field", lev); + ablastr::fields::VectorField Bfield_aux = m_fields.get_alldirs(FieldType::Bfield_aux, lev); + const auto& B_ext_lev = m_fields.get_alldirs(FieldType::B_external_particle_field, lev); amrex::MultiFab::Add(*Bfield_aux[0], *B_ext_lev[0], 0, 0, B_ext_lev[0]->nComp(), guard_cells.ng_FieldGather); amrex::MultiFab::Add(*Bfield_aux[1], *B_ext_lev[1], 0, 0, B_ext_lev[1]->nComp(), guard_cells.ng_FieldGather); amrex::MultiFab::Add(*Bfield_aux[2], *B_ext_lev[2], 0, 0, B_ext_lev[2]->nComp(), guard_cells.ng_FieldGather); @@ -100,18 +102,18 @@ WarpX::UpdateAuxilaryDataStagToNodal () #endif using ablastr::fields::Direction; - ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); - ablastr::fields::MultiLevelVectorField const& Efield_fp = m_fields.get_mr_levels_alldirs("Efield_fp", finest_level); - ablastr::fields::MultiLevelVectorField const& Efield_aux = m_fields.get_mr_levels_alldirs("Efield_aux", finest_level); - ablastr::fields::MultiLevelVectorField const& Bfield_aux = m_fields.get_mr_levels_alldirs("Bfield_aux", finest_level); + ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_fp, finest_level); + ablastr::fields::MultiLevelVectorField const& Efield_fp = m_fields.get_mr_levels_alldirs(FieldType::Efield_fp, finest_level); + ablastr::fields::MultiLevelVectorField const& Efield_aux = m_fields.get_mr_levels_alldirs(FieldType::Efield_aux, finest_level); + ablastr::fields::MultiLevelVectorField const& Bfield_aux = m_fields.get_mr_levels_alldirs(FieldType::Bfield_aux, finest_level); ablastr::fields::MultiLevelVectorField const & Bmf = WarpX::fft_do_time_averaging ? - m_fields.get_mr_levels_alldirs("Bfield_avg_fp", finest_level) : + m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_fp, finest_level) : Bfield_fp; ablastr::fields::MultiLevelVectorField const & Emf = WarpX::fft_do_time_averaging ? - m_fields.get_mr_levels_alldirs("Efield_avg_fp", finest_level) : + m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_fp, finest_level) : Efield_fp; const amrex::IntVect& Bx_stag = Bmf[0][0]->ixType().toIntVect(); @@ -194,10 +196,10 @@ WarpX::UpdateAuxilaryDataStagToNodal () { if (electromagnetic_solver_id != ElectromagneticSolverAlgo::None) { Array,3> Btmp; - if (m_fields.has("Bfield_cax", Direction{0}, lev)) { + if (m_fields.has(FieldType::Bfield_cax, Direction{0}, lev)) { for (int i = 0; i < 3; ++i) { Btmp[i] = std::make_unique( - *m_fields.get("Bfield_cax", Direction{i}, lev), amrex::make_alias, 0, 1); + *m_fields.get(FieldType::Bfield_cax, Direction{i}, lev), amrex::make_alias, 0, 1); } } else { const IntVect ngtmp = Bfield_aux[lev-1][0]->nGrowVect(); @@ -221,13 +223,13 @@ WarpX::UpdateAuxilaryDataStagToNodal () const amrex::IntVect& refinement_ratio = refRatio(lev-1); - const amrex::IntVect& Bx_fp_stag = m_fields.get("Bfield_fp",Direction{0},lev)->ixType().toIntVect(); - const amrex::IntVect& By_fp_stag = m_fields.get("Bfield_fp",Direction{1},lev)->ixType().toIntVect(); - const amrex::IntVect& Bz_fp_stag = m_fields.get("Bfield_fp",Direction{2},lev)->ixType().toIntVect(); + const amrex::IntVect& Bx_fp_stag = m_fields.get(FieldType::Bfield_fp, Direction{0}, lev)->ixType().toIntVect(); + const amrex::IntVect& By_fp_stag = m_fields.get(FieldType::Bfield_fp, Direction{1}, lev)->ixType().toIntVect(); + const amrex::IntVect& Bz_fp_stag = m_fields.get(FieldType::Bfield_fp, Direction{2}, lev)->ixType().toIntVect(); - const amrex::IntVect& Bx_cp_stag = m_fields.get("Bfield_cp",Direction{0},lev)->ixType().toIntVect(); - const amrex::IntVect& By_cp_stag = m_fields.get("Bfield_cp",Direction{1},lev)->ixType().toIntVect(); - const amrex::IntVect& Bz_cp_stag = m_fields.get("Bfield_cp",Direction{2},lev)->ixType().toIntVect(); + const amrex::IntVect& Bx_cp_stag = m_fields.get(FieldType::Bfield_cp, Direction{0}, lev)->ixType().toIntVect(); + const amrex::IntVect& By_cp_stag = m_fields.get(FieldType::Bfield_cp, Direction{1}, lev)->ixType().toIntVect(); + const amrex::IntVect& Bz_cp_stag = m_fields.get(FieldType::Bfield_cp, Direction{2}, lev)->ixType().toIntVect(); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) @@ -237,12 +239,12 @@ WarpX::UpdateAuxilaryDataStagToNodal () Array4 const& bx_aux = Bfield_aux[lev][0]->array(mfi); Array4 const& by_aux = Bfield_aux[lev][1]->array(mfi); Array4 const& bz_aux = Bfield_aux[lev][2]->array(mfi); - Array4 const& bx_fp = m_fields.get("Bfield_fp",Direction{0},lev)->const_array(mfi); - Array4 const& by_fp = m_fields.get("Bfield_fp",Direction{1},lev)->const_array(mfi); - Array4 const& bz_fp = m_fields.get("Bfield_fp",Direction{2},lev)->const_array(mfi); - Array4 const& bx_cp = m_fields.get("Bfield_cp",Direction{0},lev)->const_array(mfi); - Array4 const& by_cp = m_fields.get("Bfield_cp",Direction{1},lev)->const_array(mfi); - Array4 const& bz_cp = m_fields.get("Bfield_cp",Direction{2},lev)->const_array(mfi); + Array4 const& bx_fp = m_fields.get(FieldType::Bfield_fp, Direction{0}, lev)->const_array(mfi); + Array4 const& by_fp = m_fields.get(FieldType::Bfield_fp, Direction{1}, lev)->const_array(mfi); + Array4 const& bz_fp = m_fields.get(FieldType::Bfield_fp, Direction{2}, lev)->const_array(mfi); + Array4 const& bx_cp = m_fields.get(FieldType::Bfield_cp, Direction{0}, lev)->const_array(mfi); + Array4 const& by_cp = m_fields.get(FieldType::Bfield_cp, Direction{1}, lev)->const_array(mfi); + Array4 const& bz_cp = m_fields.get(FieldType::Bfield_cp, Direction{2}, lev)->const_array(mfi); Array4 const& bx_c = Btmp[0]->const_array(mfi); Array4 const& by_c = Btmp[1]->const_array(mfi); Array4 const& bz_c = Btmp[2]->const_array(mfi); @@ -288,10 +290,10 @@ WarpX::UpdateAuxilaryDataStagToNodal () { if (electromagnetic_solver_id != ElectromagneticSolverAlgo::None) { Array,3> Etmp; - if (m_fields.has("Efield_cax", Direction{0}, lev)) { + if (m_fields.has(FieldType::Efield_cax, Direction{0}, lev)) { for (int i = 0; i < 3; ++i) { Etmp[i] = std::make_unique( - *m_fields.get("Efield_cax", Direction{i}, lev), amrex::make_alias, 0, 1); + *m_fields.get(FieldType::Efield_cax, Direction{i}, lev), amrex::make_alias, 0, 1); } } else { const IntVect ngtmp = Efield_aux[lev-1][0]->nGrowVect(); @@ -316,13 +318,13 @@ WarpX::UpdateAuxilaryDataStagToNodal () const amrex::IntVect& refinement_ratio = refRatio(lev-1); - const amrex::IntVect& Ex_fp_stag = m_fields.get("Efield_fp",Direction{0},lev)->ixType().toIntVect(); - const amrex::IntVect& Ey_fp_stag = m_fields.get("Efield_fp",Direction{1},lev)->ixType().toIntVect(); - const amrex::IntVect& Ez_fp_stag = m_fields.get("Efield_fp",Direction{2},lev)->ixType().toIntVect(); + const amrex::IntVect& Ex_fp_stag = m_fields.get(FieldType::Efield_fp, Direction{0}, lev)->ixType().toIntVect(); + const amrex::IntVect& Ey_fp_stag = m_fields.get(FieldType::Efield_fp, Direction{1}, lev)->ixType().toIntVect(); + const amrex::IntVect& Ez_fp_stag = m_fields.get(FieldType::Efield_fp, Direction{2}, lev)->ixType().toIntVect(); - const amrex::IntVect& Ex_cp_stag = m_fields.get("Efield_cp",Direction{0},lev)->ixType().toIntVect(); - const amrex::IntVect& Ey_cp_stag = m_fields.get("Efield_cp",Direction{1},lev)->ixType().toIntVect(); - const amrex::IntVect& Ez_cp_stag = m_fields.get("Efield_cp",Direction{2},lev)->ixType().toIntVect(); + const amrex::IntVect& Ex_cp_stag = m_fields.get(FieldType::Efield_cp, Direction{0}, lev)->ixType().toIntVect(); + const amrex::IntVect& Ey_cp_stag = m_fields.get(FieldType::Efield_cp, Direction{1}, lev)->ixType().toIntVect(); + const amrex::IntVect& Ez_cp_stag = m_fields.get(FieldType::Efield_cp, Direction{2}, lev)->ixType().toIntVect(); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) @@ -332,12 +334,12 @@ WarpX::UpdateAuxilaryDataStagToNodal () Array4 const& ex_aux = Efield_aux[lev][0]->array(mfi); Array4 const& ey_aux = Efield_aux[lev][1]->array(mfi); Array4 const& ez_aux = Efield_aux[lev][2]->array(mfi); - Array4 const& ex_fp = m_fields.get("Efield_fp",Direction{0},lev)->const_array(mfi); - Array4 const& ey_fp = m_fields.get("Efield_fp",Direction{1},lev)->const_array(mfi); - Array4 const& ez_fp = m_fields.get("Efield_fp",Direction{2},lev)->const_array(mfi); - Array4 const& ex_cp = m_fields.get("Efield_cp",Direction{0},lev)->const_array(mfi); - Array4 const& ey_cp = m_fields.get("Efield_cp",Direction{1},lev)->const_array(mfi); - Array4 const& ez_cp = m_fields.get("Efield_cp",Direction{2},lev)->const_array(mfi); + Array4 const& ex_fp = m_fields.get(FieldType::Efield_fp, Direction{0}, lev)->const_array(mfi); + Array4 const& ey_fp = m_fields.get(FieldType::Efield_fp, Direction{1}, lev)->const_array(mfi); + Array4 const& ez_fp = m_fields.get(FieldType::Efield_fp, Direction{2}, lev)->const_array(mfi); + Array4 const& ex_cp = m_fields.get(FieldType::Efield_cp, Direction{0}, lev)->const_array(mfi); + Array4 const& ey_cp = m_fields.get(FieldType::Efield_cp, Direction{1}, lev)->const_array(mfi); + Array4 const& ez_cp = m_fields.get(FieldType::Efield_cp, Direction{2}, lev)->const_array(mfi); Array4 const& ex_c = Etmp[0]->const_array(mfi); Array4 const& ey_c = Etmp[1]->const_array(mfi); Array4 const& ez_c = Etmp[2]->const_array(mfi); @@ -353,9 +355,9 @@ WarpX::UpdateAuxilaryDataStagToNodal () } } else { // electrostatic - const amrex::IntVect& Ex_fp_stag = m_fields.get("Efield_fp",Direction{0},lev)->ixType().toIntVect(); - const amrex::IntVect& Ey_fp_stag = m_fields.get("Efield_fp",Direction{1},lev)->ixType().toIntVect(); - const amrex::IntVect& Ez_fp_stag = m_fields.get("Efield_fp",Direction{2},lev)->ixType().toIntVect(); + const amrex::IntVect& Ex_fp_stag = m_fields.get(FieldType::Efield_fp, Direction{0}, lev)->ixType().toIntVect(); + const amrex::IntVect& Ey_fp_stag = m_fields.get(FieldType::Efield_fp, Direction{1}, lev)->ixType().toIntVect(); + const amrex::IntVect& Ez_fp_stag = m_fields.get(FieldType::Efield_fp, Direction{2}, lev)->ixType().toIntVect(); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif @@ -364,9 +366,9 @@ WarpX::UpdateAuxilaryDataStagToNodal () Array4 const& ex_aux = Efield_aux[lev][0]->array(mfi); Array4 const& ey_aux = Efield_aux[lev][1]->array(mfi); Array4 const& ez_aux = Efield_aux[lev][2]->array(mfi); - Array4 const& ex_fp = m_fields.get("Efield_fp",Direction{0},lev)->const_array(mfi); - Array4 const& ey_fp = m_fields.get("Efield_fp",Direction{1},lev)->const_array(mfi); - Array4 const& ez_fp = m_fields.get("Efield_fp",Direction{2},lev)->const_array(mfi); + Array4 const& ex_fp = m_fields.get(FieldType::Efield_fp, Direction{0}, lev)->const_array(mfi); + Array4 const& ey_fp = m_fields.get(FieldType::Efield_fp, Direction{1}, lev)->const_array(mfi); + Array4 const& ez_fp = m_fields.get(FieldType::Efield_fp, Direction{2}, lev)->const_array(mfi); const Box& bx = mfi.growntilebox(); amrex::ParallelFor(bx, @@ -389,22 +391,22 @@ WarpX::UpdateAuxilaryDataSameType () const amrex::IntVect& ng_src = guard_cells.ng_FieldGather; using ablastr::fields::Direction; - ablastr::fields::MultiLevelVectorField Efield_fp = m_fields.get_mr_levels_alldirs("Efield_fp", finest_level); - ablastr::fields::MultiLevelVectorField Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); - ablastr::fields::MultiLevelVectorField Efield_aux = m_fields.get_mr_levels_alldirs("Efield_aux", finest_level); - ablastr::fields::MultiLevelVectorField Bfield_aux = m_fields.get_mr_levels_alldirs("Bfield_aux", finest_level); + ablastr::fields::MultiLevelVectorField Efield_fp = m_fields.get_mr_levels_alldirs(FieldType::Efield_fp, finest_level); + ablastr::fields::MultiLevelVectorField Bfield_fp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_fp, finest_level); + ablastr::fields::MultiLevelVectorField Efield_aux = m_fields.get_mr_levels_alldirs(FieldType::Efield_aux, finest_level); + ablastr::fields::MultiLevelVectorField Bfield_aux = m_fields.get_mr_levels_alldirs(FieldType::Bfield_aux, finest_level); // Level 0: Copy from fine to aux // Note: in some configurations, Efield_aux/Bfield_aux and Efield_fp/Bfield_fp are simply aliases to the // same MultiFab object. MultiFab::Copy operation automatically detects this and does nothing in this case. if (WarpX::fft_do_time_averaging) { - MultiFab::Copy(*Efield_aux[0][0], *m_fields.get("Efield_avg_fp", Direction{0}, 0), 0, 0, Efield_aux[0][0]->nComp(), ng_src); - MultiFab::Copy(*Efield_aux[0][1], *m_fields.get("Efield_avg_fp", Direction{1}, 0), 0, 0, Efield_aux[0][1]->nComp(), ng_src); - MultiFab::Copy(*Efield_aux[0][2], *m_fields.get("Efield_avg_fp", Direction{2}, 0), 0, 0, Efield_aux[0][2]->nComp(), ng_src); - MultiFab::Copy(*Bfield_aux[0][0], *m_fields.get("Bfield_avg_fp", Direction{0}, 0), 0, 0, Bfield_aux[0][0]->nComp(), ng_src); - MultiFab::Copy(*Bfield_aux[0][1], *m_fields.get("Bfield_avg_fp", Direction{1}, 0), 0, 0, Bfield_aux[0][1]->nComp(), ng_src); - MultiFab::Copy(*Bfield_aux[0][2], *m_fields.get("Bfield_avg_fp", Direction{2}, 0), 0, 0, Bfield_aux[0][2]->nComp(), ng_src); + MultiFab::Copy(*Efield_aux[0][0], *m_fields.get(FieldType::Efield_avg_fp, Direction{0}, 0), 0, 0, Efield_aux[0][0]->nComp(), ng_src); + MultiFab::Copy(*Efield_aux[0][1], *m_fields.get(FieldType::Efield_avg_fp, Direction{1}, 0), 0, 0, Efield_aux[0][1]->nComp(), ng_src); + MultiFab::Copy(*Efield_aux[0][2], *m_fields.get(FieldType::Efield_avg_fp, Direction{2}, 0), 0, 0, Efield_aux[0][2]->nComp(), ng_src); + MultiFab::Copy(*Bfield_aux[0][0], *m_fields.get(FieldType::Bfield_avg_fp, Direction{0}, 0), 0, 0, Bfield_aux[0][0]->nComp(), ng_src); + MultiFab::Copy(*Bfield_aux[0][1], *m_fields.get(FieldType::Bfield_avg_fp, Direction{1}, 0), 0, 0, Bfield_aux[0][1]->nComp(), ng_src); + MultiFab::Copy(*Bfield_aux[0][2], *m_fields.get(FieldType::Bfield_avg_fp, Direction{2}, 0), 0, 0, Bfield_aux[0][2]->nComp(), ng_src); } else { @@ -418,19 +420,19 @@ WarpX::UpdateAuxilaryDataSameType () for (int lev = 1; lev <= finest_level; ++lev) { const amrex::Periodicity& crse_period = Geom(lev-1).periodicity(); - const IntVect& ng = m_fields.get("Bfield_cp", Direction{0}, lev)->nGrowVect(); - const DistributionMapping& dm = m_fields.get("Bfield_cp", Direction{0}, lev)->DistributionMap(); + const IntVect& ng = m_fields.get(FieldType::Bfield_cp, Direction{0}, lev)->nGrowVect(); + const DistributionMapping& dm = m_fields.get(FieldType::Bfield_cp, Direction{0}, lev)->DistributionMap(); // B field { if (electromagnetic_solver_id != ElectromagneticSolverAlgo::None) { - MultiFab dBx(m_fields.get("Bfield_cp",Direction{0},lev)->boxArray(), dm, - m_fields.get("Bfield_cp",Direction{0},lev)->nComp(), ng); - MultiFab dBy(m_fields.get("Bfield_cp",Direction{1},lev)->boxArray(), dm, - m_fields.get("Bfield_cp",Direction{1},lev)->nComp(), ng); - MultiFab dBz(m_fields.get("Bfield_cp",Direction{2},lev)->boxArray(), dm, - m_fields.get("Bfield_cp",Direction{2},lev)->nComp(), ng); + MultiFab dBx(m_fields.get(FieldType::Bfield_cp, Direction{0}, lev)->boxArray(), dm, + m_fields.get(FieldType::Bfield_cp, Direction{0}, lev)->nComp(), ng); + MultiFab dBy(m_fields.get(FieldType::Bfield_cp, Direction{1}, lev)->boxArray(), dm, + m_fields.get(FieldType::Bfield_cp, Direction{1}, lev)->nComp(), ng); + MultiFab dBz(m_fields.get(FieldType::Bfield_cp, Direction{2}, lev)->boxArray(), dm, + m_fields.get(FieldType::Bfield_cp, Direction{2}, lev)->nComp(), ng); dBx.setVal(0.0); dBy.setVal(0.0); dBz.setVal(0.0); @@ -448,18 +450,18 @@ WarpX::UpdateAuxilaryDataSameType () Bfield_aux[lev - 1][2]->nComp(), ng_src, ng, WarpX::do_single_precision_comms, crse_period); - if (m_fields.has("Bfield_cax", Direction{0}, lev)) + if (m_fields.has(FieldType::Bfield_cax, Direction{0}, lev)) { - MultiFab::Copy(*m_fields.get("Bfield_cax", Direction{0}, lev), dBx, 0, 0, m_fields.get("Bfield_cax", Direction{0}, lev)->nComp(), ng); - MultiFab::Copy(*m_fields.get("Bfield_cax", Direction{1}, lev), dBy, 0, 0, m_fields.get("Bfield_cax", Direction{1}, lev)->nComp(), ng); - MultiFab::Copy(*m_fields.get("Bfield_cax", Direction{2}, lev), dBz, 0, 0, m_fields.get("Bfield_cax", Direction{2}, lev)->nComp(), ng); + MultiFab::Copy(*m_fields.get(FieldType::Bfield_cax, Direction{0}, lev), dBx, 0, 0, m_fields.get(FieldType::Bfield_cax, Direction{0}, lev)->nComp(), ng); + MultiFab::Copy(*m_fields.get(FieldType::Bfield_cax, Direction{1}, lev), dBy, 0, 0, m_fields.get(FieldType::Bfield_cax, Direction{1}, lev)->nComp(), ng); + MultiFab::Copy(*m_fields.get(FieldType::Bfield_cax, Direction{2}, lev), dBz, 0, 0, m_fields.get(FieldType::Bfield_cax, Direction{2}, lev)->nComp(), ng); } - MultiFab::Subtract(dBx, *m_fields.get("Bfield_cp",Direction{0},lev), - 0, 0, m_fields.get("Bfield_cp",Direction{0},lev)->nComp(), ng); - MultiFab::Subtract(dBy, *m_fields.get("Bfield_cp",Direction{1},lev), - 0, 0, m_fields.get("Bfield_cp",Direction{1},lev)->nComp(), ng); - MultiFab::Subtract(dBz, *m_fields.get("Bfield_cp",Direction{2},lev), - 0, 0, m_fields.get("Bfield_cp",Direction{2},lev)->nComp(), ng); + MultiFab::Subtract(dBx, *m_fields.get(FieldType::Bfield_cp, Direction{0}, lev), + 0, 0, m_fields.get(FieldType::Bfield_cp, Direction{0}, lev)->nComp(), ng); + MultiFab::Subtract(dBy, *m_fields.get(FieldType::Bfield_cp, Direction{1}, lev), + 0, 0, m_fields.get(FieldType::Bfield_cp, Direction{1}, lev)->nComp(), ng); + MultiFab::Subtract(dBz, *m_fields.get(FieldType::Bfield_cp, Direction{2}, lev), + 0, 0, m_fields.get(FieldType::Bfield_cp, Direction{2}, lev)->nComp(), ng); const amrex::IntVect& refinement_ratio = refRatio(lev-1); @@ -508,12 +510,12 @@ WarpX::UpdateAuxilaryDataSameType () { if (electromagnetic_solver_id != ElectromagneticSolverAlgo::None) { - MultiFab dEx(m_fields.get("Efield_cp",Direction{0},lev)->boxArray(), dm, - m_fields.get("Efield_cp",Direction{0},lev)->nComp(), ng); - MultiFab dEy(m_fields.get("Efield_cp",Direction{1},lev)->boxArray(), dm, - m_fields.get("Efield_cp",Direction{1},lev)->nComp(), ng); - MultiFab dEz(m_fields.get("Efield_cp",Direction{2},lev)->boxArray(), dm, - m_fields.get("Efield_cp",Direction{2},lev)->nComp(), ng); + MultiFab dEx(m_fields.get(FieldType::Efield_cp, Direction{0}, lev)->boxArray(), dm, + m_fields.get(FieldType::Efield_cp, Direction{0}, lev)->nComp(), ng); + MultiFab dEy(m_fields.get(FieldType::Efield_cp, Direction{1}, lev)->boxArray(), dm, + m_fields.get(FieldType::Efield_cp, Direction{1}, lev)->nComp(), ng); + MultiFab dEz(m_fields.get(FieldType::Efield_cp, Direction{2}, lev)->boxArray(), dm, + m_fields.get(FieldType::Efield_cp, Direction{2}, lev)->nComp(), ng); dEx.setVal(0.0); dEy.setVal(0.0); dEz.setVal(0.0); @@ -533,18 +535,18 @@ WarpX::UpdateAuxilaryDataSameType () WarpX::do_single_precision_comms, crse_period); - if (m_fields.has("Efield_cax", Direction{0}, lev)) + if (m_fields.has(FieldType::Efield_cax, Direction{0}, lev)) { - MultiFab::Copy(*m_fields.get("Efield_cax", Direction{0}, lev), dEx, 0, 0, m_fields.get("Efield_cax", Direction{0}, lev)->nComp(), ng); - MultiFab::Copy(*m_fields.get("Efield_cax", Direction{1}, lev), dEy, 0, 0, m_fields.get("Efield_cax", Direction{1}, lev)->nComp(), ng); - MultiFab::Copy(*m_fields.get("Efield_cax", Direction{2}, lev), dEz, 0, 0, m_fields.get("Efield_cax", Direction{2}, lev)->nComp(), ng); + MultiFab::Copy(*m_fields.get(FieldType::Efield_cax, Direction{0}, lev), dEx, 0, 0, m_fields.get(FieldType::Efield_cax, Direction{0}, lev)->nComp(), ng); + MultiFab::Copy(*m_fields.get(FieldType::Efield_cax, Direction{1}, lev), dEy, 0, 0, m_fields.get(FieldType::Efield_cax, Direction{1}, lev)->nComp(), ng); + MultiFab::Copy(*m_fields.get(FieldType::Efield_cax, Direction{2}, lev), dEz, 0, 0, m_fields.get(FieldType::Efield_cax, Direction{2}, lev)->nComp(), ng); } - MultiFab::Subtract(dEx, *m_fields.get("Efield_cp",Direction{0},lev), - 0, 0, m_fields.get("Efield_cp",Direction{0},lev)->nComp(), ng); - MultiFab::Subtract(dEy, *m_fields.get("Efield_cp",Direction{1},lev), - 0, 0, m_fields.get("Efield_cp",Direction{1},lev)->nComp(), ng); - MultiFab::Subtract(dEz, *m_fields.get("Efield_cp",Direction{2},lev), - 0, 0, m_fields.get("Efield_cp",Direction{2},lev)->nComp(), ng); + MultiFab::Subtract(dEx, *m_fields.get(FieldType::Efield_cp, Direction{0}, lev), + 0, 0, m_fields.get(FieldType::Efield_cp, Direction{0}, lev)->nComp(), ng); + MultiFab::Subtract(dEy, *m_fields.get(FieldType::Efield_cp, Direction{1}, lev), + 0, 0, m_fields.get(FieldType::Efield_cp, Direction{1}, lev)->nComp(), ng); + MultiFab::Subtract(dEz, *m_fields.get(FieldType::Efield_cp, Direction{2}, lev), + 0, 0, m_fields.get(FieldType::Efield_cp, Direction{2}, lev)->nComp(), ng); const amrex::IntVect& refinement_ratio = refRatio(lev-1); @@ -560,9 +562,9 @@ WarpX::UpdateAuxilaryDataSameType () Array4 const& ex_aux = Efield_aux[lev][0]->array(mfi); Array4 const& ey_aux = Efield_aux[lev][1]->array(mfi); Array4 const& ez_aux = Efield_aux[lev][2]->array(mfi); - Array4 const& ex_fp = m_fields.get("Efield_fp",Direction{0},lev)->const_array(mfi); - Array4 const& ey_fp = m_fields.get("Efield_fp",Direction{1},lev)->const_array(mfi); - Array4 const& ez_fp = m_fields.get("Efield_fp",Direction{2},lev)->const_array(mfi); + Array4 const& ex_fp = m_fields.get(FieldType::Efield_fp, Direction{0}, lev)->const_array(mfi); + Array4 const& ey_fp = m_fields.get(FieldType::Efield_fp, Direction{1}, lev)->const_array(mfi); + Array4 const& ez_fp = m_fields.get(FieldType::Efield_fp, Direction{2}, lev)->const_array(mfi); Array4 const& ex_c = dEx.const_array(mfi); Array4 const& ey_c = dEy.const_array(mfi); Array4 const& ez_c = dEz.const_array(mfi); @@ -584,9 +586,9 @@ WarpX::UpdateAuxilaryDataSameType () } else // electrostatic { - MultiFab::Copy(*Efield_aux[lev][0], *m_fields.get("Efield_fp",Direction{0},lev), 0, 0, Efield_aux[lev][0]->nComp(), Efield_aux[lev][0]->nGrowVect()); - MultiFab::Copy(*Efield_aux[lev][1], *m_fields.get("Efield_fp",Direction{1},lev), 0, 0, Efield_aux[lev][1]->nComp(), Efield_aux[lev][1]->nGrowVect()); - MultiFab::Copy(*Efield_aux[lev][2], *m_fields.get("Efield_fp",Direction{2},lev), 0, 0, Efield_aux[lev][2]->nComp(), Efield_aux[lev][2]->nGrowVect()); + MultiFab::Copy(*Efield_aux[lev][0], *m_fields.get(FieldType::Efield_fp, Direction{0}, lev), 0, 0, Efield_aux[lev][0]->nComp(), Efield_aux[lev][0]->nGrowVect()); + MultiFab::Copy(*Efield_aux[lev][1], *m_fields.get(FieldType::Efield_fp, Direction{1}, lev), 0, 0, Efield_aux[lev][1]->nComp(), Efield_aux[lev][1]->nGrowVect()); + MultiFab::Copy(*Efield_aux[lev][2], *m_fields.get(FieldType::Efield_fp, Direction{2}, lev), 0, 0, Efield_aux[lev][2]->nComp(), Efield_aux[lev][2]->nGrowVect()); } } } @@ -711,16 +713,16 @@ WarpX::FillBoundaryE (const int lev, const PatchType patch_type, const amrex::In if (patch_type == PatchType::fine) { - mf = {m_fields.get("Efield_fp",Direction{0},lev), - m_fields.get("Efield_fp",Direction{1},lev), - m_fields.get("Efield_fp",Direction{2},lev)}; + mf = {m_fields.get(FieldType::Efield_fp, Direction{0}, lev), + m_fields.get(FieldType::Efield_fp, Direction{1}, lev), + m_fields.get(FieldType::Efield_fp, Direction{2}, lev)}; period = Geom(lev).periodicity(); } else // coarse patch { - mf = {m_fields.get("Efield_cp",Direction{0},lev), - m_fields.get("Efield_cp",Direction{1},lev), - m_fields.get("Efield_cp",Direction{2},lev)}; + mf = {m_fields.get(FieldType::Efield_cp, Direction{0}, lev), + m_fields.get(FieldType::Efield_cp, Direction{1}, lev), + m_fields.get(FieldType::Efield_cp, Direction{2}, lev)}; period = Geom(lev-1).periodicity(); } @@ -732,8 +734,8 @@ WarpX::FillBoundaryE (const int lev, const PatchType patch_type, const amrex::In { const std::array mf_pml = (patch_type == PatchType::fine) ? - m_fields.get_alldirs("pml_E_fp", lev) : - m_fields.get_alldirs("pml_E_cp", lev); + m_fields.get_alldirs(FieldType::pml_E_fp, lev) : + m_fields.get_alldirs(FieldType::pml_E_cp, lev); pml[lev]->Exchange(mf_pml, mf, patch_type, do_pml_in_domain); pml[lev]->FillBoundary(mf_pml, patch_type, nodal_sync); @@ -776,16 +778,16 @@ WarpX::FillBoundaryB (const int lev, const PatchType patch_type, const amrex::In if (patch_type == PatchType::fine) { - mf = {m_fields.get("Bfield_fp",Direction{0},lev), - m_fields.get("Bfield_fp",Direction{1},lev), - m_fields.get("Bfield_fp",Direction{2},lev)}; + mf = {m_fields.get(FieldType::Bfield_fp, Direction{0}, lev), + m_fields.get(FieldType::Bfield_fp, Direction{1}, lev), + m_fields.get(FieldType::Bfield_fp, Direction{2}, lev)}; period = Geom(lev).periodicity(); } else // coarse patch { - mf = {m_fields.get("Bfield_cp",Direction{0},lev), - m_fields.get("Bfield_cp",Direction{1},lev), - m_fields.get("Bfield_cp",Direction{2},lev)}; + mf = {m_fields.get(FieldType::Bfield_cp, Direction{0}, lev), + m_fields.get(FieldType::Bfield_cp, Direction{1}, lev), + m_fields.get(FieldType::Bfield_cp, Direction{2}, lev)}; period = Geom(lev-1).periodicity(); } @@ -797,8 +799,8 @@ WarpX::FillBoundaryB (const int lev, const PatchType patch_type, const amrex::In { const std::array mf_pml = (patch_type == PatchType::fine) ? - m_fields.get_alldirs("pml_B_fp", lev) : - m_fields.get_alldirs("pml_B_cp", lev); + m_fields.get_alldirs(FieldType::pml_B_fp, lev) : + m_fields.get_alldirs(FieldType::pml_B_cp, lev); pml[lev]->Exchange(mf_pml, mf, patch_type, do_pml_in_domain); pml[lev]->FillBoundary(mf_pml, patch_type, nodal_sync); @@ -841,7 +843,7 @@ WarpX::FillBoundaryE_avg (int lev, PatchType patch_type, IntVect ng) WARPX_ABORT_WITH_MESSAGE("Averaged Galilean PSATD with PML is not yet implemented"); } - ablastr::fields::MultiLevelVectorField Efield_avg_fp = m_fields.get_mr_levels_alldirs("Efield_avg_fp", finest_level); + ablastr::fields::MultiLevelVectorField Efield_avg_fp = m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_fp, finest_level); const amrex::Periodicity& period = Geom(lev).periodicity(); if ( safe_guard_cells ){ @@ -863,7 +865,7 @@ WarpX::FillBoundaryE_avg (int lev, PatchType patch_type, IntVect ng) WARPX_ABORT_WITH_MESSAGE("Averaged Galilean PSATD with PML is not yet implemented"); } - ablastr::fields::MultiLevelVectorField Efield_avg_cp = m_fields.get_mr_levels_alldirs("Efield_avg_cp", finest_level); + ablastr::fields::MultiLevelVectorField Efield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_cp, finest_level); const amrex::Periodicity& cperiod = Geom(lev-1).periodicity(); if ( safe_guard_cells ) { @@ -901,7 +903,7 @@ WarpX::FillBoundaryB_avg (int lev, PatchType patch_type, IntVect ng) WARPX_ABORT_WITH_MESSAGE("Averaged Galilean PSATD with PML is not yet implemented"); } - ablastr::fields::MultiLevelVectorField Bfield_avg_fp = m_fields.get_mr_levels_alldirs("Bfield_avg_fp", finest_level); + ablastr::fields::MultiLevelVectorField Bfield_avg_fp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_fp, finest_level); const amrex::Periodicity& period = Geom(lev).periodicity(); if ( safe_guard_cells ) { @@ -909,7 +911,7 @@ WarpX::FillBoundaryB_avg (int lev, PatchType patch_type, IntVect ng) ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period); } else { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - ng.allLE(m_fields.get("Bfield_fp",Direction{0},lev)->nGrowVect()), + ng.allLE(m_fields.get(FieldType::Bfield_fp, Direction{0}, lev)->nGrowVect()), "Error: in FillBoundaryB, requested more guard cells than allocated"); ablastr::utils::communication::FillBoundary(*Bfield_avg_fp[lev][0], ng, WarpX::do_single_precision_comms, period); ablastr::utils::communication::FillBoundary(*Bfield_avg_fp[lev][1], ng, WarpX::do_single_precision_comms, period); @@ -923,7 +925,7 @@ WarpX::FillBoundaryB_avg (int lev, PatchType patch_type, IntVect ng) WARPX_ABORT_WITH_MESSAGE("Averaged Galilean PSATD with PML is not yet implemented"); } - ablastr::fields::MultiLevelVectorField Bfield_avg_cp = m_fields.get_mr_levels_alldirs("Bfield_avg_cp", finest_level); + ablastr::fields::MultiLevelVectorField Bfield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_cp, finest_level); const amrex::Periodicity& cperiod = Geom(lev-1).periodicity(); if ( safe_guard_cells ){ @@ -954,38 +956,38 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng, std::optionalok()) { - if (m_fields.has("pml_F_fp", lev) && m_fields.has("F_fp", lev)) { - pml[lev]->Exchange(m_fields.get("pml_F_fp", lev), m_fields.get("F_fp", lev), patch_type, do_pml_in_domain); + if (m_fields.has(FieldType::pml_F_fp, lev) && m_fields.has(FieldType::F_fp, lev)) { + pml[lev]->Exchange(m_fields.get(FieldType::pml_F_fp, lev), m_fields.get(FieldType::F_fp, lev), patch_type, do_pml_in_domain); } - if (m_fields.has("pml_F_fp", lev)) { - pml[lev]->FillBoundary(*m_fields.get("pml_F_fp", lev), patch_type, nodal_sync); + if (m_fields.has(FieldType::pml_F_fp, lev)) { + pml[lev]->FillBoundary(*m_fields.get(FieldType::pml_F_fp, lev), patch_type, nodal_sync); } } - if (m_fields.has("F_fp", lev)) + if (m_fields.has(FieldType::F_fp, lev)) { const amrex::Periodicity& period = Geom(lev).periodicity(); - const amrex::IntVect& nghost = (safe_guard_cells) ? m_fields.get("F_fp", lev)->nGrowVect() : ng; - ablastr::utils::communication::FillBoundary(*m_fields.get("F_fp", lev), nghost, WarpX::do_single_precision_comms, period, nodal_sync); + const amrex::IntVect& nghost = (safe_guard_cells) ? m_fields.get(FieldType::F_fp, lev)->nGrowVect() : ng; + ablastr::utils::communication::FillBoundary(*m_fields.get(FieldType::F_fp, lev), nghost, WarpX::do_single_precision_comms, period, nodal_sync); } } else if (patch_type == PatchType::coarse) { if (do_pml && pml[lev] && pml[lev]->ok()) { - if (m_fields.has("pml_F_cp", lev) && m_fields.has("F_cp", lev)) { - pml[lev]->Exchange(m_fields.get("pml_F_cp", lev), m_fields.get("F_cp", lev), patch_type, do_pml_in_domain); + if (m_fields.has(FieldType::pml_F_cp, lev) && m_fields.has(FieldType::F_cp, lev)) { + pml[lev]->Exchange(m_fields.get(FieldType::pml_F_cp, lev), m_fields.get(FieldType::F_cp, lev), patch_type, do_pml_in_domain); } - if (m_fields.has("pml_F_cp", lev)) { - pml[lev]->FillBoundary(*m_fields.get("pml_F_cp", lev), patch_type, nodal_sync); + if (m_fields.has(FieldType::pml_F_cp, lev)) { + pml[lev]->FillBoundary(*m_fields.get(FieldType::pml_F_cp, lev), patch_type, nodal_sync); } } - if (m_fields.has("F_cp", lev)) + if (m_fields.has(FieldType::F_cp, lev)) { const amrex::Periodicity& period = Geom(lev-1).periodicity(); - const amrex::IntVect& nghost = (safe_guard_cells) ? m_fields.get("F_cp", lev)->nGrowVect() : ng; - ablastr::utils::communication::FillBoundary(*m_fields.get("F_cp", lev), nghost, WarpX::do_single_precision_comms, period, nodal_sync); + const amrex::IntVect& nghost = (safe_guard_cells) ? m_fields.get(FieldType::F_cp, lev)->nGrowVect() : ng; + ablastr::utils::communication::FillBoundary(*m_fields.get(FieldType::F_cp, lev), nghost, WarpX::do_single_precision_comms, period, nodal_sync); } } } @@ -1006,18 +1008,18 @@ void WarpX::FillBoundaryG (int lev, PatchType patch_type, IntVect ng, std::optio { if (do_pml && pml[lev] && pml[lev]->ok()) { - if (m_fields.has("pml_G_fp",lev) && m_fields.has("G_fp",lev)) { - pml[lev]->Exchange(m_fields.get("pml_G_fp", lev), m_fields.get("G_fp", lev), patch_type, do_pml_in_domain); + if (m_fields.has(FieldType::pml_G_fp,lev) && m_fields.has(FieldType::G_fp,lev)) { + pml[lev]->Exchange(m_fields.get(FieldType::pml_G_fp, lev), m_fields.get(FieldType::G_fp, lev), patch_type, do_pml_in_domain); } - if (m_fields.has("pml_G_fp",lev)) { - pml[lev]->FillBoundary(*m_fields.get("pml_G_fp", lev), patch_type, nodal_sync); + if (m_fields.has(FieldType::pml_G_fp,lev)) { + pml[lev]->FillBoundary(*m_fields.get(FieldType::pml_G_fp, lev), patch_type, nodal_sync); } } - if (m_fields.has("G_fp",lev)) + if (m_fields.has(FieldType::G_fp,lev)) { const amrex::Periodicity& period = Geom(lev).periodicity(); - MultiFab* G_fp = m_fields.get("G_fp",lev); + MultiFab* G_fp = m_fields.get(FieldType::G_fp,lev); const amrex::IntVect& nghost = (safe_guard_cells) ? G_fp->nGrowVect() : ng; ablastr::utils::communication::FillBoundary(*G_fp, nghost, WarpX::do_single_precision_comms, period, nodal_sync); } @@ -1026,18 +1028,18 @@ void WarpX::FillBoundaryG (int lev, PatchType patch_type, IntVect ng, std::optio { if (do_pml && pml[lev] && pml[lev]->ok()) { - if (m_fields.has("pml_G_cp",lev) && m_fields.has("G_cp",lev)) { - pml[lev]->Exchange(m_fields.get("pml_G_cp", lev), m_fields.get("G_cp", lev), patch_type, do_pml_in_domain); + if (m_fields.has(FieldType::pml_G_cp,lev) && m_fields.has(FieldType::G_cp,lev)) { + pml[lev]->Exchange(m_fields.get(FieldType::pml_G_cp, lev), m_fields.get(FieldType::G_cp, lev), patch_type, do_pml_in_domain); } - if (m_fields.has("pml_G_cp", lev)) { - pml[lev]->FillBoundary(*m_fields.get("pml_G_cp", lev), patch_type, nodal_sync); + if (m_fields.has(FieldType::pml_G_cp, lev)) { + pml[lev]->FillBoundary(*m_fields.get(FieldType::pml_G_cp, lev), patch_type, nodal_sync); } } - if (m_fields.has("G_cp",lev)) + if (m_fields.has(FieldType::G_cp,lev)) { const amrex::Periodicity& period = Geom(lev-1).periodicity(); - MultiFab* G_cp = m_fields.get("G_cp",lev); + MultiFab* G_cp = m_fields.get(FieldType::G_cp,lev); const amrex::IntVect& nghost = (safe_guard_cells) ? G_cp->nGrowVect() : ng; ablastr::utils::communication::FillBoundary(*G_cp, nghost, WarpX::do_single_precision_comms, period, nodal_sync); } @@ -1056,8 +1058,8 @@ WarpX::FillBoundaryAux (IntVect ng) void WarpX::FillBoundaryAux (int lev, IntVect ng) { - ablastr::fields::MultiLevelVectorField Efield_aux = m_fields.get_mr_levels_alldirs("Efield_aux", finest_level); - ablastr::fields::MultiLevelVectorField Bfield_aux = m_fields.get_mr_levels_alldirs("Bfield_aux", finest_level); + ablastr::fields::MultiLevelVectorField Efield_aux = m_fields.get_mr_levels_alldirs(FieldType::Efield_aux, finest_level); + ablastr::fields::MultiLevelVectorField Bfield_aux = m_fields.get_mr_levels_alldirs(FieldType::Bfield_aux, finest_level); const amrex::Periodicity& period = Geom(lev).periodicity(); ablastr::utils::communication::FillBoundary(*Efield_aux[lev][0], ng, WarpX::do_single_precision_comms, period); @@ -1080,7 +1082,7 @@ WarpX::SyncCurrent (const std::string& current_fp_string) // If warpx.do_current_centering = 1, center currents from nodal grid to staggered grid if (do_current_centering) { - ablastr::fields::MultiLevelVectorField const& J_fp_nodal = m_fields.get_mr_levels_alldirs("current_fp_nodal", finest_level+1); + ablastr::fields::MultiLevelVectorField const& J_fp_nodal = m_fields.get_mr_levels_alldirs(FieldType::current_fp_nodal, finest_level+1); AMREX_ALWAYS_ASSERT_WITH_MESSAGE(finest_level <= 1, "warpx.do_current_centering=1 not supported with more than one fine levels"); @@ -1190,7 +1192,7 @@ WarpX::SyncCurrent (const std::string& current_fp_string) } }); // Now it's safe to apply filter and sumboundary on J_cp - ablastr::fields::MultiLevelVectorField const& J_cp = m_fields.get_mr_levels_alldirs("current_cp", finest_level); + ablastr::fields::MultiLevelVectorField const& J_cp = m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level); if (use_filter) { ApplyFilterJ(J_cp, lev+1, idim); @@ -1205,14 +1207,14 @@ WarpX::SyncCurrent (const std::string& current_fp_string) // filtering depends on the level. This is also done before any // same-level communication because it's easier this way to // avoid double counting. - ablastr::fields::MultiLevelVectorField const& J_cp = m_fields.get_mr_levels_alldirs("current_cp", finest_level); + ablastr::fields::MultiLevelVectorField const& J_cp = m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level); J_cp[lev][Direction{idim}]->setVal(0.0); ablastr::coarsen::average::Coarsen(*J_cp[lev][Direction{idim}], *J_fp[lev][Direction{idim}], refRatio(lev-1)); - if (m_fields.has("current_buf", Direction{idim}, lev)) + if (m_fields.has(FieldType::current_buf, Direction{idim}, lev)) { - ablastr::fields::MultiLevelVectorField const& J_buffer = m_fields.get_mr_levels_alldirs("current_buf", finest_level); + ablastr::fields::MultiLevelVectorField const& J_buffer = m_fields.get_mr_levels_alldirs(FieldType::current_buf, finest_level); IntVect const& ng = J_cp[lev][Direction{idim}]->nGrowVect(); AMREX_ASSERT(ng.allLE(J_buffer[lev][Direction{idim}]->nGrowVect())); @@ -1239,14 +1241,14 @@ WarpX::SyncCurrent (const std::string& current_fp_string) void WarpX::SyncRho () { - const ablastr::fields::MultiLevelScalarField rho_fp = m_fields.has("rho_fp", 0) ? - m_fields.get_mr_levels("rho_fp", finest_level) : + const ablastr::fields::MultiLevelScalarField rho_fp = m_fields.has(FieldType::rho_fp, 0) ? + m_fields.get_mr_levels(FieldType::rho_fp, finest_level) : ablastr::fields::MultiLevelScalarField{static_cast(finest_level+1)}; - const ablastr::fields::MultiLevelScalarField rho_cp = m_fields.has("rho_cp", 1) ? - m_fields.get_mr_levels("rho_cp", finest_level) : + const ablastr::fields::MultiLevelScalarField rho_cp = m_fields.has(FieldType::rho_cp, 1) ? + m_fields.get_mr_levels(FieldType::rho_cp, finest_level) : ablastr::fields::MultiLevelScalarField{static_cast(finest_level+1)}; - const ablastr::fields::MultiLevelScalarField rho_buf = m_fields.has("rho_buf", 1) ? - m_fields.get_mr_levels("rho_buf", finest_level) : + const ablastr::fields::MultiLevelScalarField rho_buf = m_fields.has(FieldType::rho_buf, 1) ? + m_fields.get_mr_levels(FieldType::rho_buf, finest_level) : ablastr::fields::MultiLevelScalarField{static_cast(finest_level+1)}; SyncRho(rho_fp, rho_cp, rho_buf); @@ -1525,10 +1527,10 @@ void WarpX::AddCurrentFromFineLevelandSumBoundary ( void WarpX::RestrictRhoFromFineToCoarsePatch ( const int lev ) { - if (m_fields.has("rho_fp", lev)) { - m_fields.get("rho_cp", lev)->setVal(0.0); + if (m_fields.has(FieldType::rho_fp, lev)) { + m_fields.get(FieldType::rho_cp, lev)->setVal(0.0); const IntVect& refinement_ratio = refRatio(lev-1); - ablastr::coarsen::average::Coarsen(*m_fields.get("rho_cp", lev), *m_fields.get("rho_fp", lev), refinement_ratio ); + ablastr::coarsen::average::Coarsen(*m_fields.get(FieldType::rho_cp, lev), *m_fields.get(FieldType::rho_fp, lev), refinement_ratio ); } } diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index b4deed06b4b..a0a2d4929df 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -12,6 +12,7 @@ #include "Diagnostics/ReducedDiags/MultiReducedDiags.H" #include "EmbeddedBoundary/Enabled.H" #include "EmbeddedBoundary/WarpXFaceInfoBox.H" +#include "Fields.H" #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" #include "Initialization/ExternalField.H" #include "Particles/MultiParticleContainer.H" @@ -170,8 +171,8 @@ WarpX::LoadBalance () void WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const DistributionMapping& dm) { - using ablastr::fields::Direction; + using warpx::fields::FieldType; bool const eb_enabled = EB::enabled(); if (ba == boxArray(lev)) @@ -181,7 +182,7 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi m_fields.remake_level(lev, dm); // Fine patch - ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); + ablastr::fields::MultiLevelVectorField const& Bfield_fp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_fp, finest_level); for (int idim=0; idim < 3; ++idim) { if (eb_enabled) { @@ -315,6 +316,7 @@ void WarpX::ComputeCostsHeuristic (amrex::Vector > >& a_costs) { using ablastr::fields::Direction; + using warpx::fields::FieldType; for (int lev = 0; lev <= finest_level; ++lev) { @@ -334,7 +336,7 @@ WarpX::ComputeCostsHeuristic (amrex::Vectorupdate(t_lab); - BL_ASSERT(OnSameGrids(lev, *fields.get("current_fp", Direction{0}, lev))); + BL_ASSERT(OnSameGrids(lev, *fields.get(FieldType::current_fp, Direction{0}, lev))); amrex::LayoutData* cost = WarpX::getCosts(lev); - const bool has_rho = fields.has("rho_fp", lev); - const bool has_buffer = fields.has("current_buf", lev); + const bool has_rho = fields.has(FieldType::rho_fp, lev); + const bool has_buffer = fields.has(FieldType::current_buf, lev); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) @@ -626,11 +628,11 @@ LaserParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, if (has_rho && ! skip_deposition && ! do_not_deposit) { int* AMREX_RESTRICT ion_lev = nullptr; - amrex::MultiFab* rho = fields.get("rho_fp", lev); + amrex::MultiFab* rho = fields.get(FieldType::rho_fp, lev); DepositCharge(pti, wp, ion_lev, rho, 0, 0, np_current, thread_num, lev, lev); if (has_buffer) { - amrex::MultiFab* crho = fields.get("rho_buf", lev); + amrex::MultiFab* crho = fields.get(FieldType::rho_buf, lev); DepositCharge(pti, wp, ion_lev, crho, 0, np_current, np-np_current, thread_num, lev, lev-1); } @@ -676,9 +678,9 @@ LaserParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, if (has_buffer) { // Deposit in buffers - amrex::MultiFab * cjx = fields.get("current_buf", Direction{0}, lev); - amrex::MultiFab * cjy = fields.get("current_buf", Direction{1}, lev); - amrex::MultiFab * cjz = fields.get("current_buf", Direction{2}, lev); + amrex::MultiFab * cjx = fields.get(FieldType::current_buf, Direction{0}, lev); + amrex::MultiFab * cjy = fields.get(FieldType::current_buf, Direction{1}, lev); + amrex::MultiFab * cjz = fields.get(FieldType::current_buf, Direction{2}, lev); DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, cjx, cjy, cjz, np_current, np-np_current, thread_num, lev, lev-1, dt, relative_time, push_type); @@ -688,11 +690,11 @@ LaserParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, if (has_rho && ! skip_deposition && ! do_not_deposit) { int* AMREX_RESTRICT ion_lev = nullptr; - amrex::MultiFab* rho = fields.get("rho_fp", lev); + amrex::MultiFab* rho = fields.get(FieldType::rho_fp, lev); DepositCharge(pti, wp, ion_lev, rho, 1, 0, np_current, thread_num, lev, lev); if (has_buffer) { - amrex::MultiFab* crho = fields.get("rho_buf", lev); + amrex::MultiFab* crho = fields.get(FieldType::rho_buf, lev); DepositCharge(pti, wp, ion_lev, crho, 1, np_current, np-np_current, thread_num, lev, lev-1); } diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 533ee574aba..619b54ed7ad 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -81,7 +81,7 @@ #include using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; namespace { @@ -470,11 +470,11 @@ MultiParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, fields.get(current_fp_string, Direction{0}, lev)->setVal(0.0); fields.get(current_fp_string, Direction{1}, lev)->setVal(0.0); fields.get(current_fp_string, Direction{2}, lev)->setVal(0.0); - if (fields.has("current_buf", Direction{0}, lev)) { fields.get("current_buf", Direction{0}, lev)->setVal(0.0); } - if (fields.has("current_buf", Direction{1}, lev)) { fields.get("current_buf", Direction{1}, lev)->setVal(0.0); } - if (fields.has("current_buf", Direction{2}, lev)) { fields.get("current_buf", Direction{2}, lev)->setVal(0.0); } - if (fields.has("rho_fp", lev)) { fields.get("rho_fp", lev)->setVal(0.0); } - if (fields.has("rho_buf", lev)) { fields.get("rho_buf", lev)->setVal(0.0); } + if (fields.has(FieldType::current_buf, Direction{0}, lev)) { fields.get(FieldType::current_buf, Direction{0}, lev)->setVal(0.0); } + if (fields.has(FieldType::current_buf, Direction{1}, lev)) { fields.get(FieldType::current_buf, Direction{1}, lev)->setVal(0.0); } + if (fields.has(FieldType::current_buf, Direction{2}, lev)) { fields.get(FieldType::current_buf, Direction{2}, lev)->setVal(0.0); } + if (fields.has(FieldType::rho_fp, lev)) { fields.get(FieldType::rho_fp, lev)->setVal(0.0); } + if (fields.has(FieldType::rho_buf, lev)) { fields.get(FieldType::rho_buf, lev)->setVal(0.0); } } for (auto& pc : allcontainers) { pc->Evolve(fields, lev, current_fp_string, t, dt, a_dt_type, skip_deposition, push_type); @@ -1357,12 +1357,12 @@ MultiParticleContainer::doQEDSchwinger () pc_product_pos->defineAllParticleTiles(); using ablastr::fields::Direction; - const MultiFab & Ex = *warpx.m_fields.get("Efield_aux", Direction{0}, level_0); - const MultiFab & Ey = *warpx.m_fields.get("Efield_aux", Direction{1}, level_0); - const MultiFab & Ez = *warpx.m_fields.get("Efield_aux", Direction{2}, level_0); - const MultiFab & Bx = *warpx.m_fields.get("Bfield_aux", Direction{0}, level_0); - const MultiFab & By = *warpx.m_fields.get("Bfield_aux", Direction{1}, level_0); - const MultiFab & Bz = *warpx.m_fields.get("Bfield_aux", Direction{2}, level_0); + const MultiFab & Ex = *warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, level_0); + const MultiFab & Ey = *warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, level_0); + const MultiFab & Ez = *warpx.m_fields.get(FieldType::Efield_aux, Direction{2}, level_0); + const MultiFab & Bx = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, level_0); + const MultiFab & By = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, level_0); + const MultiFab & Bz = *warpx.m_fields.get(FieldType::Bfield_aux, Direction{2}, level_0); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index d86daed4dc6..07997a61f0c 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -10,6 +10,7 @@ */ #include "PhysicalParticleContainer.H" +#include "Fields.H" #include "Filter/NCIGodfreyFilter.H" #include "Initialization/InjectorDensity.H" #include "Initialization/InjectorMomentum.H" @@ -1342,10 +1343,11 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int #ifdef AMREX_USE_EB if (EB::enabled()) { + using warpx::fields::FieldType; auto & warpx = WarpX::GetInstance(); scrapeParticlesAtEB( *this, - warpx.m_fields.get_mr_levels("distance_to_eb", warpx.finestLevel()), + warpx.m_fields.get_mr_levels(FieldType::distance_to_eb, warpx.finestLevel()), ParticleBoundaryProcess::Absorb()); } #endif @@ -1712,10 +1714,11 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, #ifdef AMREX_USE_EB if (EB::enabled()) { + using warpx::fields::FieldType; auto & warpx = WarpX::GetInstance(); scrapeParticlesAtEB( tmp_pc, - warpx.m_fields.get_mr_levels("distance_to_eb", warpx.finestLevel()), + warpx.m_fields.get_mr_levels(FieldType::distance_to_eb, warpx.finestLevel()), ParticleBoundaryProcess::Absorb()); } #endif @@ -1737,28 +1740,29 @@ PhysicalParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, PushType push_type) { using ablastr::fields::Direction; + using warpx::fields::FieldType; WARPX_PROFILE("PhysicalParticleContainer::Evolve()"); WARPX_PROFILE_VAR_NS("PhysicalParticleContainer::Evolve::GatherAndPush", blp_fg); - BL_ASSERT(OnSameGrids(lev, *fields.get("current_fp", Direction{0}, lev))); + BL_ASSERT(OnSameGrids(lev, *fields.get(FieldType::current_fp, Direction{0}, lev))); amrex::LayoutData* cost = WarpX::getCosts(lev); const iMultiFab* current_masks = WarpX::CurrentBufferMasks(lev); const iMultiFab* gather_masks = WarpX::GatherBufferMasks(lev); - const bool has_rho = fields.has("rho_fp", lev); - const bool has_cjx = fields.has("current_buf", Direction{0}, lev); - const bool has_cEx = fields.has("Efield_cax", Direction{0}, lev); + const bool has_rho = fields.has(FieldType::rho_fp, lev); + const bool has_cjx = fields.has(FieldType::current_buf, Direction{0}, lev); + const bool has_cEx = fields.has(FieldType::Efield_cax, Direction{0}, lev); const bool has_buffer = has_cEx || has_cjx; - amrex::MultiFab & Ex = *fields.get("Efield_aux", Direction{0}, lev); - amrex::MultiFab & Ey = *fields.get("Efield_aux", Direction{1}, lev); - amrex::MultiFab & Ez = *fields.get("Efield_aux", Direction{2}, lev); - amrex::MultiFab & Bx = *fields.get("Bfield_aux", Direction{0}, lev); - amrex::MultiFab & By = *fields.get("Bfield_aux", Direction{1}, lev); - amrex::MultiFab & Bz = *fields.get("Bfield_aux", Direction{2}, lev); + amrex::MultiFab & Ex = *fields.get(FieldType::Efield_aux, Direction{0}, lev); + amrex::MultiFab & Ey = *fields.get(FieldType::Efield_aux, Direction{1}, lev); + amrex::MultiFab & Ez = *fields.get(FieldType::Efield_aux, Direction{2}, lev); + amrex::MultiFab & Bx = *fields.get(FieldType::Bfield_aux, Direction{0}, lev); + amrex::MultiFab & By = *fields.get(FieldType::Bfield_aux, Direction{1}, lev); + amrex::MultiFab & Bz = *fields.get(FieldType::Bfield_aux, Direction{2}, lev); if (m_do_back_transformed_particles) { @@ -1854,11 +1858,11 @@ PhysicalParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, const int* const AMREX_RESTRICT ion_lev = (do_field_ionization)? pti.GetiAttribs(particle_icomps["ionizationLevel"]).dataPtr():nullptr; - amrex::MultiFab* rho = fields.get("rho_fp", lev); + amrex::MultiFab* rho = fields.get(FieldType::rho_fp, lev); DepositCharge(pti, wp, ion_lev, rho, 0, 0, np_current, thread_num, lev, lev); if (has_buffer){ - amrex::MultiFab* crho = fields.get("rho_buf", lev); + amrex::MultiFab* crho = fields.get(FieldType::rho_buf, lev); DepositCharge(pti, wp, ion_lev, crho, 0, np_current, np-np_current, thread_num, lev, lev-1); } @@ -1893,12 +1897,12 @@ PhysicalParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, const IntVect& ref_ratio = WarpX::RefRatio(lev-1); const Box& cbox = amrex::coarsen(box,ref_ratio); - amrex::MultiFab & cEx = *fields.get("Efield_cax", Direction{0}, lev); - amrex::MultiFab & cEy = *fields.get("Efield_cax", Direction{1}, lev); - amrex::MultiFab & cEz = *fields.get("Efield_cax", Direction{2}, lev); - amrex::MultiFab & cBx = *fields.get("Bfield_cax", Direction{0}, lev); - amrex::MultiFab & cBy = *fields.get("Bfield_cax", Direction{1}, lev); - amrex::MultiFab & cBz = *fields.get("Bfield_cax", Direction{2}, lev); + amrex::MultiFab & cEx = *fields.get(FieldType::Efield_cax, Direction{0}, lev); + amrex::MultiFab & cEy = *fields.get(FieldType::Efield_cax, Direction{1}, lev); + amrex::MultiFab & cEz = *fields.get(FieldType::Efield_cax, Direction{2}, lev); + amrex::MultiFab & cBx = *fields.get(FieldType::Bfield_cax, Direction{0}, lev); + amrex::MultiFab & cBy = *fields.get(FieldType::Bfield_cax, Direction{1}, lev); + amrex::MultiFab & cBz = *fields.get(FieldType::Bfield_cax, Direction{2}, lev); // Data on the grid FArrayBox const* cexfab = &cEx[pti]; @@ -1961,9 +1965,9 @@ PhysicalParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, if (has_buffer) { // Deposit in buffers - amrex::MultiFab * cjx = fields.get("current_buf", Direction{0}, lev); - amrex::MultiFab * cjy = fields.get("current_buf", Direction{1}, lev); - amrex::MultiFab * cjz = fields.get("current_buf", Direction{2}, lev); + amrex::MultiFab * cjx = fields.get(FieldType::current_buf, Direction{0}, lev); + amrex::MultiFab * cjy = fields.get(FieldType::current_buf, Direction{1}, lev); + amrex::MultiFab * cjz = fields.get(FieldType::current_buf, Direction{2}, lev); DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, cjx, cjy, cjz, np_current, np-np_current, thread_num, lev, lev-1, dt, relative_time, push_type); @@ -1975,7 +1979,7 @@ PhysicalParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, // Deposit charge after particle push, in component 1 of MultiFab rho. // (Skipped for electrostatic solver, as this may lead to out-of-bounds) if (WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::None) { - amrex::MultiFab* rho = fields.get("rho_fp", lev); + amrex::MultiFab* rho = fields.get(FieldType::rho_fp, lev); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(rho->nComp() >= 2, "Cannot deposit charge in rho component 1: only component 0 is allocated!"); @@ -1985,7 +1989,7 @@ PhysicalParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields, DepositCharge(pti, wp, ion_lev, rho, 1, 0, np_current, thread_num, lev, lev); if (has_buffer){ - amrex::MultiFab* crho = fields.get("rho_buf", lev); + amrex::MultiFab* crho = fields.get(FieldType::rho_buf, lev); DepositCharge(pti, wp, ion_lev, crho, 1, np_current, np-np_current, thread_num, lev, lev-1); } diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index b3e41f0d04e..36793c8619b 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -14,6 +14,7 @@ #include "Deposition/CurrentDeposition.H" #include "Deposition/SharedDepositionUtils.H" #include "EmbeddedBoundary/Enabled.H" +#include "Fields.H" #include "Pusher/GetAndSetPosition.H" #include "Pusher/UpdatePosition.H" #include "ParticleBoundaries_K.H" @@ -173,6 +174,7 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, long n, int uniqueparticles, amrex::Long id) { using namespace amrex::literals; + using warpx::fields::FieldType; WARPX_ALWAYS_ASSERT_WITH_MESSAGE((PIdx::nattribs + nattr_real - 1) <= NumRealComps(), "Too many real attributes specified"); @@ -305,7 +307,7 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, long n, auto & warpx = WarpX::GetInstance(); scrapeParticlesAtEB( *this, - warpx.m_fields.get_mr_levels("distance_to_eb", warpx.finestLevel()), + warpx.m_fields.get_mr_levels(FieldType::distance_to_eb, warpx.finestLevel()), ParticleBoundaryProcess::Absorb()); deleteInvalidParticles(); } diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index e6cabd4aa65..d5cebd69254 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -14,6 +14,7 @@ #endif #include "Initialization/ExternalField.H" #include "Particles/MultiParticleContainer.H" +#include "Fields.H" #include "Fluids/MultiFluidContainer.H" #include "Fluids/WarpXFluidContainer.H" #include "Utils/TextMsg.H" @@ -140,6 +141,7 @@ WarpX::MoveWindow (const int step, bool move_j) WARPX_PROFILE("WarpX::MoveWindow"); using ablastr::fields::Direction; + using warpx::fields::FieldType; if (step == start_moving_window_step) { amrex::Print() << Utils::TextMsg::Info("Starting moving window"); @@ -236,57 +238,57 @@ WarpX::MoveWindow (const int step, bool move_j) if (dim == 1) { Efield_parser = m_p_ext_field_params->Eyfield_parser->compile<3>(); } if (dim == 2) { Efield_parser = m_p_ext_field_params->Ezfield_parser->compile<3>(); } } - shiftMF(*m_fields.get("Bfield_fp", Direction{dim}, lev), geom[lev], num_shift, dir, lev, do_update_cost, + shiftMF(*m_fields.get(FieldType::Bfield_fp, Direction{dim}, lev), geom[lev], num_shift, dir, lev, do_update_cost, m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); - shiftMF(*m_fields.get("Efield_fp",Direction{dim},lev), geom[lev], num_shift, dir, lev, do_update_cost, + shiftMF(*m_fields.get(FieldType::Efield_fp, Direction{dim}, lev), geom[lev], num_shift, dir, lev, do_update_cost, m_p_ext_field_params->E_external_grid[dim], use_Eparser, Efield_parser); if (fft_do_time_averaging) { - ablastr::fields::MultiLevelVectorField Efield_avg_fp = m_fields.get_mr_levels_alldirs("Efield_avg_fp", finest_level); - ablastr::fields::MultiLevelVectorField Bfield_avg_fp = m_fields.get_mr_levels_alldirs("Bfield_avg_fp", finest_level); + ablastr::fields::MultiLevelVectorField Efield_avg_fp = m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_fp, finest_level); + ablastr::fields::MultiLevelVectorField Bfield_avg_fp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_fp, finest_level); shiftMF(*Bfield_avg_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost, m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); shiftMF(*Efield_avg_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost, m_p_ext_field_params-> E_external_grid[dim], use_Eparser, Efield_parser); } if (move_j) { - shiftMF(*m_fields.get("current_fp", Direction{dim}, lev), geom[lev], num_shift, dir, lev, do_update_cost); + shiftMF(*m_fields.get(FieldType::current_fp, Direction{dim}, lev), geom[lev], num_shift, dir, lev, do_update_cost); } if (pml[lev] && pml[lev]->ok()) { - amrex::MultiFab* pml_B = m_fields.get("pml_B_fp", Direction{dim}, lev); - amrex::MultiFab* pml_E = m_fields.get("pml_E_fp", Direction{dim}, lev); + amrex::MultiFab* pml_B = m_fields.get(FieldType::pml_B_fp, Direction{dim}, lev); + amrex::MultiFab* pml_E = m_fields.get(FieldType::pml_E_fp, Direction{dim}, lev); shiftMF(*pml_B, geom[lev], num_shift, dir, lev, dont_update_cost); shiftMF(*pml_E, geom[lev], num_shift, dir, lev, dont_update_cost); } #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev] && dim < 2) { - amrex::MultiFab* pml_rz_B = m_fields.get("pml_B_fp", Direction{dim}, lev); - amrex::MultiFab* pml_rz_E = m_fields.get("pml_E_fp", Direction{dim}, lev); + amrex::MultiFab* pml_rz_B = m_fields.get(FieldType::pml_B_fp, Direction{dim}, lev); + amrex::MultiFab* pml_rz_E = m_fields.get(FieldType::pml_E_fp, Direction{dim}, lev); shiftMF(*pml_rz_B, geom[lev], num_shift, dir, lev, dont_update_cost); shiftMF(*pml_rz_E, geom[lev], num_shift, dir, lev, dont_update_cost); } #endif if (lev > 0) { // coarse grid - shiftMF(*m_fields.get("Bfield_cp",Direction{dim},lev), geom[lev-1], num_shift_crse, dir, lev, do_update_cost, + shiftMF(*m_fields.get(FieldType::Bfield_cp, Direction{dim}, lev), geom[lev-1], num_shift_crse, dir, lev, do_update_cost, m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); - shiftMF(*m_fields.get("Efield_cp",Direction{dim},lev), geom[lev-1], num_shift_crse, dir, lev, do_update_cost, + shiftMF(*m_fields.get(FieldType::Efield_cp, Direction{dim}, lev), geom[lev-1], num_shift_crse, dir, lev, do_update_cost, m_p_ext_field_params->E_external_grid[dim], use_Eparser, Efield_parser); - shiftMF(*m_fields.get("Bfield_aux",Direction{dim},lev), geom[lev], num_shift, dir, lev, do_update_cost); - shiftMF(*m_fields.get("Efield_aux",Direction{dim},lev), geom[lev], num_shift, dir, lev, do_update_cost); + shiftMF(*m_fields.get(FieldType::Bfield_aux, Direction{dim}, lev), geom[lev], num_shift, dir, lev, do_update_cost); + shiftMF(*m_fields.get(FieldType::Efield_aux, Direction{dim}, lev), geom[lev], num_shift, dir, lev, do_update_cost); if (fft_do_time_averaging) { - ablastr::fields::MultiLevelVectorField Efield_avg_cp = m_fields.get_mr_levels_alldirs("Efield_avg_cp", finest_level); - ablastr::fields::MultiLevelVectorField Bfield_avg_cp = m_fields.get_mr_levels_alldirs("Bfield_avg_cp", finest_level); + ablastr::fields::MultiLevelVectorField Efield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_cp, finest_level); + ablastr::fields::MultiLevelVectorField Bfield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_cp, finest_level); shiftMF(*Bfield_avg_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost, m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); shiftMF(*Efield_avg_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost, m_p_ext_field_params->E_external_grid[dim], use_Eparser, Efield_parser); } if (move_j) { - shiftMF(*m_fields.get("current_cp", Direction{dim}, lev), geom[lev-1], num_shift_crse, dir, lev, do_update_cost); + shiftMF(*m_fields.get(FieldType::current_cp, Direction{dim}, lev), geom[lev-1], num_shift_crse, dir, lev, do_update_cost); } if (do_pml && pml[lev]->ok()) { - amrex::MultiFab* pml_B_cp = m_fields.get("pml_B_cp", Direction{dim}, lev); - amrex::MultiFab* pml_E_cp = m_fields.get("pml_E_cp", Direction{dim}, lev); + amrex::MultiFab* pml_B_cp = m_fields.get(FieldType::pml_B_cp, Direction{dim}, lev); + amrex::MultiFab* pml_E_cp = m_fields.get(FieldType::pml_E_cp, Direction{dim}, lev); shiftMF(*pml_B_cp, geom[lev-1], num_shift_crse, dir, lev, dont_update_cost); shiftMF(*pml_E_cp, geom[lev-1], num_shift_crse, dir, lev, dont_update_cost); } @@ -295,14 +297,14 @@ WarpX::MoveWindow (const int step, bool move_j) // Shift scalar field F with div(E) cleaning in valid domain // TODO: shift F from pml_rz for RZ geometry with PSATD, once implemented - if (m_fields.has("F_fp", lev)) + if (m_fields.has(FieldType::F_fp, lev)) { // Fine grid - shiftMF(*m_fields.get("F_fp", lev), geom[lev], num_shift, dir, lev, do_update_cost); + shiftMF(*m_fields.get(FieldType::F_fp, lev), geom[lev], num_shift, dir, lev, do_update_cost); if (lev > 0) { // Coarse grid - shiftMF(*m_fields.get("F_cp", lev), geom[lev-1], num_shift_crse, dir, lev, do_update_cost); + shiftMF(*m_fields.get(FieldType::F_cp, lev), geom[lev-1], num_shift_crse, dir, lev, do_update_cost); } } @@ -312,7 +314,7 @@ WarpX::MoveWindow (const int step, bool move_j) // Fine grid if (do_pml && pml[lev]->ok()) { - amrex::MultiFab* pml_F = m_fields.get("pml_F_fp", lev); + amrex::MultiFab* pml_F = m_fields.get(FieldType::pml_F_fp, lev); shiftMF(*pml_F, geom[lev], num_shift, dir, lev, dont_update_cost); } if (lev > 0) @@ -320,7 +322,7 @@ WarpX::MoveWindow (const int step, bool move_j) // Coarse grid if (do_pml && pml[lev]->ok()) { - amrex::MultiFab* pml_F = m_fields.get("pml_F_cp", lev); + amrex::MultiFab* pml_F = m_fields.get(FieldType::pml_F_cp, lev); shiftMF(*pml_F, geom[lev-1], num_shift_crse, dir, lev, dont_update_cost); } } @@ -328,14 +330,14 @@ WarpX::MoveWindow (const int step, bool move_j) // Shift scalar field G with div(B) cleaning in valid domain // TODO: shift G from pml_rz for RZ geometry with PSATD, once implemented - if (m_fields.has("G_fp", lev)) + if (m_fields.has(FieldType::G_fp, lev)) { // Fine grid - shiftMF(*m_fields.get("G_fp", lev), geom[lev], num_shift, dir, lev, do_update_cost); + shiftMF(*m_fields.get(FieldType::G_fp, lev), geom[lev], num_shift, dir, lev, do_update_cost); if (lev > 0) { // Coarse grid - shiftMF(*m_fields.get("G_cp", lev), geom[lev-1], num_shift_crse, dir, lev, do_update_cost); + shiftMF(*m_fields.get(FieldType::G_cp, lev), geom[lev-1], num_shift_crse, dir, lev, do_update_cost); } } @@ -345,7 +347,7 @@ WarpX::MoveWindow (const int step, bool move_j) // Fine grid if (do_pml && pml[lev]->ok()) { - amrex::MultiFab* pml_G = m_fields.get("pml_G_fp", lev); + amrex::MultiFab* pml_G = m_fields.get(FieldType::pml_G_fp, lev); shiftMF(*pml_G, geom[lev], num_shift, dir, lev, dont_update_cost); } if (lev > 0) @@ -353,7 +355,7 @@ WarpX::MoveWindow (const int step, bool move_j) // Coarse grid if (do_pml && pml[lev]->ok()) { - amrex::MultiFab* pml_G = m_fields.get("pml_G_cp", lev); + amrex::MultiFab* pml_G = m_fields.get(FieldType::pml_G_cp, lev); shiftMF(*pml_G, geom[lev-1], num_shift_crse, dir, lev, dont_update_cost); } } @@ -361,12 +363,12 @@ WarpX::MoveWindow (const int step, bool move_j) // Shift scalar component rho if (move_j) { - if (m_fields.has("rho_fp", lev)) { + if (m_fields.has(FieldType::rho_fp, lev)) { // Fine grid - shiftMF(*m_fields.get("rho_fp",lev), geom[lev], num_shift, dir, lev, do_update_cost); + shiftMF(*m_fields.get(FieldType::rho_fp,lev), geom[lev], num_shift, dir, lev, do_update_cost); if (lev > 0){ // Coarse grid - shiftMF(*m_fields.get("rho_cp",lev), geom[lev-1], num_shift_crse, dir, lev, do_update_cost); + shiftMF(*m_fields.get(FieldType::rho_cp,lev), geom[lev-1], num_shift_crse, dir, lev, do_update_cost); } } } @@ -464,9 +466,9 @@ WarpX::MoveWindow (const int step, bool move_j) const int lev_zero = 0; m_macroscopic_properties->InitData( Geom(lev_zero), - m_fields.get("Efield_fp",Direction{0},lev_zero)->ixType().toIntVect(), - m_fields.get("Efield_fp",Direction{1},lev_zero)->ixType().toIntVect(), - m_fields.get("Efield_fp",Direction{2},lev_zero)->ixType().toIntVect() + m_fields.get(FieldType::Efield_fp, Direction{0}, lev_zero)->ixType().toIntVect(), + m_fields.get(FieldType::Efield_fp, Direction{1}, lev_zero)->ixType().toIntVect(), + m_fields.get(FieldType::Efield_fp, Direction{2}, lev_zero)->ixType().toIntVect() ); } diff --git a/Source/WarpX.H b/Source/WarpX.H index 5575ea2036f..83b1880f2b1 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -153,8 +153,9 @@ public: MultiDiagnostics& GetMultiDiags () {return *multi_diags;} #ifdef AMREX_USE_EB ablastr::fields::MultiLevelScalarField GetDistanceToEB () { - return m_fields.get_mr_levels("distance_to_eb", finestLevel()); - } + using warpx::fields::FieldType; + return m_fields.get_mr_levels(FieldType::distance_to_eb, finestLevel()); + } #endif ParticleBoundaryBuffer& GetParticleBoundaryBuffer () { return *m_particle_boundary_buffer; } diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index fdff1bbc2b2..30585114ff7 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -92,7 +92,7 @@ #include using namespace amrex; -using namespace warpx::fields; +using warpx::fields::FieldType; int WarpX::do_moving_window = 0; int WarpX::start_moving_window_step = 0; @@ -2194,48 +2194,48 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm // const std::array dx = CellSize(lev); - m_fields.alloc_init( "Bfield_fp", Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Bfield_fp", Direction{1}, lev, amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Bfield_fp", Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_fp, Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_fp, Direction{1}, lev, amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_fp, Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Efield_fp", Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Efield_fp", Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Efield_fp", Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_fp, Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_fp, Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_fp, Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "current_fp", Direction{0}, lev, amrex::convert(ba, jx_nodal_flag), dm, ncomps, ngJ, 0.0_rt); - m_fields.alloc_init( "current_fp", Direction{1}, lev, amrex::convert(ba, jy_nodal_flag), dm, ncomps, ngJ, 0.0_rt); - m_fields.alloc_init( "current_fp", Direction{2}, lev, amrex::convert(ba, jz_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_fp, Direction{0}, lev, amrex::convert(ba, jx_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_fp, Direction{1}, lev, amrex::convert(ba, jy_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_fp, Direction{2}, lev, amrex::convert(ba, jz_nodal_flag), dm, ncomps, ngJ, 0.0_rt); if (do_current_centering) { amrex::BoxArray const& nodal_ba = amrex::convert(ba, amrex::IntVect::TheNodeVector()); - m_fields.alloc_init( "current_fp_nodal", Direction{0}, lev, nodal_ba, dm, ncomps, ngJ, 0.0_rt); - m_fields.alloc_init( "current_fp_nodal", Direction{1}, lev, nodal_ba, dm, ncomps, ngJ, 0.0_rt); - m_fields.alloc_init( "current_fp_nodal", Direction{2}, lev, nodal_ba, dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_fp_nodal, Direction{0}, lev, nodal_ba, dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_fp_nodal, Direction{1}, lev, nodal_ba, dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_fp_nodal, Direction{2}, lev, nodal_ba, dm, ncomps, ngJ, 0.0_rt); } if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Vay) { - m_fields.alloc_init( "current_fp_vay", Direction{0}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngJ, 0.0_rt); - m_fields.alloc_init( "current_fp_vay", Direction{1}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngJ, 0.0_rt); - m_fields.alloc_init( "current_fp_vay", Direction{2}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_fp_vay, Direction{0}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_fp_vay, Direction{1}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_fp_vay, Direction{2}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngJ, 0.0_rt); } if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) { - m_fields.alloc_init("vector_potential_fp_nodal", Direction{0}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngRho, 0.0_rt); - m_fields.alloc_init("vector_potential_fp_nodal", Direction{1}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngRho, 0.0_rt); - m_fields.alloc_init("vector_potential_fp_nodal", Direction{2}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngRho, 0.0_rt); + m_fields.alloc_init(FieldType::vector_potential_fp_nodal, Direction{0}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngRho, 0.0_rt); + m_fields.alloc_init(FieldType::vector_potential_fp_nodal, Direction{1}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngRho, 0.0_rt); + m_fields.alloc_init(FieldType::vector_potential_fp_nodal, Direction{2}, lev, amrex::convert(ba, rho_nodal_flag), dm, ncomps, ngRho, 0.0_rt); // Memory buffers for computing magnetostatic fields // Vector Potential A and previous step. Time buffer needed for computing dA/dt to first order - m_fields.alloc_init("vector_potential_grad_buf_e_stag", Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("vector_potential_grad_buf_e_stag", Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("vector_potential_grad_buf_e_stag", Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::vector_potential_grad_buf_e_stag, Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::vector_potential_grad_buf_e_stag, Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::vector_potential_grad_buf_e_stag, Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("vector_potential_grad_buf_b_stag", Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("vector_potential_grad_buf_b_stag", Direction{1}, lev, amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("vector_potential_grad_buf_b_stag", Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::vector_potential_grad_buf_b_stag, Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::vector_potential_grad_buf_b_stag, Direction{1}, lev, amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::vector_potential_grad_buf_b_stag, Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); } // Allocate extra multifabs needed by the kinetic-fluid hybrid algorithm. @@ -2265,38 +2265,38 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (fft_do_time_averaging) { - m_fields.alloc_init( "Bfield_avg_fp", Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Bfield_avg_fp", Direction{1}, lev, amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Bfield_avg_fp", Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_avg_fp, Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_avg_fp, Direction{1}, lev, amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_avg_fp, Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Efield_avg_fp", Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Efield_avg_fp", Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Efield_avg_fp", Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_avg_fp, Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_avg_fp, Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_avg_fp, Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); } if (EB::enabled()) { constexpr int nc_ls = 1; amrex::IntVect const ng_ls(2); //EB level set - m_fields.alloc_init("distance_to_eb", lev, amrex::convert(ba, IntVect::TheNodeVector()), dm, nc_ls, ng_ls, 0.0_rt); + m_fields.alloc_init(FieldType::distance_to_eb, lev, amrex::convert(ba, IntVect::TheNodeVector()), dm, nc_ls, ng_ls, 0.0_rt); // EB info are needed only at the finest level if (lev == maxLevel()) { if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { //! EB: Lengths of the mesh edges - m_fields.alloc_init( "edge_lengths", Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), + m_fields.alloc_init(FieldType::edge_lengths, Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); - m_fields.alloc_init( "edge_lengths", Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), + m_fields.alloc_init(FieldType::edge_lengths, Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); - m_fields.alloc_init( "edge_lengths", Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), + m_fields.alloc_init(FieldType::edge_lengths, Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); //! EB: Areas of the mesh faces - m_fields.alloc_init( "face_areas", Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), + m_fields.alloc_init(FieldType::face_areas, Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); - m_fields.alloc_init( "face_areas", Direction{1}, lev, amrex::convert(ba, By_nodal_flag), + m_fields.alloc_init(FieldType::face_areas, Direction{1}, lev, amrex::convert(ba, By_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); - m_fields.alloc_init( "face_areas", Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), + m_fields.alloc_init(FieldType::face_areas, Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); } if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { @@ -2316,11 +2316,11 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm /** EB: area_mod contains the modified areas of the mesh faces, i.e. if a face is enlarged it * contains the area of the enlarged face * This is only used for the ECT solver.*/ - m_fields.alloc_init( "area_mod", Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), + m_fields.alloc_init(FieldType::area_mod, Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); - m_fields.alloc_init( "area_mod", Direction{1}, lev, amrex::convert(ba, By_nodal_flag), + m_fields.alloc_init(FieldType::area_mod, Direction{1}, lev, amrex::convert(ba, By_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); - m_fields.alloc_init( "area_mod", Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), + m_fields.alloc_init(FieldType::area_mod, Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); m_borrowing[lev][0] = std::make_unique>( @@ -2333,11 +2333,11 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm /** Venl contains the electromotive force for every mesh face, i.e. every entry is * the corresponding entry in ECTRhofield multiplied by the total area (possibly with enlargement) * This is only used for the ECT solver.*/ - m_fields.alloc_init( "Venl", Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), + m_fields.alloc_init(FieldType::Venl, Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); - m_fields.alloc_init( "Venl", Direction{1}, lev, amrex::convert(ba, By_nodal_flag), + m_fields.alloc_init(FieldType::Venl, Direction{1}, lev, amrex::convert(ba, By_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); - m_fields.alloc_init( "Venl", Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), + m_fields.alloc_init(FieldType::Venl, Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); /** ECTRhofield is needed only by the ect @@ -2347,11 +2347,11 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm * and below). * Although it's called rho it has nothing to do with the charge density! * This is only used for the ECT solver.*/ - m_fields.alloc_init( "ECTRhofield", Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), + m_fields.alloc_init(FieldType::ECTRhofield, Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); - m_fields.alloc_init( "ECTRhofield", Direction{1}, lev, amrex::convert(ba, By_nodal_flag), + m_fields.alloc_init(FieldType::ECTRhofield, Direction{1}, lev, amrex::convert(ba, By_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); - m_fields.alloc_init( "ECTRhofield", Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), + m_fields.alloc_init(FieldType::ECTRhofield, Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); } } @@ -2374,8 +2374,8 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm } if (rho_ncomps > 0) { - m_fields.alloc_init( - "rho_fp", lev, amrex::convert(ba, rho_nodal_flag), dm, + m_fields.alloc_init(FieldType::rho_fp, + lev, amrex::convert(ba, rho_nodal_flag), dm, rho_ncomps, ngRho, 0.0_rt); } @@ -2383,28 +2383,28 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) { const IntVect ngPhi = IntVect( AMREX_D_DECL(1,1,1) ); - m_fields.alloc_init( "phi_fp", lev, amrex::convert(ba, phi_nodal_flag), dm, + m_fields.alloc_init(FieldType::phi_fp, lev, amrex::convert(ba, phi_nodal_flag), dm, ncomps, ngPhi, 0.0_rt ); } if (do_subcycling && lev == 0) { - m_fields.alloc_init("current_store", Direction{0}, lev, amrex::convert(ba,jx_nodal_flag), dm, ncomps, ngJ, 0.0_rt); - m_fields.alloc_init("current_store", Direction{1}, lev, amrex::convert(ba,jy_nodal_flag), dm, ncomps, ngJ, 0.0_rt); - m_fields.alloc_init("current_store", Direction{2}, lev, amrex::convert(ba,jz_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_store, Direction{0}, lev, amrex::convert(ba,jx_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_store, Direction{1}, lev, amrex::convert(ba,jy_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_store, Direction{2}, lev, amrex::convert(ba,jz_nodal_flag), dm, ncomps, ngJ, 0.0_rt); } if (do_dive_cleaning) { - m_fields.alloc_init( - "F_fp", lev, amrex::convert(ba, F_nodal_flag), dm, + m_fields.alloc_init(FieldType::F_fp, + lev, amrex::convert(ba, F_nodal_flag), dm, ncomps, ngF, 0.0_rt); } if (do_divb_cleaning) { - m_fields.alloc_init( - "G_fp", lev, amrex::convert(ba, G_nodal_flag), dm, + m_fields.alloc_init(FieldType::G_fp, + lev, amrex::convert(ba, G_nodal_flag), dm, ncomps, ngG, 0.0_rt); } @@ -2474,102 +2474,102 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm // Create aux multifabs on Nodal Box Array BoxArray const nba = amrex::convert(ba,IntVect::TheNodeVector()); - m_fields.alloc_init("Bfield_aux", Direction{0}, lev, nba, dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Bfield_aux", Direction{1}, lev, nba, dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Bfield_aux", Direction{2}, lev, nba, dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_aux, Direction{0}, lev, nba, dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_aux, Direction{1}, lev, nba, dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_aux, Direction{2}, lev, nba, dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Efield_aux", Direction{0}, lev, nba, dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Efield_aux", Direction{1}, lev, nba, dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Efield_aux", Direction{2}, lev, nba, dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_aux, Direction{0}, lev, nba, dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_aux, Direction{1}, lev, nba, dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_aux, Direction{2}, lev, nba, dm, ncomps, ngEB, 0.0_rt); } else if (lev == 0) { if (WarpX::fft_do_time_averaging) { - m_fields.alias_init("Bfield_aux", "Bfield_avg_fp", Direction{0}, lev, 0.0_rt); - m_fields.alias_init("Bfield_aux", "Bfield_avg_fp", Direction{1}, lev, 0.0_rt); - m_fields.alias_init("Bfield_aux", "Bfield_avg_fp", Direction{2}, lev, 0.0_rt); + m_fields.alias_init(FieldType::Bfield_aux, FieldType::Bfield_avg_fp, Direction{0}, lev, 0.0_rt); + m_fields.alias_init(FieldType::Bfield_aux, FieldType::Bfield_avg_fp, Direction{1}, lev, 0.0_rt); + m_fields.alias_init(FieldType::Bfield_aux, FieldType::Bfield_avg_fp, Direction{2}, lev, 0.0_rt); - m_fields.alias_init("Efield_aux", "Efield_avg_fp", Direction{0}, lev, 0.0_rt); - m_fields.alias_init("Efield_aux", "Efield_avg_fp", Direction{1}, lev, 0.0_rt); - m_fields.alias_init("Efield_aux", "Efield_avg_fp", Direction{2}, lev, 0.0_rt); + m_fields.alias_init(FieldType::Efield_aux, FieldType::Efield_avg_fp, Direction{0}, lev, 0.0_rt); + m_fields.alias_init(FieldType::Efield_aux, FieldType::Efield_avg_fp, Direction{1}, lev, 0.0_rt); + m_fields.alias_init(FieldType::Efield_aux, FieldType::Efield_avg_fp, Direction{2}, lev, 0.0_rt); } else { if (mypc->m_B_ext_particle_s == "read_from_file") { - m_fields.alloc_init("Bfield_aux", Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Bfield_aux", Direction{1}, lev, amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Bfield_aux", Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_aux, Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_aux, Direction{1}, lev, amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_aux, Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); } else { // In this case, the aux grid is simply an alias of the fp grid (most common case in WarpX) - m_fields.alias_init("Bfield_aux", "Bfield_fp", Direction{0}, lev, 0.0_rt); - m_fields.alias_init("Bfield_aux", "Bfield_fp", Direction{1}, lev, 0.0_rt); - m_fields.alias_init("Bfield_aux", "Bfield_fp", Direction{2}, lev, 0.0_rt); + m_fields.alias_init(FieldType::Bfield_aux, FieldType::Bfield_fp, Direction{0}, lev, 0.0_rt); + m_fields.alias_init(FieldType::Bfield_aux, FieldType::Bfield_fp, Direction{1}, lev, 0.0_rt); + m_fields.alias_init(FieldType::Bfield_aux, FieldType::Bfield_fp, Direction{2}, lev, 0.0_rt); } if (mypc->m_E_ext_particle_s == "read_from_file") { - m_fields.alloc_init("Efield_aux", Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Efield_aux", Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Efield_aux", Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_aux, Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_aux, Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_aux, Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); } else { // In this case, the aux grid is simply an alias of the fp grid (most common case in WarpX) - m_fields.alias_init("Efield_aux", "Efield_fp", Direction{0}, lev, 0.0_rt); - m_fields.alias_init("Efield_aux", "Efield_fp", Direction{1}, lev, 0.0_rt); - m_fields.alias_init("Efield_aux", "Efield_fp", Direction{2}, lev, 0.0_rt); + m_fields.alias_init(FieldType::Efield_aux, FieldType::Efield_fp, Direction{0}, lev, 0.0_rt); + m_fields.alias_init(FieldType::Efield_aux, FieldType::Efield_fp, Direction{1}, lev, 0.0_rt); + m_fields.alias_init(FieldType::Efield_aux, FieldType::Efield_fp, Direction{2}, lev, 0.0_rt); } } } else { - m_fields.alloc_init("Bfield_aux", Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Bfield_aux", Direction{1}, lev, amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Bfield_aux", Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_aux, Direction{0}, lev, amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_aux, Direction{1}, lev, amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_aux, Direction{2}, lev, amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Efield_aux", Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Efield_aux", Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Efield_aux", Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_aux, Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_aux, Direction{1}, lev, amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_aux, Direction{2}, lev, amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); } // The external fields that are read from file if (m_p_ext_field_params->B_ext_grid_type != ExternalFieldType::default_zero && m_p_ext_field_params->B_ext_grid_type != ExternalFieldType::constant) { // These fields will be added directly to the grid, i.e. to fp, and need to match the index type - m_fields.alloc_init( "Bfield_fp_external", Direction{0}, lev, - amrex::convert(ba, m_fields.get("Bfield_fp",Direction{0},lev)->ixType()), + m_fields.alloc_init(FieldType::Bfield_fp_external, Direction{0}, lev, + amrex::convert(ba, m_fields.get(FieldType::Bfield_fp,Direction{0},lev)->ixType()), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Bfield_fp_external", Direction{1}, lev, - amrex::convert(ba, m_fields.get("Bfield_fp",Direction{1},lev)->ixType()), + m_fields.alloc_init(FieldType::Bfield_fp_external, Direction{1}, lev, + amrex::convert(ba, m_fields.get(FieldType::Bfield_fp,Direction{1},lev)->ixType()), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Bfield_fp_external", Direction{2}, lev, - amrex::convert(ba, m_fields.get("Bfield_fp",Direction{2},lev)->ixType()), + m_fields.alloc_init(FieldType::Bfield_fp_external, Direction{2}, lev, + amrex::convert(ba, m_fields.get(FieldType::Bfield_fp,Direction{2},lev)->ixType()), dm, ncomps, ngEB, 0.0_rt); } if (mypc->m_B_ext_particle_s == "read_from_file") { // These fields will be added to the fields that the particles see, and need to match the index type - auto *Bfield_aux_levl_0 = m_fields.get("Bfield_aux", Direction{0}, lev); - auto *Bfield_aux_levl_1 = m_fields.get("Bfield_aux", Direction{1}, lev); - auto *Bfield_aux_levl_2 = m_fields.get("Bfield_aux", Direction{2}, lev); + auto *Bfield_aux_levl_0 = m_fields.get(FieldType::Bfield_aux, Direction{0}, lev); + auto *Bfield_aux_levl_1 = m_fields.get(FieldType::Bfield_aux, Direction{1}, lev); + auto *Bfield_aux_levl_2 = m_fields.get(FieldType::Bfield_aux, Direction{2}, lev); // Same as Bfield_fp for reading external field data - m_fields.alloc_init( "B_external_particle_field", Direction{0}, lev, amrex::convert(ba, Bfield_aux_levl_0->ixType()), + m_fields.alloc_init(FieldType::B_external_particle_field, Direction{0}, lev, amrex::convert(ba, Bfield_aux_levl_0->ixType()), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "B_external_particle_field", Direction{1}, lev, amrex::convert(ba, Bfield_aux_levl_1->ixType()), + m_fields.alloc_init(FieldType::B_external_particle_field, Direction{1}, lev, amrex::convert(ba, Bfield_aux_levl_1->ixType()), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "B_external_particle_field", Direction{2}, lev, amrex::convert(ba, Bfield_aux_levl_2->ixType()), + m_fields.alloc_init(FieldType::B_external_particle_field, Direction{2}, lev, amrex::convert(ba, Bfield_aux_levl_2->ixType()), dm, ncomps, ngEB, 0.0_rt); } if (m_p_ext_field_params->E_ext_grid_type != ExternalFieldType::default_zero && m_p_ext_field_params->E_ext_grid_type != ExternalFieldType::constant) { // These fields will be added directly to the grid, i.e. to fp, and need to match the index type - m_fields.alloc_init( "Efield_fp_external", Direction{0}, lev, amrex::convert(ba, m_fields.get("Efield_fp",Direction{0},lev)->ixType()), + m_fields.alloc_init(FieldType::Efield_fp_external, Direction{0}, lev, amrex::convert(ba, m_fields.get(FieldType::Efield_fp, Direction{0}, lev)->ixType()), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Efield_fp_external", Direction{1}, lev, amrex::convert(ba, m_fields.get("Efield_fp",Direction{1},lev)->ixType()), + m_fields.alloc_init(FieldType::Efield_fp_external, Direction{1}, lev, amrex::convert(ba, m_fields.get(FieldType::Efield_fp, Direction{1}, lev)->ixType()), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Efield_fp_external", Direction{2}, lev, amrex::convert(ba, m_fields.get("Efield_fp",Direction{2},lev)->ixType()), + m_fields.alloc_init(FieldType::Efield_fp_external, Direction{2}, lev, amrex::convert(ba, m_fields.get(FieldType::Efield_fp, Direction{2}, lev)->ixType()), dm, ncomps, ngEB, 0.0_rt); } if (mypc->m_E_ext_particle_s == "read_from_file") { // These fields will be added to the fields that the particles see, and need to match the index type - auto *Efield_aux_levl_0 = m_fields.get("Efield_aux", Direction{0}, lev); - auto *Efield_aux_levl_1 = m_fields.get("Efield_aux", Direction{1}, lev); - auto *Efield_aux_levl_2 = m_fields.get("Efield_aux", Direction{2}, lev); + auto *Efield_aux_levl_0 = m_fields.get(FieldType::Efield_aux, Direction{0}, lev); + auto *Efield_aux_levl_1 = m_fields.get(FieldType::Efield_aux, Direction{1}, lev); + auto *Efield_aux_levl_2 = m_fields.get(FieldType::Efield_aux, Direction{2}, lev); // Same as Efield_fp for reading external field data - m_fields.alloc_init( "E_external_particle_field", Direction{0}, lev, amrex::convert(ba, Efield_aux_levl_0->ixType()), + m_fields.alloc_init(FieldType::E_external_particle_field, Direction{0}, lev, amrex::convert(ba, Efield_aux_levl_0->ixType()), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "E_external_particle_field", Direction{1}, lev, amrex::convert(ba, Efield_aux_levl_1->ixType()), + m_fields.alloc_init(FieldType::E_external_particle_field, Direction{1}, lev, amrex::convert(ba, Efield_aux_levl_1->ixType()), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "E_external_particle_field", Direction{2}, lev, amrex::convert(ba, Efield_aux_levl_2->ixType()), + m_fields.alloc_init(FieldType::E_external_particle_field, Direction{2}, lev, amrex::convert(ba, Efield_aux_levl_2->ixType()), dm, ncomps, ngEB, 0.0_rt); } @@ -2583,41 +2583,41 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm const std::array cdx = CellSize(lev-1); // Create the MultiFabs for B - m_fields.alloc_init( "Bfield_cp", Direction{0}, lev, amrex::convert(cba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Bfield_cp", Direction{1}, lev, amrex::convert(cba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Bfield_cp", Direction{2}, lev, amrex::convert(cba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_cp, Direction{0}, lev, amrex::convert(cba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_cp, Direction{1}, lev, amrex::convert(cba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_cp, Direction{2}, lev, amrex::convert(cba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); // Create the MultiFabs for E - m_fields.alloc_init( "Efield_cp", Direction{0}, lev, amrex::convert(cba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Efield_cp", Direction{1}, lev, amrex::convert(cba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init( "Efield_cp", Direction{2}, lev, amrex::convert(cba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_cp, Direction{0}, lev, amrex::convert(cba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_cp, Direction{1}, lev, amrex::convert(cba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_cp, Direction{2}, lev, amrex::convert(cba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); if (fft_do_time_averaging) { - m_fields.alloc_init("Bfield_avg_cp", Direction{0}, lev, amrex::convert(cba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Bfield_avg_cp", Direction{1}, lev, amrex::convert(cba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Bfield_avg_cp", Direction{2}, lev, amrex::convert(cba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_avg_cp, Direction{0}, lev, amrex::convert(cba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_avg_cp, Direction{1}, lev, amrex::convert(cba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_avg_cp, Direction{2}, lev, amrex::convert(cba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Efield_avg_cp", Direction{0}, lev, amrex::convert(cba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Efield_avg_cp", Direction{1}, lev, amrex::convert(cba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Efield_avg_cp", Direction{2}, lev, amrex::convert(cba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_avg_cp, Direction{0}, lev, amrex::convert(cba, Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_avg_cp, Direction{1}, lev, amrex::convert(cba, Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_avg_cp, Direction{2}, lev, amrex::convert(cba, Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); } // Create the MultiFabs for the current - m_fields.alloc_init( "current_cp", Direction{0}, lev, amrex::convert(cba, jx_nodal_flag), dm, ncomps, ngJ, 0.0_rt); - m_fields.alloc_init( "current_cp", Direction{1}, lev, amrex::convert(cba, jy_nodal_flag), dm, ncomps, ngJ, 0.0_rt); - m_fields.alloc_init( "current_cp", Direction{2}, lev, amrex::convert(cba, jz_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_cp, Direction{0}, lev, amrex::convert(cba, jx_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_cp, Direction{1}, lev, amrex::convert(cba, jy_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_cp, Direction{2}, lev, amrex::convert(cba, jz_nodal_flag), dm, ncomps, ngJ, 0.0_rt); if (rho_ncomps > 0) { - m_fields.alloc_init( - "rho_cp", lev, amrex::convert(cba, rho_nodal_flag), dm, + m_fields.alloc_init(FieldType::rho_cp, + lev, amrex::convert(cba, rho_nodal_flag), dm, rho_ncomps, ngRho, 0.0_rt); } if (do_dive_cleaning) { - m_fields.alloc_init( - "F_cp", lev, amrex::convert(cba, IntVect::TheUnitVector()), dm, + m_fields.alloc_init(FieldType::F_cp, + lev, amrex::convert(cba, IntVect::TheUnitVector()), dm, ncomps, ngF, 0.0_rt); } @@ -2625,14 +2625,14 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm { if (grid_type == GridType::Collocated) { - m_fields.alloc_init( - "G_cp", lev, amrex::convert(ba, IntVect::TheUnitVector()), dm, + m_fields.alloc_init(FieldType::G_cp, + lev, amrex::convert(ba, IntVect::TheUnitVector()), dm, ncomps, ngG, 0.0_rt); } else // grid_type=staggered or grid_type=hybrid { - m_fields.alloc_init( - "G_cp", lev, amrex::convert(ba, IntVect::TheZeroVector()), dm, + m_fields.alloc_init(FieldType::G_cp, + lev, amrex::convert(ba, IntVect::TheZeroVector()), dm, ncomps, ngG, 0.0_rt); } } @@ -2692,24 +2692,24 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (aux_is_nodal) { BoxArray const& cnba = amrex::convert(cba,IntVect::TheNodeVector()); // Create the MultiFabs for B - m_fields.alloc_init("Bfield_cax", Direction{0}, lev, cnba, dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Bfield_cax", Direction{1}, lev, cnba, dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Bfield_cax", Direction{2}, lev, cnba, dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_cax, Direction{0}, lev, cnba, dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_cax, Direction{1}, lev, cnba, dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_cax, Direction{2}, lev, cnba, dm, ncomps, ngEB, 0.0_rt); // Create the MultiFabs for E - m_fields.alloc_init("Efield_cax", Direction{0}, lev, cnba, dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Efield_cax", Direction{1}, lev, cnba, dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Efield_cax", Direction{2}, lev, cnba, dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_cax, Direction{0}, lev, cnba, dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_cax, Direction{1}, lev, cnba, dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_cax, Direction{2}, lev, cnba, dm, ncomps, ngEB, 0.0_rt); } else { // Create the MultiFabs for B - m_fields.alloc_init("Bfield_cax", Direction{0}, lev, amrex::convert(cba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Bfield_cax", Direction{1}, lev, amrex::convert(cba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Bfield_cax", Direction{2}, lev, amrex::convert(cba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_cax, Direction{0}, lev, amrex::convert(cba, Bx_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_cax, Direction{1}, lev, amrex::convert(cba, By_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Bfield_cax, Direction{2}, lev, amrex::convert(cba, Bz_nodal_flag), dm, ncomps, ngEB, 0.0_rt); // Create the MultiFabs for E - m_fields.alloc_init("Efield_cax", Direction{0}, lev, amrex::convert(cba,Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Efield_cax", Direction{1}, lev, amrex::convert(cba,Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); - m_fields.alloc_init("Efield_cax", Direction{2}, lev, amrex::convert(cba,Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_cax, Direction{0}, lev, amrex::convert(cba,Ex_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_cax, Direction{1}, lev, amrex::convert(cba,Ey_nodal_flag), dm, ncomps, ngEB, 0.0_rt); + m_fields.alloc_init(FieldType::Efield_cax, Direction{2}, lev, amrex::convert(cba,Ez_nodal_flag), dm, ncomps, ngEB, 0.0_rt); } AllocInitMultiFab(gather_buffer_masks[lev], ba, dm, ncomps, amrex::IntVect(1), lev, "gather_buffer_masks"); @@ -2718,11 +2718,11 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm } if (n_current_deposition_buffer > 0) { - m_fields.alloc_init("current_buf", Direction{0}, lev, amrex::convert(cba,jx_nodal_flag), dm, ncomps, ngJ, 0.0_rt); - m_fields.alloc_init("current_buf", Direction{1}, lev, amrex::convert(cba,jy_nodal_flag), dm, ncomps, ngJ, 0.0_rt); - m_fields.alloc_init("current_buf", Direction{2}, lev, amrex::convert(cba,jz_nodal_flag), dm, ncomps, ngJ, 0.0_rt); - if (m_fields.has("rho_cp", lev)) { - m_fields.alloc_init("rho_buf", lev, amrex::convert(cba,rho_nodal_flag), dm, 2*ncomps, ngRho, 0.0_rt); + m_fields.alloc_init(FieldType::current_buf, Direction{0}, lev, amrex::convert(cba,jx_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_buf, Direction{1}, lev, amrex::convert(cba,jy_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + m_fields.alloc_init(FieldType::current_buf, Direction{2}, lev, amrex::convert(cba,jz_nodal_flag), dm, ncomps, ngJ, 0.0_rt); + if (m_fields.has(FieldType::rho_cp, lev)) { + m_fields.alloc_init(FieldType::rho_buf, lev, amrex::convert(cba,rho_nodal_flag), dm, 2*ncomps, ngRho, 0.0_rt); } AllocInitMultiFab(current_buffer_masks[lev], ba, dm, ncomps, amrex::IntVect(1), lev, "current_buffer_masks"); // Current buffer masks have 1 ghost cell, because of the fact @@ -2973,14 +2973,14 @@ WarpX::ComputeDivE(amrex::MultiFab& divE, const int lev) { if ( WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD ) { #ifdef WARPX_USE_FFT - const ablastr::fields::VectorField Efield_aux_lev = m_fields.get_alldirs("Efield_aux", lev); + const ablastr::fields::VectorField Efield_aux_lev = m_fields.get_alldirs(FieldType::Efield_aux, lev); spectral_solver_fp[lev]->ComputeSpectralDivE(lev, Efield_aux_lev, divE); #else WARPX_ABORT_WITH_MESSAGE( "ComputeDivE: PSATD requested but not compiled"); #endif } else { - const ablastr::fields::VectorField Efield_aux_lev = m_fields.get_alldirs("Efield_aux", lev); + const ablastr::fields::VectorField Efield_aux_lev = m_fields.get_alldirs(FieldType::Efield_aux, lev); m_fdtd_solver_fp[lev]->ComputeDivE(Efield_aux_lev, divE); } } @@ -3258,10 +3258,10 @@ WarpX::StoreCurrent (int lev) { using ablastr::fields::Direction; for (int idim = 0; idim < 3; ++idim) { - if (m_fields.has("current_store", Direction{idim},lev)) { - MultiFab::Copy(*m_fields.get("current_store", Direction{idim}, lev), - *m_fields.get("current_fp", Direction{idim}, lev), - 0, 0, 1, m_fields.get("current_store", Direction{idim}, lev)->nGrowVect()); + if (m_fields.has(FieldType::current_store, Direction{idim},lev)) { + MultiFab::Copy(*m_fields.get(FieldType::current_store, Direction{idim}, lev), + *m_fields.get(FieldType::current_fp, Direction{idim}, lev), + 0, 0, 1, m_fields.get(FieldType::current_store, Direction{idim}, lev)->nGrowVect()); } } } @@ -3270,12 +3270,13 @@ void WarpX::RestoreCurrent (int lev) { using ablastr::fields::Direction; + using warpx::fields::FieldType; for (int idim = 0; idim < 3; ++idim) { - if (m_fields.has("current_store", Direction{idim}, lev)) { + if (m_fields.has(FieldType::current_store, Direction{idim}, lev)) { std::swap( - *m_fields.get("current_fp", Direction{idim}, lev), - *m_fields.get("current_store", Direction{idim}, lev) + *m_fields.get(FieldType::current_fp, Direction{idim}, lev), + *m_fields.get(FieldType::current_store, Direction{idim}, lev) ); } } From f8fa6e132e4967c98374bb1d1c88848e567a7578 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sat, 21 Sep 2024 22:39:31 -0700 Subject: [PATCH 301/314] `MultiFabRegister`: Const & Clean - do not copy strings - include cleaning - finalize doxygen comments --- Source/ablastr/fields/MultiFabRegister.H | 158 ++++++------ Source/ablastr/fields/MultiFabRegister.cpp | 225 +++++++++--------- .../ablastr/fields/MultiFabRegister_string.H | 101 ++++---- 3 files changed, 237 insertions(+), 247 deletions(-) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index ec088a74863..74d82758ec6 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -1,27 +1,27 @@ -/* Copyright 2024 The ABLAST Community +/* Copyright 2024 The ABLASTR Community * - * This file is part of WarpX. + * This file is part of ABLASTR. * * License: BSD-3-Clause-LBNL - * Authors: Axel Huebl, ... + * Authors: Axel Huebl */ #ifndef ABLASTR_FIELDS_MF_REGISTER_H #define ABLASTR_FIELDS_MF_REGISTER_H -#include #include #include #include #include #include #include +#include +#include #include #include #include #include #include -#include #include #include @@ -193,12 +193,12 @@ namespace ablastr::fields amrex::DistributionMapping const & dm, int ncomp, amrex::IntVect const & ngrow, - std::optional initial_value = std::nullopt, + std::optional initial_value = std::nullopt, bool remake = true, bool redistribute_on_remake = true ) { - return alloc_init( + return alloc_init( getExtractedName(name), level, ba, @@ -238,12 +238,12 @@ namespace ablastr::fields amrex::DistributionMapping const & dm, int ncomp, amrex::IntVect const & ngrow, - std::optional initial_value = std::nullopt, + std::optional initial_value = std::nullopt, bool remake = true, bool redistribute_on_remake = true ) { - return alloc_init( + return alloc_init( getExtractedName(name), dir, level, @@ -274,10 +274,10 @@ namespace ablastr::fields N new_name, A alias_name, int level, - std::optional initial_value = std::nullopt + std::optional initial_value = std::nullopt ) { - return alias_init( + return alias_init( getExtractedName(new_name), getExtractedName(alias_name), level, @@ -304,10 +304,10 @@ namespace ablastr::fields A alias_name, Direction dir, int level, - std::optional initial_value = std::nullopt + std::optional initial_value = std::nullopt ) { - return alias_init( + return alias_init( getExtractedName(new_name), getExtractedName(alias_name), dir, @@ -345,7 +345,7 @@ namespace ablastr::fields int level ) const { - return has( + return has( getExtractedName(name), level ); @@ -366,7 +366,7 @@ namespace ablastr::fields int level ) const { - return has( + return has( getExtractedName(name), dir, level @@ -386,7 +386,7 @@ namespace ablastr::fields int level ) const { - return has_vector( + return has_vector( getExtractedName(name), level ); @@ -396,8 +396,8 @@ namespace ablastr::fields * * This throws a runtime error if the requested field is not present. * - * @param name the name to check if registered - * @param level the MR level to check + * @param name the name of the field + * @param level the MR level * @return a non-owning pointer to the MultiFab (field) */ template @@ -407,7 +407,7 @@ namespace ablastr::fields int level ) { - return get( + return get( getExtractedName(name), level ); @@ -417,9 +417,9 @@ namespace ablastr::fields * * This throws a runtime error if the requested field is not present. * - * @param name the name to check if registered + * @param name the name of the field * @param dir the field component for vector fields ("direction" of the unit vector) - * @param level the MR level to check + * @param level the MR level * @return a non-owning pointer to the MultiFab (field) */ template @@ -430,7 +430,7 @@ namespace ablastr::fields int level ) { - return get( + return get( getExtractedName(name), dir, level @@ -441,8 +441,8 @@ namespace ablastr::fields * * This throws a runtime error if the requested field is not present. * - * @param name the name to check if registered - * @param level the MR level to check + * @param name the name of the field + * @param level the MR level * @return a non-owning pointer to the MultiFab (field) */ template @@ -452,7 +452,7 @@ namespace ablastr::fields int level ) const { - return get( + return get( getExtractedName(name), level ); @@ -462,9 +462,9 @@ namespace ablastr::fields * * This throws a runtime error if the requested field is not present. * - * @param name the name to check if registered + * @param name the name of the field * @param dir the field component for vector fields ("direction" of the unit vector) - * @param level the MR level to check + * @param level the MR level * @return a non-owning pointer to the MultiFab (field) */ template @@ -475,20 +475,20 @@ namespace ablastr::fields int level ) const { - return get( + return get( getExtractedName(name), dir, level ); } - /** title + /** Return the MultiFab of a scalar field on all MR levels. * - * Same as get above, but returns all levels for a name. + * This throws a runtime error if the requested field is not present. * - * @param name ... - * @param finest_level ... - * @return ... + * @param name the name of the field + * @param finest_level the highest MR level to return + * @return non-owning pointers to the MultiFab (field) on all levels */ //@{ template @@ -498,7 +498,7 @@ namespace ablastr::fields int finest_level ) { - return get_mr_levels( + return get_mr_levels( getExtractedName(name), finest_level ); @@ -510,7 +510,7 @@ namespace ablastr::fields int finest_level ) const { - return get_mr_levels( + return get_mr_levels( getExtractedName(name), finest_level ); @@ -521,9 +521,9 @@ namespace ablastr::fields * * Same as get above, but returns all levels for a name. * - * @param name ... - * @param level ... - * @return ... + * @param name the name of the field + * @param level the MR level + * @return non-owning pointers to all components of a vector field */ //@{ template @@ -533,7 +533,7 @@ namespace ablastr::fields int level ) { - return get_alldirs( + return get_alldirs( getExtractedName(name), level ); @@ -545,7 +545,7 @@ namespace ablastr::fields int level ) const { - return get_alldirs( + return get_alldirs( getExtractedName(name), level ); @@ -557,9 +557,9 @@ namespace ablastr::fields * Out loop: MR levels. * Inner loop: directions (components). * - * @param name ... - * @param finest_level ... - * @return ... + * @param name the name of the field + * @param finest_level the highest MR level to return + * @return non-owning pointers to all components of a vector field on all MR levels */ //@{ template @@ -569,7 +569,7 @@ namespace ablastr::fields int finest_level ) { - return get_mr_levels_alldirs( + return get_mr_levels_alldirs( getExtractedName(name), finest_level ); @@ -581,30 +581,24 @@ namespace ablastr::fields int finest_level ) const { - return get_mr_levels_alldirs( + return get_mr_levels_alldirs( getExtractedName(name), finest_level ); } //@} - /** title + /** List the internal names of all registered fields. * - * body body - * body - * - * @return ... + * @return all currently allocated and registered fields */ [[nodiscard]] std::vector list () const; - /** title - * - * body body - * body + /** Deallocate and remove a scalar field. * - * @param name ... - * @param level ... + * @param name the name of the field + * @param level the MR level */ template void @@ -613,17 +607,14 @@ namespace ablastr::fields int level ) { - erase(getExtractedName(name), level); + erase(getExtractedName(name), level); } - /** title - * - * body body - * body + /** Deallocate and remove a vector field component. * - * @param name ... - * @param dir ... - * @param level ... + * @param name the name of the field + * @param dir the field component for vector fields ("direction" of the unit vector) + * @param level the MR level */ template void @@ -633,7 +624,7 @@ namespace ablastr::fields int level ) { - erase(getExtractedName(name), dir, level); + erase(getExtractedName(name), dir, level); } /** Erase all MultiFabs on a specific MR level. @@ -660,14 +651,11 @@ namespace ablastr::fields amrex::DistributionMapping const & new_dm ); - /** title - * - * body body - * body + /** Create the register name of scalar field and MR level * - * @param name ... - * @param level ... - * @return ... + * @param name the name of the field + * @param level the MR level + * @return internal name of the field in the register */ [[nodiscard]] std::string mf_name ( @@ -675,15 +663,12 @@ namespace ablastr::fields int level ) const; - /** title + /** Create the register name of vector field, component direction and MR level * - * body body - * body - * - * @param name ... - * @param dir ... - * @param level ... - * @return ... + * @param name the name of the field + * @param dir the field component for vector fields ("direction" of the unit vector) + * @param level the MR level + * @return internal name of the field in the register */ [[nodiscard]] std::string mf_name ( @@ -695,11 +680,11 @@ namespace ablastr::fields private: amrex::MultiFab * internal_get ( - const std::string& key + std::string const & key ); [[nodiscard]] amrex::MultiFab const * internal_get ( - const std::string& key + std::string const & key ) const; /** data storage: ownership and lifetime control */ @@ -707,15 +692,18 @@ namespace ablastr::fields std::string, MultiFabOwner > m_mf_register; + + /** the three directions of a vector field */ + std::vector m_all_dirs = {Direction{0}, Direction{1}, Direction{2}}; }; - /** TODO: temporary, remove me + /** Little temporary helper function to pass temporary MultiFabs as VectorField. * - * @return + * @return pointers to externally managed vector field components (3 MultiFab) */ VectorField a2m ( - const std::array< std::unique_ptr, 3 > & old_vectorfield + std::array< std::unique_ptr, 3 > const & old_vectorfield ); } // namespace ablastr::fields diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 27407ad70f2..2f3b4fb3a97 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -1,29 +1,35 @@ -/* Copyright 2024 The ABLAST Community +/* Copyright 2024 The ABLASTR Community * - * This file is part of WarpX. + * This file is part of ABLASTR. * * License: BSD-3-Clause-LBNL - * Authors: Axel Huebl, ... + * Authors: Axel Huebl */ #include "MultiFabRegister.H" -#include "ablastr/utils/TextMsg.H" +#include -#include +#include +#include +#include +#include +#include +#include +#include namespace ablastr::fields { template<> amrex::MultiFab* - MultiFabRegister::alloc_init ( - std::string name, + MultiFabRegister::alloc_init ( + std::string const & name, int level, amrex::BoxArray const & ba, amrex::DistributionMapping const & dm, int ncomp, amrex::IntVect const & ngrow, - std::optional initial_value, + std::optional initial_value, bool remake, bool redistribute_on_remake ) @@ -34,12 +40,12 @@ namespace ablastr::fields } // fully qualified name - name = mf_name(name, level); + std::string const internal_name = mf_name(name, level); // allocate - const auto tag = amrex::MFInfo().SetTag(name); + const auto tag = amrex::MFInfo().SetTag(internal_name); auto [it, success] = m_mf_register.emplace( - name, + internal_name, MultiFabOwner{ {ba, dm, ncomp, ngrow, tag}, std::nullopt, // scalar: no direction @@ -50,10 +56,10 @@ namespace ablastr::fields } ); if (!success) { - throw std::runtime_error("MultiFabRegister::alloc_init failed for " + name); + throw std::runtime_error("MultiFabRegister::alloc_init failed for " + internal_name); } - // a short-hand alias for the code below + // a shorthand alias for the code below amrex::MultiFab & mf = it->second.m_mf; // initialize with value @@ -66,15 +72,15 @@ namespace ablastr::fields template<> amrex::MultiFab* - MultiFabRegister::alloc_init ( - std::string name, + MultiFabRegister::alloc_init ( + std::string const & name, Direction dir, int level, amrex::BoxArray const & ba, amrex::DistributionMapping const & dm, int ncomp, amrex::IntVect const & ngrow, - std::optional initial_value, + std::optional initial_value, bool remake, bool redistribute_on_remake ) @@ -89,12 +95,12 @@ namespace ablastr::fields } // fully qualified name - name = mf_name(name, dir, level); + std::string const internal_name = mf_name(name, dir, level); // allocate - const auto tag = amrex::MFInfo().SetTag(name); + const auto tag = amrex::MFInfo().SetTag(internal_name); auto [it, success] = m_mf_register.emplace( - name, + internal_name, MultiFabOwner{ {ba, dm, ncomp, ngrow, tag}, dir, @@ -105,10 +111,10 @@ namespace ablastr::fields } ); if (!success) { - throw std::runtime_error("MultiFabRegister::alloc_init failed for " + name); + throw std::runtime_error("MultiFabRegister::alloc_init failed for " + internal_name); } - // a short-hand alias for the code below + // a shorthand alias for the code below amrex::MultiFab & mf = it->second.m_mf; // initialize with value @@ -136,11 +142,11 @@ namespace ablastr::fields template<> amrex::MultiFab* - MultiFabRegister::alias_init ( - std::string new_name, - std::string alias_name, + MultiFabRegister::alias_init ( + std::string const & new_name, + std::string const & alias_name, int level, - std::optional initial_value + std::optional initial_value ) { // checks @@ -160,30 +166,30 @@ namespace ablastr::fields } // fully qualified name - new_name = mf_name(new_name, level); - alias_name = mf_name(alias_name, level); + std::string const internal_new_name = mf_name(new_name, level); + std::string const internal_alias_name = mf_name(alias_name, level); - MultiFabOwner & alias = m_mf_register[alias_name]; + MultiFabOwner & alias = m_mf_register[internal_alias_name]; amrex::MultiFab & mf_alias = alias.m_mf; // allocate auto [it, success] = m_mf_register.emplace( - new_name, + internal_new_name, MultiFabOwner{ {mf_alias, amrex::make_alias, 0, mf_alias.nComp()}, std::nullopt, // scalar: no direction level, alias.m_remake, alias.m_redistribute_on_remake, - alias_name + internal_alias_name } ); if (!success) { - throw std::runtime_error("MultiFabRegister::alias_init failed for " + new_name); + throw std::runtime_error("MultiFabRegister::alias_init failed for " + internal_new_name); } - // a short-hand alias for the code below + // a shorthand alias for the code below amrex::MultiFab & mf = it->second.m_mf; // initialize with value @@ -196,12 +202,12 @@ namespace ablastr::fields template<> amrex::MultiFab* - MultiFabRegister::alias_init ( - std::string new_name, - std::string alias_name, + MultiFabRegister::alias_init ( + std::string const & new_name, + std::string const & alias_name, Direction dir, int level, - std::optional initial_value + std::optional initial_value ) { // checks @@ -221,26 +227,26 @@ namespace ablastr::fields } // fully qualified name - new_name = mf_name(new_name, dir, level); - alias_name = mf_name(alias_name, dir, level); + std::string const internal_new_name = mf_name(new_name, dir, level); + std::string const internal_alias_name = mf_name(alias_name, dir, level); - MultiFabOwner & alias = m_mf_register[alias_name]; + MultiFabOwner & alias = m_mf_register[internal_alias_name]; amrex::MultiFab & mf_alias = alias.m_mf; // allocate auto [it, success] = m_mf_register.emplace( - new_name, + internal_new_name, MultiFabOwner{ {mf_alias, amrex::make_alias, 0, mf_alias.nComp()}, dir, level, alias.m_remake, alias.m_redistribute_on_remake, - alias_name + internal_alias_name } ); if (!success) { - throw std::runtime_error("MultiFabRegister::alias_init failed for " + new_name); + throw std::runtime_error("MultiFabRegister::alias_init failed for " + internal_new_name); } // a short-hand alias for the code below @@ -312,40 +318,39 @@ namespace ablastr::fields template<> bool - MultiFabRegister::has ( - std::string name, + MultiFabRegister::has ( + std::string const & name, int level ) const { - name = mf_name(name, level); + std::string const internal_name = mf_name(name, level); - return m_mf_register.count(name) > 0; + return m_mf_register.count(internal_name) > 0; } template<> bool - MultiFabRegister::has ( - std::string name, + MultiFabRegister::has ( + std::string const & name, Direction dir, int level ) const { - name = mf_name(name, dir, level); + std::string const internal_name = mf_name(name, dir, level); - return m_mf_register.count(name) > 0; + return m_mf_register.count(internal_name) > 0; } template<> bool - MultiFabRegister::has_vector ( - std::string name, + MultiFabRegister::has_vector ( + std::string const & name, int level ) const { - const std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; - unsigned long count = 0; - for (const Direction& dir : all_dirs) { + for (Direction const & dir : m_all_dirs) + { std::string const internal_name = mf_name(name, dir, level); count += m_mf_register.count(internal_name); } @@ -355,7 +360,7 @@ namespace ablastr::fields amrex::MultiFab* MultiFabRegister::internal_get ( - const std::string& key + std::string const & key ) { if (m_mf_register.count(key) == 0) { @@ -370,7 +375,7 @@ namespace ablastr::fields amrex::MultiFab const * MultiFabRegister::internal_get ( - const std::string& key + std::string const & key ) const { if (m_mf_register.count(key) == 0) { @@ -385,54 +390,54 @@ namespace ablastr::fields template<> amrex::MultiFab* - MultiFabRegister::get ( - std::string name, + MultiFabRegister::get ( + std::string const & name, int level ) { - name = mf_name(name, level); - return internal_get(name); + std::string const internal_name = mf_name(name, level); + return internal_get(internal_name); } template<> amrex::MultiFab* - MultiFabRegister::get ( - std::string name, + MultiFabRegister::get ( + std::string const & name, Direction dir, int level ) { - name = mf_name(name, dir, level); - return internal_get(name); + std::string const internal_name = mf_name(name, dir, level); + return internal_get(internal_name); } template<> amrex::MultiFab const * - MultiFabRegister::get ( - std::string name, + MultiFabRegister::get ( + std::string const & name, int level ) const { - name = mf_name(name, level); - return internal_get(name); + std::string const internal_name = mf_name(name, level); + return internal_get(internal_name); } template<> amrex::MultiFab const * - MultiFabRegister::get ( - std::string name, + MultiFabRegister::get ( + std::string const & name, Direction dir, int level ) const { - name = mf_name(name, dir, level); - return internal_get(name); + std::string const internal_name = mf_name(name, dir, level); + return internal_get(internal_name); } template<> MultiLevelScalarField - MultiFabRegister::get_mr_levels ( - std::string name, + MultiFabRegister::get_mr_levels ( + std::string const & name, int finest_level ) { @@ -447,8 +452,8 @@ namespace ablastr::fields template<> ConstMultiLevelScalarField - MultiFabRegister::get_mr_levels ( - std::string name, + MultiFabRegister::get_mr_levels ( + std::string const & name, int finest_level ) const { @@ -463,19 +468,16 @@ namespace ablastr::fields template<> VectorField - MultiFabRegister::get_alldirs ( - std::string name, + MultiFabRegister::get_alldirs ( + std::string const & name, int level ) { - // TODO: Technically, we should search field_on_level via std::unique_copy - const std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; - // insert a new level VectorField vectorField; // insert components - for (const Direction& dir : all_dirs) + for (Direction const & dir : m_all_dirs) { vectorField[dir] = get(name, dir, level); } @@ -484,19 +486,16 @@ namespace ablastr::fields template<> ConstVectorField - MultiFabRegister::get_alldirs ( - std::string name, + MultiFabRegister::get_alldirs ( + std::string const & name, int level ) const { - // TODO: Technically, we should search field_on_level via std::unique_copy - const std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; - // insert a new level ConstVectorField vectorField; // insert components - for (const Direction& dir: all_dirs) + for (Direction const & dir : m_all_dirs) { vectorField[dir] = get(name, dir, level); } @@ -505,24 +504,21 @@ namespace ablastr::fields template<> MultiLevelVectorField - MultiFabRegister::get_mr_levels_alldirs ( - std::string name, + MultiFabRegister::get_mr_levels_alldirs ( + std::string const & name, int finest_level ) { MultiLevelVectorField field_on_level; field_on_level.reserve(finest_level+1); - // TODO: Technically, we should search field_on_level via std::unique_copy - const std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; - for (int lvl = 0; lvl <= finest_level; lvl++) { // insert a new level field_on_level.push_back(VectorField{}); // insert components - for (const Direction& dir : all_dirs) + for (Direction const & dir : m_all_dirs) { field_on_level[lvl][dir] = get(name, dir, lvl); } @@ -532,24 +528,21 @@ namespace ablastr::fields template<> ConstMultiLevelVectorField - MultiFabRegister::get_mr_levels_alldirs ( - std::string name, + MultiFabRegister::get_mr_levels_alldirs ( + std::string const & name, int finest_level ) const { ConstMultiLevelVectorField field_on_level; field_on_level.reserve(finest_level+1); - // TODO: Technically, we should search field_on_level via std::unique_copy - const std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; - for (int lvl = 0; lvl <= finest_level; lvl++) { // insert a new level field_on_level.push_back(ConstVectorField{}); // insert components - for (const Direction& dir : all_dirs) + for (Direction const & dir : m_all_dirs) { field_on_level[lvl][dir] = get(name, dir, lvl); } @@ -569,33 +562,33 @@ namespace ablastr::fields template<> void - MultiFabRegister::erase ( - std::string name, + MultiFabRegister::erase ( + std::string const & name, int level ) { - name = mf_name(name, level); + std::string const internal_name = mf_name(name, level); - if (m_mf_register.count(name) != 1) { - throw std::runtime_error("MultiFabRegister::erase name does not exist in register: " + name); + if (m_mf_register.count(internal_name) != 1) { + throw std::runtime_error("MultiFabRegister::erase name does not exist in register: " + internal_name); } - m_mf_register.erase(name); + m_mf_register.erase(internal_name); } template<> void - MultiFabRegister::erase ( - std::string name, + MultiFabRegister::erase ( + std::string const & name, Direction dir, int level ) { - name = mf_name(name, dir, level); + std::string const internal_name = mf_name(name, dir, level); - if (m_mf_register.count(name) != 1) { - throw std::runtime_error("MultiFabRegister::erase name does not exist in register: " + name); + if (m_mf_register.count(internal_name) != 1) { + throw std::runtime_error("MultiFabRegister::erase name does not exist in register: " + internal_name); } - m_mf_register.erase(name); + m_mf_register.erase(internal_name); } void @@ -645,15 +638,15 @@ namespace ablastr::fields VectorField a2m ( - const std::array< std::unique_ptr, 3 > & old_vectorfield + std::array< std::unique_ptr, 3 > const & old_vectorfield ) { - const std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; + std::vector const all_dirs = {Direction{0}, Direction{1}, Direction{2}}; VectorField field_on_level; // insert components - for (const auto dir : {0, 1, 2}) + for (auto const dir : {0, 1, 2}) { field_on_level[Direction{dir}] = old_vectorfield[dir].get(); } diff --git a/Source/ablastr/fields/MultiFabRegister_string.H b/Source/ablastr/fields/MultiFabRegister_string.H index f4139c7a540..8ef8ccb2b2f 100644 --- a/Source/ablastr/fields/MultiFabRegister_string.H +++ b/Source/ablastr/fields/MultiFabRegister_string.H @@ -1,6 +1,6 @@ -/* Copyright 2024 The ABLAST Community +/* Copyright 2024 The ABLASTR Community * - * This file is part of WarpX. + * This file is part of ABLASTR. * It adds unique string key access to the MultiFabRegister. * * License: BSD-3-Clause-LBNL @@ -11,162 +11,171 @@ #include "MultiFabRegister.H" +#include +#include +#include +#include +#include + +#include +#include + namespace ablastr::fields { template<> amrex::MultiFab* - MultiFabRegister::alloc_init ( - std::string name, + MultiFabRegister::alloc_init ( + std::string const & name, int level, amrex::BoxArray const & ba, amrex::DistributionMapping const & dm, int ncomp, amrex::IntVect const & ngrow, - std::optional initial_value, + std::optional initial_value, bool remake, bool redistribute_on_remake ); template<> amrex::MultiFab* - MultiFabRegister::alloc_init ( - std::string name, + MultiFabRegister::alloc_init ( + std::string const & name, Direction dir, int level, amrex::BoxArray const & ba, amrex::DistributionMapping const & dm, int ncomp, amrex::IntVect const & ngrow, - std::optional initial_value, + std::optional initial_value, bool remake, bool redistribute_on_remake ); template<> amrex::MultiFab* - MultiFabRegister::alias_init ( - std::string new_name, - std::string alias_name, + MultiFabRegister::alias_init ( + std::string const & new_name, + std::string const & alias_name, int level, - std::optional initial_value + std::optional initial_value ); template<> amrex::MultiFab* - MultiFabRegister::alias_init ( - std::string new_name, - std::string alias_name, + MultiFabRegister::alias_init ( + std::string const & new_name, + std::string const & alias_name, Direction dir, int level, - std::optional initial_value + std::optional initial_value ); template<> bool - MultiFabRegister::has ( - std::string name, + MultiFabRegister::has ( + std::string const & name, int level ) const; template<> bool - MultiFabRegister::has ( - std::string name, + MultiFabRegister::has ( + std::string const & name, Direction dir, int level ) const; template<> bool - MultiFabRegister::has_vector ( - std::string name, + MultiFabRegister::has_vector ( + std::string const & name, int level ) const; template<> amrex::MultiFab* - MultiFabRegister::get ( - std::string name, + MultiFabRegister::get ( + std::string const & name, int level ); template<> amrex::MultiFab* - MultiFabRegister::get ( - std::string name, + MultiFabRegister::get ( + std::string const & name, Direction dir, int level ); template<> amrex::MultiFab const* - MultiFabRegister::get ( - std::string name, + MultiFabRegister::get ( + std::string const & name, int level ) const; template<> amrex::MultiFab const * - MultiFabRegister::get ( - std::string name, + MultiFabRegister::get ( + std::string const & name, Direction dir, int level ) const; template<> MultiLevelScalarField - MultiFabRegister::get_mr_levels ( - std::string name, + MultiFabRegister::get_mr_levels ( + std::string const & name, int finest_level ); template<> ConstMultiLevelScalarField - MultiFabRegister::get_mr_levels ( - std::string name, + MultiFabRegister::get_mr_levels ( + std::string const & name, int finest_level ) const; template<> VectorField - MultiFabRegister::get_alldirs ( - std::string name, + MultiFabRegister::get_alldirs ( + std::string const & name, int level ); template<> ConstVectorField - MultiFabRegister::get_alldirs ( - std::string name, + MultiFabRegister::get_alldirs ( + std::string const & name, int level ) const; template<> MultiLevelVectorField - MultiFabRegister::get_mr_levels_alldirs ( - std::string name, + MultiFabRegister::get_mr_levels_alldirs ( + std::string const & name, int finest_level ); template<> ConstMultiLevelVectorField - MultiFabRegister::get_mr_levels_alldirs ( - std::string name, + MultiFabRegister::get_mr_levels_alldirs ( + std::string const & name, int finest_level ) const; template<> void - MultiFabRegister::erase ( - std::string name, + MultiFabRegister::erase ( + std::string const & name, int level ); template<> void - MultiFabRegister::erase ( - std::string name, + MultiFabRegister::erase ( + std::string const & name, Direction dir, int level ); From 339e909e1a2270dac1ec94eca7e7cba6157da55c Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sun, 22 Sep 2024 11:00:41 -0700 Subject: [PATCH 302/314] Clang-Tidy --- Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp | 2 +- .../MacroscopicProperties/MacroscopicProperties.cpp | 1 - Source/FieldSolver/WarpXPushFieldsEM.cpp | 8 ++++---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp b/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp index d3592c6c6f2..050b18d3b9d 100644 --- a/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp +++ b/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp @@ -29,7 +29,6 @@ #include using namespace amrex; -using warpx::fields::FieldType; // constructor @@ -106,6 +105,7 @@ void ChargeOnEB::ComputeDiags (const int step) int const lev = 0; // get MultiFab data at lev + using warpx::fields::FieldType; const amrex::MultiFab & Ex = *warpx.m_fields.get(FieldType::Efield_fp, Direction{0}, lev); const amrex::MultiFab & Ey = *warpx.m_fields.get(FieldType::Efield_fp, Direction{1}, lev); const amrex::MultiFab & Ez = *warpx.m_fields.get(FieldType::Efield_fp, Direction{2}, lev); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp index 583e0a43903..18c010d9385 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp @@ -23,7 +23,6 @@ #include using namespace amrex; -using warpx::fields::FieldType; MacroscopicProperties::MacroscopicProperties () { diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index db2dcea65ac..fd786dc65ba 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -330,7 +330,7 @@ void WarpX::PSATDForwardTransformJ ( idx_jz = (J_in_time == JInTime::Linear) ? static_cast(Idx.Jz_new) : static_cast(Idx.Jz_mid); if (m_fields.has_vector(J_fp_string, lev)) { - ablastr::fields::VectorField J_fp = m_fields.get_alldirs(J_fp_string, lev); + ablastr::fields::VectorField const J_fp = m_fields.get_alldirs(J_fp_string, lev); ForwardTransformVect(lev, *spectral_solver_fp[lev], J_fp, idx_jx, idx_jy, idx_jz); } @@ -343,7 +343,7 @@ void WarpX::PSATDForwardTransformJ ( idx_jz = (J_in_time == JInTime::Linear) ? static_cast(Idx.Jz_new) : static_cast(Idx.Jz_mid); if (m_fields.has_vector(J_cp_string, lev)) { - ablastr::fields::VectorField J_cp = m_fields.get_alldirs(J_cp_string, lev); + ablastr::fields::VectorField const J_cp = m_fields.get_alldirs(J_cp_string, lev); ForwardTransformVect(lev, *spectral_solver_cp[lev], J_cp, idx_jx, idx_jy, idx_jz); } } @@ -400,7 +400,7 @@ void WarpX::PSATDBackwardTransformJ ( idx_jz = static_cast(Idx.Jz_mid); if (m_fields.has_vector(J_fp_string, lev)) { - ablastr::fields::VectorField J_fp = m_fields.get_alldirs(J_fp_string, lev); + ablastr::fields::VectorField const J_fp = m_fields.get_alldirs(J_fp_string, lev); BackwardTransformVect(lev, *spectral_solver_fp[lev], J_fp, idx_jx, idx_jy, idx_jz, m_fill_guards_current); } @@ -416,7 +416,7 @@ void WarpX::PSATDBackwardTransformJ ( idx_jz = static_cast(Idx.Jz_mid); if (m_fields.has_vector(J_cp_string, lev)) { - ablastr::fields::VectorField J_cp = m_fields.get_alldirs(J_cp_string, lev); + ablastr::fields::VectorField const J_cp = m_fields.get_alldirs(J_cp_string, lev); BackwardTransformVect(lev, *spectral_solver_cp[lev], J_cp, idx_jx, idx_jy, idx_jz, m_fill_guards_current); } From c6439b9b4724a29fec360e4153e3477f45e3401b Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sun, 22 Sep 2024 23:28:18 -0700 Subject: [PATCH 303/314] Clang-Tidy RZ --- Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp index c0532163232..dfd64fe5af9 100644 --- a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp @@ -59,7 +59,7 @@ #include using namespace amrex; -using warpx::fields::FieldType; + ColliderRelevant::ColliderRelevant (const std::string& rd_name) : ReducedDiags{rd_name} @@ -443,6 +443,7 @@ void ColliderRelevant::ComputeDiags (int step) const int lev = 0; // define variables in preparation for field gathering + using warpx::fields::FieldType; const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(lev, 0)); const amrex::MultiFab & Ex = *warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev); const amrex::MultiFab & Ey = *warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, lev); From f07e1d4ed212e82d7394ec8cd1db969f07661b0a Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 23 Sep 2024 10:37:01 -0700 Subject: [PATCH 304/314] MSVC: Disable for now --- .github/workflows/windows.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index c850970bca6..fc75ccb0141 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -10,7 +10,9 @@ jobs: build_win_msvc: name: MSVC C++17 w/o MPI runs-on: windows-latest - if: github.event.pull_request.draft == false + # disabled due to issues in #5230 + if: 0 + #if: github.event.pull_request.draft == false steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -54,14 +56,7 @@ jobs: run: | $env:PATH += ';C:/Program Files (x86)/WarpX/bin/' - $job = Start-ThreadJob -Name "test" -ScriptBlock { & "python3" "Examples/Tests/gaussian_beam/inputs_test_3d_gaussian_beam_picmi.py" } - Start-Sleep -Seconds 120 - Stop-Job $job.Id - - $jobOutput = $job | Receive-Job - $jobOutput - - cat Backtrace.0 + python3 Examples/Tests/gaussian_beam/inputs_test_3d_gaussian_beam_picmi.py # JSON writes are currently very slow (50min) with MSVC # --diagformat=openpmd From de0f8c0abe044818733a37be63be72acb160a97c Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Mon, 23 Sep 2024 14:22:28 -0700 Subject: [PATCH 305/314] Apply suggestions from code review Co-authored-by: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> --- Source/WarpX.cpp | 4 ++-- Source/ablastr/fields/MultiFabRegister.H | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 30585114ff7..60374133a52 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2626,13 +2626,13 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (grid_type == GridType::Collocated) { m_fields.alloc_init(FieldType::G_cp, - lev, amrex::convert(ba, IntVect::TheUnitVector()), dm, + lev, amrex::convert(cba, IntVect::TheUnitVector()), dm, ncomps, ngG, 0.0_rt); } else // grid_type=staggered or grid_type=hybrid { m_fields.alloc_init(FieldType::G_cp, - lev, amrex::convert(ba, IntVect::TheZeroVector()), dm, + lev, amrex::convert(cba, IntVect::TheZeroVector()), dm, ncomps, ngG, 0.0_rt); } } diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 74d82758ec6..d5d64201a00 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -124,7 +124,7 @@ namespace ablastr::fields /** owned (i)MultiFab */ amrex::MultiFab m_mf; - /** Components (base vector directions) of this (i)MultiFab */ + /** Components (base vector directions) of this MultiFab */ std::optional m_dir = std::nullopt; /** the MR level of this (i)MultiFab */ @@ -153,7 +153,7 @@ namespace ablastr::fields is_alias () const { return !m_owner.empty(); } }; - /** This is a register of fields aka amrex::(i)MultiFabs. + /** This is a register of fields aka amrex::MultiFabs. * * This is owned by a simulation instance. All used fields should be registered here. * Internally, this contains @see MultiFabOwner values. From 9de7fbc16325b13e7d5843ec8194d2718b8ff418 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Mon, 23 Sep 2024 14:25:19 -0700 Subject: [PATCH 306/314] Remove unused function alloc_like --- Source/Python/MultiFabRegister.cpp | 7 ------- Source/ablastr/fields/MultiFabRegister.H | 16 ---------------- Source/ablastr/fields/MultiFabRegister.cpp | 15 --------------- 3 files changed, 38 deletions(-) diff --git a/Source/Python/MultiFabRegister.cpp b/Source/Python/MultiFabRegister.cpp index 6ae86ebd61b..fcf38a1a6db 100644 --- a/Source/Python/MultiFabRegister.cpp +++ b/Source/Python/MultiFabRegister.cpp @@ -96,13 +96,6 @@ void init_MultiFabRegister (py::module & m) py::arg("initial_value") ) - .def("alloc_like", - &MultiFabRegister::alloc_like, - py::arg("other_name"), - py::arg("other_level"), - py::arg("level") - ) - .def("has", py::overload_cast< std::string, diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index d5d64201a00..7d902471a8d 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -316,22 +316,6 @@ namespace ablastr::fields ); } - /** Allocate a new MultiFab (field) with the same size and distribution as another. - * - * @todo this is reserved for future use and not yet implemented. - * @todo we will also add an overload with dir. - * - * @param new_name new name - * @param other_name the other multifab - * @param level the MR level to represent - */ - void - alloc_like ( - std::string const & new_name, - std::string const & other_name, - int level - ); - /** Check if a scalar MultiFab (field) is registered. * * @param name the name to check if registered diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 2f3b4fb3a97..74b6fbe1d2c 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -125,21 +125,6 @@ namespace ablastr::fields return &mf; } - void - MultiFabRegister::alloc_like ( - std::string const & /* new_name */, - std::string const & /* other_name */, - int /* level */ - ) - { - // other_name = mf_name(other_name, other_level); - - throw std::runtime_error("MultiFabRegister::alloc_like not yet implemented"); - - // Checks - // TODO: does the other_name already exist? error - } - template<> amrex::MultiFab* MultiFabRegister::alias_init ( From bff1720e82676cad2d6bb87e41070d2d4eff483d Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Mon, 23 Sep 2024 14:46:15 -0700 Subject: [PATCH 307/314] Simplify templated functions alloc_init, alias_init, has --- Source/ablastr/fields/MultiFabRegister.H | 75 ++++++- Source/ablastr/fields/MultiFabRegister.cpp | 21 +- .../ablastr/fields/MultiFabRegister_string.H | 185 ------------------ 3 files changed, 73 insertions(+), 208 deletions(-) delete mode 100644 Source/ablastr/fields/MultiFabRegister_string.H diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 7d902471a8d..38b4b905d3d 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -198,7 +198,7 @@ namespace ablastr::fields bool redistribute_on_remake = true ) { - return alloc_init( + return internal_alloc_init( getExtractedName(name), level, ba, @@ -243,7 +243,7 @@ namespace ablastr::fields bool redistribute_on_remake = true ) { - return alloc_init( + return internal_alloc_init( getExtractedName(name), dir, level, @@ -277,7 +277,7 @@ namespace ablastr::fields std::optional initial_value = std::nullopt ) { - return alias_init( + return internal_alias_init( getExtractedName(new_name), getExtractedName(alias_name), level, @@ -307,7 +307,7 @@ namespace ablastr::fields std::optional initial_value = std::nullopt ) { - return alias_init( + return internal_alias_init( getExtractedName(new_name), getExtractedName(alias_name), dir, @@ -329,7 +329,7 @@ namespace ablastr::fields int level ) const { - return has( + return internal_has( getExtractedName(name), level ); @@ -350,7 +350,7 @@ namespace ablastr::fields int level ) const { - return has( + return internal_has( getExtractedName(name), dir, level @@ -370,7 +370,7 @@ namespace ablastr::fields int level ) const { - return has_vector( + return internal_has_vector( getExtractedName(name), level ); @@ -671,6 +671,65 @@ namespace ablastr::fields std::string const & key ) const; + amrex::MultiFab* + internal_alloc_init ( + std::string const & name, + int level, + amrex::BoxArray const & ba, + amrex::DistributionMapping const & dm, + int ncomp, + amrex::IntVect const & ngrow, + std::optional initial_value = std::nullopt, + bool remake = true, + bool redistribute_on_remake = true + ); + amrex::MultiFab* + internal_alloc_init ( + std::string const & name, + Direction dir, + int level, + amrex::BoxArray const & ba, + amrex::DistributionMapping const & dm, + int ncomp, + amrex::IntVect const & ngrow, + std::optional initial_value = std::nullopt, + bool remake = true, + bool redistribute_on_remake = true + ); + + amrex::MultiFab* + internal_alias_init ( + std::string const & new_name, + std::string const & alias_name, + int level, + std::optional initial_value = std::nullopt + ); + amrex::MultiFab* + internal_alias_init ( + std::string const & new_name, + std::string const & alias_name, + Direction dir, + int level, + std::optional initial_value = std::nullopt + ); + + [[nodiscard]] bool + internal_has ( + std::string const & name, + int level + ) const; + [[nodiscard]] bool + internal_has ( + std::string const & name, + Direction dir, + int level + ) const; + [[nodiscard]] bool + internal_has_vector ( + std::string const & name, + int level + ) const; + /** data storage: ownership and lifetime control */ std::map< std::string, @@ -692,6 +751,4 @@ namespace ablastr::fields } // namespace ablastr::fields -#include "MultiFabRegister_string.H" - #endif // ABLASTR_FIELDS_MF_REGISTER_H diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 74b6fbe1d2c..949044a8235 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -20,9 +20,8 @@ namespace ablastr::fields { - template<> amrex::MultiFab* - MultiFabRegister::alloc_init ( + MultiFabRegister::internal_alloc_init ( std::string const & name, int level, amrex::BoxArray const & ba, @@ -70,9 +69,8 @@ namespace ablastr::fields return &mf; } - template<> amrex::MultiFab* - MultiFabRegister::alloc_init ( + MultiFabRegister::internal_alloc_init ( std::string const & name, Direction dir, int level, @@ -125,9 +123,8 @@ namespace ablastr::fields return &mf; } - template<> amrex::MultiFab* - MultiFabRegister::alias_init ( + MultiFabRegister::internal_alias_init ( std::string const & new_name, std::string const & alias_name, int level, @@ -185,9 +182,8 @@ namespace ablastr::fields return &mf; } - template<> amrex::MultiFab* - MultiFabRegister::alias_init ( + MultiFabRegister::internal_alias_init ( std::string const & new_name, std::string const & alias_name, Direction dir, @@ -301,9 +297,8 @@ namespace ablastr::fields } } - template<> bool - MultiFabRegister::has ( + MultiFabRegister::internal_has ( std::string const & name, int level ) const @@ -313,9 +308,8 @@ namespace ablastr::fields return m_mf_register.count(internal_name) > 0; } - template<> bool - MultiFabRegister::has ( + MultiFabRegister::internal_has ( std::string const & name, Direction dir, int level @@ -326,9 +320,8 @@ namespace ablastr::fields return m_mf_register.count(internal_name) > 0; } - template<> bool - MultiFabRegister::has_vector ( + MultiFabRegister::internal_has_vector ( std::string const & name, int level ) const diff --git a/Source/ablastr/fields/MultiFabRegister_string.H b/Source/ablastr/fields/MultiFabRegister_string.H deleted file mode 100644 index 8ef8ccb2b2f..00000000000 --- a/Source/ablastr/fields/MultiFabRegister_string.H +++ /dev/null @@ -1,185 +0,0 @@ -/* Copyright 2024 The ABLASTR Community - * - * This file is part of ABLASTR. - * It adds unique string key access to the MultiFabRegister. - * - * License: BSD-3-Clause-LBNL - * Authors: Axel Huebl - */ -#ifndef ABLASTR_FIELDS_MF_REGISTER_STRING_H -#define ABLASTR_FIELDS_MF_REGISTER_STRING_H - -#include "MultiFabRegister.H" - -#include -#include -#include -#include -#include - -#include -#include - - -namespace ablastr::fields -{ - template<> - amrex::MultiFab* - MultiFabRegister::alloc_init ( - std::string const & name, - int level, - amrex::BoxArray const & ba, - amrex::DistributionMapping const & dm, - int ncomp, - amrex::IntVect const & ngrow, - std::optional initial_value, - bool remake, - bool redistribute_on_remake - ); - - template<> - amrex::MultiFab* - MultiFabRegister::alloc_init ( - std::string const & name, - Direction dir, - int level, - amrex::BoxArray const & ba, - amrex::DistributionMapping const & dm, - int ncomp, - amrex::IntVect const & ngrow, - std::optional initial_value, - bool remake, - bool redistribute_on_remake - ); - - template<> - amrex::MultiFab* - MultiFabRegister::alias_init ( - std::string const & new_name, - std::string const & alias_name, - int level, - std::optional initial_value - ); - - template<> - amrex::MultiFab* - MultiFabRegister::alias_init ( - std::string const & new_name, - std::string const & alias_name, - Direction dir, - int level, - std::optional initial_value - ); - - template<> - bool - MultiFabRegister::has ( - std::string const & name, - int level - ) const; - - template<> - bool - MultiFabRegister::has ( - std::string const & name, - Direction dir, - int level - ) const; - - template<> - bool - MultiFabRegister::has_vector ( - std::string const & name, - int level - ) const; - - template<> - amrex::MultiFab* - MultiFabRegister::get ( - std::string const & name, - int level - ); - - template<> - amrex::MultiFab* - MultiFabRegister::get ( - std::string const & name, - Direction dir, - int level - ); - - template<> - amrex::MultiFab const* - MultiFabRegister::get ( - std::string const & name, - int level - ) const; - - template<> - amrex::MultiFab const * - MultiFabRegister::get ( - std::string const & name, - Direction dir, - int level - ) const; - - template<> - MultiLevelScalarField - MultiFabRegister::get_mr_levels ( - std::string const & name, - int finest_level - ); - - template<> - ConstMultiLevelScalarField - MultiFabRegister::get_mr_levels ( - std::string const & name, - int finest_level - ) const; - - template<> - VectorField - MultiFabRegister::get_alldirs ( - std::string const & name, - int level - ); - - template<> - ConstVectorField - MultiFabRegister::get_alldirs ( - std::string const & name, - int level - ) const; - - template<> - MultiLevelVectorField - MultiFabRegister::get_mr_levels_alldirs ( - std::string const & name, - int finest_level - ); - - template<> - ConstMultiLevelVectorField - MultiFabRegister::get_mr_levels_alldirs ( - std::string const & name, - int finest_level - ) const; - - template<> - void - MultiFabRegister::erase ( - std::string const & name, - int level - ); - - template<> - void - MultiFabRegister::erase ( - std::string const & name, - Direction dir, - int level - ); - -} // namespace ablastr::fields - -#endif // ABLASTR_FIELDS_MF_REGISTER_STRING_H From a4d3cd83e500eea97366ea5efa3543fd5706d5f2 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Mon, 23 Sep 2024 15:22:07 -0700 Subject: [PATCH 308/314] Fix compilation issues --- Source/ablastr/fields/MultiFabRegister.H | 53 ++++++++++++++++++++++ Source/ablastr/fields/MultiFabRegister.cpp | 42 +++++++---------- 2 files changed, 69 insertions(+), 26 deletions(-) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 38b4b905d3d..258002faebe 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -730,6 +730,59 @@ namespace ablastr::fields int level ) const; + amrex::MultiFab * + internal_get ( + std::string const & name, + int level + ); + amrex::MultiFab const * + internal_get ( + std::string const & name, + int level + ) const; + amrex::MultiFab * + internal_get ( + std::string const & name, + Direction dir, + int level + ); + amrex::MultiFab const * + internal_get ( + std::string const & name, + Direction dir, + int level + ) const; + MultiLevelScalarField + internal_get_mr_levels ( + std::string const & name, + int finest_level + ); + ConstMultiLevelScalarField + internal_get_mr_levels ( + std::string const & name, + int finest_level + ) const; + VectorField + internal_get_alldirs ( + std::string const & name, + int level + ); + ConstVectorField + internal_get_alldirs ( + std::string const & name, + int level + ) const; + MultiLevelVectorField + internal_get_mr_levels_alldirs ( + std::string const & name, + int finest_level + ); + ConstMultiLevelVectorField + internal_get_mr_levels_alldirs ( + std::string const & name, + int finest_level + ) const; + /** data storage: ownership and lifetime control */ std::map< std::string, diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 949044a8235..94f398542bd 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -366,9 +366,8 @@ namespace ablastr::fields return &mf; } - template<> amrex::MultiFab* - MultiFabRegister::get ( + MultiFabRegister::internal_get ( std::string const & name, int level ) @@ -377,9 +376,8 @@ namespace ablastr::fields return internal_get(internal_name); } - template<> amrex::MultiFab* - MultiFabRegister::get ( + MultiFabRegister::internal_get ( std::string const & name, Direction dir, int level @@ -389,9 +387,8 @@ namespace ablastr::fields return internal_get(internal_name); } - template<> amrex::MultiFab const * - MultiFabRegister::get ( + MultiFabRegister::internal_get ( std::string const & name, int level ) const @@ -400,9 +397,8 @@ namespace ablastr::fields return internal_get(internal_name); } - template<> amrex::MultiFab const * - MultiFabRegister::get ( + MultiFabRegister::internal_get ( std::string const & name, Direction dir, int level @@ -412,9 +408,8 @@ namespace ablastr::fields return internal_get(internal_name); } - template<> MultiLevelScalarField - MultiFabRegister::get_mr_levels ( + MultiFabRegister::internal_get_mr_levels ( std::string const & name, int finest_level ) @@ -423,14 +418,13 @@ namespace ablastr::fields field_on_level.reserve(finest_level+1); for (int lvl = 0; lvl <= finest_level; lvl++) { - field_on_level.push_back(get(name, lvl)); + field_on_level.push_back(internal_get(name, lvl)); } return field_on_level; } - template<> ConstMultiLevelScalarField - MultiFabRegister::get_mr_levels ( + MultiFabRegister::internal_get_mr_levels ( std::string const & name, int finest_level ) const @@ -439,14 +433,13 @@ namespace ablastr::fields field_on_level.reserve(finest_level+1); for (int lvl = 0; lvl <= finest_level; lvl++) { - field_on_level.push_back(get(name, lvl)); + field_on_level.push_back(internal_get(name, lvl)); } return field_on_level; } - template<> VectorField - MultiFabRegister::get_alldirs ( + MultiFabRegister::internal_get_alldirs ( std::string const & name, int level ) @@ -457,14 +450,13 @@ namespace ablastr::fields // insert components for (Direction const & dir : m_all_dirs) { - vectorField[dir] = get(name, dir, level); + vectorField[dir] = internal_get(name, dir, level); } return vectorField; } - template<> ConstVectorField - MultiFabRegister::get_alldirs ( + MultiFabRegister::internal_get_alldirs ( std::string const & name, int level ) const @@ -475,14 +467,13 @@ namespace ablastr::fields // insert components for (Direction const & dir : m_all_dirs) { - vectorField[dir] = get(name, dir, level); + vectorField[dir] = internal_get(name, dir, level); } return vectorField; } - template<> MultiLevelVectorField - MultiFabRegister::get_mr_levels_alldirs ( + MultiFabRegister::internal_get_mr_levels_alldirs ( std::string const & name, int finest_level ) @@ -498,15 +489,14 @@ namespace ablastr::fields // insert components for (Direction const & dir : m_all_dirs) { - field_on_level[lvl][dir] = get(name, dir, lvl); + field_on_level[lvl][dir] = internal_get(name, dir, lvl); } } return field_on_level; } - template<> ConstMultiLevelVectorField - MultiFabRegister::get_mr_levels_alldirs ( + MultiFabRegister::internal_get_mr_levels_alldirs ( std::string const & name, int finest_level ) const @@ -522,7 +512,7 @@ namespace ablastr::fields // insert components for (Direction const & dir : m_all_dirs) { - field_on_level[lvl][dir] = get(name, dir, lvl); + field_on_level[lvl][dir] = internal_get(name, dir, lvl); } } return field_on_level; From 68abffc25810842b3d94ac7270ad651df9e1f942 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Mon, 23 Sep 2024 15:26:25 -0700 Subject: [PATCH 309/314] Additional compilation fixes --- Source/ablastr/fields/MultiFabRegister.H | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 258002faebe..76d0d35335e 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -391,7 +391,7 @@ namespace ablastr::fields int level ) { - return get( + return internal_get( getExtractedName(name), level ); @@ -414,7 +414,7 @@ namespace ablastr::fields int level ) { - return get( + return internal_get( getExtractedName(name), dir, level @@ -436,7 +436,7 @@ namespace ablastr::fields int level ) const { - return get( + return internal_get( getExtractedName(name), level ); @@ -459,7 +459,7 @@ namespace ablastr::fields int level ) const { - return get( + return internal_get( getExtractedName(name), dir, level @@ -482,7 +482,7 @@ namespace ablastr::fields int finest_level ) { - return get_mr_levels( + return internal_get_mr_levels( getExtractedName(name), finest_level ); @@ -494,7 +494,7 @@ namespace ablastr::fields int finest_level ) const { - return get_mr_levels( + return internal_get_mr_levels( getExtractedName(name), finest_level ); @@ -517,7 +517,7 @@ namespace ablastr::fields int level ) { - return get_alldirs( + return internal_get_alldirs( getExtractedName(name), level ); @@ -529,7 +529,7 @@ namespace ablastr::fields int level ) const { - return get_alldirs( + return internal_get_alldirs( getExtractedName(name), level ); @@ -553,7 +553,7 @@ namespace ablastr::fields int finest_level ) { - return get_mr_levels_alldirs( + return internal_get_mr_levels_alldirs( getExtractedName(name), finest_level ); @@ -565,7 +565,7 @@ namespace ablastr::fields int finest_level ) const { - return get_mr_levels_alldirs( + return internal_get_mr_levels_alldirs( getExtractedName(name), finest_level ); From cc5443f4969bf58dfbd97aea1610e6f014ff0a55 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Mon, 23 Sep 2024 15:39:12 -0700 Subject: [PATCH 310/314] Remove last template --- Source/ablastr/fields/MultiFabRegister.H | 16 ++++++++++++++-- Source/ablastr/fields/MultiFabRegister.cpp | 6 ++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 76d0d35335e..b67784aa94c 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -591,7 +591,7 @@ namespace ablastr::fields int level ) { - erase(getExtractedName(name), level); + internal_erase(getExtractedName(name), level); } /** Deallocate and remove a vector field component. @@ -608,7 +608,7 @@ namespace ablastr::fields int level ) { - erase(getExtractedName(name), dir, level); + internal_erase(getExtractedName(name), dir, level); } /** Erase all MultiFabs on a specific MR level. @@ -783,6 +783,18 @@ namespace ablastr::fields int finest_level ) const; + void + internal_erase ( + std::string const & name, + int level + ); + void + internal_erase ( + std::string const & name, + Direction dir, + int level + ); + /** data storage: ownership and lifetime control */ std::map< std::string, diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 94f398542bd..6002c9f9811 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -528,9 +528,8 @@ namespace ablastr::fields return names; } - template<> void - MultiFabRegister::erase ( + MultiFabRegister::internal_erase ( std::string const & name, int level ) @@ -543,9 +542,8 @@ namespace ablastr::fields m_mf_register.erase(internal_name); } - template<> void - MultiFabRegister::erase ( + MultiFabRegister::internal_erase ( std::string const & name, Direction dir, int level From 4eb99a56c00e99feffb6d2db1f0c37e43f060a20 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 23 Sep 2024 15:55:39 -0700 Subject: [PATCH 311/314] `MultiFabRegister::mf_name` Keep Direction Convention Keep `[x]`, `[y]`, `[z]` suffixes for now to avoid breakage for Python users. --- Source/ablastr/fields/MultiFabRegister.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index 6002c9f9811..106a3aede79 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -592,11 +592,16 @@ namespace ablastr::fields int level ) const { - // Add the suffix "[dir=dir]" + // Add the suffix for the direction [x] or [y] or [z] + // note: since Cartesian is not correct for all our supported geometries, + // in the future we might want to break this to "[dir=0/1/2]". + // This will be a breaking change for (Python) users that rely on that string. + constexpr int x_in_ascii = 120; + std::string const component_name{char(x_in_ascii + dir.dir)}; return mf_name( name - .append("[dir=") - .append(std::to_string(dir.dir)) + .append("[") + .append(component_name) .append("]"), level ); From b795a3dc71b1a69fb80565fe1150603611c5889d Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Mon, 23 Sep 2024 16:47:44 -0700 Subject: [PATCH 312/314] Add documentation for the fields --- Source/Fields.H | 62 ++++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/Source/Fields.H b/Source/Fields.H index 5e82e524fa2..00d1872a049 100644 --- a/Source/Fields.H +++ b/Source/Fields.H @@ -19,23 +19,23 @@ namespace warpx::fields { AMREX_ENUM(FieldType, None, - Efield_aux, - Bfield_aux, - Efield_fp, - Bfield_fp, - Efield_fp_external, - Bfield_fp_external, - current_fp, - current_fp_nodal, - current_fp_vay, - current_buf, - current_store, - rho_buf, - rho_fp, - F_fp, - G_fp, - phi_fp, - vector_potential_fp, + Efield_aux, // Field that the particles gather from. Obtained from Efield_fp (and Efield_cp when using MR); see UpdateAuxilaryData + Bfield_aux, // Field that the particles gather from. Obtained from Bfield_fp (and Bfield_cp when using MR); see UpdateAuxilaryData + Efield_fp, // The field that is updated by the field solver at each timestep + Bfield_fp, // The field that is updated by the field solver at each timestep + Efield_fp_external, // Stores grid particle fields provided by the user as through an openPMD file + Bfield_fp_external, // Stores grid particle fields provided by the user as through an openPMD file + current_fp, // The current that is used as a source for the field solver + current_fp_nodal, // Only used when using nodal current deposition + current_fp_vay, // Only used when using Vay current deposition + current_buf, // Particles that are close to the edge of the MR patch (i.e. in the deposition buffer) deposit to this field. + current_store, // Only used when doing subcycling with mesh refinement, for book-keeping of currents + rho_buf, // Particles that are close to the edge of the MR patch (i.e. in the deposition buffer) deposit to this field. + rho_fp, // The charge density that is used as a source for the field solver (mostly for labframe electrostatic and PSATD) + F_fp, // Used for divE cleaning + G_fp, // Used for divB cleaning + phi_fp, // Obtained by the Poisson solver, for labframe electrostatic + vector_potential_fp, // Obtained by the magnetostatic solver vector_potential_fp_nodal, vector_potential_grad_buf_e_stag, vector_potential_grad_buf_b_stag, @@ -44,19 +44,19 @@ namespace warpx::fields hybrid_current_fp_temp, hybrid_current_fp_ampere, hybrid_current_fp_external, - Efield_cp, - Bfield_cp, - current_cp, - rho_cp, - F_cp, - G_cp, - Efield_cax, - Bfield_cax, - E_external_particle_field, - B_external_particle_field, - distance_to_eb, - edge_lengths, - face_areas, + Efield_cp, // Only used with MR. The field that is updated by the field solver at each timestep, on the coarse patch of each level + Bfield_cp, // Only used with MR. The field that is updated by the field solver at each timestep, on the coarse patch of each level + current_cp, // Only used with MR. The current that is used as a source for the field solver, on the coarse patch of each level + rho_cp, // Only used with MR. The charge density that is used as a source for the field solver, on the coarse patch of each level + F_cp, // Only used with MR. Used for divE cleaning, on the coarse patch of each level + G_cp, // Only used with MR. Used for divB cleaning, on the coarse patch of each level + Efield_cax, // Only used with MR. Particles that are close to the edge of the MR patch (i.e. in the gather buffer) gather from this field + Bfield_cax, // Only used with MR. Particles that are close to the edge of the MR patch (i.e. in the gather buffer) gather from this field + E_external_particle_field, // Stores external particle fields provided by the user as through an openPMD file + B_external_particle_field, // Stores external particle fields provided by the user as through an openPMD file + distance_to_eb, // Only used with embedded boundaries (EB). Stores the distance to the nearest EB + edge_lengths, // Only used with embedded boundaries (EB). Indicates the length of the cell edge that is covered by the EB, in SI units + face_areas, // Only used with embedded boundaries (EB). Indicates the area of the cell face that is covered by the EB, in SI units area_mod, pml_E_fp, pml_B_fp, @@ -73,7 +73,7 @@ namespace warpx::fields Bfield_avg_fp, Efield_avg_cp, Bfield_avg_cp, - Bold, + Bold, // Stores the value of B at the beginning of the timestep, for the implicit solver ECTRhofield, Venl ); From cbc0c821018a8388b8c91f9c4d0ff2e5a94aa445 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 23 Sep 2024 16:51:06 -0700 Subject: [PATCH 313/314] Cleanup Tests --- Examples/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/Examples/CMakeLists.txt b/Examples/CMakeLists.txt index 8bb02088a16..fe1da3d08e6 100644 --- a/Examples/CMakeLists.txt +++ b/Examples/CMakeLists.txt @@ -129,7 +129,6 @@ function(add_warpx_test set(runtime_params "amrex.abort_on_unused_inputs = 1" "amrex.throw_exception = 1" - "amrex.signal_handling = 1" "warpx.always_warn_immediately = 1" "warpx.do_dynamic_scheduling = 0" "warpx.serialize_initial_conditions = 1" From fb9e7dd1a2946c1f293cd25f2061d78cf5d1d71f Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 23 Sep 2024 16:52:11 -0700 Subject: [PATCH 314/314] More `has_vector` Co-authored-by: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> --- Source/BoundaryConditions/PML.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index 4ac9a1588e5..91d821d6646 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -1060,10 +1060,10 @@ PML::CopyJtoPMLs ( { using ablastr::fields::Direction; - bool const has_j_fp = fields.has(FieldType::current_fp, Direction{0}, lev); - bool const has_pml_j_fp = fields.has(FieldType::pml_j_fp, Direction{0}, lev); - bool const has_j_cp = fields.has(FieldType::current_cp, Direction{0}, lev); - bool const has_pml_j_cp = fields.has(FieldType::pml_j_cp, Direction{0}, lev); + bool const has_j_fp = fields.has_vector(FieldType::current_fp, lev); + bool const has_pml_j_fp = fields.has_vector(FieldType::pml_j_fp, lev); + bool const has_j_cp = fields.has_vector(FieldType::current_cp, lev); + bool const has_pml_j_cp = fields.has_vector(FieldType::pml_j_cp, lev); if (patch_type == PatchType::fine && has_pml_j_fp && has_j_fp) {