Skip to content

Commit

Permalink
MultiFabRegister Experiments
Browse files Browse the repository at this point in the history 
This performs two experiments on the MultiFabRegister:
- const correctness for nested containers
  (note: `-Wreturn-local-addr` warnings persist!)
- type safety for `MultiLevelVectorField` between
  level and direction (component)
  • Loading branch information
@ax3l
ax3l committed Sep 24, 2024
1 parent 31734d3 commit 08d5d72
Show file tree
Hide file tree
Showing 65 changed files with 1,437 additions and 1,316 deletions.
244 changes: 122 additions & 122 deletions Source/BoundaryConditions/PML.cpp
View file

Large diffs are not rendered by default.

60 changes: 30 additions & 30 deletions Source/BoundaryConditions/PML_RZ.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@
#include <memory>

using namespace amrex::literals;
using warpx::fields::FieldType;
using ablastr::fields::Direction;
using namespace warpx::fields;
using ablastr::fields::Dir;

PML_RZ::PML_RZ (int lev, amrex::BoxArray const& grid_ba, amrex::DistributionMapping const& grid_dm,
amrex::Geometry const* geom, int ncell, int do_pml_in_domain)
Expand All @@ -48,22 +48,22 @@ PML_RZ::PML_RZ (int lev, amrex::BoxArray const& grid_ba, amrex::DistributionMapp
bool const remake = false;
bool const redistribute_on_remake = false;

amrex::MultiFab const& Er_fp = *warpx.m_fields.get(FieldType::Efield_fp, Direction{0}, lev);
amrex::MultiFab const& Et_fp = *warpx.m_fields.get(FieldType::Efield_fp, Direction{1}, lev);
amrex::MultiFab const& Er_fp = *warpx.m_fields.get(FieldType::Efield_fp, 0_dir, lev);
amrex::MultiFab const& Et_fp = *warpx.m_fields.get(FieldType::Efield_fp, 1_dir, lev);
amrex::BoxArray const ba_Er = amrex::convert(grid_ba, Er_fp.ixType().toIntVect());
amrex::BoxArray const ba_Et = amrex::convert(grid_ba, Et_fp.ixType().toIntVect());
warpx.m_fields.alloc_init(FieldType::pml_E_fp, Direction{0}, lev, ba_Er, grid_dm, Er_fp.nComp(), Er_fp.nGrowVect(), 0.0_rt,
warpx.m_fields.alloc_init(FieldType::pml_E_fp, 0_dir, lev, ba_Er, grid_dm, Er_fp.nComp(), Er_fp.nGrowVect(), 0.0_rt,
remake, redistribute_on_remake);
warpx.m_fields.alloc_init(FieldType::pml_E_fp, Direction{1}, lev, ba_Et, grid_dm, Et_fp.nComp(), Et_fp.nGrowVect(), 0.0_rt,
warpx.m_fields.alloc_init(FieldType::pml_E_fp, 1_dir, lev, ba_Et, grid_dm, Et_fp.nComp(), Et_fp.nGrowVect(), 0.0_rt,
remake, redistribute_on_remake);

amrex::MultiFab const& Br_fp = *warpx.m_fields.get(FieldType::Bfield_fp,Direction{0},lev);
amrex::MultiFab const& Bt_fp = *warpx.m_fields.get(FieldType::Bfield_fp,Direction{1},lev);
amrex::MultiFab const& Br_fp = *warpx.m_fields.get(FieldType::Bfield_fp,0_dir,lev);
amrex::MultiFab const& Bt_fp = *warpx.m_fields.get(FieldType::Bfield_fp,1_dir,lev);
amrex::BoxArray const ba_Br = amrex::convert(grid_ba, Br_fp.ixType().toIntVect());
amrex::BoxArray const ba_Bt = amrex::convert(grid_ba, Bt_fp.ixType().toIntVect());
warpx.m_fields.alloc_init(FieldType::pml_B_fp, Direction{0}, lev, ba_Br, grid_dm, Br_fp.nComp(), Br_fp.nGrowVect(), 0.0_rt,
warpx.m_fields.alloc_init(FieldType::pml_B_fp, 0_dir, lev, ba_Br, grid_dm, Br_fp.nComp(), Br_fp.nGrowVect(), 0.0_rt,
remake, redistribute_on_remake);
warpx.m_fields.alloc_init(FieldType::pml_B_fp, Direction{1}, lev, ba_Bt, grid_dm, Bt_fp.nComp(), Bt_fp.nGrowVect(), 0.0_rt,
warpx.m_fields.alloc_init(FieldType::pml_B_fp, 1_dir, lev, ba_Bt, grid_dm, Bt_fp.nComp(), Bt_fp.nGrowVect(), 0.0_rt,
remake, redistribute_on_remake);

}
Expand All @@ -77,8 +77,8 @@ PML_RZ::ApplyDamping (amrex::MultiFab* Et_fp, amrex::MultiFab* Ez_fp,
amrex::Real const dr = m_geom->CellSize(0);
amrex::Real const cdt_over_dr = PhysConst::c*dt/dr;

amrex::MultiFab* pml_Et = fields.get(FieldType::pml_E_fp, Direction{1}, 0);
amrex::MultiFab* pml_Bt = fields.get(FieldType::pml_B_fp, Direction{1}, 0);
amrex::MultiFab* pml_Et = fields.get(FieldType::pml_E_fp, 1_dir, 0);
amrex::MultiFab* pml_Bt = fields.get(FieldType::pml_B_fp, 1_dir, 0);

#ifdef AMREX_USE_OMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
Expand Down Expand Up @@ -132,8 +132,8 @@ PML_RZ::ApplyDamping (amrex::MultiFab* Et_fp, amrex::MultiFab* Ez_fp,
void
PML_RZ::FillBoundaryE (ablastr::fields::MultiFabRegister& fields, PatchType patch_type, std::optional<bool> nodal_sync)
{
amrex::MultiFab * pml_Er = fields.get(FieldType::pml_E_fp, Direction{0}, 0);
amrex::MultiFab * pml_Et = fields.get(FieldType::pml_E_fp, Direction{1}, 0);
amrex::MultiFab * pml_Er = fields.get(FieldType::pml_E_fp, 0_dir, 0);
amrex::MultiFab * pml_Et = fields.get(FieldType::pml_E_fp, 1_dir, 0);

if (patch_type == PatchType::fine && pml_Er->nGrowVect().max() > 0)
{
Expand All @@ -148,8 +148,8 @@ PML_RZ::FillBoundaryB (ablastr::fields::MultiFabRegister& fields, PatchType patc
{
if (patch_type == PatchType::fine)
{
amrex::MultiFab * pml_Br = fields.get(FieldType::pml_B_fp, Direction{0}, 0);
amrex::MultiFab * pml_Bt = fields.get(FieldType::pml_B_fp, Direction{1}, 0);
amrex::MultiFab * pml_Br = fields.get(FieldType::pml_B_fp, 0_dir, 0);
amrex::MultiFab * pml_Bt = fields.get(FieldType::pml_B_fp, 1_dir, 0);

amrex::Periodicity const& period = m_geom->periodicity();
const amrex::Vector<amrex::MultiFab*> mf = {pml_Br, pml_Bt};
Expand All @@ -160,22 +160,22 @@ PML_RZ::FillBoundaryB (ablastr::fields::MultiFabRegister& fields, PatchType patc
void
PML_RZ::CheckPoint (ablastr::fields::MultiFabRegister& fields, std::string const& dir) const
{
if (fields.has(FieldType::pml_E_fp, Direction{0}, 0)) {
amrex::VisMF::AsyncWrite(*fields.get(FieldType::pml_E_fp, Direction{0}, 0), dir+"_Er_fp");
amrex::VisMF::AsyncWrite(*fields.get(FieldType::pml_E_fp, Direction{1}, 0), dir+"_Et_fp");
amrex::VisMF::AsyncWrite(*fields.get(FieldType::pml_B_fp, Direction{0}, 0), dir+"_Br_fp");
amrex::VisMF::AsyncWrite(*fields.get(FieldType::pml_B_fp, Direction{1}, 0), dir+"_Bt_fp");
if (fields.has(FieldType::pml_E_fp, 0_dir, 0)) {
amrex::VisMF::AsyncWrite(*fields.get(FieldType::pml_E_fp, 0_dir, 0), dir+"_Er_fp");
amrex::VisMF::AsyncWrite(*fields.get(FieldType::pml_E_fp, 1_dir, 0), dir+"_Et_fp");
amrex::VisMF::AsyncWrite(*fields.get(FieldType::pml_B_fp, 0_dir, 0), dir+"_Br_fp");
amrex::VisMF::AsyncWrite(*fields.get(FieldType::pml_B_fp, 1_dir, 0), dir+"_Bt_fp");
}
}

void
PML_RZ::Restart (ablastr::fields::MultiFabRegister& fields, std::string const& dir)
{
if (fields.has(FieldType::pml_E_fp, Direction{0}, 0)) {
amrex::VisMF::Read(*fields.get(FieldType::pml_E_fp, Direction{0}, 0), dir+"_Er_fp");
amrex::VisMF::Read(*fields.get(FieldType::pml_E_fp, Direction{1}, 0), dir+"_Et_fp");
amrex::VisMF::Read(*fields.get(FieldType::pml_B_fp, Direction{0}, 0), dir+"_Br_fp");
amrex::VisMF::Read(*fields.get(FieldType::pml_B_fp, Direction{1}, 0), dir+"_Bt_fp");
if (fields.has(FieldType::pml_E_fp, 0_dir, 0)) {
amrex::VisMF::Read(*fields.get(FieldType::pml_E_fp, 0_dir, 0), dir+"_Er_fp");
amrex::VisMF::Read(*fields.get(FieldType::pml_E_fp, 1_dir, 0), dir+"_Et_fp");
amrex::VisMF::Read(*fields.get(FieldType::pml_B_fp, 0_dir, 0), dir+"_Br_fp");
amrex::VisMF::Read(*fields.get(FieldType::pml_B_fp, 1_dir, 0), dir+"_Bt_fp");
}
}

Expand All @@ -196,10 +196,10 @@ PML_RZ::PushPMLPSATDSinglePatchRZ (
ablastr::fields::MultiFabRegister& fields)
{
SpectralFieldIndex const& Idx = solver.m_spectral_index;
amrex::MultiFab * pml_Er = fields.get(FieldType::pml_E_fp, Direction{0}, 0);
amrex::MultiFab * pml_Et = fields.get(FieldType::pml_E_fp, Direction{1}, 0);
amrex::MultiFab * pml_Br = fields.get(FieldType::pml_B_fp, Direction{0}, 0);
amrex::MultiFab * pml_Bt = fields.get(FieldType::pml_B_fp, Direction{1}, 0);
amrex::MultiFab * pml_Er = fields.get(FieldType::pml_E_fp, 0_dir, 0);
amrex::MultiFab * pml_Et = fields.get(FieldType::pml_E_fp, 1_dir, 0);
amrex::MultiFab * pml_Br = fields.get(FieldType::pml_B_fp, 0_dir, 0);
amrex::MultiFab * pml_Bt = fields.get(FieldType::pml_B_fp, 1_dir, 0);

// Perform forward Fourier transforms
solver.ForwardTransform(lev, *pml_Er, Idx.Er_pml, *pml_Et, Idx.Et_pml);
Expand Down
76 changes: 38 additions & 38 deletions Source/BoundaryConditions/WarpXEvolvePML.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,11 +67,11 @@ WarpX::DampPML (const int lev, PatchType patch_type)
#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT)
if (pml_rz[lev]) {
using ablastr::fields::Direction;
using warpx::fields::FieldType;
pml_rz[lev]->ApplyDamping( m_fields.get(FieldType::Efield_fp, Direction{1}, lev),
m_fields.get(FieldType::Efield_fp, Direction{2}, lev),
m_fields.get(FieldType::Bfield_fp, Direction{1}, lev),
m_fields.get(FieldType::Bfield_fp, Direction{2}, lev),
using namespace warpx::fields;
pml_rz[lev]->ApplyDamping( m_fields.get(FieldType::Efield_fp, 1_dir, lev),
m_fields.get(FieldType::Efield_fp, 2_dir, lev),
m_fields.get(FieldType::Bfield_fp, 1_dir, lev),
m_fields.get(FieldType::Bfield_fp, 2_dir, lev),
dt[lev], m_fields);
}
#endif
Expand All @@ -88,19 +88,19 @@ WarpX::DampPML_Cartesian (const int lev, PatchType patch_type)

if (pml[lev]->ok())
{
using warpx::fields::FieldType;
using namespace warpx::fields;

const auto& pml_E = (patch_type == PatchType::fine) ? m_fields.get_alldirs(FieldType::pml_E_fp, lev) : m_fields.get_alldirs(FieldType::pml_E_cp, lev);
const auto& pml_B = (patch_type == PatchType::fine) ? m_fields.get_alldirs(FieldType::pml_B_fp, lev) : m_fields.get_alldirs(FieldType::pml_B_cp, lev);
const auto& sigba = (patch_type == PatchType::fine) ? pml[lev]->GetMultiSigmaBox_fp() : pml[lev]->GetMultiSigmaBox_cp();

const amrex::IntVect Ex_stag = pml_E[0]->ixType().toIntVect();
const amrex::IntVect Ey_stag = pml_E[1]->ixType().toIntVect();
const amrex::IntVect Ez_stag = pml_E[2]->ixType().toIntVect();
const amrex::IntVect Ex_stag = pml_E[0_dir]->ixType().toIntVect();
const amrex::IntVect Ey_stag = pml_E[1_dir]->ixType().toIntVect();
const amrex::IntVect Ez_stag = pml_E[2_dir]->ixType().toIntVect();

const amrex::IntVect Bx_stag = pml_B[0]->ixType().toIntVect();
const amrex::IntVect By_stag = pml_B[1]->ixType().toIntVect();
const amrex::IntVect Bz_stag = pml_B[2]->ixType().toIntVect();
const amrex::IntVect Bx_stag = pml_B[0_dir]->ixType().toIntVect();
const amrex::IntVect By_stag = pml_B[1_dir]->ixType().toIntVect();
const amrex::IntVect Bz_stag = pml_B[2_dir]->ixType().toIntVect();

amrex::IntVect F_stag;
if (m_fields.has(FieldType::pml_F_fp, lev)) {
Expand All @@ -119,21 +119,21 @@ WarpX::DampPML_Cartesian (const int lev, PatchType patch_type)
#ifdef AMREX_USE_OMP
#pragma omp parallel if (Gpu::notInLaunchRegion())
#endif
for ( MFIter mfi(*pml_E[0], TilingIfNotGPU()); mfi.isValid(); ++mfi )
for ( MFIter mfi(*pml_E[0_dir], TilingIfNotGPU()); mfi.isValid(); ++mfi )
{
const Box& tex = mfi.tilebox( pml_E[0]->ixType().toIntVect() );
const Box& tey = mfi.tilebox( pml_E[1]->ixType().toIntVect() );
const Box& tez = mfi.tilebox( pml_E[2]->ixType().toIntVect() );
const Box& tbx = mfi.tilebox( pml_B[0]->ixType().toIntVect() );
const Box& tby = mfi.tilebox( pml_B[1]->ixType().toIntVect() );
const Box& tbz = mfi.tilebox( pml_B[2]->ixType().toIntVect() );

auto const& pml_Exfab = pml_E[0]->array(mfi);
auto const& pml_Eyfab = pml_E[1]->array(mfi);
auto const& pml_Ezfab = pml_E[2]->array(mfi);
auto const& pml_Bxfab = pml_B[0]->array(mfi);
auto const& pml_Byfab = pml_B[1]->array(mfi);
auto const& pml_Bzfab = pml_B[2]->array(mfi);
const Box& tex = mfi.tilebox( pml_E[0_dir]->ixType().toIntVect() );
const Box& tey = mfi.tilebox( pml_E[1_dir]->ixType().toIntVect() );
const Box& tez = mfi.tilebox( pml_E[2_dir]->ixType().toIntVect() );
const Box& tbx = mfi.tilebox( pml_B[0_dir]->ixType().toIntVect() );
const Box& tby = mfi.tilebox( pml_B[1_dir]->ixType().toIntVect() );
const Box& tbz = mfi.tilebox( pml_B[2_dir]->ixType().toIntVect() );

auto const& pml_Exfab = pml_E[0_dir]->array(mfi);
auto const& pml_Eyfab = pml_E[1_dir]->array(mfi);
auto const& pml_Ezfab = pml_E[2_dir]->array(mfi);
auto const& pml_Bxfab = pml_B[0_dir]->array(mfi);
auto const& pml_Byfab = pml_B[1_dir]->array(mfi);
auto const& pml_Bzfab = pml_B[2_dir]->array(mfi);

amrex::Real const * AMREX_RESTRICT sigma_fac_x = sigba[mfi].sigma_fac[0].data();
#if defined(WARPX_DIM_3D)
Expand Down Expand Up @@ -258,7 +258,7 @@ WarpX::DampJPML (int lev, PatchType patch_type)

if (pml[lev]->ok())
{
using warpx::fields::FieldType;
using namespace warpx::fields;

const auto& pml_j = (patch_type == PatchType::fine) ? m_fields.get_alldirs(FieldType::pml_j_fp, lev) : m_fields.get_alldirs(FieldType::pml_j_cp, lev);
const auto& sigba = (patch_type == PatchType::fine) ? pml[lev]->GetMultiSigmaBox_fp()
Expand All @@ -267,11 +267,11 @@ WarpX::DampJPML (int lev, PatchType patch_type)
#ifdef AMREX_USE_OMP
#pragma omp parallel if (Gpu::notInLaunchRegion())
#endif
for ( MFIter mfi(*pml_j[0], TilingIfNotGPU()); mfi.isValid(); ++mfi )
for ( MFIter mfi(*pml_j[0_dir], TilingIfNotGPU()); mfi.isValid(); ++mfi )
{
auto const& pml_jxfab = pml_j[0]->array(mfi);
auto const& pml_jyfab = pml_j[1]->array(mfi);
auto const& pml_jzfab = pml_j[2]->array(mfi);
auto const& pml_jxfab = pml_j[0_dir]->array(mfi);
auto const& pml_jyfab = pml_j[1_dir]->array(mfi);
auto const& pml_jzfab = pml_j[2_dir]->array(mfi);
const Real* sigma_cumsum_fac_j_x = sigba[mfi].sigma_cumsum_fac[0].data();
const Real* sigma_star_cumsum_fac_j_x = sigba[mfi].sigma_star_cumsum_fac[0].data();
#if defined(WARPX_DIM_3D)
Expand All @@ -291,16 +291,16 @@ WarpX::DampJPML (int lev, PatchType patch_type)
if (EB::enabled()) {
const auto &pml_edge_lenghts = m_fields.get_alldirs(FieldType::pml_edge_lengths, lev);

eb_lxfab = pml_edge_lenghts[0]->array(mfi);
eb_lyfab = pml_edge_lenghts[1]->array(mfi);
eb_lzfab = pml_edge_lenghts[2]->array(mfi);
eb_lxfab = pml_edge_lenghts[0_dir]->array(mfi);
eb_lyfab = pml_edge_lenghts[1_dir]->array(mfi);
eb_lzfab = pml_edge_lenghts[2_dir]->array(mfi);
} else {
amrex::ignore_unused(eb_lxfab, eb_lyfab, eb_lzfab);
}

const Box& tjx = mfi.tilebox( pml_j[0]->ixType().toIntVect() );
const Box& tjy = mfi.tilebox( pml_j[1]->ixType().toIntVect() );
const Box& tjz = mfi.tilebox( pml_j[2]->ixType().toIntVect() );
const Box& tjx = mfi.tilebox( pml_j[0_dir]->ixType().toIntVect() );
const Box& tjy = mfi.tilebox( pml_j[1_dir]->ixType().toIntVect() );
const Box& tjz = mfi.tilebox( pml_j[2_dir]->ixType().toIntVect() );

int const x_lo = sigba[mfi].sigma_cumsum_fac[0].lo();
#if defined(WARPX_DIM_3D)
Expand Down Expand Up @@ -354,7 +354,7 @@ WarpX::DampJPML (int lev, PatchType patch_type)
void
WarpX::CopyJPML ()
{
using ablastr::fields::Direction;
using ablastr::fields::Dir;

for (int lev = 0; lev <= finest_level; ++lev)
{
Expand Down
Loading

0 comments on commit 08d5d72

Please sign in to comment.