From 5eed0f8bfc878212c3d7e60760fc086fba6a3412 Mon Sep 17 00:00:00 2001
From: Roland Haas <rhaas@illinois.edu>
Date: Sun, 23 Jul 2023 13:34:35 -0500
Subject: [PATCH] CarpetX: only multi-thread MFIter if AMReX supports it

---
 CarpetX/src/driver.cxx    | 2 ++
 CarpetX/src/reduction.cxx | 2 ++
 CarpetX/src/valid.cxx     | 4 ++++
 3 files changed, 8 insertions(+)

diff --git a/CarpetX/src/driver.cxx b/CarpetX/src/driver.cxx
index cfea78b58..32d4f0ca1 100644
--- a/CarpetX/src/driver.cxx
+++ b/CarpetX/src/driver.cxx
@@ -1573,7 +1573,9 @@ void CactusAmrCore::ErrorEst(const int level, amrex::TagBoxArray &tags,
                    []() { return "ErrorEst"; });
   std::size_t npoints_set = 0, npoints_clear = 0, npoints_total = 0;
   auto mfitinfo = amrex::MFItInfo().SetDynamic(true).EnableTiling();
+#ifdef AMREX_USE_OMP
 #pragma omp parallel
+#endif
   for (amrex::MFIter mfi(*leveldata.fab, mfitinfo); mfi.isValid(); ++mfi) {
     GridPtrDesc1 grid(leveldata, groupdata, mfi);
 
diff --git a/CarpetX/src/reduction.cxx b/CarpetX/src/reduction.cxx
index 0acf55e30..23bec122c 100644
--- a/CarpetX/src/reduction.cxx
+++ b/CarpetX/src/reduction.cxx
@@ -201,7 +201,9 @@ reduction<CCTK_REAL, dim> reduce(int gi, int vi, int tl) {
       // TODO: check that multi-threading actually helps (and we are
       // not dominated by memory latency)
       // TODO: document required version of OpenMP to use custom reductions
+#ifdef AMREX_USE_OMP
 #pragma omp parallel reduction(reduction : red)
+#endif
       for (amrex::MFIter mfi(mfab, mfitinfo); mfi.isValid(); ++mfi) {
         const amrex::Box &bx = mfi.tilebox(); // current tile (without ghosts)
         const vect<int, dim> tmin{bx.smallEnd(0), bx.smallEnd(1),
diff --git a/CarpetX/src/valid.cxx b/CarpetX/src/valid.cxx
index 8fc815253..26328cdb1 100644
--- a/CarpetX/src/valid.cxx
+++ b/CarpetX/src/valid.cxx
@@ -485,7 +485,9 @@ calculate_checksums(const vector<vector<vector<valid_t> > > &will_write) {
   assert(active_levels);
   active_levels->loop([&](auto &restrict leveldata) {
     auto mfitinfo = amrex::MFItInfo().SetDynamic(true).EnableTiling();
+#ifdef AMREX_USE_OMP
 #pragma omp parallel
+#endif
     for (amrex::MFIter mfi(*leveldata.fab, mfitinfo); mfi.isValid(); ++mfi) {
 
       for (const auto &groupdataptr : leveldata.groupdata) {
@@ -563,7 +565,9 @@ void check_checksums(const checksums_t &checksums,
   assert(active_levels);
   active_levels->loop([&](auto &restrict leveldata) {
     auto mfitinfo = amrex::MFItInfo().SetDynamic(true).EnableTiling();
+#ifdef AMREX_USE_OMP
 #pragma omp parallel
+#endif
     for (amrex::MFIter mfi(*leveldata.fab, mfitinfo); mfi.isValid(); ++mfi) {
 
       for (const auto &groupdataptr : leveldata.groupdata) {