Merge pull request #1466 from germasch/pr/mpidummy

MPICommOnly / mpidummy cleanup; no-MPI_Init support
ornladios · Jun 3, 2019 · 2ab43e8 · 2ab43e8
2 parents 3e2361f + 9d87fd6
commit 2ab43e8
Show file tree

Hide file tree

Showing 38 changed files with 411 additions and 255 deletions.
diff --git a/bindings/C/c/adios2_c_operator.h b/bindings/C/c/adios2_c_operator.h
@@ -13,8 +13,6 @@
 
 #include "adios2_c_types.h"
 
-#include "adios2/ADIOSMPICommOnly.h"
-
 #ifdef __cplusplus
 extern "C" {
 #endif

diff --git a/bindings/CXX11/cxx11/ADIOS.cpp b/bindings/CXX11/cxx11/ADIOS.cpp
@@ -25,14 +25,13 @@ ADIOS::ADIOS(MPI_Comm comm, const bool debugMode) : ADIOS("", comm, debugMode)
 {
 }
 
-#else
+#endif
 ADIOS::ADIOS(const std::string &configFile, const bool debugMode)
 : m_ADIOS(std::make_shared<core::ADIOS>(configFile, debugMode, "C++"))
 {
 }
 
 ADIOS::ADIOS(const bool debugMode) : ADIOS("", debugMode) {}
-#endif
 
 ADIOS::operator bool() const noexcept { return m_ADIOS ? true : false; }
 

diff --git a/bindings/CXX11/cxx11/ADIOS.h b/bindings/CXX11/cxx11/ADIOS.h
@@ -64,9 +64,9 @@ class ADIOS
      * @exception std::invalid_argument in debugMode = true if user input is
      * incorrect
      */
-    ADIOS(const std::string &configFile = "", MPI_Comm comm = MPI_COMM_SELF,
+    ADIOS(const std::string &configFile, MPI_Comm comm,
           const bool debugMode = true);
-#else
+#endif
 
     /**
      * Starting point for non-MPI serial apps. Creates an ADIOS object allowing
@@ -87,7 +87,6 @@ class ADIOS
      * incorrect
      */
     ADIOS(const bool debugMode = true);
-#endif
 
     /** object inspection true: valid object, false: invalid object */
     explicit operator bool() const noexcept;

diff --git a/bindings/Python/py11ADIOS.h b/bindings/Python/py11ADIOS.h
@@ -17,7 +17,6 @@
 #include <memory> //std::shared_ptr
 #include <string>
 
-#include "adios2/ADIOSMPICommOnly.h"
 #include "adios2/core/ADIOS.h"
 
 namespace adios2

diff --git a/bindings/Python/py11File.h b/bindings/Python/py11File.h
@@ -13,7 +13,6 @@
 
 #include <pybind11/numpy.h>
 
-#include "adios2/ADIOSMPICommOnly.h"
 #include "adios2/ADIOSTypes.h"
 #include "adios2/core/Stream.h"
 

diff --git a/source/adios2/ADIOSMPI.h b/source/adios2/ADIOSMPI.h
@@ -8,12 +8,7 @@
 
 #include "adios2/ADIOSConfig.h"
 
-#ifdef ADIOS2_HAVE_MPI
-#include <mpi.h>
-#else
-#include "adios2/helper/mpidummy.h"
-using namespace adios2::helper::mpi;
-#endif
+#include "adios2/helper/mpiwrap.h"
 
 #include <climits> //UXXX_MAX
 #include <cstdint> //SIZE_MAX

diff --git a/source/adios2/ADIOSMPICommOnly.h b/source/adios2/ADIOSMPICommOnly.h
diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt
@@ -186,10 +186,10 @@ if(ADIOS2_HAVE_MPI)
     engine/insitumpi/InSituMPIFunctions.cpp engine/insitumpi/InSituMPISchedules.cpp
   )
   target_link_libraries(adios2 PUBLIC MPI::MPI_C)
-else()
-  target_sources(adios2 PRIVATE helper/mpidummy.cpp)
 endif()
 
+target_sources(adios2 PRIVATE helper/mpidummy.cpp helper/mpiwrap.cpp)
+
 if(ADIOS2_HAVE_HDF5)
   if(HDF5_C_INCLUDE_DIRS)
     target_include_directories(adios2 PRIVATE ${HDF5_C_INCLUDE_DIRS})
@@ -223,7 +223,7 @@ set_target_properties(adios2 PROPERTIES
   SOVERSION ${ADIOS2_VERSION_MAJOR}
 )
 
-install(FILES ADIOSMacros.h ADIOSTypes.h ADIOSTypes.inl ADIOSMPICommOnly.h
+install(FILES ADIOSMacros.h ADIOSTypes.h ADIOSTypes.inl
   DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/adios2
 )
 

diff --git a/source/adios2/core/ADIOS.h b/source/adios2/core/ADIOS.h
@@ -19,7 +19,7 @@
 /// \endcond
 
 #include "adios2/ADIOSConfig.h"
-#include "adios2/ADIOSMPICommOnly.h"
+#include "adios2/ADIOSMPI.h"
 #include "adios2/ADIOSTypes.h"
 #include "adios2/core/Operator.h"
 

diff --git a/source/adios2/core/Engine.cpp b/source/adios2/core/Engine.cpp
@@ -69,7 +69,7 @@ void Engine::Close(const int transportIndex)
 
     if (transportIndex == -1)
     {
-        helper::CheckMPIReturn(MPI_Comm_free(&m_MPIComm),
+        helper::CheckMPIReturn(SMPI_Comm_free(&m_MPIComm),
                                "freeing comm in Engine " + m_Name +
                                    ", in call to Close");
         m_IsClosed = true;

diff --git a/source/adios2/core/Engine.h b/source/adios2/core/Engine.h
@@ -24,7 +24,6 @@
 /// \endcond
 
 #include "adios2/ADIOSConfig.h"
-#include "adios2/ADIOSMPICommOnly.h"
 #include "adios2/ADIOSMacros.h"
 #include "adios2/ADIOSTypes.h"
 #include "adios2/core/IO.h"

diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp
@@ -438,7 +438,7 @@ Engine &IO::Open(const std::string &name, const Mode mode,
     }
 
     MPI_Comm mpiComm;
-    MPI_Comm_dup(mpiComm_orig, &mpiComm);
+    SMPI_Comm_dup(mpiComm_orig, &mpiComm);
     std::shared_ptr<Engine> engine;
     const bool isDefaultEngine = m_EngineType.empty() ? true : false;
     std::string engineTypeLC = m_EngineType;

diff --git a/source/adios2/core/IO.h b/source/adios2/core/IO.h
@@ -21,7 +21,6 @@
 /// \endcond
 
 #include "adios2/ADIOSConfig.h"
-#include "adios2/ADIOSMPICommOnly.h"
 #include "adios2/ADIOSMacros.h"
 #include "adios2/ADIOSTypes.h"
 #include "adios2/core/ADIOS.h"

diff --git a/source/adios2/core/Stream.h b/source/adios2/core/Stream.h
@@ -13,7 +13,6 @@
 
 #include <memory> //std::shared_ptr
 
-#include "adios2/ADIOSMPICommOnly.h"
 #include "adios2/ADIOSMacros.h"
 #include "adios2/ADIOSTypes.h"
 #include "adios2/core/ADIOS.h"

diff --git a/source/adios2/engine/hdf5/HDF5WriterP.h b/source/adios2/engine/hdf5/HDF5WriterP.h
@@ -14,7 +14,6 @@
 #include <hdf5.h>
 
 #include "adios2/ADIOSConfig.h"
-#include "adios2/ADIOSMPICommOnly.h"
 #include "adios2/core/Engine.h"
 #include "adios2/core/IO.h"
 #include "adios2/toolkit/interop/hdf5/HDF5Common.h"

diff --git a/source/adios2/helper/adiosMPIFunctions.inl b/source/adios2/helper/adiosMPIFunctions.inl
@@ -27,8 +27,8 @@ std::vector<T> GatherValues(const T source, MPI_Comm mpiComm,
                             const int rankDestination)
 {
     int rank, size;
-    MPI_Comm_rank(mpiComm, &rank);
-    MPI_Comm_size(mpiComm, &size);
+    SMPI_Comm_rank(mpiComm, &rank);
+    SMPI_Comm_size(mpiComm, &size);
 
     std::vector<T> output;
 
@@ -47,7 +47,7 @@ template <class T>
 std::vector<T> AllGatherValues(const T source, MPI_Comm mpiComm)
 {
     int size;
-    MPI_Comm_size(mpiComm, &size);
+    SMPI_Comm_size(mpiComm, &size);
     std::vector<T> output(size);
 
     T sourceCopy = source; // so we can have an address for rvalues
@@ -67,7 +67,7 @@ void GathervVectors(const std::vector<T> &in, std::vector<T> &out,
     size_t gatheredSize = 0;
 
     int rank;
-    MPI_Comm_rank(mpiComm, &rank);
+    SMPI_Comm_rank(mpiComm, &rank);
 
     if (rank == rankDestination) // pre-allocate vector
     {

diff --git a/source/adios2/helper/adiosMPIFunctions.tcc b/source/adios2/helper/adiosMPIFunctions.tcc
@@ -32,15 +32,15 @@ size_t BroadcastValue(const size_t &input, MPI_Comm mpiComm,
                       const int rankSource)
 {
     int rank;
-    MPI_Comm_rank(mpiComm, &rank);
+    SMPI_Comm_rank(mpiComm, &rank);
     size_t output = 0;
 
     if (rank == rankSource)
     {
         output = input;
     }
 
-    MPI_Bcast(&output, 1, ADIOS2_MPI_SIZE_T, rankSource, mpiComm);
+    SMPI_Bcast(&output, 1, ADIOS2_MPI_SIZE_T, rankSource, mpiComm);
 
     return output;
 }
@@ -50,7 +50,7 @@ std::string BroadcastValue(const std::string &input, MPI_Comm mpiComm,
                            const int rankSource)
 {
     int rank;
-    MPI_Comm_rank(mpiComm, &rank);
+    SMPI_Comm_rank(mpiComm, &rank);
     const size_t inputSize = input.size();
     const size_t length = BroadcastValue(inputSize, mpiComm, rankSource);
     std::string output;
@@ -64,8 +64,8 @@ std::string BroadcastValue(const std::string &input, MPI_Comm mpiComm,
         output.resize(length);
     }
 
-    MPI_Bcast(const_cast<char *>(output.data()), static_cast<int>(length),
-              MPI_CHAR, rankSource, mpiComm);
+    SMPI_Bcast(const_cast<char *>(output.data()), static_cast<int>(length),
+               MPI_CHAR, rankSource, mpiComm);
 
     return output;
 }
@@ -77,8 +77,8 @@ unsigned int ReduceValues(const unsigned int source, MPI_Comm mpiComm,
 {
     unsigned int sourceLocal = source;
     unsigned int reduceValue = 0;
-    MPI_Reduce(&sourceLocal, &reduceValue, 1, MPI_UNSIGNED, operation,
-               rankDestination, mpiComm);
+    SMPI_Reduce(&sourceLocal, &reduceValue, 1, MPI_UNSIGNED, operation,
+                rankDestination, mpiComm);
     return reduceValue;
 }
 
@@ -88,8 +88,8 @@ unsigned long int ReduceValues(const unsigned long int source, MPI_Comm mpiComm,
 {
     unsigned long int sourceLocal = source;
     unsigned long int reduceValue = 0;
-    MPI_Reduce(&sourceLocal, &reduceValue, 1, MPI_UNSIGNED_LONG, operation,
-               rankDestination, mpiComm);
+    SMPI_Reduce(&sourceLocal, &reduceValue, 1, MPI_UNSIGNED_LONG, operation,
+                rankDestination, mpiComm);
     return reduceValue;
 }
 
@@ -100,8 +100,8 @@ unsigned long long int ReduceValues(const unsigned long long int source,
 {
     unsigned long long int sourceLocal = source;
     unsigned long long int reduceValue = 0;
-    MPI_Reduce(&sourceLocal, &reduceValue, 1, MPI_UNSIGNED_LONG_LONG, operation,
-               rankDestination, mpiComm);
+    SMPI_Reduce(&sourceLocal, &reduceValue, 1, MPI_UNSIGNED_LONG_LONG,
+                operation, rankDestination, mpiComm);
     return reduceValue;
 }
 
@@ -111,7 +111,7 @@ void BroadcastVector(std::vector<char> &vector, MPI_Comm mpiComm,
                      const int rankSource)
 {
     int size;
-    MPI_Comm_size(mpiComm, &size);
+    SMPI_Comm_size(mpiComm, &size);
 
     if (size == 1)
     {
@@ -121,7 +121,7 @@ void BroadcastVector(std::vector<char> &vector, MPI_Comm mpiComm,
     // First Broadcast the size, then the contents
     size_t inputSize = BroadcastValue(vector.size(), mpiComm, rankSource);
     int rank;
-    MPI_Comm_rank(mpiComm, &rank);
+    SMPI_Comm_rank(mpiComm, &rank);
 
     if (rank != rankSource)
     {
@@ -133,8 +133,8 @@ void BroadcastVector(std::vector<char> &vector, MPI_Comm mpiComm,
     char *buffer = vector.data();
     while (inputSize > 0)
     {
-        MPI_Bcast(buffer, static_cast<int>(blockSize), MPI_CHAR, rankSource,
-                  mpiComm);
+        SMPI_Bcast(buffer, static_cast<int>(blockSize), MPI_CHAR, rankSource,
+                   mpiComm);
         buffer += blockSize;
         inputSize -= blockSize;
         blockSize = (inputSize > MAXBCASTSIZE ? MAXBCASTSIZE : inputSize);
@@ -149,8 +149,8 @@ void GatherArrays(const char *source, const size_t sourceCount,
 {
     int countsInt = static_cast<int>(sourceCount);
     int result =
-        MPI_Gather(const_cast<char *>(source), countsInt, MPI_CHAR, destination,
-                   countsInt, MPI_CHAR, rankDestination, mpiComm);
+        SMPI_Gather(const_cast<char *>(source), countsInt, MPI_CHAR,
+                    destination, countsInt, MPI_CHAR, rankDestination, mpiComm);
 
     if (result != MPI_SUCCESS)
     {
@@ -165,9 +165,9 @@ void GatherArrays(const size_t *source, const size_t sourceCount,
                   const int rankDestination)
 {
     int countsInt = static_cast<int>(sourceCount);
-    int result = MPI_Gather(const_cast<size_t *>(source), countsInt,
-                            ADIOS2_MPI_SIZE_T, destination, countsInt,
-                            ADIOS2_MPI_SIZE_T, rankDestination, mpiComm);
+    int result = SMPI_Gather(const_cast<size_t *>(source), countsInt,
+                             ADIOS2_MPI_SIZE_T, destination, countsInt,
+                             ADIOS2_MPI_SIZE_T, rankDestination, mpiComm);
 
     if (result != MPI_SUCCESS)
     {
@@ -202,7 +202,7 @@ void GathervArrays(const char *source, const size_t sourceCount,
 {
     int result = 0;
     int rank;
-    MPI_Comm_rank(mpiComm, &rank);
+    SMPI_Comm_rank(mpiComm, &rank);
 
     std::vector<int> countsInt, displacementsInt;
 
@@ -213,9 +213,10 @@ void GathervArrays(const char *source, const size_t sourceCount,
     }
 
     int sourceCountInt = static_cast<int>(sourceCount);
-    result = MPI_Gatherv(const_cast<char *>(source), sourceCountInt, MPI_CHAR,
-                         destination, countsInt.data(), displacementsInt.data(),
-                         MPI_CHAR, rankDestination, mpiComm);
+    result =
+        SMPI_Gatherv(const_cast<char *>(source), sourceCountInt, MPI_CHAR,
+                     destination, countsInt.data(), displacementsInt.data(),
+                     MPI_CHAR, rankDestination, mpiComm);
 
     if (result != MPI_SUCCESS)
     {
@@ -232,7 +233,7 @@ void GathervArrays(const size_t *source, const size_t sourceCount,
 {
     int result = 0;
     int rank;
-    MPI_Comm_rank(mpiComm, &rank);
+    SMPI_Comm_rank(mpiComm, &rank);
 
     std::vector<int> countsInt =
         NewVectorTypeFromArray<size_t, int>(counts, countsSize);
@@ -242,10 +243,10 @@ void GathervArrays(const size_t *source, const size_t sourceCount,
 
     int sourceCountInt = static_cast<int>(sourceCount);
 
-    result = MPI_Gatherv(const_cast<size_t *>(source), sourceCountInt,
-                         ADIOS2_MPI_SIZE_T, destination, countsInt.data(),
-                         displacementsInt.data(), ADIOS2_MPI_SIZE_T,
-                         rankDestination, mpiComm);
+    result = SMPI_Gatherv(const_cast<size_t *>(source), sourceCountInt,
+                          ADIOS2_MPI_SIZE_T, destination, countsInt.data(),
+                          displacementsInt.data(), ADIOS2_MPI_SIZE_T,
+                          rankDestination, mpiComm);
 
     if (result != MPI_SUCCESS)
     {