Synchronize on make_graph_executable

PaRSEC needs to make sure that no AMs are delivered before the callbacks are registered, which happens in make_graph_executable. So we add a hook that becomes a barrier for PaRSEC and is empty for madness. Signed-off-by: Joseph Schuchart <[email protected]>
TESSEorg · Jun 23, 2023 · a7dc7fb · a7dc7fb
1 parent 90b546c
commit a7dc7fb
Show file tree

Hide file tree

Showing 5 changed files with 17 additions and 3 deletions.
diff --git a/ttg/ttg/func.h b/ttg/ttg/func.h
@@ -78,8 +78,12 @@ namespace ttg {
   template <typename... TTBasePtrs>
   inline std::enable_if_t<(std::is_convertible_v<decltype(*(std::declval<TTBasePtrs>())), TTBase &> && ...), bool>
   make_graph_executable(TTBasePtrs &&...tts) {
-    return ttg::make_traverse([](auto &&x) { std::forward<decltype(x)>(x)->make_executable(); })(
-        std::forward<TTBasePtrs>(tts)...);
+    auto traverse = ttg::make_traverse([](auto &&x) { std::forward<decltype(x)>(x)->make_executable(); });
+    auto ret = traverse(std::forward<TTBasePtrs>(tts)...);
+    // make sure everyone has traversed the TT
+    auto world = [&](auto&& tt0, auto&&... tts) { return tt0->get_world(); }(std::forward<TTBasePtrs>(tts)...);
+    TTG_IMPL_NS::make_executable_hook(world);
+    return ret;
   }
 
   /// \brief Connect output terminal to successor input terminal

diff --git a/ttg/ttg/madness/fwd.h b/ttg/ttg/madness/fwd.h
@@ -20,6 +20,8 @@ namespace ttg_madness {
 
   class WorldImpl;
 
+  inline void make_executable_hook(ttg::World&);
+
   inline void ttg_initialize(int argc, char **argv, int num_threads = -1);
 
   inline void ttg_finalize();

diff --git a/ttg/ttg/madness/ttg.h b/ttg/ttg/madness/ttg.h
@@ -116,6 +116,8 @@ namespace ttg_madness {
 #endif
   };
 
+  inline void make_executable_hook(ttg::World& world) { }
+
   inline void ttg_initialize(int argc, char **argv, int num_threads) {
     if (num_threads < 1) num_threads = ttg::detail::num_threads();
     ::madness::World &madworld = ::madness::initialize(argc, argv, num_threads, /* quiet = */ true);

diff --git a/ttg/ttg/parsec/fwd.h b/ttg/ttg/parsec/fwd.h
@@ -22,6 +22,8 @@ namespace ttg_parsec {
 
   class WorldImpl;
 
+  inline void make_executable_hook(ttg::World&);
+
   inline void ttg_initialize(int argc, char **argv, int num_threads = -1, parsec_context_s * = nullptr);
 
   inline void ttg_finalize();

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
@@ -469,7 +469,7 @@ namespace ttg_parsec {
 #endif
   };
 
-  static void unregister_parsec_tags(void *_)
+  inline void unregister_parsec_tags(void *_)
   {
     if(NULL != parsec_ce.tag_unregister) {
       parsec_ce.tag_unregister(WorldImpl::parsec_ttg_tag());
@@ -1035,6 +1035,10 @@ namespace ttg_parsec {
     value = result;
   }
 
+  inline void make_executable_hook(ttg::World& world) {
+    MPI_Barrier(world.impl().comm());
+  }
+
   /// broadcast
   /// @tparam T a serializable type
   template <typename T>