From a7dc7fb84575851d78845b4bdce4286897b74157 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Thu, 22 Jun 2023 17:59:57 -0400 Subject: [PATCH] Synchronize on make_graph_executable PaRSEC needs to make sure that no AMs are delivered before the callbacks are registered, which happens in make_graph_executable. So we add a hook that becomes a barrier for PaRSEC and is empty for madness. Signed-off-by: Joseph Schuchart --- ttg/ttg/func.h | 8 ++++++-- ttg/ttg/madness/fwd.h | 2 ++ ttg/ttg/madness/ttg.h | 2 ++ ttg/ttg/parsec/fwd.h | 2 ++ ttg/ttg/parsec/ttg.h | 6 +++++- 5 files changed, 17 insertions(+), 3 deletions(-) diff --git a/ttg/ttg/func.h b/ttg/ttg/func.h index 61615e631..e383ebbc8 100644 --- a/ttg/ttg/func.h +++ b/ttg/ttg/func.h @@ -78,8 +78,12 @@ namespace ttg { template inline std::enable_if_t<(std::is_convertible_v())), TTBase &> && ...), bool> make_graph_executable(TTBasePtrs &&...tts) { - return ttg::make_traverse([](auto &&x) { std::forward(x)->make_executable(); })( - std::forward(tts)...); + auto traverse = ttg::make_traverse([](auto &&x) { std::forward(x)->make_executable(); }); + auto ret = traverse(std::forward(tts)...); + // make sure everyone has traversed the TT + auto world = [&](auto&& tt0, auto&&... tts) { return tt0->get_world(); }(std::forward(tts)...); + TTG_IMPL_NS::make_executable_hook(world); + return ret; } /// \brief Connect output terminal to successor input terminal diff --git a/ttg/ttg/madness/fwd.h b/ttg/ttg/madness/fwd.h index abcb771c5..af050f9a1 100644 --- a/ttg/ttg/madness/fwd.h +++ b/ttg/ttg/madness/fwd.h @@ -20,6 +20,8 @@ namespace ttg_madness { class WorldImpl; + inline void make_executable_hook(ttg::World&); + inline void ttg_initialize(int argc, char **argv, int num_threads = -1); inline void ttg_finalize(); diff --git a/ttg/ttg/madness/ttg.h b/ttg/ttg/madness/ttg.h index c7580f26c..383d33c1b 100644 --- a/ttg/ttg/madness/ttg.h +++ b/ttg/ttg/madness/ttg.h @@ -116,6 +116,8 @@ namespace ttg_madness { #endif }; + inline void make_executable_hook(ttg::World& world) { } + inline void ttg_initialize(int argc, char **argv, int num_threads) { if (num_threads < 1) num_threads = ttg::detail::num_threads(); ::madness::World &madworld = ::madness::initialize(argc, argv, num_threads, /* quiet = */ true); diff --git a/ttg/ttg/parsec/fwd.h b/ttg/ttg/parsec/fwd.h index 959dbdaac..4511a6231 100644 --- a/ttg/ttg/parsec/fwd.h +++ b/ttg/ttg/parsec/fwd.h @@ -22,6 +22,8 @@ namespace ttg_parsec { class WorldImpl; + inline void make_executable_hook(ttg::World&); + inline void ttg_initialize(int argc, char **argv, int num_threads = -1, parsec_context_s * = nullptr); inline void ttg_finalize(); diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h index 5a43e5ce8..b88874b0e 100644 --- a/ttg/ttg/parsec/ttg.h +++ b/ttg/ttg/parsec/ttg.h @@ -469,7 +469,7 @@ namespace ttg_parsec { #endif }; - static void unregister_parsec_tags(void *_) + inline void unregister_parsec_tags(void *_) { if(NULL != parsec_ce.tag_unregister) { parsec_ce.tag_unregister(WorldImpl::parsec_ttg_tag()); @@ -1035,6 +1035,10 @@ namespace ttg_parsec { value = result; } + inline void make_executable_hook(ttg::World& world) { + MPI_Barrier(world.impl().comm()); + } + /// broadcast /// @tparam T a serializable type template