From 835955e64e3882c32ef1a355e94a907fb4a2ddce Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 13 Oct 2022 16:21:17 -0400
Subject: [PATCH 001/259] Mock of ttg::make_device_tt and views

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/make_device_tt.h | 51 ++++++++++++++++++++++++++++++
 ttg/ttg/make_tt.h        |  2 ++
 ttg/ttg/view.h           | 68 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 121 insertions(+)
 create mode 100644 ttg/ttg/make_device_tt.h
 create mode 100644 ttg/ttg/view.h

diff --git a/ttg/ttg/make_device_tt.h b/ttg/ttg/make_device_tt.h
new file mode 100644
index 000000000..63f4c78eb
--- /dev/null
+++ b/ttg/ttg/make_device_tt.h
@@ -0,0 +1,51 @@
+// to be #include'd within runtime::ttg namespace
+
+#ifndef TTG_MAKE_DEVICE_TT_H
+#define TTG_MAKE_DEVICE_TT_H
+
+#include <tuple>
+#include <vector>
+
+#include "ttg/edge.h"
+
+template <typename DeviceType,
+          typename keyT,
+          typename DevViewFuncT,
+          typename DevKernelFuncT,
+          typename DevOutFuncT,
+          typename... input_edge_valuesT,
+          typename... output_edgesT>
+auto make_tt(DevViewFuncT &&view_func,
+             DevKernelFuncT &&kernel_func,
+             DevOutFuncT &&out_func,
+             const std::tuple<ttg::Edge<keyT, input_edge_valuesT>...> &inedges,
+             const std::tuple<output_edgesT...> &outedges, const std::string &name = "wrapper",
+             const std::vector<std::string> &innames = std::vector<std::string>(sizeof...(input_edge_valuesT), "input"),
+             const std::vector<std::string> &outnames = std::vector<std::string>(sizeof...(output_edgesT), "output")) {
+
+  // TODO:
+}
+
+
+
+template <typename DeviceType,
+          typename keyT,
+          typename HostFuncT,
+          typename DevViewFuncT,
+          typename DevKernelFuncT,
+          typename DevOutFuncT,
+          typename... input_edge_valuesT,
+          typename... output_edgesT>
+auto make_tt(HostFuncT &&host_func,
+             DevViewFuncT &&view_func,
+             DevKernelFuncT &&kernel_func,
+             DevOutFuncT &&out_func,
+             const std::tuple<ttg::Edge<keyT, input_edge_valuesT>...> &inedges,
+             const std::tuple<output_edgesT...> &outedges, const std::string &name = "wrapper",
+             const std::vector<std::string> &innames = std::vector<std::string>(sizeof...(input_edge_valuesT), "input"),
+             const std::vector<std::string> &outnames = std::vector<std::string>(sizeof...(output_edgesT), "output")) {
+
+  // TODO:
+}
+
+#endif // TTG_MAKE_DEVICE_TT_H
diff --git a/ttg/ttg/make_tt.h b/ttg/ttg/make_tt.h
index eabd3c34e..491d32e01 100644
--- a/ttg/ttg/make_tt.h
+++ b/ttg/ttg/make_tt.h
@@ -517,4 +517,6 @@ template <typename keyT, typename funcT, typename... input_edge_valuesT, typenam
   return make_tt<keyT>(std::forward<funcT>(func), inedges, outedges, name, innames, outnames);
 }
 
+#include "ttg/make_device_tt.h"
+
 #endif  // TTG_MAKE_TT_H
diff --git a/ttg/ttg/view.h b/ttg/ttg/view.h
new file mode 100644
index 000000000..1eee6856e
--- /dev/null
+++ b/ttg/ttg/view.h
@@ -0,0 +1,68 @@
+#ifndef TTG_VIEW_H
+#define TTG_VIEW_H
+
+#include <array>
+#include "ttg/util/iovec.h"
+
+namespace ttg {
+
+  namespace detail {
+
+    template<typename T>
+    struct typed_iov {
+      T* ptr;
+      std::size_t size;
+    };
+
+    template<typename... Ts>
+    struct typed_iovs {
+      std::tuple<typed_iov<Ts...>> iovs;
+    };
+
+  } // namespace detail
+
+  template<typename HostT, typename... DevTs>
+  struct view_t {
+
+    view_t(HostT& obj, detail::typed_iovs<DevTs...> iovs)
+    : m_obj(obj)
+    , m_iovs(iovs)
+    { }
+
+    template<std::size_t i>
+    auto get_device_ptr() {
+      return std::get<i>(m_iovs).ptr;
+    }
+
+    template<std::size_t i>
+    std::size_t get_device_size() {
+      return std::get<i>(m_iovs).size;
+    }
+
+    HostT& get_host_object() {
+      return m_obj;
+    }
+
+  private:
+    HostT& m_obj;
+    detail::typed_iovs<DevTs...> m_iovs;
+
+  };
+
+  template<typename HostT, typename... DevTs>
+  auto make_view(HostT&& obj, detail::typed_iov<DevTs...> iovs) {
+    /* TODO: allocate memory on the device and transfer the data to it */
+    return view_t(obj, std::move(iovs));
+  }
+
+  template<typename HostT, typename... DevTs>
+  auto new_view(HostT&& obj, detail::typed_iov<DevTs...> iovs) {
+    /* TODO: allocate memory on the device, no copying needed */
+    return view_t(obj, std::move(iovs));
+  }
+
+
+} // namespace ttg
+
+
+#endif // TTG_VIEW_H

From 7b6319b0fa80eca3b74664e6a93ef78cea959cef Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 14 Oct 2022 14:34:00 -0400
Subject: [PATCH 002/259] Add new header to CMake files

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/CMakeLists.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ttg/CMakeLists.txt b/ttg/CMakeLists.txt
index 96429ceca..b77a264f8 100644
--- a/ttg/CMakeLists.txt
+++ b/ttg/CMakeLists.txt
@@ -53,6 +53,8 @@ set(ttg-impl-headers
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/traverse.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/world.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/make_tt.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/ttg/make_device_tt.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/ttg/view.h
     )
 set(ttg-headers
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg.h

From 25d8c3c343f5a0d038ee387320ccdffc49303b01 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 20 Oct 2022 15:57:00 -0400
Subject: [PATCH 003/259] Add device_mock example to compile the ttg device
 mock API

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/CMakeLists.txt                       |   2 +
 examples/device_mock/device_mock.cc           | 194 ++++++++++++++++++
 ttg/ttg/make_device_tt.h                      |  49 ++---
 ttg/ttg/parsec/ttg.h                          |   2 +
 .../serialization/splitmd_data_descriptor.h   |  12 +-
 ttg/ttg/util/iovec.h                          |  21 ++
 ttg/ttg/view.h                                |  35 ++--
 7 files changed, 262 insertions(+), 53 deletions(-)
 create mode 100644 examples/device_mock/device_mock.cc
 create mode 100644 ttg/ttg/util/iovec.h

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 4fc51ce3d..2ca2e3214 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -37,6 +37,8 @@ add_ttg_executable(testing_dtrtri potrf/testing_dtrtri.cc LINK_LIBRARIES lapackp
 add_ttg_executable(testing_dlauum potrf/testing_dlauum.cc LINK_LIBRARIES lapackpp)
 add_ttg_executable(testing_dpoinv potrf/testing_dpoinv.cc LINK_LIBRARIES lapackpp)
 
+add_ttg_executable(device_mock device_mock/device_mock.cc LINK_LIBRARIES MADworld)
+
 if (OpenMP_CXX_FOUND AND TARGET std::execution)
         add_ttg_executable(fw-apsp-df floyd-warshall/floyd_warshall_df.cc LINK_LIBRARIES OpenMP::OpenMP_CXX std::execution MADworld)
 endif ()
diff --git a/examples/device_mock/device_mock.cc b/examples/device_mock/device_mock.cc
new file mode 100644
index 000000000..9dd11e832
--- /dev/null
+++ b/examples/device_mock/device_mock.cc
@@ -0,0 +1,194 @@
+//#define TTG_USE_PARSEC 1
+
+#ifdef TTG_USE_PARSEC
+#endif // TTG_USE_PARSEC
+
+#define USE_PARSEC_PROF_API 0
+
+#include <ttg.h>
+#include "../matrixtile.h"
+
+#include <parsec.h>
+#include <parsec/data_internal.h>
+#include <parsec/data_dist/matrix/matrix.h>
+#include <parsec/data_dist/matrix/sym_two_dim_rectangle_cyclic.h>
+#include <parsec/data_dist/matrix/two_dim_rectangle_cyclic.h>
+// needed for madness::hashT and xterm_debug
+#include <madness/world/world.h>
+
+using Key2 = std::tuple<int, int>;
+
+using Key3 = std::tuple<int, int, int>;
+
+/* number of tiles */
+#define KT 100
+
+static void
+dplasma_dprint_tile( int m, int n,
+                     const parsec_tiled_matrix_dc_t* descA,
+                     const double *M );
+
+
+template<typename T>
+auto make_gemm(ttg::Edge<Key2, MatrixTile<T>>& A,
+               ttg::Edge<Key2, MatrixTile<T>>& B,
+               ttg::Edge<Key2, MatrixTile<T>>& output_result)
+{
+
+  ttg::Edge<Key3, MatrixTile<double>> C;
+  auto f_cpu = [=](const Key3& key,
+                   const MatrixTile<T>&  A,
+                   const MatrixTile<T>&  B,
+                   MatrixTile<T>&  C,
+                   std::tuple<ttg::Out<Key2, MatrixTile<T>>,
+                              ttg::Out<Key3, MatrixTile<T>>>& out)
+  {
+    int m = std::get<0>(key);
+    int n = std::get<1>(key);
+    int k = std::get<2>(key);
+
+    /*
+    if(k == 0) {
+      dlprng(C.data(), 1789, A.mb()*B.nb());
+    }
+    dgemm(A.data(), A.mb(), A.nb(),
+          B.data(), B.mb(), B.nb(),
+          1.0,
+          C.data(), C.nb(), C.nb());
+    */
+
+    if( k == KT-1 || C.data()[0] < 1e-9 ) {
+      ttg::send<0>(Key2{m, n}, std::move(C));
+    } else {
+      ttg::send<1>(Key3{m, n, k+1}, std::move(C));
+    }
+  };
+
+  auto f_gpu_host_views = [=](const Key3& key,
+                              const MatrixTile<T>& A,
+                                const MatrixTile<T>& B,
+                              MatrixTile<T>& C)
+  {
+    ttg::View<const MatrixTile<T>, const T> dev_A = ttg::make_view( A, std::make_tuple(ttg::span(A.data(), A.size())) );
+    ttg::View<const MatrixTile<T>, const T> dev_B = ttg::make_view( B, std::make_tuple(ttg::span(B.data(), B.size())) );
+    ttg::View<MatrixTile<T>, T> dev_C;
+    ttg::View<T, T> dev_tmp;
+    T *host_tmp = new(T);
+    dev_tmp = ttg::new_view( *host_tmp, std::make_tuple(ttg::span(host_tmp, 1)) ); // dev_tmp is a promise of 1 T on the device, associated with host_tmp
+
+    int k = std::get<2>(key);
+    if(0 == k) {
+      // view_new tells the runtime system that the device view needs to be allocated but doesn't need to be
+      // initialized with C.data(). However, C.data() is still associated with the device memory, so if the
+      // runtime system evicts that data from the device, it will be first copied back into C.data().
+      dev_C = ttg::new_view( C, std::make_tuple(ttg::span(C.data(), C.size())) );
+    } else {
+      dev_C = ttg::make_view( C, std::make_tuple(ttg::span(C.data(), C.size())) );
+    }
+
+    return std::make_tuple(dev_A, dev_B, dev_C, dev_tmp);
+  };
+
+  auto f_gpu_kernel = [=](const Key3& key,
+                                ttg::View<const MatrixTile<T>, const T>& dev_A,
+                                ttg::View<const MatrixTile<T>, const T>& dev_B,
+                                ttg::View<MatrixTile<T>, T>& dev_C,
+                          ttg::View<T, T>& dev_tmp)
+  {
+    int k = std::get<2>(key);
+
+    const MatrixTile<T>&  A = dev_A.get_host_object();
+    const MatrixTile<T>&  B = dev_B.get_host_object();
+    MatrixTile<T>&  C = dev_C.get_host_object();
+    T& host_tmp = dev_tmp.get_host_object();
+    auto beta = 1.0;
+    if(k == 0) {
+        //cublasDplrng(dev_C.get(0), C.mb(), C.nb());
+    }
+
+    /*
+    cublasDgemm(dev_A.get(0), A.mb(), A.nb(),
+                dev_B.get(0), B.mb(), B.nb(),
+                beta,
+                dev_C.get(0), C.mb(), C.nb());
+
+    cudaMemcpyAsync(&dev_C.get(0)[0], host_tmp, sizeof(T), cudaDeviceToHost);
+    */
+  };
+
+  auto f_gpu_output_flows = [=](const Key3& key,
+                                            const MatrixTile<T>&  A,
+                                            const MatrixTile<T>&  B,
+                                            MatrixTile<T>&  C,
+                                    T& host_tmp,
+                                  std::tuple<ttg::Out<Key2, MatrixTile<T>>,
+                                           ttg::Out<Key3, MatrixTile<T>>>& out)
+  {
+    int m = std::get<0>(key);
+    int n = std::get<1>(key);
+    int k = std::get<2>(key);
+
+    if( k == KT-1 || host_tmp < 1e-9 ) {
+        ttg::send<0>(Key2{m, n}, std::move(C));
+    } else {
+      ttg::send<1>(Key3{m, n, k+1}, std::move(C));
+    }
+    delete &host_tmp;
+  };
+
+  /* If we only have GPU */
+  auto gemm_tt = ttg::make_device_tt(f_gpu_host_views, f_gpu_kernel, f_gpu_output_flows, ttg::ExecutionSpace::CUDA,
+                                     ttg::edges(A, B), ttg::edges(output_result, C),
+                                     "GEMM", {"A", "B"}, {"output_result", "C"});
+
+#if 0
+  /* Alternative: to get both type of tasklets: */
+  auto gemm_tt = ttg::make_device_tt(f_cpu, f_gpu_host_views, f_gpu_kernel, f_gpu_output_flows, ttg::ExecutionSpace::CUDA,
+                                     ttg::edges(A, B), ttg::edges(output_result, C),
+                                     "GEMM", {"A", "B"}, {"output_result", "C"});
+#endif
+  return gemm_tt;
+}
+
+int main(int argc, char **argv)
+{
+
+  std::chrono::time_point<std::chrono::high_resolution_clock> beg, end;
+  int N = 1024;
+  int M = N;
+  int NB = 128;
+  int check = 0;
+  int nthreads = -1;
+  const char* prof_filename = nullptr;
+
+  if (argc > 1) {
+    N = M = atoi(argv[1]);
+  }
+
+  if (argc > 2) {
+    NB = atoi(argv[2]);
+  }
+
+  if (argc > 3) {
+    check = atoi(argv[3]);
+  }
+
+  if (argc > 4) {
+    nthreads = atoi(argv[4]);
+  }
+
+  ttg::initialize(argc, argv, nthreads);
+
+  auto world = ttg::default_execution_context();
+
+  ttg::Edge<Key2, MatrixTile<double>> edge_a, edge_b;
+  ttg::Edge<Key2, MatrixTile<double>> edge_out;
+
+  auto gemm_tt = make_gemm(edge_a, edge_b, edge_out);
+
+
+
+  ttg::finalize();
+  return 0;
+}
+
diff --git a/ttg/ttg/make_device_tt.h b/ttg/ttg/make_device_tt.h
index 63f4c78eb..6639f679f 100644
--- a/ttg/ttg/make_device_tt.h
+++ b/ttg/ttg/make_device_tt.h
@@ -1,51 +1,46 @@
 // to be #include'd within runtime::ttg namespace
 
-#ifndef TTG_MAKE_DEVICE_TT_H
-#define TTG_MAKE_DEVICE_TT_H
 
-#include <tuple>
-#include <vector>
-
-#include "ttg/edge.h"
-
-template <typename DeviceType,
-          typename keyT,
+template <typename keyT,
           typename DevViewFuncT,
           typename DevKernelFuncT,
           typename DevOutFuncT,
           typename... input_edge_valuesT,
           typename... output_edgesT>
-auto make_tt(DevViewFuncT &&view_func,
-             DevKernelFuncT &&kernel_func,
-             DevOutFuncT &&out_func,
-             const std::tuple<ttg::Edge<keyT, input_edge_valuesT>...> &inedges,
-             const std::tuple<output_edgesT...> &outedges, const std::string &name = "wrapper",
-             const std::vector<std::string> &innames = std::vector<std::string>(sizeof...(input_edge_valuesT), "input"),
-             const std::vector<std::string> &outnames = std::vector<std::string>(sizeof...(output_edgesT), "output")) {
+auto make_device_tt(DevViewFuncT &&view_func,
+                    DevKernelFuncT &&kernel_func,
+                    DevOutFuncT &&out_func,
+                    ttg::ExecutionSpace space,
+                    const std::tuple<ttg::Edge<keyT, input_edge_valuesT>...> &inedges,
+                    const std::tuple<output_edgesT...> &outedges, const std::string &name = "wrapper",
+                    const std::vector<std::string> &innames = std::vector<std::string>(sizeof...(input_edge_valuesT), "input"),
+                    const std::vector<std::string> &outnames = std::vector<std::string>(sizeof...(output_edgesT), "output")) {
 
   // TODO:
+
+  return ttg::Void();
 }
 
 
 
-template <typename DeviceType,
-          typename keyT,
+template <typename keyT,
           typename HostFuncT,
           typename DevViewFuncT,
           typename DevKernelFuncT,
           typename DevOutFuncT,
           typename... input_edge_valuesT,
           typename... output_edgesT>
-auto make_tt(HostFuncT &&host_func,
-             DevViewFuncT &&view_func,
-             DevKernelFuncT &&kernel_func,
-             DevOutFuncT &&out_func,
-             const std::tuple<ttg::Edge<keyT, input_edge_valuesT>...> &inedges,
-             const std::tuple<output_edgesT...> &outedges, const std::string &name = "wrapper",
-             const std::vector<std::string> &innames = std::vector<std::string>(sizeof...(input_edge_valuesT), "input"),
-             const std::vector<std::string> &outnames = std::vector<std::string>(sizeof...(output_edgesT), "output")) {
+auto make_device_tt(HostFuncT &&host_func,
+                    DevViewFuncT &&view_func,
+                    DevKernelFuncT &&kernel_func,
+                    DevOutFuncT &&out_func,
+                    ttg::ExecutionSpace space,
+                    const std::tuple<ttg::Edge<keyT, input_edge_valuesT>...> &inedges,
+                    const std::tuple<output_edgesT...> &outedges, const std::string &name = "wrapper",
+                    const std::vector<std::string> &innames = std::vector<std::string>(sizeof...(input_edge_valuesT), "input"),
+                    const std::vector<std::string> &outnames = std::vector<std::string>(sizeof...(output_edgesT), "output")) {
 
   // TODO:
+  return ttg::Void();
 }
 
-#endif // TTG_MAKE_DEVICE_TT_H
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index b88874b0e..ef56f4e82 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -92,6 +92,8 @@ void parsec_taskpool_termination_detected(parsec_taskpool_t *tp);
 int parsec_add_fetch_runtime_task(parsec_taskpool_t *tp, int tasks);
 }
 
+#include "ttg/view.h"
+
 namespace ttg_parsec {
   typedef void (*static_set_arg_fct_type)(void *, size_t, ttg::TTBase *);
   typedef std::pair<static_set_arg_fct_type, ttg::TTBase *> static_set_arg_fct_call_t;
diff --git a/ttg/ttg/serialization/splitmd_data_descriptor.h b/ttg/ttg/serialization/splitmd_data_descriptor.h
index 8edfe20d6..46bdb7b76 100644
--- a/ttg/ttg/serialization/splitmd_data_descriptor.h
+++ b/ttg/ttg/serialization/splitmd_data_descriptor.h
@@ -3,20 +3,10 @@
 
 #include <type_traits>
 #include "ttg/util/meta.h"
+#include "ttg/util/iovec.h"
 
 namespace ttg {
 
-  /**
-   * Used to describe transfer payload in types using the \sa SplitMetadataDescriptor.
-   * @c data Pointer to the data to be read from / written to.
-   * @c num_bytes The number of bytes to read from / write to the memory location
-   *                   \sa data.
-   */
-  struct iovec {
-    size_t num_bytes;
-    void* data;
-  };
-
   /**
    * SplitMetadataDescriptor is a serialization descriptor provided by the user
    * for a user-specified type. It should contain the following public member
diff --git a/ttg/ttg/util/iovec.h b/ttg/ttg/util/iovec.h
new file mode 100644
index 000000000..2a8d672b3
--- /dev/null
+++ b/ttg/ttg/util/iovec.h
@@ -0,0 +1,21 @@
+#ifndef TTG_UTIL_IOVEC_H_
+#define TTG_UTIL_IOVEC_H_
+
+#include <cstdint>
+
+namespace ttg {
+
+  /**
+   * Used to describe transfer payload in types using the \sa SplitMetadataDescriptor.
+   * @member data Pointer to the data to be read from / written to.
+   * @member num_bytes The number of bytes to read from / write to the memory location
+   *                   \sa data.
+   */
+  struct iovec {
+    std::size_t num_bytes;
+    void* data;
+  };
+
+} // ttg
+
+#endif // TTG_UTIL_IOVEC_H_
diff --git a/ttg/ttg/view.h b/ttg/ttg/view.h
index 1eee6856e..73a8bf5e0 100644
--- a/ttg/ttg/view.h
+++ b/ttg/ttg/view.h
@@ -22,43 +22,48 @@ namespace ttg {
   } // namespace detail
 
   template<typename HostT, typename... DevTs>
-  struct view_t {
+  struct View {
 
-    view_t(HostT& obj, detail::typed_iovs<DevTs...> iovs)
-    : m_obj(obj)
-    , m_iovs(iovs)
+    View()
+    : m_obj(nullptr)
+    , m_spans(ttg::span<DevTs>(nullptr, std::size_t{0})...)
+    {}
+
+    View(HostT& obj, std::tuple<ttg::span<DevTs>...> spans)
+    : m_obj(&obj)
+    , m_spans(std::move(spans))
     { }
 
     template<std::size_t i>
     auto get_device_ptr() {
-      return std::get<i>(m_iovs).ptr;
+      return std::get<i>(m_spans).data();
     }
 
     template<std::size_t i>
     std::size_t get_device_size() {
-      return std::get<i>(m_iovs).size;
+      return std::get<i>(m_spans).size();
     }
 
     HostT& get_host_object() {
-      return m_obj;
+      return *m_obj;
     }
 
   private:
-    HostT& m_obj;
-    detail::typed_iovs<DevTs...> m_iovs;
+    HostT* m_obj;
+    std::tuple<ttg::span<DevTs>...> m_spans;
 
   };
 
-  template<typename HostT, typename... DevTs>
-  auto make_view(HostT&& obj, detail::typed_iov<DevTs...> iovs) {
+  template<typename HostT, typename... Spans>
+  auto make_view(HostT& obj, std::tuple<Spans...> spans) {
     /* TODO: allocate memory on the device and transfer the data to it */
-    return view_t(obj, std::move(iovs));
+    return View(obj, std::move(spans));
   }
 
-  template<typename HostT, typename... DevTs>
-  auto new_view(HostT&& obj, detail::typed_iov<DevTs...> iovs) {
+  template<typename HostT, typename... Spans>
+  auto new_view(HostT& obj, std::tuple<Spans...> spans) {
     /* TODO: allocate memory on the device, no copying needed */
-    return view_t(obj, std::move(iovs));
+    return View(obj, std::move(spans));
   }
 
 

From 831bc0f09a790b6dda79263611846437481e7315 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Thu, 20 Oct 2022 16:03:24 -0400
Subject: [PATCH 004/259] [example/device_mock] added missing #include

---
 examples/device_mock/device_mock.cc | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/examples/device_mock/device_mock.cc b/examples/device_mock/device_mock.cc
index 9dd11e832..ff37bf922 100644
--- a/examples/device_mock/device_mock.cc
+++ b/examples/device_mock/device_mock.cc
@@ -1,11 +1,6 @@
-//#define TTG_USE_PARSEC 1
-
-#ifdef TTG_USE_PARSEC
-#endif // TTG_USE_PARSEC
-
-#define USE_PARSEC_PROF_API 0
 
 #include <ttg.h>
+#include <ttg/view.h>
 #include "../matrixtile.h"
 
 #include <parsec.h>

From 5618e86b1c6275841efa76a1ddb7d3c1cb55b30b Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Thu, 20 Oct 2022 16:04:13 -0400
Subject: [PATCH 005/259] [example/device_mock] declare as off-limits to
 clang-format

---
 examples/device_mock/device_mock.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/device_mock/device_mock.cc b/examples/device_mock/device_mock.cc
index ff37bf922..ca180a596 100644
--- a/examples/device_mock/device_mock.cc
+++ b/examples/device_mock/device_mock.cc
@@ -1,3 +1,4 @@
+// clang-format off
 
 #include <ttg.h>
 #include <ttg/view.h>
@@ -186,4 +187,3 @@ int main(int argc, char **argv)
   ttg::finalize();
   return 0;
 }
-

From 1e8fb69270a839f4c1ef77cb341f0f9c8cbbe555 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Thu, 20 Oct 2022 16:23:36 -0400
Subject: [PATCH 006/259] [example/device_mock] cleanup

---
 examples/device_mock/device_mock.cc | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/examples/device_mock/device_mock.cc b/examples/device_mock/device_mock.cc
index ca180a596..971d1c3d1 100644
--- a/examples/device_mock/device_mock.cc
+++ b/examples/device_mock/device_mock.cc
@@ -19,12 +19,6 @@ using Key3 = std::tuple<int, int, int>;
 /* number of tiles */
 #define KT 100
 
-static void
-dplasma_dprint_tile( int m, int n,
-                     const parsec_tiled_matrix_dc_t* descA,
-                     const double *M );
-
-
 template<typename T>
 auto make_gemm(ttg::Edge<Key2, MatrixTile<T>>& A,
                ttg::Edge<Key2, MatrixTile<T>>& B,

From 4d2b9373a7818e3c3818962c2965c0a5c416dd9a Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 20 Oct 2022 17:35:43 -0400
Subject: [PATCH 007/259] Partial implementation of ttg::make_device_tt

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/make_device_tt.h | 49 ++++++++++++++++++++++++++++++++++++++--
 ttg/ttg/view.h           |  4 ++++
 2 files changed, 51 insertions(+), 2 deletions(-)

diff --git a/ttg/ttg/make_device_tt.h b/ttg/ttg/make_device_tt.h
index 6639f679f..65e9740f0 100644
--- a/ttg/ttg/make_device_tt.h
+++ b/ttg/ttg/make_device_tt.h
@@ -22,6 +22,38 @@ auto make_device_tt(DevViewFuncT &&view_func,
 }
 
 
+namespace detail {
+  template<typename FuncT, typename keyT, typename... Args, std::size_t... Is>
+  void invoke_unpacked_views(FuncT&& func, const keyT& key, std::tuple<Args...>& views, std::index_sequence<Is...>) {
+    func(key, std::get<Is>(views)...);
+  }
+
+  template<typename HostT, typename... DevTs, std::size_t... Is>
+  void allocate_view_on_device(ttg::View<HostT, DevTs...>& view, std::index_sequence<Is...>) {
+
+    /* TODO: allocate memory on device */
+
+    /* TODO: copy data to device */
+
+    allocate_view_on_device(view, std::index_sequence<Is...>);
+  }
+
+  template<typename... ViewTs, std::size_t I, std::size_t... Is>
+  void allocate_on_device(std::tuple<ViewTs...>& views, std::index_sequence<I, Is...>) {
+
+    auto& view = std::get<I>(views);
+    allocate_view_on_device(view, std::make_index_sequence<view.size()>());
+
+    allocate_on_device<ViewTs..., Is...>();
+  }
+
+  template<typename... ViewTs, std::size_t... Is>
+  void allocate_on_device(std::tuple<ViewTs...>& views, std::index_sequence<Is...>) {
+
+  }
+
+} // namespace detail
+
 
 template <typename keyT,
           typename HostFuncT,
@@ -40,7 +72,20 @@ auto make_device_tt(HostFuncT &&host_func,
                     const std::vector<std::string> &innames = std::vector<std::string>(sizeof...(input_edge_valuesT), "input"),
                     const std::vector<std::string> &outnames = std::vector<std::string>(sizeof...(output_edgesT), "output")) {
 
-  // TODO:
-  return ttg::Void();
+
+
+  auto taskfn = [=](const keyT& key, auto... args){
+    auto views = view_func(key, args...);
+    /* 1) allocate memory on device */
+    auto device_view = views;
+    /* 2) move data from views to device */
+    /* 3) call kernel function */
+    detail::invoke_unpacked_views(key, views, std::index_sequence_for<decltype(views)>());
+    /* 4) move data back out into host objects */
+    /* 5) call output function */
+    out_func(key, args...);
+  };
+
+  return make_tt<keyT>(taskfn, inedges, outedges, innames, outnames);
 }
 
diff --git a/ttg/ttg/view.h b/ttg/ttg/view.h
index 73a8bf5e0..13c6b0e2a 100644
--- a/ttg/ttg/view.h
+++ b/ttg/ttg/view.h
@@ -48,6 +48,10 @@ namespace ttg {
       return *m_obj;
     }
 
+    constexpr static std::size_t size() {
+      return std::tuple_size_v<decltype(m_spans)>;
+    }
+
   private:
     HostT* m_obj;
     std::tuple<ttg::span<DevTs>...> m_spans;

From 11c4d4d769b2e3a595c695921cce1c5934250634 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Wed, 26 Oct 2022 07:04:35 -0400
Subject: [PATCH 008/259] make device_mock compile again, and no need for
 MAWworld for no reason

---
 examples/device_mock/device_mock.cc | 2 --
 ttg/ttg/make_device_tt.h            | 2 +-
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/examples/device_mock/device_mock.cc b/examples/device_mock/device_mock.cc
index 971d1c3d1..e0d6d6c07 100644
--- a/examples/device_mock/device_mock.cc
+++ b/examples/device_mock/device_mock.cc
@@ -9,8 +9,6 @@
 #include <parsec/data_dist/matrix/matrix.h>
 #include <parsec/data_dist/matrix/sym_two_dim_rectangle_cyclic.h>
 #include <parsec/data_dist/matrix/two_dim_rectangle_cyclic.h>
-// needed for madness::hashT and xterm_debug
-#include <madness/world/world.h>
 
 using Key2 = std::tuple<int, int>;
 
diff --git a/ttg/ttg/make_device_tt.h b/ttg/ttg/make_device_tt.h
index 65e9740f0..008285935 100644
--- a/ttg/ttg/make_device_tt.h
+++ b/ttg/ttg/make_device_tt.h
@@ -35,7 +35,7 @@ namespace detail {
 
     /* TODO: copy data to device */
 
-    allocate_view_on_device(view, std::index_sequence<Is...>);
+    allocate_view_on_device(view, std::index_sequence<Is...>{});
   }
 
   template<typename... ViewTs, std::size_t I, std::size_t... Is>

From fa6359ac7ac60eda3121d69fa3cd776a8fa0485b Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 26 Oct 2022 11:45:26 -0400
Subject: [PATCH 009/259] Encapsulate callables passed to make_device_tt mock
 in a regular task

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/make_device_tt.h | 270 ++++++++++++++++++++++++++++++++-------
 ttg/ttg/view.h           |  27 +++-
 2 files changed, 249 insertions(+), 48 deletions(-)

diff --git a/ttg/ttg/make_device_tt.h b/ttg/ttg/make_device_tt.h
index 008285935..f5425e334 100644
--- a/ttg/ttg/make_device_tt.h
+++ b/ttg/ttg/make_device_tt.h
@@ -1,6 +1,170 @@
+#ifndef TTG_MAKE_DEVICE_TT_H
+#define TTG_MAKE_DEVICE_TT_H
+
 // to be #include'd within runtime::ttg namespace
 
 
+namespace detail {
+#ifdef TTG_USE_CUDA
+  inline thread_local cudaStream_t* ts_stream = nullptr;
+#endif // TTG_USE_CUDA
+
+  template<typename FuncT, typename keyT, typename... Args, std::size_t... Is>
+  inline void invoke_with_unpacked_views(FuncT&& func, const keyT& key, std::tuple<Args...>& views, std::index_sequence<Is...>) {
+#ifdef TTG_USE_CUDA
+    func(key, std::get<Is>(views)..., ts_stream);
+#else // TTG_USE_CUDA
+    func(key, std::get<Is>(views)...);
+#endif // TTG_USE_CUDA
+  }
+
+  /* TODO: extract host objects from views */
+  template<typename ViewTupleT>
+  struct host_obj_type;
+
+  template<typename... ViewTs>
+  struct host_obj_type<std::tuple<ViewTs...>> {
+    using type = std::tuple<typename ViewTs::host_type...>;
+  };
+
+  template<typename ViewTupleT>
+  using host_obj_type_t = typename host_obj_type<ViewTupleT>::type;
+
+  template<typename FuncT, typename keyT, typename... ViewTs, typename  std::size_t... Is>
+  inline void invoke_out_with_unpacked_views(FuncT&& func, const keyT& key, std::tuple<ViewTs...> views, std::index_sequence<Is...>) {
+    func(key, std::get<Is>(views).get_host_object()...);
+  }
+
+  template<typename HostT, typename... DevTs, std::size_t I, std::size_t... Is>
+  inline void create_view_on_device(const ttg::View<HostT, DevTs...>& view, std::tuple<ttg::span<DevTs>...>& dev_spans, std::index_sequence<I, Is...>) {
+
+    /* fill in pointers for the device -- we're relying on managed memory for this simple wrapper */
+    typename std::tuple_element_t<I, typename ttg::View<HostT, DevTs...>::span_tuple_type>::element_type *ptr;
+    size_t size;
+    ptr = view.template get_device_ptr<I>();
+    size = view.template get_device_size<I>();
+    //cudaMalloc(&ptr, span.size_bytes());
+    std::get<I>(dev_spans) = ttg::span(ptr, size);
+
+    /* copy data to device */
+    //cudaMemcpy(ptr, span.data(), span.size_bytes(), cudaMemcpyHostToDevice);
+#ifdef TTG_USE_CUDA
+    cudaMemPrefetchAsync(span.data(), span.size_bytes(), 0, *ts_stream);
+#endif // TTG_USE_CUDA
+    if constexpr(sizeof...(Is) > 0) {
+      create_view_on_device(view, dev_spans, std::index_sequence<Is...>{});
+    }
+  }
+
+  template<typename... ViewTs, std::size_t I, std::size_t... Is>
+  inline void create_on_device(std::tuple<ViewTs...>& views, std::tuple<ViewTs...>& dev_views, std::index_sequence<I, Is...>) {
+
+    using view_tuple_t = typename std::tuple<ViewTs...>;
+    auto& view = std::get<I>(views);
+    typename std::tuple_element_t<I, view_tuple_t>::span_tuple_type dev_spans;
+    create_view_on_device(view, dev_spans, std::make_index_sequence<std::tuple_element_t<I, view_tuple_t>::size()>());
+
+    /* set the view for the device */
+    std::get<I>(dev_views) = ttg::make_view(view.get_host_object(), dev_spans);
+    if constexpr(sizeof...(Is) > 0) {
+      create_on_device(views, dev_views, std::index_sequence<Is...>{});
+    }
+  }
+
+  template<typename HostT, typename... ViewTs, typename... DevTs, std::size_t I, std::size_t... Is>
+  inline void sync_view_to_host(const ttg::View<HostT, ViewTs...>& view, std::tuple<ttg::span<DevTs>...>& dev_spans, std::index_sequence<I, Is...>) {
+    /* prefetch back to host */
+
+    void *ptr;
+    auto& span = std::get<I>(dev_spans);
+
+    /* prefetch data from device */
+#ifdef TTG_USE_CUDA
+    cudaMemPrefetchAsync(span.data(), span.size_bytes(), cudaCpuDeviceId, *ts_stream);
+#endif // TTG_USE_CUDA
+
+    if constexpr(sizeof...(Is) > 0) {
+      sync_view_to_host(view, dev_spans, std::index_sequence<Is...>{});
+    }
+  }
+
+  template<typename... ViewTs, std::size_t I, std::size_t... Is>
+  inline void sync_back_to_host(std::tuple<ViewTs...>& views, std::tuple<ViewTs...>& dev_views, std::index_sequence<I, Is...>) {
+
+    auto& view = std::get<I>(views);
+    using view_type = std::remove_reference_t<decltype(view)>;
+    typename std::tuple_element_t<I, std::tuple<ViewTs...>>::span_tuple_type dev_spans;
+    sync_view_to_host(view, dev_spans, std::make_index_sequence<view_type::size()>());
+
+    if constexpr(sizeof...(Is) > 0) {
+      sync_back_to_host(views, dev_views, std::index_sequence<Is...>{});
+    }
+  }
+
+  template<typename keyT,
+           bool have_outterm_tuple,
+           typename DevViewFuncT,
+           typename DevKernelFuncT,
+           typename DevOutFuncT,
+           typename... input_edge_valuesT,
+           typename... output_edgesT,
+           typename... Args>
+  auto make_device_tt_helper(DevViewFuncT &&view_func,
+                             DevKernelFuncT &&kernel_func,
+                             DevOutFuncT &&out_func,
+                             ttg::ExecutionSpace space,
+                             const std::tuple<ttg::Edge<keyT, input_edge_valuesT>...> &inedges,
+                             const std::tuple<output_edgesT...> &outedges,
+                             const std::string &name,
+                             const std::vector<std::string> &innames,
+                             const std::vector<std::string> &outnames,
+                             const ttg::typelist<Args...>& full_input_args) {
+
+    using output_terminals_type = typename ttg::edges_to_output_terminals<std::tuple<output_edgesT...>>::type;
+
+    auto taskfn = [=](const keyT& key, Args... args) mutable {
+
+#ifdef TTG_USE_CUDA
+      if (nullptr == ts_stream) {
+        ts_stream = new cudaStream_t();
+        cudaStreamCreate(ts_stream);
+      }
+#endif // TTG_USE_CUDA
+
+      auto views = view_func(key, std::forward<Args>(args)...);
+      using view_tuple_t = std::remove_reference_t<decltype(views)>;
+      constexpr std::size_t view_tuple_size = std::tuple_size_v<view_tuple_t>;
+      /* 1) allocate memory on device */
+      auto device_views = views;
+      /* 2) move data from views to device */
+      if constexpr(std::tuple_size_v<view_tuple_t> > 0) {
+        create_on_device(views, device_views, std::make_index_sequence<view_tuple_size>());
+      }
+      /* 3) call kernel function */
+      detail::invoke_with_unpacked_views(kernel_func, key, device_views, std::make_index_sequence<view_tuple_size>());
+      /* 4) move data back out into host objects */
+      if constexpr(std::tuple_size_v<view_tuple_t> > 0) {
+        sync_back_to_host(views, device_views, std::make_index_sequence<view_tuple_size>());
+      }
+  #ifdef TTG_USE_CUDA
+      /* wait for the */
+      cudaStreamSynchronize(*ts_stream);
+  #endif // TTG_USE_CUDA
+      /* 5) call output function */
+      detail::invoke_out_with_unpacked_views(out_func, key, views, std::make_index_sequence<view_tuple_size>());
+    };
+
+  using wrapT = typename CallableWrapTTArgsAsTypelist<decltype(taskfn), have_outterm_tuple, keyT, output_terminals_type,
+                                                      ttg::typelist<Args...>>::type;
+
+  return std::make_unique<wrapT>(std::move(taskfn), inedges, outedges, name, innames, outnames);
+
+  }
+
+
+} // namespace detail
+
+
 template <typename keyT,
           typename DevViewFuncT,
           typename DevKernelFuncT,
@@ -16,45 +180,73 @@ auto make_device_tt(DevViewFuncT &&view_func,
                     const std::vector<std::string> &innames = std::vector<std::string>(sizeof...(input_edge_valuesT), "input"),
                     const std::vector<std::string> &outnames = std::vector<std::string>(sizeof...(output_edgesT), "output")) {
 
-  // TODO:
-
-  return ttg::Void();
-}
-
-
-namespace detail {
-  template<typename FuncT, typename keyT, typename... Args, std::size_t... Is>
-  void invoke_unpacked_views(FuncT&& func, const keyT& key, std::tuple<Args...>& views, std::index_sequence<Is...>) {
-    func(key, std::get<Is>(views)...);
-  }
-
-  template<typename HostT, typename... DevTs, std::size_t... Is>
-  void allocate_view_on_device(ttg::View<HostT, DevTs...>& view, std::index_sequence<Is...>) {
+  using output_terminals_type = typename ttg::edges_to_output_terminals<std::tuple<output_edgesT...>>::type;
 
-    /* TODO: allocate memory on device */
+  constexpr auto void_key = ttg::meta::is_void_v<keyT>;
 
-    /* TODO: copy data to device */
+  // gross list of candidate argument types
+  using gross_candidate_func_args_t = ttg::meta::typelist<
+      ttg::meta::candidate_argument_bindings_t<std::add_const_t<keyT>>,
+      ttg::meta::candidate_argument_bindings_t<typename ttg::Edge<keyT, input_edge_valuesT>::value_type>...,
+      ttg::meta::typelist<output_terminals_type &, void>>;
 
-    allocate_view_on_device(view, std::index_sequence<Is...>{});
-  }
+  // net list of candidate argument types excludes the empty typelists for void arguments
+  using candidate_func_args_t = ttg::meta::filter_t<gross_candidate_func_args_t, ttg::meta::typelist_is_not_empty>;
 
-  template<typename... ViewTs, std::size_t I, std::size_t... Is>
-  void allocate_on_device(std::tuple<ViewTs...>& views, std::index_sequence<I, Is...>) {
+  // gross argument typelist for invoking func, can include void for optional args
+  constexpr static auto func_is_generic = ttg::meta::is_generic_callable_v<DevViewFuncT>;
+  using gross_func_args_t = decltype(ttg::meta::compute_arg_binding_types_r<void>(view_func, candidate_func_args_t{}));
+  constexpr auto DETECTED_HOW_TO_INVOKE_GENERIC_FUNC =
+      func_is_generic ? !std::is_same_v<gross_func_args_t, ttg::typelist<>> : true;
+  static_assert(DETECTED_HOW_TO_INVOKE_GENERIC_FUNC,
+                "ttd::make_tt(func, inedges, ...): could not detect how to invoke generic callable func, either the "
+                "signature of func "
+                "is faulty, or inedges does match the expected list of types, or both");
 
-    auto& view = std::get<I>(views);
-    allocate_view_on_device(view, std::make_index_sequence<view.size()>());
+  // net argument typelist
+  using func_args_t = ttg::meta::drop_void_t<gross_func_args_t>;
+  constexpr auto num_args = std::tuple_size_v<func_args_t>;
 
-    allocate_on_device<ViewTs..., Is...>();
-  }
+  // if given task id, make sure it's passed via const lvalue ref
+  constexpr bool TASK_ID_PASSED_AS_CONST_LVALUE_REF =
+      !void_key ? ttg::meta::probe_first_v<ttg::meta::is_const_lvalue_reference, true, func_args_t> : true;
+  static_assert(TASK_ID_PASSED_AS_CONST_LVALUE_REF,
+                "ttg::make_tt(func, ...): if given to func, the task id must be passed by const lvalue ref");
 
-  template<typename... ViewTs, std::size_t... Is>
-  void allocate_on_device(std::tuple<ViewTs...>& views, std::index_sequence<Is...>) {
+  // if given out-terminal tuple, make sure it's passed via nonconst lvalue ref
+  constexpr bool have_outterm_tuple =
+      func_is_generic ? !ttg::meta::is_last_void_v<gross_func_args_t>
+                      : ttg::meta::probe_last_v<ttg::meta::decays_to_output_terminal_tuple, false, gross_func_args_t>;
+  constexpr bool OUTTERM_TUPLE_PASSED_AS_NONCONST_LVALUE_REF =
+      have_outterm_tuple ? ttg::meta::probe_last_v<ttg::meta::is_nonconst_lvalue_reference, false, func_args_t> : true;
+  static_assert(
+      OUTTERM_TUPLE_PASSED_AS_NONCONST_LVALUE_REF,
+      "ttg::make_tt(func, ...): if given to func, the output terminal tuple must be passed by nonconst lvalue ref");
 
-  }
-
-} // namespace detail
+  // TT needs actual types of arguments to func ... extract them and pass to CallableWrapTTArgs
+  using input_edge_value_types = ttg::meta::typelist<std::decay_t<input_edge_valuesT>...>;
+  // input_args_t = {input_valuesT&&...}
+  using input_args_t = typename ttg::meta::take_first_n<
+      typename ttg::meta::drop_first_n<func_args_t, std::size_t(void_key ? 0 : 1)>::type,
+      std::tuple_size_v<func_args_t> - (void_key ? 0 : 1) - (have_outterm_tuple ? 1 : 0)>::type;
+  constexpr auto NO_ARGUMENTS_PASSED_AS_NONCONST_LVALUE_REF =
+      !ttg::meta::is_any_nonconst_lvalue_reference_v<input_args_t>;
+  static_assert(
+      NO_ARGUMENTS_PASSED_AS_NONCONST_LVALUE_REF,
+      "ttg::make_tt(func, inedges, outedges): one or more arguments to func can only be passed by nonconst lvalue "
+      "ref; this is illegal, should only pass arguments as const lavlue ref or (nonconst) rvalue ref");
+  using decayed_input_args_t = ttg::meta::decayed_typelist_t<input_args_t>;
+  // 3. full_input_args_t = edge-types with non-void types replaced by input_args_t
+  using full_input_args_t = ttg::meta::replace_nonvoid_t<input_edge_value_types, input_args_t>;
 
+  return detail::make_device_tt_helper<keyT, have_outterm_tuple>(std::forward<DevViewFuncT>(view_func),
+                                                                 std::forward<DevKernelFuncT>(kernel_func),
+                                                                 std::forward<DevOutFuncT>(out_func),
+                                                                 space, inedges, outedges, name, innames, outnames,
+                                                                 full_input_args_t{});
+}
 
+#if 0
 template <typename keyT,
           typename HostFuncT,
           typename DevViewFuncT,
@@ -72,20 +264,8 @@ auto make_device_tt(HostFuncT &&host_func,
                     const std::vector<std::string> &innames = std::vector<std::string>(sizeof...(input_edge_valuesT), "input"),
                     const std::vector<std::string> &outnames = std::vector<std::string>(sizeof...(output_edgesT), "output")) {
 
-
-
-  auto taskfn = [=](const keyT& key, auto... args){
-    auto views = view_func(key, args...);
-    /* 1) allocate memory on device */
-    auto device_view = views;
-    /* 2) move data from views to device */
-    /* 3) call kernel function */
-    detail::invoke_unpacked_views(key, views, std::index_sequence_for<decltype(views)>());
-    /* 4) move data back out into host objects */
-    /* 5) call output function */
-    out_func(key, args...);
-  };
-
-  return make_tt<keyT>(taskfn, inedges, outedges, innames, outnames);
+  /* drop the host function */
+  return make_device_tt(view_func, kernel_func, out_func, space, inedges, outedges, name, innames, outnames);
 }
-
+#endif // 0
+#endif // TTG_MAKE_DEVICE_TT_H
diff --git a/ttg/ttg/view.h b/ttg/ttg/view.h
index 13c6b0e2a..1723482c7 100644
--- a/ttg/ttg/view.h
+++ b/ttg/ttg/view.h
@@ -24,23 +24,40 @@ namespace ttg {
   template<typename HostT, typename... DevTs>
   struct View {
 
+    using span_tuple_type = std::tuple<ttg::span<DevTs>...>;
+    using host_type = HostT;
+
+    using view_type = View<HostT, DevTs...>;
+
     View()
     : m_obj(nullptr)
     , m_spans(ttg::span<DevTs>(nullptr, std::size_t{0})...)
     {}
 
-    View(HostT& obj, std::tuple<ttg::span<DevTs>...> spans)
+    View(HostT& obj, span_tuple_type spans)
     : m_obj(&obj)
     , m_spans(std::move(spans))
     { }
 
+    View(view_type&&) = default;
+
+    View(const view_type&) = default;
+
+    view_type& operator=(view_type&&) = default;
+    view_type& operator=(const view_type&) = default;
+
     template<std::size_t i>
     auto get_device_ptr() {
       return std::get<i>(m_spans).data();
     }
 
     template<std::size_t i>
-    std::size_t get_device_size() {
+    const auto get_device_ptr() const {
+      return std::get<i>(m_spans).data();
+    }
+
+    template<std::size_t i>
+    std::size_t get_device_size() const {
       return std::get<i>(m_spans).size();
     }
 
@@ -48,13 +65,17 @@ namespace ttg {
       return *m_obj;
     }
 
+    const HostT& get_host_object() const {
+      return *m_obj;
+    }
+
     constexpr static std::size_t size() {
       return std::tuple_size_v<decltype(m_spans)>;
     }
 
   private:
     HostT* m_obj;
-    std::tuple<ttg::span<DevTs>...> m_spans;
+    span_tuple_type m_spans;
 
   };
 

From 0317069de76edaaafe594392450c014a729d3b59 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 26 Oct 2022 11:50:49 -0400
Subject: [PATCH 010/259] Use the new multiindex key type in device mock
 example

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/device_mock/device_mock.cc | 49 +++++++++++++++--------------
 1 file changed, 26 insertions(+), 23 deletions(-)

diff --git a/examples/device_mock/device_mock.cc b/examples/device_mock/device_mock.cc
index e0d6d6c07..21bbbc7b3 100644
--- a/examples/device_mock/device_mock.cc
+++ b/examples/device_mock/device_mock.cc
@@ -10,16 +10,19 @@
 #include <parsec/data_dist/matrix/sym_two_dim_rectangle_cyclic.h>
 #include <parsec/data_dist/matrix/two_dim_rectangle_cyclic.h>
 
-using Key2 = std::tuple<int, int>;
+#include "ttg/util/meta.h"
 
-using Key3 = std::tuple<int, int, int>;
+#include <ttg/util/multiindex.h>
+
+using Key2 = ttg::MultiIndex<2>;
+using Key3 = ttg::MultiIndex<3>;
 
 /* number of tiles */
 #define KT 100
 
 template<typename T>
-auto make_gemm(ttg::Edge<Key2, MatrixTile<T>>& A,
-               ttg::Edge<Key2, MatrixTile<T>>& B,
+auto make_gemm(ttg::Edge<Key3, MatrixTile<T>>& A,
+               ttg::Edge<Key3, MatrixTile<T>>& B,
                ttg::Edge<Key2, MatrixTile<T>>& output_result)
 {
 
@@ -31,9 +34,9 @@ auto make_gemm(ttg::Edge<Key2, MatrixTile<T>>& A,
                    std::tuple<ttg::Out<Key2, MatrixTile<T>>,
                               ttg::Out<Key3, MatrixTile<T>>>& out)
   {
-    int m = std::get<0>(key);
-    int n = std::get<1>(key);
-    int k = std::get<2>(key);
+    int m = key[0];
+    int n = key[1];
+    int k = key[2];
 
     /*
     if(k == 0) {
@@ -54,8 +57,8 @@ auto make_gemm(ttg::Edge<Key2, MatrixTile<T>>& A,
 
   auto f_gpu_host_views = [=](const Key3& key,
                               const MatrixTile<T>& A,
-                                const MatrixTile<T>& B,
-                              MatrixTile<T>& C)
+                              const MatrixTile<T>& B,
+                              MatrixTile<T>&& C)
   {
     ttg::View<const MatrixTile<T>, const T> dev_A = ttg::make_view( A, std::make_tuple(ttg::span(A.data(), A.size())) );
     ttg::View<const MatrixTile<T>, const T> dev_B = ttg::make_view( B, std::make_tuple(ttg::span(B.data(), B.size())) );
@@ -64,7 +67,7 @@ auto make_gemm(ttg::Edge<Key2, MatrixTile<T>>& A,
     T *host_tmp = new(T);
     dev_tmp = ttg::new_view( *host_tmp, std::make_tuple(ttg::span(host_tmp, 1)) ); // dev_tmp is a promise of 1 T on the device, associated with host_tmp
 
-    int k = std::get<2>(key);
+    int k = key[2];
     if(0 == k) {
       // view_new tells the runtime system that the device view needs to be allocated but doesn't need to be
       // initialized with C.data(). However, C.data() is still associated with the device memory, so if the
@@ -83,7 +86,7 @@ auto make_gemm(ttg::Edge<Key2, MatrixTile<T>>& A,
                                 ttg::View<MatrixTile<T>, T>& dev_C,
                           ttg::View<T, T>& dev_tmp)
   {
-    int k = std::get<2>(key);
+    int k = key[2];
 
     const MatrixTile<T>&  A = dev_A.get_host_object();
     const MatrixTile<T>&  B = dev_B.get_host_object();
@@ -105,16 +108,14 @@ auto make_gemm(ttg::Edge<Key2, MatrixTile<T>>& A,
   };
 
   auto f_gpu_output_flows = [=](const Key3& key,
-                                            const MatrixTile<T>&  A,
-                                            const MatrixTile<T>&  B,
-                                            MatrixTile<T>&  C,
-                                    T& host_tmp,
-                                  std::tuple<ttg::Out<Key2, MatrixTile<T>>,
-                                           ttg::Out<Key3, MatrixTile<T>>>& out)
+                                const MatrixTile<T>&  A,
+                                const MatrixTile<T>&  B,
+                                MatrixTile<T>&  C,
+                                T& host_tmp)
   {
-    int m = std::get<0>(key);
-    int n = std::get<1>(key);
-    int k = std::get<2>(key);
+    int m = key[0];
+    int n = key[1];
+    int k = key[2];
 
     if( k == KT-1 || host_tmp < 1e-9 ) {
         ttg::send<0>(Key2{m, n}, std::move(C));
@@ -124,9 +125,11 @@ auto make_gemm(ttg::Edge<Key2, MatrixTile<T>>& A,
     delete &host_tmp;
   };
 
+  //ttg::meta::type_printer<decltype(ttg::edges(output_result, C))> x;
+
   /* If we only have GPU */
-  auto gemm_tt = ttg::make_device_tt(f_gpu_host_views, f_gpu_kernel, f_gpu_output_flows, ttg::ExecutionSpace::CUDA,
-                                     ttg::edges(A, B), ttg::edges(output_result, C),
+  auto gemm_tt = ttg::make_device_tt<Key3>(f_gpu_host_views, f_gpu_kernel, f_gpu_output_flows, ttg::ExecutionSpace::CUDA,
+                                     ttg::edges(A, B, C), ttg::edges(output_result, C),
                                      "GEMM", {"A", "B"}, {"output_result", "C"});
 
 #if 0
@@ -169,7 +172,7 @@ int main(int argc, char **argv)
 
   auto world = ttg::default_execution_context();
 
-  ttg::Edge<Key2, MatrixTile<double>> edge_a, edge_b;
+  ttg::Edge<Key3, MatrixTile<double>> edge_a, edge_b;
   ttg::Edge<Key2, MatrixTile<double>> edge_out;
 
   auto gemm_tt = make_gemm(edge_a, edge_b, edge_out);

From ffef761c5503d753e7f0663cc7796aa533c63dcb Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 26 Oct 2022 11:53:34 -0400
Subject: [PATCH 011/259] Remove superfluous todos

The views on the device will be handled by the backend, not in ttg::make_view

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/view.h | 2 --
 1 file changed, 2 deletions(-)

diff --git a/ttg/ttg/view.h b/ttg/ttg/view.h
index 1723482c7..404f453b1 100644
--- a/ttg/ttg/view.h
+++ b/ttg/ttg/view.h
@@ -81,13 +81,11 @@ namespace ttg {
 
   template<typename HostT, typename... Spans>
   auto make_view(HostT& obj, std::tuple<Spans...> spans) {
-    /* TODO: allocate memory on the device and transfer the data to it */
     return View(obj, std::move(spans));
   }
 
   template<typename HostT, typename... Spans>
   auto new_view(HostT& obj, std::tuple<Spans...> spans) {
-    /* TODO: allocate memory on the device, no copying needed */
     return View(obj, std::move(spans));
   }
 

From 65668f4fae53691151dd234afbc540118d299e58 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 26 Oct 2022 15:28:49 -0400
Subject: [PATCH 012/259] Provide a scope for views and replace ttg::span with
 ttg::ViewSpan

[ttg|std]::span is not a good fit because it carries a size template
argument that we don't care about but makes handling more difficult.
Plus, we need to encode the scope for each span.

Three scopes are available:
1) SyncIn: copy host data to device before invoking the kernel callable.
2) SyncOut: copy the device data back to the host before invoking the output callable.
3) Allocate: allocate but do not synchronize in or out.

Both SyncIn and SyncOut will allocate sufficient memory on the device.

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/device_mock/device_mock.cc | 16 +++--
 ttg/ttg/make_device_tt.h            | 43 +++++++------
 ttg/ttg/util/meta.h                 |  9 +++
 ttg/ttg/view.h                      | 94 ++++++++++++++++++++---------
 4 files changed, 108 insertions(+), 54 deletions(-)

diff --git a/examples/device_mock/device_mock.cc b/examples/device_mock/device_mock.cc
index 21bbbc7b3..daa6decbd 100644
--- a/examples/device_mock/device_mock.cc
+++ b/examples/device_mock/device_mock.cc
@@ -60,21 +60,25 @@ auto make_gemm(ttg::Edge<Key3, MatrixTile<T>>& A,
                               const MatrixTile<T>& B,
                               MatrixTile<T>&& C)
   {
-    ttg::View<const MatrixTile<T>, const T> dev_A = ttg::make_view( A, std::make_tuple(ttg::span(A.data(), A.size())) );
-    ttg::View<const MatrixTile<T>, const T> dev_B = ttg::make_view( B, std::make_tuple(ttg::span(B.data(), B.size())) );
+    // The default ViewScope::SyncIn scope tells the runtime that the data should be copied
+    // to the device before the kernel callable is invoked.
+    ttg::View<const MatrixTile<T>, const T> dev_A = ttg::make_view( A, ttg::ViewSpan(A.data(), A.size()) );
+    ttg::View<const MatrixTile<T>, const T> dev_B = ttg::make_view( B, ttg::ViewSpan(B.data(), B.size()) );
     ttg::View<MatrixTile<T>, T> dev_C;
     ttg::View<T, T> dev_tmp;
     T *host_tmp = new(T);
-    dev_tmp = ttg::new_view( *host_tmp, std::make_tuple(ttg::span(host_tmp, 1)) ); // dev_tmp is a promise of 1 T on the device, associated with host_tmp
+    // ViewScope::SyncOut tells the runtime system that the view should be synchronized back to the
+    // host before invoking the output callable.
+    dev_tmp = ttg::make_view( *host_tmp, ttg::ViewSpan(host_tmp, 1, ttg::ViewScope::SyncOut) );
 
     int k = key[2];
     if(0 == k) {
-      // view_new tells the runtime system that the device view needs to be allocated but doesn't need to be
+      // ViewScope::Allocate tells the runtime system that the device view needs to be allocated but doesn't need to be
       // initialized with C.data(). However, C.data() is still associated with the device memory, so if the
       // runtime system evicts that data from the device, it will be first copied back into C.data().
-      dev_C = ttg::new_view( C, std::make_tuple(ttg::span(C.data(), C.size())) );
+      dev_C = ttg::make_view( C, ttg::ViewSpan(C.data(), C.size(), ttg::ViewScope::Allocate) );
     } else {
-      dev_C = ttg::make_view( C, std::make_tuple(ttg::span(C.data(), C.size())) );
+      dev_C = ttg::make_view( C, ttg::ViewSpan(C.data(), C.size()) );
     }
 
     return std::make_tuple(dev_A, dev_B, dev_C, dev_tmp);
diff --git a/ttg/ttg/make_device_tt.h b/ttg/ttg/make_device_tt.h
index f5425e334..dbfbdc933 100644
--- a/ttg/ttg/make_device_tt.h
+++ b/ttg/ttg/make_device_tt.h
@@ -36,7 +36,9 @@ namespace detail {
   }
 
   template<typename HostT, typename... DevTs, std::size_t I, std::size_t... Is>
-  inline void create_view_on_device(const ttg::View<HostT, DevTs...>& view, std::tuple<ttg::span<DevTs>...>& dev_spans, std::index_sequence<I, Is...>) {
+  inline void create_view_on_device(const ttg::View<HostT, DevTs...>& view,
+                                    std::tuple<ttg::ViewSpan<DevTs>...>& dev_spans,
+                                    std::index_sequence<I, Is...>) {
 
     /* fill in pointers for the device -- we're relying on managed memory for this simple wrapper */
     typename std::tuple_element_t<I, typename ttg::View<HostT, DevTs...>::span_tuple_type>::element_type *ptr;
@@ -44,18 +46,26 @@ namespace detail {
     ptr = view.template get_device_ptr<I>();
     size = view.template get_device_size<I>();
     //cudaMalloc(&ptr, span.size_bytes());
-    std::get<I>(dev_spans) = ttg::span(ptr, size);
+    std::get<I>(dev_spans) = ttg::ViewSpan(ptr, size, view.template get_scope<I>());
 
     /* copy data to device */
     //cudaMemcpy(ptr, span.data(), span.size_bytes(), cudaMemcpyHostToDevice);
+    if (view.template get_span<I>().is_sync_in()) {
 #ifdef TTG_USE_CUDA
-    cudaMemPrefetchAsync(span.data(), span.size_bytes(), 0, *ts_stream);
+      cudaMemPrefetchAsync(span.data(), span.size_bytes(), 0, *ts_stream);
 #endif // TTG_USE_CUDA
+    }
+
     if constexpr(sizeof...(Is) > 0) {
       create_view_on_device(view, dev_spans, std::index_sequence<Is...>{});
     }
   }
 
+  template<typename HostT, typename... ViewSpanTs, std::size_t... Is>
+  auto make_view_from_tuple(HostT& obj, std::tuple<ttg::ViewSpan<ViewSpanTs>...>& spans, std::index_sequence<Is...>) {
+    return ttg::make_view(obj, std::get<Is>(spans)...);
+  }
+
   template<typename... ViewTs, std::size_t I, std::size_t... Is>
   inline void create_on_device(std::tuple<ViewTs...>& views, std::tuple<ViewTs...>& dev_views, std::index_sequence<I, Is...>) {
 
@@ -65,39 +75,36 @@ namespace detail {
     create_view_on_device(view, dev_spans, std::make_index_sequence<std::tuple_element_t<I, view_tuple_t>::size()>());
 
     /* set the view for the device */
-    std::get<I>(dev_views) = ttg::make_view(view.get_host_object(), dev_spans);
+    std::get<I>(dev_views) = make_view_from_tuple(view.get_host_object(), dev_spans, std::make_index_sequence<std::tuple_size_v<decltype(dev_spans)>>{});
     if constexpr(sizeof...(Is) > 0) {
       create_on_device(views, dev_views, std::index_sequence<Is...>{});
     }
   }
 
-  template<typename HostT, typename... ViewTs, typename... DevTs, std::size_t I, std::size_t... Is>
-  inline void sync_view_to_host(const ttg::View<HostT, ViewTs...>& view, std::tuple<ttg::span<DevTs>...>& dev_spans, std::index_sequence<I, Is...>) {
+  template<typename HostT, typename... DevTs, std::size_t I, std::size_t... Is>
+  inline void sync_view_to_host(ttg::View<HostT, DevTs...>& dev_view, std::index_sequence<I, Is...>) {
     /* prefetch back to host */
-
-    void *ptr;
-    auto& span = std::get<I>(dev_spans);
+    auto span = dev_view.template get_span<I>();
 
     /* prefetch data from device */
+    if (span.is_sync_out()) {
 #ifdef TTG_USE_CUDA
-    cudaMemPrefetchAsync(span.data(), span.size_bytes(), cudaCpuDeviceId, *ts_stream);
+      cudaMemPrefetchAsync(span.data(), span.size_bytes(), cudaCpuDeviceId, *ts_stream);
 #endif // TTG_USE_CUDA
+    }
 
     if constexpr(sizeof...(Is) > 0) {
-      sync_view_to_host(view, dev_spans, std::index_sequence<Is...>{});
+      sync_view_to_host(dev_view, std::index_sequence<Is...>{});
     }
   }
 
   template<typename... ViewTs, std::size_t I, std::size_t... Is>
-  inline void sync_back_to_host(std::tuple<ViewTs...>& views, std::tuple<ViewTs...>& dev_views, std::index_sequence<I, Is...>) {
+  inline void sync_back_to_host(std::tuple<ViewTs...>& dev_views, std::index_sequence<I, Is...>) {
 
-    auto& view = std::get<I>(views);
-    using view_type = std::remove_reference_t<decltype(view)>;
-    typename std::tuple_element_t<I, std::tuple<ViewTs...>>::span_tuple_type dev_spans;
-    sync_view_to_host(view, dev_spans, std::make_index_sequence<view_type::size()>());
+    sync_view_to_host(std::get<I>(dev_views), std::make_index_sequence<std::tuple_element_t<I, std::tuple<ViewTs...>>::size()>());
 
     if constexpr(sizeof...(Is) > 0) {
-      sync_back_to_host(views, dev_views, std::index_sequence<Is...>{});
+      sync_back_to_host(dev_views, std::index_sequence<Is...>{});
     }
   }
 
@@ -144,7 +151,7 @@ namespace detail {
       detail::invoke_with_unpacked_views(kernel_func, key, device_views, std::make_index_sequence<view_tuple_size>());
       /* 4) move data back out into host objects */
       if constexpr(std::tuple_size_v<view_tuple_t> > 0) {
-        sync_back_to_host(views, device_views, std::make_index_sequence<view_tuple_size>());
+        sync_back_to_host(device_views, std::make_index_sequence<view_tuple_size>());
       }
   #ifdef TTG_USE_CUDA
       /* wait for the */
diff --git a/ttg/ttg/util/meta.h b/ttg/ttg/util/meta.h
index c19776118..37b43e4f4 100644
--- a/ttg/ttg/util/meta.h
+++ b/ttg/ttg/util/meta.h
@@ -549,6 +549,15 @@ namespace ttg {
     template <typename T>
     constexpr bool is_tuple_v = is_tuple<T>::value;
 
+    template <typename>
+    struct is_span : std::false_type {};
+
+    template <typename T, std::size_t S>
+    struct is_span<ttg::span<T, S>> : std::true_type {};
+
+    template <typename T>
+    constexpr bool is_span_v = is_span<T>::value;
+
     template <template <class> class Pred, typename TupleT, std::size_t I, std::size_t... Is>
     struct predicate_index_seq_helper;
 
diff --git a/ttg/ttg/view.h b/ttg/ttg/view.h
index 404f453b1..8e5722d40 100644
--- a/ttg/ttg/view.h
+++ b/ttg/ttg/view.h
@@ -6,33 +6,64 @@
 
 namespace ttg {
 
-  namespace detail {
+  enum class ViewScope {
+    Allocate  = 0x0,
+    SyncIn    = 0x1,
+    SyncOut   = 0x2
+  };
+
+  template<typename T>
+  struct ViewSpan {
+
+    using element_type = T;
+    using value_type   = std::remove_cv_t<T>;
+
+    constexpr ViewSpan() = default;
+    constexpr ViewSpan(T* ptr, std::size_t size, ViewScope scope = ViewScope::SyncIn)
+    : m_ptr(ptr)
+    , m_size(size)
+    , m_scope(scope)
+    { }
+
+    constexpr std::size_t size() const {
+      return m_size;
+    }
 
-    template<typename T>
-    struct typed_iov {
-      T* ptr;
-      std::size_t size;
-    };
+    constexpr T* data() const {
+      return m_ptr;
+    }
+
+    constexpr bool is_allocate() const {
+      return (m_scope == ViewScope::Allocate);
+    }
 
-    template<typename... Ts>
-    struct typed_iovs {
-      std::tuple<typed_iov<Ts...>> iovs;
-    };
+    constexpr bool is_sync_in() const {
+      return !!(static_cast<int>(m_scope) & static_cast<int>(ViewScope::SyncIn));
+    }
 
-  } // namespace detail
+    constexpr bool is_sync_out() const {
+      return !!(static_cast<int>(m_scope) & static_cast<int>(ViewScope::SyncOut));
+    }
+
+    constexpr ViewScope scope() const {
+      return m_scope;
+    }
 
-  template<typename HostT, typename... DevTs>
+  private:
+    T *m_ptr = nullptr;
+    std::size_t m_size = 0;
+    ViewScope m_scope = ViewScope::Allocate;
+  };
+
+  template<typename HostT, typename... DevTypeTs>
   struct View {
 
-    using span_tuple_type = std::tuple<ttg::span<DevTs>...>;
+    using span_tuple_type  = std::tuple<ttg::ViewSpan<DevTypeTs>...>;
     using host_type = HostT;
 
-    using view_type = View<HostT, DevTs...>;
+    using view_type = View<HostT, DevTypeTs...>;
 
-    View()
-    : m_obj(nullptr)
-    , m_spans(ttg::span<DevTs>(nullptr, std::size_t{0})...)
-    {}
+    constexpr View() noexcept = default;
 
     View(HostT& obj, span_tuple_type spans)
     : m_obj(&obj)
@@ -61,6 +92,11 @@ namespace ttg {
       return std::get<i>(m_spans).size();
     }
 
+    template<std::size_t i>
+    auto get_span() const {
+      return std::get<i>(m_spans);
+    }
+
     HostT& get_host_object() {
       return *m_obj;
     }
@@ -69,27 +105,25 @@ namespace ttg {
       return *m_obj;
     }
 
+    template<std::size_t i>
+    ViewScope get_scope() const {
+      return std::get<i>(m_spans).scope();
+    }
+
     constexpr static std::size_t size() {
       return std::tuple_size_v<decltype(m_spans)>;
     }
 
   private:
-    HostT* m_obj;
-    span_tuple_type m_spans;
-
+    HostT* m_obj = nullptr;
+    span_tuple_type m_spans{};
   };
 
-  template<typename HostT, typename... Spans>
-  auto make_view(HostT& obj, std::tuple<Spans...> spans) {
-    return View(obj, std::move(spans));
+  template<typename HostT, typename... ViewSpanTs>
+  auto make_view(HostT& obj, ViewSpan<ViewSpanTs>... spans) {
+    return View(obj, std::make_tuple(std::move(spans)...));
   }
 
-  template<typename HostT, typename... Spans>
-  auto new_view(HostT& obj, std::tuple<Spans...> spans) {
-    return View(obj, std::move(spans));
-  }
-
-
 } // namespace ttg
 
 

From 6f5e782635c5affa8427079bd6ccf1b3d422cfa0 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 26 Oct 2022 15:53:21 -0400
Subject: [PATCH 013/259] Add missing view include in madness backend

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/madness/ttg.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ttg/ttg/madness/ttg.h b/ttg/ttg/madness/ttg.h
index 383d33c1b..e40fa25f0 100644
--- a/ttg/ttg/madness/ttg.h
+++ b/ttg/ttg/madness/ttg.h
@@ -21,6 +21,7 @@
 #include "ttg/util/macro.h"
 #include "ttg/util/meta.h"
 #include "ttg/util/meta/callable.h"
+#include "ttg/view.h"
 #include "ttg/util/void.h"
 #include "ttg/world.h"
 

From 7510b543bd5349891cb1dda5338a644cfa94ff6a Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 26 Oct 2022 15:54:43 -0400
Subject: [PATCH 014/259] Rename CUDA precompile guards:

- TTG_HAVE_CUDA enables CUDA support: synchronizing the stream between
  the kernel call and the output call.
- TTG_USE_CUDA_PREFETCH enables support for prefetching based on the view scope.

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/make_device_tt.h | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/ttg/ttg/make_device_tt.h b/ttg/ttg/make_device_tt.h
index dbfbdc933..94977624f 100644
--- a/ttg/ttg/make_device_tt.h
+++ b/ttg/ttg/make_device_tt.h
@@ -5,17 +5,17 @@
 
 
 namespace detail {
-#ifdef TTG_USE_CUDA
+#ifdef TTG_HAVE_CUDA
   inline thread_local cudaStream_t* ts_stream = nullptr;
-#endif // TTG_USE_CUDA
+#endif // TTG_HAVE_CUDA
 
   template<typename FuncT, typename keyT, typename... Args, std::size_t... Is>
   inline void invoke_with_unpacked_views(FuncT&& func, const keyT& key, std::tuple<Args...>& views, std::index_sequence<Is...>) {
-#ifdef TTG_USE_CUDA
+#ifdef TTG_HAVE_CUDA
     func(key, std::get<Is>(views)..., ts_stream);
-#else // TTG_USE_CUDA
+#else // TTG_HAVE_CUDA
     func(key, std::get<Is>(views)...);
-#endif // TTG_USE_CUDA
+#endif // TTG_HAVE_CUDA
   }
 
   /* TODO: extract host objects from views */
@@ -51,9 +51,9 @@ namespace detail {
     /* copy data to device */
     //cudaMemcpy(ptr, span.data(), span.size_bytes(), cudaMemcpyHostToDevice);
     if (view.template get_span<I>().is_sync_in()) {
-#ifdef TTG_USE_CUDA
+#if defined(TTG_HAVE_CUDA) && defined(TTG_USE_CUDA_PREFETCH)
       cudaMemPrefetchAsync(span.data(), span.size_bytes(), 0, *ts_stream);
-#endif // TTG_USE_CUDA
+#endif // TTG_USE_CUDA_PREFETCH
     }
 
     if constexpr(sizeof...(Is) > 0) {
@@ -88,9 +88,9 @@ namespace detail {
 
     /* prefetch data from device */
     if (span.is_sync_out()) {
-#ifdef TTG_USE_CUDA
+#if defined(TTG_HAVE_CUDA) && defined(TTG_USE_CUDA_PREFETCH)
       cudaMemPrefetchAsync(span.data(), span.size_bytes(), cudaCpuDeviceId, *ts_stream);
-#endif // TTG_USE_CUDA
+#endif // TTG_USE_CUDA_PREFETCH
     }
 
     if constexpr(sizeof...(Is) > 0) {
@@ -131,12 +131,12 @@ namespace detail {
 
     auto taskfn = [=](const keyT& key, Args... args) mutable {
 
-#ifdef TTG_USE_CUDA
+#ifdef TTG_HAVE_CUDA
       if (nullptr == ts_stream) {
         ts_stream = new cudaStream_t();
         cudaStreamCreate(ts_stream);
       }
-#endif // TTG_USE_CUDA
+#endif // TTG_HAVE_CUDA
 
       auto views = view_func(key, std::forward<Args>(args)...);
       using view_tuple_t = std::remove_reference_t<decltype(views)>;
@@ -153,10 +153,10 @@ namespace detail {
       if constexpr(std::tuple_size_v<view_tuple_t> > 0) {
         sync_back_to_host(device_views, std::make_index_sequence<view_tuple_size>());
       }
-  #ifdef TTG_USE_CUDA
+  #ifdef TTG_HAVE_CUDA
       /* wait for the */
       cudaStreamSynchronize(*ts_stream);
-  #endif // TTG_USE_CUDA
+  #endif // TTG_HAVE_CUDA
       /* 5) call output function */
       detail::invoke_out_with_unpacked_views(out_func, key, views, std::make_index_sequence<view_tuple_size>());
     };

From de2ad298dbc90c7b525b23f35f0518ba0b0460bf Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 26 Oct 2022 17:09:28 -0400
Subject: [PATCH 015/259] Remove noexcept from from default View ctor

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/view.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ttg/ttg/view.h b/ttg/ttg/view.h
index 8e5722d40..0b4fe92ee 100644
--- a/ttg/ttg/view.h
+++ b/ttg/ttg/view.h
@@ -63,7 +63,7 @@ namespace ttg {
 
     using view_type = View<HostT, DevTypeTs...>;
 
-    constexpr View() noexcept = default;
+    constexpr View() = default;
 
     View(HostT& obj, span_tuple_type spans)
     : m_obj(&obj)

From 3c2186bede86ef7771bf5ba24c41a15dcf06d6eb Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 26 Oct 2022 17:58:08 -0400
Subject: [PATCH 016/259] Add missing terminal name in mock device example

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/device_mock/device_mock.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/device_mock/device_mock.cc b/examples/device_mock/device_mock.cc
index daa6decbd..055b0b1f6 100644
--- a/examples/device_mock/device_mock.cc
+++ b/examples/device_mock/device_mock.cc
@@ -134,7 +134,7 @@ auto make_gemm(ttg::Edge<Key3, MatrixTile<T>>& A,
   /* If we only have GPU */
   auto gemm_tt = ttg::make_device_tt<Key3>(f_gpu_host_views, f_gpu_kernel, f_gpu_output_flows, ttg::ExecutionSpace::CUDA,
                                      ttg::edges(A, B, C), ttg::edges(output_result, C),
-                                     "GEMM", {"A", "B"}, {"output_result", "C"});
+                                     "GEMM", {"A", "B", "C"}, {"output_result", "C"});
 
 #if 0
   /* Alternative: to get both type of tasklets: */

From a04a434fe4782922ef0c7d8ce548d7e1aa781495 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 26 Oct 2022 21:45:36 -0400
Subject: [PATCH 017/259] Add a fence to the mock device test to make the
 termdet happy

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/device_mock/device_mock.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/device_mock/device_mock.cc b/examples/device_mock/device_mock.cc
index 055b0b1f6..8722591e7 100644
--- a/examples/device_mock/device_mock.cc
+++ b/examples/device_mock/device_mock.cc
@@ -181,7 +181,7 @@ int main(int argc, char **argv)
 
   auto gemm_tt = make_gemm(edge_a, edge_b, edge_out);
 
-
+  ttg::fence();
 
   ttg::finalize();
   return 0;

From de0d3c40a07a5ef1bdfac1dc4f82841505c07d79 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 26 Oct 2022 22:26:32 -0400
Subject: [PATCH 018/259] Add iovec.h to list of installed files.

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ttg/CMakeLists.txt b/ttg/CMakeLists.txt
index b77a264f8..651028e1e 100644
--- a/ttg/CMakeLists.txt
+++ b/ttg/CMakeLists.txt
@@ -17,6 +17,7 @@ set(ttg-util-headers
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/future.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/hash.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/hash/std/pair.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/iovec.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/macro.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/meta.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/meta/callable.h

From d26b43e0b0517673b4f5fee62a164ca23b5af753 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Thu, 27 Oct 2022 10:10:11 -0400
Subject: [PATCH 019/259] slight reorg of contrib dox + added device design
 notes

---
 doc/dox/contrib/CI-Administration-Notes.md    |  2 +-
 doc/dox/contrib/Design-Device.md              | 45 +++++++++++++++++++
 .../contrib}/Pull-terminal-design-doc.md      |  6 +--
 doc/dox/contrib/top.md                        | 13 ++++--
 4 files changed, 56 insertions(+), 10 deletions(-)
 create mode 100644 doc/dox/contrib/Design-Device.md
 rename doc/{ => dox/contrib}/Pull-terminal-design-doc.md (98%)

diff --git a/doc/dox/contrib/CI-Administration-Notes.md b/doc/dox/contrib/CI-Administration-Notes.md
index 32542b970..c5dd9430a 100644
--- a/doc/dox/contrib/CI-Administration-Notes.md
+++ b/doc/dox/contrib/CI-Administration-Notes.md
@@ -4,7 +4,7 @@
 * TTG uses GitHub Actions (GHA) for its CI service
 * GHA CI configuration is in file `.github/workflows/cmake.yml`. Only Linux and MacOS builds are currently supported.
 * Unlike earlier CI setups, there is no need to cache TTG prerequisites; default system-wide packages are used for most prerequisites, and the rest is compiled from source every time. 
-* Doxygen documentation deployment uses a Github token that is defined as variable `GH_TTG_TOKEN` in GHA's TTG repo settings' [secrets](https://github.com/TESSEorg/ttg/settings/secrets/actions).
+* Doxygen documentation deployment uses a GitHub token that is defined as variable `GH_TTG_TOKEN` in GHA's TTG repo settings' [secrets](https://github.com/TESSEorg/ttg/settings/secrets/actions).
 
 # Debugging GitHub Actions jobs
 
diff --git a/doc/dox/contrib/Design-Device.md b/doc/dox/contrib/Design-Device.md
new file mode 100644
index 000000000..c6e92a82c
--- /dev/null
+++ b/doc/dox/contrib/Design-Device.md
@@ -0,0 +1,45 @@
+# Device Task Design {#Design-Device}
+
+### problem statement
+TTG must be able to execute general user-defined graphs on machines with heterogeneous execution and address spaces, e.g., using multiple processes each having multiple CPU threads + device streams, with each thread/stream preferring or limited to a specific address range.
+
+### key concerns
+The key issues are how to manage:
+- the asynchrony of the device programming models, and
+- the heterogeneity of the address space.
+
+There are multiple "solutions" to each issue, hence there are many possible designs. I'll discuss each issue first, then outline the aggregate designs we are pursuing.
+
+#### Memory:
+- *Unified (UM)*: where available, use single address space (unified memory visible to both host and device executors; it is also possible to use pinned host memory for device calls)
+    - pro: simplifies memory management by removing the capacity limitation
+    - con: still needs user cooperation: all compute data must be allocated on UM heap, this impacts the design of
+      user data types, e.g. making them allocator aware, etc.
+    - con: the user will likely needs to use pooled memory management for efficiency reasons (e.g., TiledArray uses Umpire)
+    - con: still necessary to provide hints to the kernel driver managing UM.
+    - con: reasoning about UM driver performance is difficult, its details are opaque and platform dependent.
+- *Native (NM)*: using "native" device memory.
+    - pro: simpler performance model due to greatest amount of control (by runtime) over execution
+    - pro: can work with stack-capable data types
+    - con: The amount is limited, hence this memory must be explicitly managed (akin to how a cache is managed).
+
+#### Asynchrony
+- *Stages*: decompose tasks into stages, with runtime-managed periods between stages for managing asynchrony of the actions scheduled by each stage
+    - pro: most explicit, easier to reason about
+    - con: most verbose; device-capable tasks look very different from host tasks
+- *"Threads"*: use conventional mechanisms (threads for current backends, coroutines/fibers usable).
+    - pro: least host/device dichotomy
+        - threads: tasks are ordinary functions
+        - fibers/coroutines: task functions "look" like ordinary functions (and can be made almost like normal functions using macros)
+    - con: potentially lower performance
+        - threads: due to the need to context switch to "yield" to other tasks
+        - fibers/coroutines: implementation details are more complex and usually involve heap allocation (C++20 coro)
+    - con: runtime is still responsible for managing the executor space heterogeneity (control where to launch a task) and asynchrony (events/host callbacks).
+
+#### other considerations
+
+- it's not possible to manage memory from the device code, hence all program logic, including _device-capable_ tasks, must execute on host executors. In principle if we restricted ourselves to a single-source language (SYLC-extended C++) we could write device capable tasks directly as device code, but current language limitations mandate wrapping everything into host code.
+
+#### Current designs
+- *UM+threads*: use UM for memory management + threads for asynchrony
+- *NM+stages*: use Parsec's native memory pool manager + stage-decomposed tasks
diff --git a/doc/Pull-terminal-design-doc.md b/doc/dox/contrib/Pull-terminal-design-doc.md
similarity index 98%
rename from doc/Pull-terminal-design-doc.md
rename to doc/dox/contrib/Pull-terminal-design-doc.md
index 377c42b22..1af4ed11b 100644
--- a/doc/Pull-terminal-design-doc.md
+++ b/doc/dox/contrib/Pull-terminal-design-doc.md
@@ -1,4 +1,4 @@
-# Pull Terminals - Design Notes
+# Pull Terminals Design Notes {#Design-Pull}
 
 ### Motivation
 
@@ -49,7 +49,3 @@
 
 - Should Pull Op be able to send data to multiple successors? Use cases?
 - Cholesky - why pull ops are needed?
-
-
-
-
diff --git a/doc/dox/contrib/top.md b/doc/dox/contrib/top.md
index 87fd1edd8..3cf224b2f 100644
--- a/doc/dox/contrib/top.md
+++ b/doc/dox/contrib/top.md
@@ -1,7 +1,12 @@
 # Contributor Guide {#contribguide}
 
-* [TTG Build Intrastructure](@ref TTG-Build-Infrastructure)
+* Development
+  - [TTG Build Intrastructure](@ref TTG-Build-Infrastructure)
+  - [Documenting TTG](@ref Documenting-TTG)
+  - [Recommended Workflow Elements](@ref Recommended-Workflow-Elements)
+  - [CodingStandard](@ref Coding-Standard)
+* Design Notes
+  - [Pull Terminals](@ref Design-Pull)
+  - [Device Tasks](@ref Design-Device)
+* Maintenance
   - [Managing Continuous Integration (CI)](@ref CI-Administration-Notes)
-* [Documenting TTG](@ref Documenting-TTG)
-* [Recommended Workflow Elements](@ref Recommended-Workflow-Elements)
-* [CodingStandard](@ref Coding-Standard)

From 816b6f2e7df4123edcdbb05724769cfea2caed63 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Thu, 27 Oct 2022 13:49:35 -0400
Subject: [PATCH 020/259] updated device design notes to point out the need for
 Views in both UM and DM models

---
 doc/dox/contrib/Design-Device.md | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/doc/dox/contrib/Design-Device.md b/doc/dox/contrib/Design-Device.md
index c6e92a82c..b46357e98 100644
--- a/doc/dox/contrib/Design-Device.md
+++ b/doc/dox/contrib/Design-Device.md
@@ -11,18 +11,26 @@ The key issues are how to manage:
 There are multiple "solutions" to each issue, hence there are many possible designs. I'll discuss each issue first, then outline the aggregate designs we are pursuing.
 
 #### Memory:
-- *Unified (UM)*: where available, use single address space (unified memory visible to both host and device executors; it is also possible to use pinned host memory for device calls)
+- *Unified Memory (UM)*: where available, use single address space (unified memory visible to both host and device executors; it is also possible to use pinned host memory for device calls)
     - pro: simplifies memory management by removing the capacity limitation
     - con: still needs user cooperation: all compute data must be allocated on UM heap, this impacts the design of
       user data types, e.g. making them allocator aware, etc.
     - con: the user will likely needs to use pooled memory management for efficiency reasons (e.g., TiledArray uses Umpire)
     - con: still necessary to provide hints to the kernel driver managing UM.
     - con: reasoning about UM driver performance is difficult, its details are opaque and platform dependent.
-- *Native (NM)*: using "native" device memory.
+- *Device Memory (DM)*: using "native" device memory.
     - pro: simpler performance model due to greatest amount of control (by runtime) over execution
     - pro: can work with stack-capable data types
     - con: The amount is limited, hence this memory must be explicitly managed (akin to how a cache is managed).
 
+Additional memory-related concerns common to both models:
+- only partial state needs to be transferred to/from the device
+    - which part of the state will differ from algorithm to algorithm, hence encoding/constructing such representation cannot use constexpr code (such as traits)
+    - the need for _explicit_ handling of object's partial state is shared by both models
+        - UM: such optimization may seem automatic (only the pages of the data actually used on the device are transfered) but in practice the data must be explicitly prefetched, hence partial state transfers are not automatic; furthermore, the unit of UM transfer is a page (4k or more), which is too coarse for many applications
+        - DM: serialization of an entire object (which can leverage standard RDMA-like serialization), transfering partial state requires explicit annotation 
+    - hence it makes sense to make representation of object's partial state (`View`) a first-class concept in both models.
+
 #### Asynchrony
 - *Stages*: decompose tasks into stages, with runtime-managed periods between stages for managing asynchrony of the actions scheduled by each stage
     - pro: most explicit, easier to reason about

From ace8e175c59fa4684c4e1f1c4107e71df4256b43 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Fri, 28 Oct 2022 09:32:14 -0400
Subject: [PATCH 021/259] make TTG compilable with C++20

---
 CMakeLists.txt                          |  7 ++++++-
 cmake/modules/FindCXXStdExecution.cmake |  3 ---
 ttg/ttg/util/span.h                     | 13 -------------
 3 files changed, 6 insertions(+), 17 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 158842853..22bbdc503 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -29,7 +29,12 @@ project(ttg
     LANGUAGES CXX
     HOMEPAGE_URL "https://tesseorg.github.io/ttg/")
 
-set(CMAKE_CXX_EXTENSIONS OFF)
+if (NOT CMAKE_CXX_STANDARD)
+    set(CMAKE_CXX_STANDARD "17" CACHE STRING "The C++ standard")
+endif()
+if (NOT CMAKE_CXX_EXTENSIONS)
+  set(CMAKE_CXX_EXTENSIONS OFF CACHE BOOL "The C++ standard extensions allowed?")
+endif()
 
 list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/modules/")
 include(GNUInstallDirs)
diff --git a/cmake/modules/FindCXXStdExecution.cmake b/cmake/modules/FindCXXStdExecution.cmake
index 059368d1f..b86cb8ba4 100644
--- a/cmake/modules/FindCXXStdExecution.cmake
+++ b/cmake/modules/FindCXXStdExecution.cmake
@@ -69,9 +69,6 @@ cmake_push_check_state()
 
 set(CMAKE_REQUIRED_QUIET ${CXXStdExecution_FIND_QUIETLY})
 
-# All of our tests required C++17 or later
-set(CMAKE_CXX_STANDARD 17)
-
 set(CXXStdExecution_FOUND FALSE)
 
 # We have execution header, but how do we use it? Do link checks
diff --git a/ttg/ttg/util/span.h b/ttg/ttg/util/span.h
index 812322c3b..f551ead39 100644
--- a/ttg/ttg/util/span.h
+++ b/ttg/ttg/util/span.h
@@ -1,17 +1,6 @@
 #ifndef TTG_UTIL_SPAN_H
 #define TTG_UTIL_SPAN_H
 
-#if __cplusplus >= 202002L
-
-/* Use the std:: implementation available from C++20 onwards */
-#include <span>
-namespace ttg {
-  template <class T, std::size_t Extent = std::dynamic_extent>
-  using span = std::span<T, Extent>;
-}  // namespace ttg
-
-#else  // __cplusplus >= 202002L
-
 /* if TCB_SPAN_NAMESPACE_NAME already defined someone imported TCB span somewhere,
  * then reset it here to the ttg namespace, don't forget to revert back */
 #ifdef TCB_SPAN_NAMESPACE_NAME
@@ -533,6 +522,4 @@ namespace std {
 
 #endif
 
-#endif  // __cplusplus >= 202002L
-
 #endif  // TTG_UTIL_SPAN_H

From b59677a7017a276ef2ae73351ae746b6115f2e7e Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Fri, 28 Oct 2022 17:26:42 -0400
Subject: [PATCH 022/259] avoid infinite recursion in
 ttg::{set_size,finalize}<i> that do not take outterm as an argument

---
 ttg/ttg/func.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ttg/ttg/func.h b/ttg/ttg/func.h
index e383ebbc8..e1394be89 100644
--- a/ttg/ttg/func.h
+++ b/ttg/ttg/func.h
@@ -519,7 +519,7 @@ namespace ttg {
 
   template <std::size_t i>
   inline void set_size(const std::size_t size) {
-    set_size<i>(size);
+    set_size(i, size);
   }
 
   /// \brief Finalize streaming input terminals connecting to the given output terminal for tasks
@@ -574,7 +574,7 @@ namespace ttg {
 
   template <std::size_t i>
   inline void finalize() {
-    finalize<i>();
+    finalize(i);
   }
 
 }  // namespace ttg

From 557c423660c1443cdf3ca9844e0d5de8c33baead Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Mon, 31 Oct 2022 10:07:03 -0400
Subject: [PATCH 023/259] ttg_parsec::TT::static_op{,_noarg} return
 parsec_hook_return_t in preparation for suspendable tasks

---
 ttg/ttg/parsec/ttg.h | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index ef56f4e82..01d5b8c0f 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -481,7 +481,7 @@ namespace ttg_parsec {
 
   namespace detail {
 
-    typedef void (*parsec_static_op_t)(void *);  // static_op will be cast to this type
+    typedef parsec_hook_return_t (*parsec_static_op_t)(void *);  // static_op will be cast to this type
 
     const parsec_symbol_t parsec_taskclass_param0 = {
       .flags = PARSEC_SYMBOL_IS_STANDALONE|PARSEC_SYMBOL_IS_GLOBAL,
@@ -1268,7 +1268,7 @@ namespace ttg_parsec {
     }
 
     template <ttg::ExecutionSpace Space>
-    static void static_op(parsec_task_t *parsec_task) {
+    static parsec_hook_return_t static_op(parsec_task_t *parsec_task) {
       task_t *task = (task_t*)parsec_task;
       ttT *baseobj = task->tt;
       derivedT *obj = static_cast<derivedT *>(baseobj);
@@ -1302,10 +1302,13 @@ namespace ttg_parsec {
         else
           ttg::trace(obj->get_world().rank(), ":", obj->get_name(), " : done executing");
       }
+
+      // if op is coro-based this will be something else
+      return PARSEC_HOOK_RETURN_DONE;
     }
 
     template <ttg::ExecutionSpace Space>
-    static void static_op_noarg(parsec_task_t *parsec_task) {
+    static parsec_hook_return_t static_op_noarg(parsec_task_t *parsec_task) {
       task_t *task = static_cast<task_t*>(parsec_task);
       ttT *baseobj = (ttT *)task->object_ptr;
       derivedT *obj = (derivedT *)task->object_ptr;
@@ -1318,6 +1321,9 @@ namespace ttg_parsec {
       } else
         abort();
       parsec_ttg_caller = NULL;
+
+      // if op is coro-based this will be something else
+      return PARSEC_HOOK_RETURN_DONE;
     }
 
    protected:

From aa7c1b731251f85ca5e675bc97d0f50987087069 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Mon, 31 Oct 2022 10:07:16 -0400
Subject: [PATCH 024/259] cleanup

---
 cmake/modules/FindCXXStdExecution.cmake | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmake/modules/FindCXXStdExecution.cmake b/cmake/modules/FindCXXStdExecution.cmake
index b86cb8ba4..acb6b1730 100644
--- a/cmake/modules/FindCXXStdExecution.cmake
+++ b/cmake/modules/FindCXXStdExecution.cmake
@@ -9,8 +9,8 @@
 FindCXXStdExecution
 ##############
 
-This module supports the C++17 standard library's execution utilities. Use the
-:imp-target:`std::execution` imported target to
+This module supports the C++17 standard library's execution utilities. Link your target to the
+:imp-target:`std::execution` imported target to provide standard C++ execution API.
 
 Imported Targets
 ****************
@@ -84,7 +84,7 @@ string(CONFIGURE [[
   }
   ]] code @ONLY)
 
-# Try to compile a simple filesystem program without any linker flags
+# Try to compile a simple execution program without any linker flags
 check_cxx_source_compiles("${code}" CXX_EXECUTION_NO_LINK_NEEDED)
 
 set(CXXStdExecution_CAN_LINK ${CXX_EXECUTION_NO_LINK_NEEDED})

From 302b74510f99f0b66e611ffafa320a106ae5ddef Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Mon, 31 Oct 2022 10:08:42 -0400
Subject: [PATCH 025/259] find <coroutine>

---
 CMakeLists.txt                          |   3 +
 cmake/modules/FindCXXStdCoroutine.cmake | 203 ++++++++++++++++++++++++
 ttg/CMakeLists.txt                      |   5 +
 ttg/ttg/config.in.h                     |  14 ++
 4 files changed, 225 insertions(+)
 create mode 100644 cmake/modules/FindCXXStdCoroutine.cmake
 create mode 100644 ttg/ttg/config.in.h

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 22bbdc503..6f9e2dec3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -83,6 +83,9 @@ endif (BUILD_TESTING)
 include(FindOrFetchBoost)
 # Cereal
 #include(FindOrFetchCereal)
+# C++ coroutines
+find_package(CXXStdCoroutine REQUIRED COMPONENTS Final Experimental)
+
 
 ##########################
 #### prerequisite runtimes
diff --git a/cmake/modules/FindCXXStdCoroutine.cmake b/cmake/modules/FindCXXStdCoroutine.cmake
new file mode 100644
index 000000000..8bf8facb9
--- /dev/null
+++ b/cmake/modules/FindCXXStdCoroutine.cmake
@@ -0,0 +1,203 @@
+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
+
+# This is copied from:
+#   https://github.com/vector-of-bool/CMakeCM/blob/master/modules/FindFilesystem.cmake
+
+#[=======================================================================[.rst:
+
+FindCXXStdCoroutine
+##############
+
+This module supports the C++20 standard library's coroutine utilities. Link your target to the
+:imp-target:`std::coroutine` imported target to provide standard C++ coroutine API.
+
+Options
+*******
+
+The ``COMPONENTS`` argument to this module supports the following values:
+
+.. find-component:: Experimental
+    :name: coro.Experimental
+
+    Allows the module to find the "experimental" version of the
+    Coroutine library. This is the library that should be used with the
+    ``std::experimental::coroutine`` namespace.
+
+.. find-component:: Final
+    :name: coro.Final
+
+    Finds the final C++20 standard version of the coroutine library.
+
+If no components are provided, behaves as if the
+:find-component:`coro.Final` component was specified.
+
+If both :find-component:`coro.Experimental` and :find-component:`coro.Final` are
+provided, first looks for ``Final``, and falls back to ``Experimental`` in case
+of failure. If ``Final`` is found, :imp-target:`std::coroutine` and all
+:ref:`variables <coro.variables>` will refer to the ``Final`` version.
+
+
+Imported Targets
+****************
+
+.. imp-target:: std::coroutine
+
+    The ``std::coroutine`` imported target is defined when any requested
+    version of the C++ coroutine library has been found, whether it is
+    *Experimental* or *Final*.
+
+    If no version of the coroutine library is available, this target will not
+    be defined.
+
+    .. note::
+        This target has ``cxx_std_20`` as an ``INTERFACE``
+        :ref:`compile language standard feature <req-lang-standards>`. Linking
+        to this target will automatically enable C++20 if no later standard
+        version is already required on the linking target.
+
+
+.. coro.variables:
+
+Variables
+*********
+
+.. variable:: CXX_COROUTINE_IS_EXPERIMENTAL
+
+    Set to ``TRUE`` when the :find-component:`coro.Experimental` version of C++
+    coroutine library was found, otherwise ``FALSE``.
+
+.. variable:: CXX_COROUTINE_HAVE_CORO
+
+    Set to ``TRUE`` when a coroutine header was found.
+
+.. variable:: CXX_COROUTINE_HEADER
+
+    Set to either ``coroutine`` or ``experimental/coroutine`` depending on
+    whether :find-component:`coro.Final` or :find-component:`coro.Experimental` was
+    found.
+
+.. variable:: CXX_COROUTINE_NAMESPACE
+
+    Set to either ``std::coroutine`` or ``std::experimental::coroutine``
+    depending on whether :find-component:`coro.Final` or
+    :find-component:`coro.Experimental` was found.
+
+
+Examples
+********
+
+Using `find_package(Coroutine)` with no component arguments:
+
+.. code-block:: cmake
+
+    find_package(Coroutine REQUIRED)
+
+    add_executable(my-program main.cpp)
+    target_link_libraries(my-program PRIVATE std::coroutine)
+
+
+#]=======================================================================]
+
+
+if(TARGET std::coroutine)
+  # This module has already been processed. Don't do it again.
+  return()
+endif()
+
+include(CMakePushCheckState)
+include(CheckIncludeFileCXX)
+include(CheckCXXSourceCompiles)
+
+cmake_push_check_state()
+
+set(CMAKE_REQUIRED_QUIET ${CXXStdCoroutine_FIND_QUIETLY})
+
+# Normalize and check the component list we were given
+set(CXXStdCoroutines_want_components ${CXXStdCoroutine_FIND_COMPONENTS})
+if(CXXStdCoroutine_FIND_COMPONENTS STREQUAL "")
+  set(CXXStdCoroutines_want_components Final)
+endif()
+
+# Warn on any unrecognized components
+set(CXXStdCoroutines_extra_components ${CXXStdCoroutines_want_components})
+list(REMOVE_ITEM CXXStdCoroutines_extra_components Final Experimental)
+foreach(component IN LISTS CXXStdCoroutines_extra_components)
+  message(WARNING "Extraneous find_package component for CXXStdCoroutine: ${component}")
+endforeach()
+
+# clang may need to use -stdlib=c++ to have coroutines
+# gcc/libstdc++ needs -fcoroutines
+set(CXXStdCoroutines_find_options "" "-stdlib=libc++" "-fcoroutines")
+set(CXXStdCoroutines_want_components_ordered "${CXXStdCoroutines_want_components}")
+list(SORT CXXStdCoroutines_want_components_ordered ORDER DESCENDING)  # Final before Experimental
+
+foreach(component IN LISTS CXXStdCoroutines_want_components_ordered)
+  if(component STREQUAL "Final")
+    set(_coro_header coroutine)
+    set(_coro_namespace std)
+  else()
+    set(_coro_header experimental/coroutine)
+    set(_coro_namespace std::experimental)
+  endif()
+  foreach(option IN LISTS CXXStdCoroutines_find_options)
+    cmake_push_check_state()
+    list(APPEND CMAKE_REQUIRED_FLAGS "${option}")
+    check_include_file_cxx("${_coro_header}" _CXX_COROUTINE_HAVE_HEADER)
+    mark_as_advanced(_CXX_COROUTINE_HAVE_HEADER)
+    cmake_pop_check_state()
+    if(_CXX_COROUTINE_HAVE_HEADER)
+      set(CXXStdCoroutines_option "${option}" CACHE STRING "The compiler option needed to use <coroutine>")
+      mark_as_advanced(CXXStdCoroutines_option)
+      # We found a header, skip rest of tests
+      break()
+    else()
+      unset(_CXX_COROUTINE_HAVE_HEADER CACHE)
+    endif()
+  endforeach()  # options
+  if (_CXX_COROUTINE_HAVE_HEADER)
+    break()
+  endif()
+endforeach() # components
+
+set(CXX_COROUTINE_HAVE_CORO ${_CXX_COROUTINE_HAVE_HEADER} CACHE BOOL "TRUE if we have the C++ coroutine headers")
+set(CXX_COROUTINE_HEADER ${_coro_header} CACHE STRING "The header that should be included to obtain the coroutine APIs")
+set(CXX_COROUTINE_NAMESPACE ${_coro_namespace} CACHE STRING "The C++ namespace that contains the coroutine APIs")
+
+set(_found FALSE)
+
+if(CXX_COROUTINE_HAVE_CORO)
+  # We have some coroutine library available. Do compile checks
+  string(CONFIGURE [[
+        #include <@CXX_COROUTINE_HEADER@>
+
+        int main() {
+            auto x = @CXX_COROUTINE_NAMESPACE@::suspend_always{};
+            return 0;
+        }
+    ]] code @ONLY)
+
+  cmake_push_check_state()
+  if (DEFINED CXXStdCoroutines_option)
+    list(APPEND CMAKE_REQUIRED_FLAGS "${CXXStdCoroutines_option}")
+  endif()
+  check_cxx_source_compiles("${code}" CXX_COROUTINE_NO_LINK_NEEDED)
+  cmake_pop_check_state()
+
+  if(CXX_COROUTINE_NO_LINK_NEEDED)
+    add_library(std::coroutine INTERFACE IMPORTED GLOBAL)
+    target_compile_features(std::coroutine INTERFACE cxx_std_20)
+    if (DEFINED CXXStdCoroutines_option)
+      target_compile_options(std::coroutine INTERFACE "${CXXStdCoroutines_option}")
+    endif()
+    set(_found TRUE)
+  endif()
+endif()
+
+cmake_pop_check_state()
+
+set(CXXStdCoroutine_FOUND ${_found} CACHE BOOL "TRUE if we can compile and link a program using std::coroutine" FORCE)
+
+if(CXXStdCoroutine_FIND_REQUIRED AND NOT TARGET std::coroutine)
+  message(FATAL_ERROR "Cannot Compile simple program using std::coroutine")
+endif()
diff --git a/ttg/CMakeLists.txt b/ttg/CMakeLists.txt
index 651028e1e..1f703fc0e 100644
--- a/ttg/CMakeLists.txt
+++ b/ttg/CMakeLists.txt
@@ -38,6 +38,10 @@ set(ttg-base-headers
 file(GLOB_RECURSE ttg-external-headers $<$<VERSION_GREATER_EQUAL:${CMAKE_VERSION},3.12>:CONFIGURE_DEPENDS>
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/external/boost/*
     )
+configure_file(
+        ${CMAKE_CURRENT_SOURCE_DIR}/ttg/config.in.h
+        ${CMAKE_CURRENT_BINARY_DIR}/ttg/config.h
+)
 set(ttg-impl-headers
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/broadcast.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/edge.h
@@ -56,6 +60,7 @@ set(ttg-impl-headers
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/make_tt.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/make_device_tt.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/view.h
+        ${CMAKE_CURRENT_BINARY_DIR}/ttg/config.h
     )
 set(ttg-headers
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg.h
diff --git a/ttg/ttg/config.in.h b/ttg/ttg/config.in.h
new file mode 100644
index 000000000..87113b4e5
--- /dev/null
+++ b/ttg/ttg/config.in.h
@@ -0,0 +1,14 @@
+//
+// Created by Eduard Valeyev on 10/31/22.
+//
+
+#ifndef TTG_CONFIG_IN_H
+#define TTG_CONFIG_IN_H
+
+/** the C++ header containing the coroutine API */
+#define TTG_CXX_COROUTINE_HEADER <@CXX_COROUTINE_HEADER@>
+
+/** the C++ namespace containing the coroutine API */
+#define TTG_CXX_COROUTINE_NAMESPACE @CXX_COROUTINE_NAMESPACE@
+
+#endif  // TTG_CONFIG_IN_H

From 402e47d7c5722843f6d1ffb10166a7ce1015a906 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Mon, 31 Oct 2022 10:27:03 -0400
Subject: [PATCH 026/259] added an example of a task body being a (trivial)
 coroutine

---
 tests/unit/CMakeLists.txt    |  13 +++-
 tests/unit/fibonacci-coro.cc | 116 +++++++++++++++++++++++++++++++++++
 ttg/ttg/util/coroutine.h     |  25 ++++++++
 3 files changed, 153 insertions(+), 1 deletion(-)
 create mode 100644 tests/unit/fibonacci-coro.cc
 create mode 100644 ttg/ttg/util/coroutine.h

diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt
index 2016872e2..bb0df6255 100644
--- a/tests/unit/CMakeLists.txt
+++ b/tests/unit/CMakeLists.txt
@@ -1,7 +1,18 @@
 include(AddTTGExecutable)
 
 # TT unit test: core TTG ops
-add_ttg_executable(core-unittests-ttg "fibonacci.cc;ranges.cc;tt.cc;unit_main.cpp" LINK_LIBRARIES "Catch2::Catch2")
+set(ut_src
+        fibonacci.cc
+        ranges.cc
+        tt.cc
+        unit_main.cpp)
+set(ut_libs Catch2::Catch2)
+if (TARGET std::coroutine)
+    list(APPEND ut_src fibonacci-coro.cc)
+    list(APPEND ut_libs std::coroutine)
+endif()
+add_ttg_executable(core-unittests-ttg "${ut_src}" LINK_LIBRARIES "${ut_libs}")
+
 
 # serialization test: probes serialization via all supported serialization methods (MADNESS, Boost::serialization, cereal) that are available
 add_executable(serialization "serialization.cc;unit_main.cpp")
diff --git a/tests/unit/fibonacci-coro.cc b/tests/unit/fibonacci-coro.cc
new file mode 100644
index 000000000..63cc97a1b
--- /dev/null
+++ b/tests/unit/fibonacci-coro.cc
@@ -0,0 +1,116 @@
+#include <catch2/catch.hpp>
+
+#include "ttg.h"
+
+#include "ttg/serialization/std/pair.h"
+#include "ttg/util/hash/std/pair.h"
+
+#include "ttg/util/coroutine.h"
+
+constexpr int64_t N = 1000;
+
+TEST_CASE("Fibonacci-coroutines", "[fib][core]") {
+  // compute the reference result
+  int reference_result = 0;
+  {
+    // recursive lambda pattern from http://pedromelendez.com/blog/2015/07/16/recursive-lambdas-in-c14/
+    auto compute_reference_result = [&reference_result](int f_np1, int f_n) {
+      auto impl = [&reference_result](int f_np1, int f_n, const auto &impl_ref) -> void {
+        assert(f_n < N);
+        reference_result += f_n;
+        if (f_np1 < N) {
+          const auto f_np2 = f_np1 + f_n;
+          impl_ref(f_np2, f_np1, impl_ref);
+        }
+      };
+      impl(f_np1, f_n, impl);
+    };
+    compute_reference_result(1, 0);
+  }
+
+  SECTION("shared-memory") {
+    if (ttg::default_execution_context().size() == 1) {
+      ttg::Edge<int, int> F2F;
+      ttg::Edge<void, int> F2P;
+
+      // N.B. wrap a trivial (nonsuspending) coroutine using make_tt!
+      auto fib_op = ttg::make_tt(
+          // computes next value: F_{n+2} = F_{n+1} + F_{n}, seeded by F_1 = 1, F_0 = 0
+          // N.B. can't autodeduce return type, must add "-> void"
+          [](const int &F_n_plus_1, const int &F_n) -> void {
+            // on 1 process the right order of sends can avoid the race iff reductions are inline (on-current-thread)
+            // and not async (nthread>1):
+            // - send<1> will call wc->set_arg which will eagerly reduce the argument
+            // - send<0> then will call wa->set_arg which will create task for key F_np2 ... that can potentially call
+            // finalize<1> in the other clause
+            // - reversing the order of sends will create a race between wc->set_arg->send<1> executing on this thread
+            // and wa->set_arg->finalize<1> executing in thread pool
+            // - there is no way to detect the "undesired" outcome of the race without keeping expired TTArgs from the
+            // cache there is no way currently to avoid race if there is more than 1 process ... need to track the
+            // number of messages that the reducing terminal will receive, that's what distributed example demonstrates.
+            // The order of operations will still matter.
+            if (F_n_plus_1 < N) {
+              const auto F_n_plus_2 = F_n_plus_1 + F_n;
+              ttg::sendv<1>(F_n_plus_1);
+              ttg::send<0>(F_n_plus_2, F_n_plus_1);
+            } else
+              ttg::finalize<1>();
+
+            // N.B. return void just as normal TT op
+            co_return;
+          },
+          ttg::edges(F2F), ttg::edges(F2F, F2P));
+      auto print_op = ttg::make_tt(
+          [reference_result](const int &value, std::tuple<> &out) {
+            ttg::print("sum of Fibonacci numbers up to ", N, " = ", value);
+            CHECK(value == reference_result);
+          },
+          ttg::edges(F2P), ttg::edges());
+      print_op->set_input_reducer<0>([](int &a, const int &b) { a = a + b; });
+      make_graph_executable(fib_op);
+      if (ttg::default_execution_context().rank() == 0) fib_op->invoke(1, 0);
+      ttg::ttg_fence(ttg::default_execution_context());
+    }
+  }
+
+  // in distributed memory we must count how many messages the reducer will receive
+  SECTION("distributed-memory") {
+    ttg::Edge<int, std::pair<int, int>> F2F;
+    ttg::Edge<void, int> F2P;
+    const auto nranks = ttg::default_execution_context().size();
+
+    auto fib_op = ttg::make_tt(
+        // computes next value: F_{n+2} = F_{n+1} + F_{n}, seeded by F_1 = 1, F_0 = 0
+        [](const int &n, const std::pair<int, int> &F_np1_n) {
+          const auto &[F_n_plus_1, F_n] = F_np1_n;
+          if (F_n_plus_1 < N) {
+            const auto F_n_plus_2 = F_n_plus_1 + F_n;
+            ttg::print("sent ", F_n_plus_1, " to fib reducer");
+            ttg::sendv<1>(F_n_plus_1);
+            ttg::send<0>(n + 1, std::make_pair(F_n_plus_2, F_n_plus_1));
+          } else {
+            // how many messages the reducer should expect to receive
+            ttg::set_size<1>(n);
+            ttg::print("fib reducer will expect ", n, " messages");
+          }
+        },
+        ttg::edges(F2F), ttg::edges(F2F, F2P));
+    auto print_op = ttg::make_tt(
+        [reference_result](const int &value, std::tuple<> &out) {
+          ttg::print("sum of Fibonacci numbers up to ", N, " = ", value);
+          CHECK(value == reference_result);
+        },
+        ttg::edges(F2P), ttg::edges());
+    // move all fib tasks to last rank, all reductions will happen on 0 => for some reason no reductions occur!
+    fib_op->set_keymap([=](const auto &key) { return nranks - 1; });
+    fib_op->set_trace_instance(true);
+    print_op->set_input_reducer<0>([](int &a, const int &b) {
+      ttg::print("fib reducer: current value = ", a, ", incremented by ", b, " set to ", a + b);
+      a = a + b;
+    });
+    make_graph_executable(fib_op);
+    ttg::ttg_fence(ttg::default_execution_context());
+    if (ttg::default_execution_context().rank() == 0) fib_op->invoke(0, std::make_pair(1, 0));
+    ttg::ttg_fence(ttg::default_execution_context());
+  }
+}  // TEST_CAST("Fibonacci")
diff --git a/ttg/ttg/util/coroutine.h b/ttg/ttg/util/coroutine.h
new file mode 100644
index 000000000..c4c219eb5
--- /dev/null
+++ b/ttg/ttg/util/coroutine.h
@@ -0,0 +1,25 @@
+//
+// Created by Eduard Valeyev on 10/31/22.
+//
+
+#ifndef TTG_UTIL_COROUTINE_H
+#define TTG_UTIL_COROUTINE_H
+
+#include "ttg/config.h"
+#include TTG_CXX_COROUTINE_HEADER
+
+// coroutines in TTG can return void
+template <typename... Args>
+struct TTG_CXX_COROUTINE_NAMESPACE::coroutine_traits<void, Args...> {
+  struct promise_type {
+    auto get_return_object() noexcept { return this; }
+
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_never initial_suspend() const noexcept { return {}; }
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_never final_suspend() const noexcept { return {}; }
+
+    void return_void() noexcept {}
+    void unhandled_exception() noexcept {}
+  };
+};
+
+#endif  // TTG_UTIL_COROUTINE_H

From 76b52bd360c6fcd65085244555a84d52237485fc Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Mon, 31 Oct 2022 10:33:19 -0400
Subject: [PATCH 027/259] [ci] try using gcc-10, first to provide support  C++
 corotuines,  for ubuntu runners

---
 .github/workflows/cmake.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index bd9281d0b..4367ff3a0 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -13,7 +13,7 @@ jobs:
         os : [ macos-latest, ubuntu-20.04 ]
         include:
           - os: ubuntu-20.04
-            cxx: /usr/bin/g++-9
+            cxx: /usr/bin/g++-10
           - os: macos-latest
             cxx: clang++
 
@@ -50,7 +50,7 @@ jobs:
         wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null
         sudo apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main"
         sudo apt-get update
-        sudo apt-get -y install ninja-build g++-9 liblapack-dev libboost-dev libboost-serialization-dev libeigen3-dev openmpi-bin libopenmpi-dev libtbb-dev ccache flex bison cmake
+        sudo apt-get -y install ninja-build g++-10 liblapack-dev libboost-dev libboost-serialization-dev libeigen3-dev openmpi-bin libopenmpi-dev libtbb-dev ccache flex bison cmake
 
     - name: Create Build Environment
       # Some projects don't allow in-source building, so create a separate build directory

From 328445721fdbd03381667ab3be4ffcb6e8e5902a Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Mon, 31 Oct 2022 10:42:12 -0400
Subject: [PATCH 028/259] [ci] try dumping CMake{Output,Error}.log if cmake
 fails

---
 .github/workflows/cmake.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index 4367ff3a0..18d81817e 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -99,7 +99,8 @@ jobs:
       # Note the current convention is to use the -S and -B options here to specify source 
       # and build directories, but this is only available with CMake 3.13 and higher.  
       # The CMake binaries on the Github Actions machines are (as of this writing) 3.12
-      run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE $BUILD_CONFIG
+      run: |
+        cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE $BUILD_CONFIG || (cat CMakeFiles/CMakeOutput.log && cat CMakeFiles/CMakeError.log)
 
     - name: Build
       working-directory: ${{github.workspace}}/build

From 05994324de59092d4ea71639319eff69ccb961cb Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Mon, 31 Oct 2022 10:50:17 -0400
Subject: [PATCH 029/259] coroutines are optional, but CI will try to use them

---
 .github/workflows/cmake.yml | 1 +
 CMakeLists.txt              | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index 18d81817e..b25f773ed 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -36,6 +36,7 @@ jobs:
         -DMPIEXEC_PREFLAGS='--bind-to;none;--allow-run-as-root'
         -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/install
         -DTTG_EXAMPLES=ON
+        -DCMAKE_CXX_STANDARD=20
 
     steps:
     - uses: actions/checkout@v2
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6f9e2dec3..045a74d90 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -84,7 +84,7 @@ include(FindOrFetchBoost)
 # Cereal
 #include(FindOrFetchCereal)
 # C++ coroutines
-find_package(CXXStdCoroutine REQUIRED COMPONENTS Final Experimental)
+find_package(CXXStdCoroutine COMPONENTS Final Experimental)
 
 
 ##########################

From 96b19aa96f00ef98d0b5b559990ff712892c305b Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Mon, 31 Oct 2022 11:31:44 -0400
Subject: [PATCH 030/259] gcc-12 does not seem to like co_return in a function
 returning void

---
 tests/unit/fibonacci-coro.cc | 10 ++++++++--
 ttg/ttg/util/coroutine.h     | 16 ++++++++++++++++
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/tests/unit/fibonacci-coro.cc b/tests/unit/fibonacci-coro.cc
index 63cc97a1b..39014ff21 100644
--- a/tests/unit/fibonacci-coro.cc
+++ b/tests/unit/fibonacci-coro.cc
@@ -36,8 +36,14 @@ TEST_CASE("Fibonacci-coroutines", "[fib][core]") {
       // N.B. wrap a trivial (nonsuspending) coroutine using make_tt!
       auto fib_op = ttg::make_tt(
           // computes next value: F_{n+2} = F_{n+1} + F_{n}, seeded by F_1 = 1, F_0 = 0
-          // N.B. can't autodeduce return type, must add "-> void"
-          [](const int &F_n_plus_1, const int &F_n) -> void {
+          // N.B. can't autodeduce return type, must explicitly declare the return type
+          [](const int &F_n_plus_1, const int &F_n)
+#ifdef __clang__
+              -> void
+#else  // gcc gets confused by co_return; statement in function returning void, use ttg::Void
+              -> ttg::Void
+#endif
+          {
             // on 1 process the right order of sends can avoid the race iff reductions are inline (on-current-thread)
             // and not async (nthread>1):
             // - send<1> will call wc->set_arg which will eagerly reduce the argument
diff --git a/ttg/ttg/util/coroutine.h b/ttg/ttg/util/coroutine.h
index c4c219eb5..ff9fa7b08 100644
--- a/ttg/ttg/util/coroutine.h
+++ b/ttg/ttg/util/coroutine.h
@@ -22,4 +22,20 @@ struct TTG_CXX_COROUTINE_NAMESPACE::coroutine_traits<void, Args...> {
   };
 };
 
+#include "ttg/util/void.h"
+
+// coroutines in TTG can return Void
+template <typename... Args>
+struct TTG_CXX_COROUTINE_NAMESPACE::coroutine_traits<ttg::Void, Args...> {
+  struct promise_type {
+    auto get_return_object() noexcept { return this; }
+
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_never initial_suspend() const noexcept { return {}; }
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_never final_suspend() const noexcept { return {}; }
+
+    void return_void() noexcept {}
+    void unhandled_exception() noexcept {}
+  };
+};
+
 #endif  // TTG_UTIL_COROUTINE_H

From 1b32e81e4ff70148e260bf810def64ae50827d7d Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Mon, 31 Oct 2022 12:31:09 -0400
Subject: [PATCH 031/259] updated design notes w/ tentative coroutine-based
 sketch

---
 doc/dox/contrib/Design-Device.md | 43 ++++++++++++++++++++++++++------
 1 file changed, 36 insertions(+), 7 deletions(-)

diff --git a/doc/dox/contrib/Design-Device.md b/doc/dox/contrib/Design-Device.md
index b46357e98..492a61f28 100644
--- a/doc/dox/contrib/Design-Device.md
+++ b/doc/dox/contrib/Design-Device.md
@@ -1,16 +1,16 @@
 # Device Task Design {#Design-Device}
 
-### problem statement
+## problem statement
 TTG must be able to execute general user-defined graphs on machines with heterogeneous execution and address spaces, e.g., using multiple processes each having multiple CPU threads + device streams, with each thread/stream preferring or limited to a specific address range.
 
-### key concerns
+## key concerns
 The key issues are how to manage:
 - the asynchrony of the device programming models, and
 - the heterogeneity of the address space.
 
 There are multiple "solutions" to each issue, hence there are many possible designs. I'll discuss each issue first, then outline the aggregate designs we are pursuing.
 
-#### Memory:
+### Memory:
 - *Unified Memory (UM)*: where available, use single address space (unified memory visible to both host and device executors; it is also possible to use pinned host memory for device calls)
     - pro: simplifies memory management by removing the capacity limitation
     - con: still needs user cooperation: all compute data must be allocated on UM heap, this impacts the design of
@@ -31,7 +31,7 @@ Additional memory-related concerns common to both models:
         - DM: serialization of an entire object (which can leverage standard RDMA-like serialization), transfering partial state requires explicit annotation 
     - hence it makes sense to make representation of object's partial state (`View`) a first-class concept in both models.
 
-#### Asynchrony
+### Asynchrony
 - *Stages*: decompose tasks into stages, with runtime-managed periods between stages for managing asynchrony of the actions scheduled by each stage
     - pro: most explicit, easier to reason about
     - con: most verbose; device-capable tasks look very different from host tasks
@@ -44,10 +44,39 @@ Additional memory-related concerns common to both models:
         - fibers/coroutines: implementation details are more complex and usually involve heap allocation (C++20 coro)
     - con: runtime is still responsible for managing the executor space heterogeneity (control where to launch a task) and asynchrony (events/host callbacks).
 
-#### other considerations
+### other considerations
 
 - it's not possible to manage memory from the device code, hence all program logic, including _device-capable_ tasks, must execute on host executors. In principle if we restricted ourselves to a single-source language (SYLC-extended C++) we could write device capable tasks directly as device code, but current language limitations mandate wrapping everything into host code.
 
-#### Current designs
+## Current designs
 - *UM+threads*: use UM for memory management + threads for asynchrony
-- *NM+stages*: use Parsec's native memory pool manager + stage-decomposed tasks
+- *DM+stages*: use Parsec's device memory pool manager + stage-decomposed tasks
+- *?M+coroutines*: UM/DM for memory + C++20 coroutines for handling the asynchrony
+
+### Coroutine-based design
+C++ coroutines were introduced in C++20. The design is fairly convoluted, with many customization points, and our understanding of the best practices is evolving.
+The basic idea is to have user compose device tasks as a single coroutine that gets decomposed by the compiler into the stages that in the *DM+stages* design user had to express as individual lambdas:
+```cpp
+make_tt([](auto& key, auto& data1, auto& data2) -> ttg::coroutine {
+    // stage 1
+    ConstView view1(data1);  // gets added aww's list of work items
+    ConstView view2(data2);  // gets added aww's list of work items
+    double data3;
+    View view3(data3, NewView | SyncView_D2H);
+    co_await sync_views(view1, view2, view3);  // runtime processes these work items
+    
+    // stage 2
+    cublasDdot(view1.device_ptr(), view2.device_ptr(), view3.device_ptr());
+    co_await;  // syncs view3; since transfers and kernels execute in different streams the runtime will sync kernel stream, then launch transfers, then resume here
+
+    if (data3 >= 0.)
+        send<0>(data1);
+    else
+        send<0>(data2);
+    co_return;  // processes sends and destroys coroutine+aww
+}, ...);
+```
+
+The advantages of this design over the *DM+stages* design are:
+- the number and type of stages is completely arbitrary, and
+- it is possible to make the device code for simple cases as similar to the host case as possible.

From ca6ba972fdba33abe474c9ac29a967c856b5bb51 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Mon, 31 Oct 2022 16:32:33 -0400
Subject: [PATCH 032/259] started sketching out resumable_task

---
 doc/dox/contrib/Design-Device.md | 10 ++++----
 ttg/ttg/util/coroutine.h         | 25 ++++++++++++++++++
 ttg/ttg/view.h                   | 44 +++++++++++++++++++++++++++++++-
 3 files changed, 73 insertions(+), 6 deletions(-)

diff --git a/doc/dox/contrib/Design-Device.md b/doc/dox/contrib/Design-Device.md
index 492a61f28..cf2e7f1b8 100644
--- a/doc/dox/contrib/Design-Device.md
+++ b/doc/dox/contrib/Design-Device.md
@@ -57,13 +57,13 @@ Additional memory-related concerns common to both models:
 C++ coroutines were introduced in C++20. The design is fairly convoluted, with many customization points, and our understanding of the best practices is evolving.
 The basic idea is to have user compose device tasks as a single coroutine that gets decomposed by the compiler into the stages that in the *DM+stages* design user had to express as individual lambdas:
 ```cpp
-make_tt([](auto& key, auto& data1, auto& data2) -> ttg::coroutine {
+make_tt([](auto& key, auto& data1, auto& data2) -> ttg::resumable_task {
     // stage 1
-    ConstView view1(data1);  // gets added aww's list of work items
-    ConstView view2(data2);  // gets added aww's list of work items
+    ConstView view1(data1);
+    ConstView view2(data2);
     double data3;
     View view3(data3, NewView | SyncView_D2H);
-    co_await sync_views(view1, view2, view3);  // runtime processes these work items
+    co_await sync_views(data1, data2, data3);  // creates list of transfers to be fulfilled by the runtime
     
     // stage 2
     cublasDdot(view1.device_ptr(), view2.device_ptr(), view3.device_ptr());
@@ -73,7 +73,7 @@ make_tt([](auto& key, auto& data1, auto& data2) -> ttg::coroutine {
         send<0>(data1);
     else
         send<0>(data2);
-    co_return;  // processes sends and destroys coroutine+aww
+    co_return;  // processes sends and destroys coroutine
 }, ...);
 ```
 
diff --git a/ttg/ttg/util/coroutine.h b/ttg/ttg/util/coroutine.h
index ff9fa7b08..eeef29033 100644
--- a/ttg/ttg/util/coroutine.h
+++ b/ttg/ttg/util/coroutine.h
@@ -8,6 +8,8 @@
 #include "ttg/config.h"
 #include TTG_CXX_COROUTINE_HEADER
 
+#include "ttg/view.h"
+
 // coroutines in TTG can return void
 template <typename... Args>
 struct TTG_CXX_COROUTINE_NAMESPACE::coroutine_traits<void, Args...> {
@@ -38,4 +40,27 @@ struct TTG_CXX_COROUTINE_NAMESPACE::coroutine_traits<ttg::Void, Args...> {
   };
 };
 
+namespace ttg {
+
+  struct resumable_task_state;
+  struct resumable_task : std::coroutine_handle<resumable_task_state> {
+    using promise_type = struct resumable_task_state;
+  };
+
+  struct resumable_task_state {
+    resumable_task get_return_object() { return {resumable_task::from_promise(*this)}; }
+    std::suspend_never initial_suspend() noexcept { return {}; }
+    std::suspend_never final_suspend() noexcept { return {}; }
+    resumable_task_state& return_value(std::initializer_list<HDSpan> views) {
+      // push views onto work queue
+      return *this;
+    }
+    void unhandled_exception() {}
+
+   private:
+    ViewSpan<std::byte> views[20];
+  };
+
+}  // namespace ttg
+
 #endif  // TTG_UTIL_COROUTINE_H
diff --git a/ttg/ttg/view.h b/ttg/ttg/view.h
index 0b4fe92ee..842081f40 100644
--- a/ttg/ttg/view.h
+++ b/ttg/ttg/view.h
@@ -124,7 +124,49 @@ namespace ttg {
     return View(obj, std::make_tuple(std::move(spans)...));
   }
 
-} // namespace ttg
+  /// std::span mirrored between host and device memory
+  struct HDSpan {
+    HDSpan() = default;
+    HDSpan(std::byte* ptr, std::size_t nbytes) {
+      ptrs_[0] = ptr;
+      nbytes_ = nbytes;
+      last_touched_space_ = 0;
+    }
+
+    std::size_t nbytes() const { return nbytes_; }
+
+    const std::byte* host_data() const { return ptrs_[0]; }
+
+    std::byte* host_data() {
+      last_touched_space_ = 0;
+      return ptrs_[0];
+    }
+
+    const std::byte* device_data() const { return ptrs_[1]; }
+
+    std::byte* device_data() {
+      last_touched_space_ = 1;
+      return ptrs_[1];
+    }
+
+    void mark_synched() { last_touched_space_ = 2; }
+
+   private:
+    std::array<std::byte*, 2> ptrs_ = {nullptr, nullptr};
+    std::size_t nbytes_ = 0;
+    std::size_t last_touched_space_ = 2;
+  };
+
+  /// set of std::span's mirrored between host and device memory
+  template <std::size_t N = 10>
+  struct HDSpans {
+    HDSpans() = default;
+
+   private:
+    std::array<HDSpan, N> spans_;
+  };
+
+}  // namespace ttg
 
 
 #endif // TTG_VIEW_H

From 6cbb6043f7398712cad6cb1565665a4ecc8070cd Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Tue, 1 Nov 2022 09:12:55 -0400
Subject: [PATCH 033/259] resumable_task is starting to come together, trivial
 co_await works

---
 tests/unit/fibonacci-coro.cc |  10 +--
 ttg/ttg/util/coroutine.h     | 125 +++++++++++++++++++++++++----------
 2 files changed, 93 insertions(+), 42 deletions(-)

diff --git a/tests/unit/fibonacci-coro.cc b/tests/unit/fibonacci-coro.cc
index 39014ff21..2ec9dd52c 100644
--- a/tests/unit/fibonacci-coro.cc
+++ b/tests/unit/fibonacci-coro.cc
@@ -37,13 +37,7 @@ TEST_CASE("Fibonacci-coroutines", "[fib][core]") {
       auto fib_op = ttg::make_tt(
           // computes next value: F_{n+2} = F_{n+1} + F_{n}, seeded by F_1 = 1, F_0 = 0
           // N.B. can't autodeduce return type, must explicitly declare the return type
-          [](const int &F_n_plus_1, const int &F_n)
-#ifdef __clang__
-              -> void
-#else  // gcc gets confused by co_return; statement in function returning void, use ttg::Void
-              -> ttg::Void
-#endif
-          {
+          [](const int &F_n_plus_1, const int &F_n) -> ttg::resumable_task {
             // on 1 process the right order of sends can avoid the race iff reductions are inline (on-current-thread)
             // and not async (nthread>1):
             // - send<1> will call wc->set_arg which will eagerly reduce the argument
@@ -57,6 +51,8 @@ TEST_CASE("Fibonacci-coroutines", "[fib][core]") {
             // The order of operations will still matter.
             if (F_n_plus_1 < N) {
               const auto F_n_plus_2 = F_n_plus_1 + F_n;
+              // cool, if there are no events to wait for co_await is no-op
+              co_await ttg::resumable_task_events{};
               ttg::sendv<1>(F_n_plus_1);
               ttg::send<0>(F_n_plus_2, F_n_plus_1);
             } else
diff --git a/ttg/ttg/util/coroutine.h b/ttg/ttg/util/coroutine.h
index eeef29033..5e221448b 100644
--- a/ttg/ttg/util/coroutine.h
+++ b/ttg/ttg/util/coroutine.h
@@ -8,59 +8,114 @@
 #include "ttg/config.h"
 #include TTG_CXX_COROUTINE_HEADER
 
-#include "ttg/view.h"
+#include <algorithm>
+#include <array>
 
-// coroutines in TTG can return void
-template <typename... Args>
-struct TTG_CXX_COROUTINE_NAMESPACE::coroutine_traits<void, Args...> {
-  struct promise_type {
-    auto get_return_object() noexcept { return this; }
-
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_never initial_suspend() const noexcept { return {}; }
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_never final_suspend() const noexcept { return {}; }
+namespace ttg {
 
-    void return_void() noexcept {}
-    void unhandled_exception() noexcept {}
-  };
-};
+  struct resumable_task_state;
 
-#include "ttg/util/void.h"
+  template <std::size_t N>
+  struct resumable_task_events;
 
-// coroutines in TTG can return Void
-template <typename... Args>
-struct TTG_CXX_COROUTINE_NAMESPACE::coroutine_traits<ttg::Void, Args...> {
-  struct promise_type {
-    auto get_return_object() noexcept { return this; }
+  /// represents a one-time event
+  struct event {
+    void finish() { finished_ = true; }
 
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_never initial_suspend() const noexcept { return {}; }
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_never final_suspend() const noexcept { return {}; }
+    /// @return true if the event has occurred
+    bool finished() const { return finished_; }
 
-    void return_void() noexcept {}
-    void unhandled_exception() noexcept {}
+   private:
+    std::atomic<bool> finished_ = false;
   };
-};
 
-namespace ttg {
+  /// task that can be resumed after some events occur
+  struct resumable_task : TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<resumable_task_state> {
+    using base_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<resumable_task_state>;
 
-  struct resumable_task_state;
-  struct resumable_task : std::coroutine_handle<resumable_task_state> {
     using promise_type = struct resumable_task_state;
+
+    base_type handle() { return *this; }
+
+    /// @return true if ready to resume
+    inline bool ready() const;
   };
 
+  /// encapsulates the state of the coroutine object visible to the outside world
+  /// @note this is the `promise_type` for resumable_task coroutine
   struct resumable_task_state {
-    resumable_task get_return_object() { return {resumable_task::from_promise(*this)}; }
-    std::suspend_never initial_suspend() noexcept { return {}; }
-    std::suspend_never final_suspend() noexcept { return {}; }
-    resumable_task_state& return_value(std::initializer_list<HDSpan> views) {
-      // push views onto work queue
-      return *this;
-    }
+    constexpr static inline std::size_t MaxNumEvents = 20;
+    using handle_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<resumable_task_state>;
+
+    resumable_task get_return_object() { return resumable_task{handle_type::from_promise(*this)}; }
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_never initial_suspend() noexcept { return {}; }
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_never final_suspend() noexcept { return {}; }
+    void return_void() {}
     void unhandled_exception() {}
 
+    /// @return true if ready to resume
+    constexpr bool ready() const {
+      for (std::size_t e = 0; e != nevents_; ++e)
+        if (!events_storage_[e]->finished()) return false;
+      return true;
+    }
+
+   private:
+    std::array<event*, MaxNumEvents> events_storage_;
+    std::size_t nevents_;
+
+    template <std::size_t N>
+    friend struct resumable_task_events;
+
+    void reset_events() {
+      std::fill(events_storage_.begin(), events_storage_.begin() + nevents_, nullptr);
+      nevents_ = 0;
+    }
+
+    template <std::size_t N>
+    void set_events(const std::array<event*, N> events) {
+      static_assert(N <= MaxNumEvents);
+      std::copy(events.begin(), events.end(), events_storage_.begin());
+      nevents_ = N;
+    }
+  };
+
+  bool resumable_task::ready() const { return base_type::promise().ready(); }
+
+  /// statically-sized sequence of events on whose completion progress of a given task depends on
+  /// @note this is the `Awaitable` for resumable_task coroutine
+  template <std::size_t N = 0>
+  struct resumable_task_events {
+   private:
+    template <std::size_t... I>
+    constexpr bool await_ready(std::index_sequence<I...>) const {
+      return (std::get<I>(events_.finished()) && ...);
+    }
+
+   public:
+    template <typename... Events>
+    constexpr resumable_task_events(Events&&... events) : events_{(&events)...} {}
+
+    constexpr bool await_ready() const { return await_ready(std::make_index_sequence<N>{}); }
+
+    void await_suspend(TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<resumable_task_state> pending_task) {
+      pending_task_ = pending_task;
+      pending_task_.promise().set_events(events_);
+    }
+
+    void await_resume() {
+      if (pending_task_) {
+        pending_task_.promise().reset_events();
+        pending_task_ = {};
+      }
+    }
+
    private:
-    ViewSpan<std::byte> views[20];
+    std::array<event*, N> events_;
+    TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<resumable_task_state> pending_task_;
   };
 
+  static_assert(resumable_task_events<0>{}.await_ready() == true);
 }  // namespace ttg
 
 #endif  // TTG_UTIL_COROUTINE_H

From 0467ab800b9af2cf8384a382f7867e1c4d11fbbe Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Tue, 1 Nov 2022 10:54:13 -0400
Subject: [PATCH 034/259] moar resumable_task ... final_suspend always suspends
 to avoid coros that never suspend

---
 tests/unit/fibonacci-coro.cc |  4 +++
 ttg/ttg/make_tt.h            | 58 ++++++++++++++++++++++++++++++++----
 ttg/ttg/util/coroutine.h     | 33 +++++++++++++++++---
 3 files changed, 85 insertions(+), 10 deletions(-)

diff --git a/tests/unit/fibonacci-coro.cc b/tests/unit/fibonacci-coro.cc
index 2ec9dd52c..e8ed5d1ba 100644
--- a/tests/unit/fibonacci-coro.cc
+++ b/tests/unit/fibonacci-coro.cc
@@ -58,6 +58,10 @@ TEST_CASE("Fibonacci-coroutines", "[fib][core]") {
             } else
               ttg::finalize<1>();
 
+            // to test coro-based task lifecycle introduce fake events
+            ttg::event null_event;
+            co_await ttg::resumable_task_events{null_event};
+
             // N.B. return void just as normal TT op
             co_return;
           },
diff --git a/ttg/ttg/make_tt.h b/ttg/ttg/make_tt.h
index 491d32e01..855706066 100644
--- a/ttg/ttg/make_tt.h
+++ b/ttg/ttg/make_tt.h
@@ -144,14 +144,60 @@ class CallableWrapTTArgs
   template <typename Key, typename Tuple, std::size_t... S>
   void call_func(Key &&key, Tuple &&args_tuple, output_terminalsT &out, std::index_sequence<S...>) {
     using func_args_t = ttg::meta::tuple_concat_t<std::tuple<const Key &>, input_refs_tuple_type, output_edges_type>;
-    if constexpr (funcT_receives_outterm_tuple)
-      func(std::forward<Key>(key),
-           baseT::template get<S, std::tuple_element_t<S + 1, func_args_t>>(std::forward<Tuple>(args_tuple))..., out);
-    else {
+    if constexpr (funcT_receives_outterm_tuple) {
+      using func_result_type = decltype(func(
+          std::forward<Key>(key),
+          baseT::template get<S, std::tuple_element_t<S + 1, func_args_t>>(std::forward<Tuple>(args_tuple))..., out));
+      if constexpr (std::is_void_v<func_result_type>)
+        func(std::forward<Key>(key),
+             baseT::template get<S, std::tuple_element_t<S + 1, func_args_t>>(std::forward<Tuple>(args_tuple))..., out);
+      else {
+        auto ret = func(
+            std::forward<Key>(key),
+            baseT::template get<S, std::tuple_element_t<S + 1, func_args_t>>(std::forward<Tuple>(args_tuple))..., out);
+        if (ret.completed())
+          ret.destroy();
+        else {
+          // in general will return the task handle ... now just mark events finished
+          auto events = ret.events();
+          for (auto &event_ptr : events) {
+            event_ptr->finish();
+          }
+          // in general will resume somewhere else ... in our test this proceeds to the end
+          assert(ret.ready());
+          ret.resume();
+          assert(ret.completed());
+          ret.destroy();
+        }
+      }
+    } else {
       auto old_output_tls_ptr = this->outputs_tls_ptr_accessor();
       this->set_outputs_tls_ptr();
-      func(std::forward<Key>(key),
-           baseT::template get<S, std::tuple_element_t<S + 1, func_args_t>>(std::forward<Tuple>(args_tuple))...);
+      using func_return_type =
+          decltype(func(std::forward<Key>(key), baseT::template get<S, std::tuple_element_t<S + 1, func_args_t>>(
+                                                    std::forward<Tuple>(args_tuple))...));
+      if constexpr (std::is_void_v<func_return_type>) {
+        func(std::forward<Key>(key),
+             baseT::template get<S, std::tuple_element_t<S + 1, func_args_t>>(std::forward<Tuple>(args_tuple))...);
+      } else {
+        auto ret =
+            func(std::forward<Key>(key),
+                 baseT::template get<S, std::tuple_element_t<S + 1, func_args_t>>(std::forward<Tuple>(args_tuple))...);
+        if (ret.completed())
+          ret.destroy();
+        else {
+          // in general will return the task handle ... now just mark events finished
+          auto events = ret.events();
+          for (auto &event_ptr : events) {
+            event_ptr->finish();
+          }
+          // in general will resume somewhere else ... in our test this proceeds to the end
+          assert(ret.ready());
+          ret.resume();
+          assert(ret.completed());
+          ret.destroy();
+        }
+      }
       this->set_outputs_tls_ptr(old_output_tls_ptr);
     }
   }
diff --git a/ttg/ttg/util/coroutine.h b/ttg/ttg/util/coroutine.h
index 5e221448b..f52857d64 100644
--- a/ttg/ttg/util/coroutine.h
+++ b/ttg/ttg/util/coroutine.h
@@ -39,6 +39,12 @@ namespace ttg {
 
     /// @return true if ready to resume
     inline bool ready() const;
+
+    /// @return true if task completed and can be destroyed
+    inline bool completed() const;
+
+    /// @return ttg::span of events that this task depends on
+    inline ttg::span<event*> events();
   };
 
   /// encapsulates the state of the coroutine object visible to the outside world
@@ -48,8 +54,15 @@ namespace ttg {
     using handle_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<resumable_task_state>;
 
     resumable_task get_return_object() { return resumable_task{handle_type::from_promise(*this)}; }
+
+    /// @note start task eagerly
     TTG_CXX_COROUTINE_NAMESPACE::suspend_never initial_suspend() noexcept { return {}; }
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_never final_suspend() noexcept { return {}; }
+
+    /// @note suspend task before destroying it so the runtime can know that the task is completed
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_always final_suspend() noexcept {
+      completed_ = true;
+      return {};
+    }
     void return_void() {}
     void unhandled_exception() {}
 
@@ -60,9 +73,15 @@ namespace ttg {
       return true;
     }
 
+    /// @return true if the task is completed
+    constexpr bool completed() const { return completed_; }
+
+    ttg::span<event*> events() { return ttg::span(events_storage_.data(), nevents_); }
+
    private:
     std::array<event*, MaxNumEvents> events_storage_;
     std::size_t nevents_;
+    bool completed_ = false;
 
     template <std::size_t N>
     friend struct resumable_task_events;
@@ -81,15 +100,17 @@ namespace ttg {
   };
 
   bool resumable_task::ready() const { return base_type::promise().ready(); }
+  bool resumable_task::completed() const { return base_type::promise().completed(); }
+  ttg::span<event*> resumable_task::events() { return base_type::promise().events(); }
 
   /// statically-sized sequence of events on whose completion progress of a given task depends on
   /// @note this is the `Awaitable` for resumable_task coroutine
-  template <std::size_t N = 0>
+  template <std::size_t N>
   struct resumable_task_events {
    private:
     template <std::size_t... I>
     constexpr bool await_ready(std::index_sequence<I...>) const {
-      return (std::get<I>(events_.finished()) && ...);
+      return (std::get<I>(events_)->finished() && ...);
     }
 
    public:
@@ -113,7 +134,11 @@ namespace ttg {
    private:
     std::array<event*, N> events_;
     TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<resumable_task_state> pending_task_;
-  };
+  };  // resumable_task_events
+
+  // deduce the number of events properly
+  template <typename... Events>
+  resumable_task_events(Events&&...) -> resumable_task_events<sizeof...(Events)>;
 
   static_assert(resumable_task_events<0>{}.await_ready() == true);
 }  // namespace ttg

From c7fcef8b9a5ed17fdf15f09eeb6bee3db2f90011 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Wed, 2 Nov 2022 08:57:50 -0400
Subject: [PATCH 035/259] implemented introspection of return type of
 generic/non-generic callables

---
 tests/unit/tt.cc             | 15 ++++++----
 ttg/ttg/util/meta.h          | 10 +++++++
 ttg/ttg/util/meta/callable.h | 55 +++++++++++++++++++++++++++++++-----
 3 files changed, 68 insertions(+), 12 deletions(-)

diff --git a/tests/unit/tt.cc b/tests/unit/tt.cc
index f77a483ef..3952d84fd 100644
--- a/tests/unit/tt.cc
+++ b/tests/unit/tt.cc
@@ -298,6 +298,9 @@ TEST_CASE("TemplateTask", "[core]") {
         // OK: all of {auto&&, auto&, const auto&} bind to const T&
         static_assert(std::is_invocable<decltype(func0), int, const float &, const float &, const float &,
                                         const float &, std::tuple<> &>::value);
+        static_assert(std::is_same_v<std::invoke_result_t<decltype(func0), int, const float &, const float &,
+                                                          const float &, const float &, std::tuple<> &>,
+                                     void>);
         // OK: ditto
         static_assert(std::is_void_v<decltype(tt_i_iv::func0(std::declval<const int &>(), std::declval<const float &>(),
                                                              std::declval<const float &>()))>);
@@ -323,14 +326,15 @@ TEST_CASE("TemplateTask", "[core]") {
                                ttg::typelist<ttg::typelist<int>, ttg::typelist<float &&, const float &>,
                                              ttg::typelist<float &&>, ttg::typelist<float &&, const float &>,
                                              ttg::typelist<float &&, const float &>, ttg::typelist<std::tuple<> &>>{})),
-                           ttg::typelist<>>);
+                           ttg::typelist<ttg::typelist<>, ttg::typelist<>>>);
         static_assert(
             std::is_same_v<
                 decltype(compute_arg_binding_types(
                     func0, ttg::typelist<ttg::typelist<int>, ttg::typelist<float &&, const float &>,
                                          ttg::typelist<float &&, const float &>, ttg::typelist<float &&, const float &>,
                                          ttg::typelist<float &&, const float &>, ttg::typelist<std::tuple<> &>>{})),
-                ttg::typelist<int, float &&, const float &, float &&, float &&, std::tuple<> &>>);
+                ttg::typelist<ttg::typelist<void>,
+                              ttg::typelist<int, float &&, const float &, float &&, float &&, std::tuple<> &>>>);
         // voids are skipped
         static_assert(
             std::is_same_v<
@@ -339,7 +343,8 @@ TEST_CASE("TemplateTask", "[core]") {
                                          ttg::typelist<float &&, const float &>, ttg::typelist<float &&, const float &>,
                                          ttg::typelist<float &&, const float &>, ttg::typelist<float &&, const float &>,
                                          ttg::typelist<void, std::tuple<> &>, ttg::typelist<void>>{})),
-                ttg::typelist<int, void, float &&, const float &, float &&, float &&, std::tuple<> &, void>>);
+                ttg::typelist<ttg::typelist<void>, ttg::typelist<int, void, float &&, const float &, float &&, float &&,
+                                                                 std::tuple<> &, void>>>);
 
         // test introspection of generic arguments by the runtime (i.e. contents of TT::input_args_type) and
         // the deduced types inside the function body
@@ -425,10 +430,10 @@ TEST_CASE("TemplateTask", "[core]") {
     static_assert(ttg::meta::is_generic_callable_v<decltype(g)>);
     auto [f_is_generic, f_args_t_v] = ttg::meta::callable_args<decltype(f)>;
     CHECK(!f_is_generic);
-    static_assert(std::is_same_v<decltype(f_args_t_v), ttg::typelist<int &>>);
+    static_assert(std::is_same_v<decltype(f_args_t_v), std::pair<ttg::typelist<void>, ttg::typelist<int &>>>);
     auto [g_is_generic, g_args_t_v] = ttg::meta::callable_args<decltype(g)>;
     CHECK(g_is_generic);
-    static_assert(std::is_same_v<decltype(g_args_t_v), ttg::typelist<>>);
+    static_assert(std::is_same_v<decltype(g_args_t_v), std::pair<ttg::typelist<>, ttg::typelist<>>>);
 
     {
       static_assert(!ttg::meta::is_generic_callable_v<decltype(&args_pmf::X::f)>);
diff --git a/ttg/ttg/util/meta.h b/ttg/ttg/util/meta.h
index 37b43e4f4..3637f66ae 100644
--- a/ttg/ttg/util/meta.h
+++ b/ttg/ttg/util/meta.h
@@ -866,6 +866,16 @@ namespace ttg {
     constexpr bool is_invocable_typelist_r_v<ReturnType, Callable, ttg::typelist<Args...>> =
         std::is_invocable_r_v<ReturnType, Callable, Args...>;
 
+    ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+    // detects the return result of a Callable when invoked with the arguments given as a typelist
+    ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+    template <typename Callable, typename Typelist>
+    struct invoke_result_typelist {};
+    template <typename Callable, typename... Args>
+    struct invoke_result_typelist<Callable, ttg::typelist<Args...>> : std::invoke_result<Callable, Args...> {};
+    template <class F, class... ArgTypes>
+    using invoke_result_typelist_t = typename invoke_result_typelist<F, ArgTypes...>::type;
+
   }  // namespace meta
 }  // namespace ttg
 
diff --git a/ttg/ttg/util/meta/callable.h b/ttg/ttg/util/meta/callable.h
index 7cdafd445..41b6f1672 100644
--- a/ttg/ttg/util/meta/callable.h
+++ b/ttg/ttg/util/meta/callable.h
@@ -31,12 +31,22 @@ namespace ttg::meta {
   template <typename Callable>
   constexpr inline bool is_generic_callable_v = is_generic_callable<Callable>::value;
 
+  /// callable_args<Callable> detects whether `Callable` is generic or not, and in the latter case
+  /// detects (using Boost.CallableTraits) its return and argument types.
+  /// callable_args<Callable> is a constexpr value of type
+  /// `std::pair<bool,ttg::typelist<ttg::typelist<ReturnType>,ttg::typelist<ArgTypes>>>`
+  /// where:
+  /// - the boolean indicates whether `Callable` is generic (true) or not (false),
+  /// - `ReturnType` is the return type of the `Callable` if it's generic, empty otherwise
+  /// - `ArgTypes`  is the argument types of the `Callable` if it's generic, empty otherwise
+  /// \tparam Callable a callable type
   template <typename Callable, typename Enabler = void>
-  constexpr std::pair<bool, ttg::typelist<>> callable_args = {true, {}};
+  constexpr std::pair<bool, std::pair<ttg::typelist<>, ttg::typelist<>>> callable_args = {true, {}};
 
   template <typename Callable>
   constexpr auto callable_args<Callable, ttg::meta::void_t<boost::callable_traits::args_t<Callable, ttg::typelist>>> =
-      std::pair<bool, boost::callable_traits::args_t<Callable, ttg::typelist>>{false, {}};
+      std::pair<bool, std::pair<ttg::typelist<boost::callable_traits::return_type_t<Callable>>,
+                                boost::callable_traits::args_t<Callable, ttg::typelist>>>{false, {}};
 
   //////////////////////////////////////
   // generic callables
@@ -57,6 +67,19 @@ namespace ttg::meta {
     return idx;
   }
 
+  /// detects argument and return types of a generic callable (\p func)
+  /// by trying each combination of types (\p argument_type_lists) for the respective arguments starting with
+  /// the combination corresponding to the given \p Ordinal
+  /// \tparam Ordinal a nonnegative integer specifying the ordinal of the type combination from \p Typelists to try;
+  ///                 maximum value is `(std::tuple_size_v<Typelists> * ...)`
+  /// \tparam Func a generic callable type
+  /// \tparam Typelists a pack of ttg::typelist's each of which specifies candidate types for the respective
+  ///         argument of \p Func
+  /// \param func a generic callable
+  /// \param argument_type_lists a ttg::typelist<Typelists...>
+  /// @note iterates over \p argument_type_lists in "row-major" order (i.e. last list in \p argument_type_lists
+  ///       is iterated first, etc.; the maxim
+  /// @return an object of type `ttg::typelist<ttg::typelist<ReturnType>,ttg::typelist<ArgTypes>>`
   template <std::size_t Ordinal, typename Func, typename... Typelists, std::size_t... ArgIdx>
   auto compute_arg_binding_types_impl(Func& func, typelist<Typelists...> argument_type_lists,
                                       std::index_sequence<ArgIdx...> arg_idx = {}) {
@@ -67,18 +90,34 @@ namespace ttg::meta {
     constexpr auto tensor_size = (extents[ArgIdx] * ...);
     static_assert(tensor_size >= Ordinal);
     if constexpr (tensor_size == Ordinal) {
-      return typelist<>{};
+      return typelist<typelist<>, typelist<>>{};
     } else {
       constexpr auto idx = ordinal2index(Ordinal, extents);
       auto args = typelist<std::tuple_element_t<idx[ArgIdx], std::tuple_element_t<ArgIdx, arg_typelists_t>>...>{};
-      if constexpr (is_invocable_typelist_v<Func, drop_void_t<decltype(args)>>) {
-        return args;
+      using args_sans_void_t = drop_void_t<decltype(args)>;
+      if constexpr (is_invocable_typelist_v<Func, args_sans_void_t>) {
+        using return_type = invoke_result_typelist_t<Func, args_sans_void_t>;
+        return ttg::typelist<ttg::typelist<return_type>, decltype(args)>{};
       } else {
         return compute_arg_binding_types_impl<Ordinal + 1>(func, argument_type_lists, arg_idx);
       }
     }
   }
 
+  /// detects argument types of a generic callable (\p func)
+  /// by trying each combination of types (\p argument_type_lists) for the respective arguments starting with
+  /// the combination corresponding to the given \p Ordinal . The callable is expected to return \p ReturnType
+  /// \tparam Ordinal a nonnegative integer specifying the ordinal of the type combination from \p Typelists to try;
+  ///                 maximum value is `(std::tuple_size_v<Typelists> * ...)`
+  /// \tparam ReturnType the expected return type of \p Func
+  /// \tparam Func a generic callable type
+  /// \tparam Typelists a pack of ttg::typelist's each of which specifies candidate types for the respective
+  ///         argument of \p Func
+  /// \param func a generic callable
+  /// \param argument_type_lists a ttg::typelist<Typelists...>
+  /// @note iterates over \p argument_type_lists in "row-major" order (i.e. last list in \p argument_type_lists
+  ///       is iterated first, etc.; the maxim
+  /// @return an object of type `ttg::typelist<ArgTypes>`
   template <std::size_t Ordinal, typename ReturnType, typename Func, typename... Typelists, std::size_t... ArgIdx>
   auto compute_arg_binding_types_r_impl(Func& func, typelist<Typelists...> argument_type_lists,
                                         std::index_sequence<ArgIdx...> arg_idx = {}) {
@@ -105,7 +144,9 @@ namespace ttg::meta {
   /// @tparam Typelists a pack of typelists encoding how each argument can be invoked
   /// @param func a reference to callable of type @p Func
   /// @param argument_type_lists a list of possible types to try for each argument; can contain `void`
-  /// @return a ttg::typelist encoding:
+  /// @return a `ttg::typelist<ttg::typelist<ReturnType>,ttg::typelist<ArgsTypes>>` where
+  ///          `ReturnType` is the return type of \p func and
+  ///         `ArgsTypes` encodes:
   ///         - the exact argument bindings used by `Func`, if @p func is a nongeneric callable;
   ///         - the first invocable combination of argument types discovered by row-major iteration, if @p func is a
   ///         generic callable
@@ -138,7 +179,7 @@ namespace ttg::meta {
       return compute_arg_binding_types_r_impl<0, ReturnType>(func, argument_type_lists,
                                                              std::make_index_sequence<sizeof...(Typelists)>{});
     } else {
-      return is_generic__args.second;
+      return is_generic__args.second.second;
     }
   }
 

From dc077e0e4b2374f2b20e11689502271461c12ee5 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Wed, 2 Nov 2022 11:18:39 -0400
Subject: [PATCH 036/259] fixup

---
 ttg/ttg/util/typelist.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ttg/ttg/util/typelist.h b/ttg/ttg/util/typelist.h
index 0f2363b12..72126a62b 100644
--- a/ttg/ttg/util/typelist.h
+++ b/ttg/ttg/util/typelist.h
@@ -26,7 +26,7 @@ namespace ttg {
       constexpr auto size() const { return sizeof...(Ts); }
 
       template <std::size_t I>
-      constexpr auto get() {
+      constexpr auto get() const {
         return type_identity<std::tuple_element_t<I, std::tuple<Ts...>>>{};
       }
     };

From 7a94b9ec49ce39b1a41c663096c5e880532c8524 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Wed, 2 Nov 2022 11:39:28 -0400
Subject: [PATCH 037/259] make_tt extracts return type name from the callable

---
 ttg/ttg/make_tt.h | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/ttg/ttg/make_tt.h b/ttg/ttg/make_tt.h
index 855706066..387e0270b 100644
--- a/ttg/ttg/make_tt.h
+++ b/ttg/ttg/make_tt.h
@@ -493,7 +493,10 @@ auto make_tt(funcT &&func, const std::tuple<ttg::Edge<keyT, input_edge_valuesT>.
 
   // gross argument typelist for invoking func, can include void for optional args
   constexpr static auto func_is_generic = ttg::meta::is_generic_callable_v<funcT>;
-  using gross_func_args_t = decltype(ttg::meta::compute_arg_binding_types_r<void>(func, candidate_func_args_t{}));
+  using return_type_typelist_and_gross_func_args_t =
+      decltype(ttg::meta::compute_arg_binding_types(func, candidate_func_args_t{}));
+  using func_return_t = std::tuple_element_t<0, std::tuple_element_t<0, return_type_typelist_and_gross_func_args_t>>;
+  using gross_func_args_t = std::tuple_element_t<1, return_type_typelist_and_gross_func_args_t>;
   constexpr auto DETECTED_HOW_TO_INVOKE_GENERIC_FUNC =
       func_is_generic ? !std::is_same_v<gross_func_args_t, ttg::typelist<>> : true;
   static_assert(DETECTED_HOW_TO_INVOKE_GENERIC_FUNC,

From 4f3e3769d72fb732cef3986b144cc4d49c2e8a18 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Wed, 2 Nov 2022 15:42:27 -0400
Subject: [PATCH 038/259] return type propagated further up the call stack to
 allow proper support for coroutines

also plan to support functions that do `return std::pair(key,value);` or `return value` (no key) or `return std::tuple{key};` instead of send
---
 ttg/CMakeLists.txt       |  20 +++--
 ttg/ttg/madness/ttg.h    |   5 +-
 ttg/ttg/make_device_tt.h |   5 +-
 ttg/ttg/make_tt.h        | 164 +++++++++++++++++++++++----------------
 ttg/ttg/parsec/ttg.h     |   3 +
 ttg/ttg/util/coroutine.h |   5 ++
 6 files changed, 128 insertions(+), 74 deletions(-)

diff --git a/ttg/CMakeLists.txt b/ttg/CMakeLists.txt
index 1f703fc0e..8945bc581 100644
--- a/ttg/CMakeLists.txt
+++ b/ttg/CMakeLists.txt
@@ -76,6 +76,7 @@ set(ttg-sources
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/env.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/version.cc
     )
+
 # extract git metadata
 include(GetGitMetadata)
 vgkit_cmake_git_metadata()
@@ -83,11 +84,7 @@ vgkit_cmake_git_metadata()
 set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/version.cc
         PROPERTIES
         COMPILE_DEFINITIONS "TTG_GIT_REVISION=\"${TTG_GIT_REVISION}\";TTG_GIT_DESCRIPTION=\"${TTG_GIT_DESCRIPTION}\"")
-set(ttg-public-headers ${ttg-headers};${ttg-impl-headers};${ttg-base-headers};${ttg-util-headers})
-if (NOT TTG_IGNORE_BUNDLED_EXTERNALS)
-  list(APPEND ttg-sources ${ttg-external-headers})
-  list(APPEND ttg-public-headers ${ttg-external-headers})
-endif()
+
 # optional dependencies
 if (TARGET Boost::boost)
   list(APPEND ttg-deps Boost::boost)
@@ -101,6 +98,19 @@ endif(TARGET TTG_Libunwind)
 if (TTG_ENABLE_TRACE)
   list(APPEND ttg-defs "TTG_ENABLE_TRACE=1")
 endif (TTG_ENABLE_TRACE)
+if (TARGET std::coroutine)
+  list(APPEND ttg-deps std::coroutine)
+  list(APPEND ttg-defs "TTG_HAS_COROUTINE=1")
+  list(APPEND ttg-util-headers
+          ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/coroutine.h
+          )
+endif ()
+
+set(ttg-public-headers ${ttg-headers};${ttg-impl-headers};${ttg-base-headers};${ttg-util-headers})
+if (NOT TTG_IGNORE_BUNDLED_EXTERNALS)
+  list(APPEND ttg-sources ${ttg-external-headers})
+  list(APPEND ttg-public-headers ${ttg-external-headers})
+endif()
 
 add_ttg_library(ttg "${ttg-sources}" PUBLIC_HEADER "${ttg-public-headers}" LINK_LIBRARIES "${ttg-deps}" INCLUDE_DIRECTORIES "${ttg-incs}" COMPILE_DEFINITIONS "${ttg-defs}")
 
diff --git a/ttg/ttg/madness/ttg.h b/ttg/ttg/madness/ttg.h
index e40fa25f0..8f2929860 100644
--- a/ttg/ttg/madness/ttg.h
+++ b/ttg/ttg/madness/ttg.h
@@ -21,9 +21,12 @@
 #include "ttg/util/macro.h"
 #include "ttg/util/meta.h"
 #include "ttg/util/meta/callable.h"
-#include "ttg/view.h"
 #include "ttg/util/void.h"
+#include "ttg/view.h"
 #include "ttg/world.h"
+#ifdef TTG_HAS_COROUTINE
+#include "ttg/util/coroutine.h"
+#endif
 
 #include <array>
 #include <cassert>
diff --git a/ttg/ttg/make_device_tt.h b/ttg/ttg/make_device_tt.h
index 94977624f..09f096d83 100644
--- a/ttg/ttg/make_device_tt.h
+++ b/ttg/ttg/make_device_tt.h
@@ -1,3 +1,4 @@
+// clang-format off
 #ifndef TTG_MAKE_DEVICE_TT_H
 #define TTG_MAKE_DEVICE_TT_H
 
@@ -161,7 +162,7 @@ namespace detail {
       detail::invoke_out_with_unpacked_views(out_func, key, views, std::make_index_sequence<view_tuple_size>());
     };
 
-  using wrapT = typename CallableWrapTTArgsAsTypelist<decltype(taskfn), have_outterm_tuple, keyT, output_terminals_type,
+  using wrapT = typename CallableWrapTTArgsAsTypelist<decltype(taskfn), void, have_outterm_tuple, keyT, output_terminals_type,
                                                       ttg::typelist<Args...>>::type;
 
   return std::make_unique<wrapT>(std::move(taskfn), inedges, outedges, name, innames, outnames);
@@ -276,3 +277,5 @@ auto make_device_tt(HostFuncT &&host_func,
 }
 #endif // 0
 #endif // TTG_MAKE_DEVICE_TT_H
+
+// clang-format on
diff --git a/ttg/ttg/make_tt.h b/ttg/ttg/make_tt.h
index 387e0270b..861222ed2 100644
--- a/ttg/ttg/make_tt.h
+++ b/ttg/ttg/make_tt.h
@@ -122,15 +122,17 @@ struct CallableWrapTTUnwrapTypelist<funcT, funcT_receives_outterm_tuple, keyT, o
 
 // Class to wrap a callable with signature
 //
-// case 1 (keyT != void): void op(auto&& key, input_valuesT&&..., std::tuple<output_terminalsT...>&)
-// case 2 (keyT == void): void op(input_valuesT&&..., std::tuple<output_terminalsT...>&)
+// case 1 (keyT != void): returnT op(auto&& key, input_valuesT&&..., std::tuple<output_terminalsT...>&)
+// case 2 (keyT == void): returnT op(input_valuesT&&..., std::tuple<output_terminalsT...>&)
 //
-template <typename funcT, bool funcT_receives_outterm_tuple, typename keyT, typename output_terminalsT,
-          typename... input_valuesT>
+// returnT is void for funcT = synchronous (ordinary) function and the appropriate return type for funcT=coroutine
+template <typename funcT, typename returnT, bool funcT_receives_outterm_tuple, typename keyT,
+          typename output_terminalsT, typename... input_valuesT>
 class CallableWrapTTArgs
-    : public TT<keyT, output_terminalsT,
-                CallableWrapTTArgs<funcT, funcT_receives_outterm_tuple, keyT, output_terminalsT, input_valuesT...>,
-                ttg::typelist<input_valuesT...>> {
+    : public TT<
+          keyT, output_terminalsT,
+          CallableWrapTTArgs<funcT, returnT, funcT_receives_outterm_tuple, keyT, output_terminalsT, input_valuesT...>,
+          ttg::typelist<input_valuesT...>> {
   using baseT = typename CallableWrapTTArgs::ttT;
 
   using input_values_tuple_type = typename baseT::input_values_tuple_type;
@@ -141,64 +143,91 @@ class CallableWrapTTArgs
   using noref_funcT = std::remove_reference_t<funcT>;
   std::conditional_t<std::is_function_v<noref_funcT>, std::add_pointer_t<noref_funcT>, noref_funcT> func;
 
+  using op_return_type =
+#ifdef TTG_HAS_COROUTINE
+      std::conditional_t<std::is_same_v<returnT, ttg::resumable_task>, ttg::resumable_task_state *, void>;
+#else   // TTG_HAS_COROUTINE
+      void;
+#endif  // TTG_HAS_COROUTINE
+
+  /// @return pointer to the coroutine state (promise) if funcT is a coroutine and it did not complete, else void
   template <typename Key, typename Tuple, std::size_t... S>
-  void call_func(Key &&key, Tuple &&args_tuple, output_terminalsT &out, std::index_sequence<S...>) {
+  auto call_func(Key &&key, Tuple &&args_tuple, output_terminalsT &out, std::index_sequence<S...>) {
     using func_args_t = ttg::meta::tuple_concat_t<std::tuple<const Key &>, input_refs_tuple_type, output_edges_type>;
+
+    auto process_return = [&out](auto &&ret) {
+      static_assert(std::is_same_v<std::remove_reference_t<decltype(ret)>, returnT>,
+                    "CallableWrapTTArgs<funcT,returnT,...>: returnT does not match the actual return type of funcT");
+      if constexpr (!std::is_void_v<returnT>) {  // protect from compiling for void returnT
+#ifdef TTG_HAS_COROUTINE
+        if constexpr (std::is_same_v<returnT, ttg::resumable_task>) {
+          ttg::resumable_task_state *result_value = nullptr;
+          if (ret.completed()) {
+            ret.destroy();
+          } else {
+            // in general will return the coroutine promise ptr
+            result_value = &ret.promise();
+            // BUT ... now just mark events finished
+            auto events = ret.events();
+            for (auto &event_ptr : events) {
+              event_ptr->finish();
+            }
+            // in general will resume somewhere else ... in our test this proceeds to the end
+            assert(ret.ready());
+            ret.resume();
+            assert(ret.completed());
+            ret.destroy();
+            result_value = nullptr;
+          }
+          return result_value;
+        } else
+#endif
+        {
+          static_assert(std::tuple_size_v<std::remove_reference_t<decltype(out)>> == 1,
+                        "CallableWrapTTArgs<funcT,returnT,funcT_receives_outterm_tuple=true,...): funcT can return a "
+                        "value only if there is only 1 out terminal");
+          static_assert(std::tuple_size_v<returnT> <= 2,
+                        "CallableWrapTTArgs<funcT,returnT,funcT_receives_outterm_tuple=true,...): funcT can return a "
+                        "value only if it is a plain value (then sent with null key), a tuple-like containing a single "
+                        "key (hence value is void), or a tuple-like containing a key and a value");
+          if constexpr (std::tuple_size_v<returnT> == 0)
+            std::get<0>(out).sendv(std::move(ret));
+          else if constexpr (std::tuple_size_v<returnT> == 1)
+            std::get<0>(out).sendk(std::move(std::get<0>(ret)));
+          else if constexpr (std::tuple_size_v<returnT> == 2)
+            std::get<0>(out).send(std::move(std::get<0>(ret)), std::move(std::get<1>(ret)));
+          return;
+        }
+      }
+    };
+
     if constexpr (funcT_receives_outterm_tuple) {
-      using func_result_type = decltype(func(
-          std::forward<Key>(key),
-          baseT::template get<S, std::tuple_element_t<S + 1, func_args_t>>(std::forward<Tuple>(args_tuple))..., out));
-      if constexpr (std::is_void_v<func_result_type>)
+      if constexpr (std::is_void_v<returnT>) {
         func(std::forward<Key>(key),
              baseT::template get<S, std::tuple_element_t<S + 1, func_args_t>>(std::forward<Tuple>(args_tuple))..., out);
-      else {
+        return;
+      } else {
         auto ret = func(
             std::forward<Key>(key),
             baseT::template get<S, std::tuple_element_t<S + 1, func_args_t>>(std::forward<Tuple>(args_tuple))..., out);
-        if (ret.completed())
-          ret.destroy();
-        else {
-          // in general will return the task handle ... now just mark events finished
-          auto events = ret.events();
-          for (auto &event_ptr : events) {
-            event_ptr->finish();
-          }
-          // in general will resume somewhere else ... in our test this proceeds to the end
-          assert(ret.ready());
-          ret.resume();
-          assert(ret.completed());
-          ret.destroy();
-        }
+
+        return process_return(std::move(ret));
       }
     } else {
       auto old_output_tls_ptr = this->outputs_tls_ptr_accessor();
       this->set_outputs_tls_ptr();
-      using func_return_type =
-          decltype(func(std::forward<Key>(key), baseT::template get<S, std::tuple_element_t<S + 1, func_args_t>>(
-                                                    std::forward<Tuple>(args_tuple))...));
-      if constexpr (std::is_void_v<func_return_type>) {
+      if constexpr (std::is_void_v<returnT>) {
         func(std::forward<Key>(key),
              baseT::template get<S, std::tuple_element_t<S + 1, func_args_t>>(std::forward<Tuple>(args_tuple))...);
+        this->set_outputs_tls_ptr(old_output_tls_ptr);
+        return;
       } else {
         auto ret =
             func(std::forward<Key>(key),
                  baseT::template get<S, std::tuple_element_t<S + 1, func_args_t>>(std::forward<Tuple>(args_tuple))...);
-        if (ret.completed())
-          ret.destroy();
-        else {
-          // in general will return the task handle ... now just mark events finished
-          auto events = ret.events();
-          for (auto &event_ptr : events) {
-            event_ptr->finish();
-          }
-          // in general will resume somewhere else ... in our test this proceeds to the end
-          assert(ret.ready());
-          ret.resume();
-          assert(ret.completed());
-          ret.destroy();
-        }
+        this->set_outputs_tls_ptr(old_output_tls_ptr);
+        return process_return(std::move(ret));
       }
-      this->set_outputs_tls_ptr(old_output_tls_ptr);
     }
   }
 
@@ -260,54 +289,55 @@ class CallableWrapTTArgs
   template <typename Key, typename ArgsTuple>
   std::enable_if_t<std::is_same_v<ArgsTuple, input_refs_tuple_type> &&
                        !ttg::meta::is_empty_tuple_v<input_refs_tuple_type> && !ttg::meta::is_void_v<Key>,
-                   void>
+                   op_return_type>
   op(Key &&key, ArgsTuple &&args_tuple, output_terminalsT &out) {
     assert(&out == &baseT::get_output_terminals());
-    call_func(std::forward<Key>(key), std::forward<ArgsTuple>(args_tuple), out,
-              std::make_index_sequence<std::tuple_size_v<ArgsTuple>>{});
+    return call_func(std::forward<Key>(key), std::forward<ArgsTuple>(args_tuple), out,
+                     std::make_index_sequence<std::tuple_size_v<ArgsTuple>>{});
   };
 
   template <typename ArgsTuple, typename Key = keyT>
   std::enable_if_t<std::is_same_v<ArgsTuple, input_refs_tuple_type> &&
                        !ttg::meta::is_empty_tuple_v<input_refs_tuple_type> && ttg::meta::is_void_v<Key>,
-                   void>
+                   op_return_type>
   op(ArgsTuple &&args_tuple, output_terminalsT &out) {
     assert(&out == &baseT::get_output_terminals());
-    call_func(std::forward<ArgsTuple>(args_tuple), out, std::make_index_sequence<std::tuple_size_v<ArgsTuple>>{});
+    return call_func(std::forward<ArgsTuple>(args_tuple), out,
+                     std::make_index_sequence<std::tuple_size_v<ArgsTuple>>{});
   };
 
   template <typename Key, typename ArgsTuple = input_refs_tuple_type>
-  std::enable_if_t<ttg::meta::is_empty_tuple_v<ArgsTuple> && !ttg::meta::is_void_v<Key>, void> op(
+  std::enable_if_t<ttg::meta::is_empty_tuple_v<ArgsTuple> && !ttg::meta::is_void_v<Key>, op_return_type> op(
       Key &&key, output_terminalsT &out) {
     assert(&out == &baseT::get_output_terminals());
-    call_func(std::forward<Key>(key), out);
+    return call_func(std::forward<Key>(key), out);
   };
 
   template <typename Key = keyT, typename ArgsTuple = input_refs_tuple_type>
-  std::enable_if_t<ttg::meta::is_empty_tuple_v<ArgsTuple> && ttg::meta::is_void_v<Key>, void> op(
+  std::enable_if_t<ttg::meta::is_empty_tuple_v<ArgsTuple> && ttg::meta::is_void_v<Key>, op_return_type> op(
       output_terminalsT &out) {
     assert(&out == &baseT::get_output_terminals());
-    call_func(out);
+    return call_func(out);
   };
 };
 
-template <typename funcT, bool funcT_receives_outterm_tuple, typename keyT, typename output_terminalsT,
-          typename input_values_typelistT>
+template <typename funcT, typename returnT, bool funcT_receives_outterm_tuple, typename keyT,
+          typename output_terminalsT, typename input_values_typelistT>
 struct CallableWrapTTArgsAsTypelist;
 
-template <typename funcT, bool funcT_receives_outterm_tuple, typename keyT, typename output_terminalsT,
-          typename... input_valuesT>
-struct CallableWrapTTArgsAsTypelist<funcT, funcT_receives_outterm_tuple, keyT, output_terminalsT,
+template <typename funcT, typename returnT, bool funcT_receives_outterm_tuple, typename keyT,
+          typename output_terminalsT, typename... input_valuesT>
+struct CallableWrapTTArgsAsTypelist<funcT, returnT, funcT_receives_outterm_tuple, keyT, output_terminalsT,
                                     std::tuple<input_valuesT...>> {
-  using type = CallableWrapTTArgs<funcT, funcT_receives_outterm_tuple, keyT, output_terminalsT,
+  using type = CallableWrapTTArgs<funcT, returnT, funcT_receives_outterm_tuple, keyT, output_terminalsT,
                                   std::remove_reference_t<input_valuesT>...>;
 };
 
-template <typename funcT, bool funcT_receives_outterm_tuple, typename keyT, typename output_terminalsT,
-          typename... input_valuesT>
-struct CallableWrapTTArgsAsTypelist<funcT, funcT_receives_outterm_tuple, keyT, output_terminalsT,
+template <typename funcT, typename returnT, bool funcT_receives_outterm_tuple, typename keyT,
+          typename output_terminalsT, typename... input_valuesT>
+struct CallableWrapTTArgsAsTypelist<funcT, returnT, funcT_receives_outterm_tuple, keyT, output_terminalsT,
                                     ttg::meta::typelist<input_valuesT...>> {
-  using type = CallableWrapTTArgs<funcT, funcT_receives_outterm_tuple, keyT, output_terminalsT,
+  using type = CallableWrapTTArgs<funcT, returnT, funcT_receives_outterm_tuple, keyT, output_terminalsT,
                                   std::remove_reference_t<input_valuesT>...>;
 };
 
@@ -542,8 +572,8 @@ auto make_tt(funcT &&func, const std::tuple<ttg::Edge<keyT, input_edge_valuesT>.
   using decayed_input_args_t = ttg::meta::decayed_typelist_t<input_args_t>;
   // 3. full_input_args_t = edge-types with non-void types replaced by input_args_t
   using full_input_args_t = ttg::meta::replace_nonvoid_t<input_edge_value_types, input_args_t>;
-  using wrapT = typename CallableWrapTTArgsAsTypelist<funcT, have_outterm_tuple, keyT, output_terminals_type,
-                                                      full_input_args_t>::type;
+  using wrapT = typename CallableWrapTTArgsAsTypelist<funcT, func_return_t, have_outterm_tuple, keyT,
+                                                      output_terminals_type, full_input_args_t>::type;
 
   return std::make_unique<wrapT>(std::forward<funcT>(func), inedges, outedges, name, innames, outnames);
 }
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 01d5b8c0f..d467136af 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -28,6 +28,9 @@
 #include "ttg/util/print.h"
 #include "ttg/util/trace.h"
 #include "ttg/util/typelist.h"
+#ifdef TTG_HAS_COROUTINE
+# include "ttg/util/coroutine.h"
+#endif
 
 #include "ttg/serialization/data_descriptor.h"
 
diff --git a/ttg/ttg/util/coroutine.h b/ttg/ttg/util/coroutine.h
index f52857d64..d05b62aae 100644
--- a/ttg/ttg/util/coroutine.h
+++ b/ttg/ttg/util/coroutine.h
@@ -50,6 +50,11 @@ namespace ttg {
   /// encapsulates the state of the coroutine object visible to the outside world
   /// @note this is the `promise_type` for resumable_task coroutine
   struct resumable_task_state {
+    resumable_task_state() noexcept = default;
+    // noncopyiable
+    resumable_task_state(const resumable_task_state&) = delete;
+    resumable_task_state& operator=(const resumable_task_state&) = delete;
+
     constexpr static inline std::size_t MaxNumEvents = 20;
     using handle_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<resumable_task_state>;
 

From ba15731e1e19a54e2b93323661e348dcc6be59e3 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Wed, 2 Nov 2022 16:13:06 -0400
Subject: [PATCH 039/259] ttg_parsec::TT::static_op now receives nonnull ptr to
 the coroutine state, if it suspended, ready to implement logic for dealing
 with suspended coroutines

---
 ttg/ttg/parsec/ttg.h | 35 ++++++++++++++++++++++++++---------
 1 file changed, 26 insertions(+), 9 deletions(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index d467136af..826831096 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1214,13 +1214,27 @@ namespace ttg_parsec {
 
    private:
     /// dispatches a call to derivedT::op if Space == Host, otherwise to derivedT::op_cuda if Space == CUDA
+    /// @return void*; if called a synchronous function or a coroutine that completed will return nullptr,
+    ///         else points to the state (promise) of the suspended coroutine
     template <ttg::ExecutionSpace Space, typename... Args>
-    void op(Args &&...args) {
+    void* op(Args &&...args) {
       derivedT *derived = static_cast<derivedT *>(this);
-      if constexpr (Space == ttg::ExecutionSpace::Host)
-        derived->op(std::forward<Args>(args)...);
-      else if constexpr (Space == ttg::ExecutionSpace::CUDA)
-        derived->op_cuda(std::forward<Args>(args)...);
+      if constexpr (Space == ttg::ExecutionSpace::Host) {
+        using return_type = decltype(derived->op(std::forward<Args>(args)...));
+        if constexpr (std::is_same_v<return_type,void>) {
+          derived->op(std::forward<Args>(args)...);
+          return nullptr;
+        }
+        else return derived->op(std::forward<Args>(args)...);
+      }
+      else if constexpr (Space == ttg::ExecutionSpace::CUDA) {
+        using return_type = decltype(derived->op_cuda(std::forward<Args>(args)...));
+        if constexpr (std::is_same_v<return_type,void>) {
+          derived->op_cuda(std::forward<Args>(args)...);
+          return nullptr;
+        }
+        else return derived->op_cuda(std::forward<Args>(args)...);
+      }
       else
         abort();
     }
@@ -1284,16 +1298,17 @@ namespace ttg_parsec {
           ttg::trace(obj->get_world().rank(), ":", obj->get_name(), " : executing");
       }
 
+      void* suspended_task_state = nullptr;  // if non-null, must resubmit the task
       if constexpr (!ttg::meta::is_void_v<keyT> && !ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
         auto input = make_tuple_of_ref_from_array(task, std::make_index_sequence<numinvals>{});
-        baseobj->template op<Space>(task->key, std::move(input), obj->output_terminals);
+        suspended_task_state = baseobj->template op<Space>(task->key, std::move(input), obj->output_terminals);
       } else if constexpr (!ttg::meta::is_void_v<keyT> && ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
-        baseobj->template op<Space>(task->key, obj->output_terminals);
+        suspended_task_state = baseobj->template op<Space>(task->key, obj->output_terminals);
       } else if constexpr (ttg::meta::is_void_v<keyT> && !ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
         auto input = make_tuple_of_ref_from_array(task, std::make_index_sequence<numinvals>{});
-        baseobj->template op<Space>(std::move(input), obj->output_terminals);
+        suspended_task_state = baseobj->template op<Space>(std::move(input), obj->output_terminals);
       } else if constexpr (ttg::meta::is_void_v<keyT> && ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
-        baseobj->template op<Space>(obj->output_terminals);
+        suspended_task_state = baseobj->template op<Space>(obj->output_terminals);
       } else {
         abort();
       }
@@ -1307,6 +1322,8 @@ namespace ttg_parsec {
       }
 
       // if op is coro-based this will be something else
+      // for now cannot handle suspended tasks
+      if (suspended_task_state) std::abort();
       return PARSEC_HOOK_RETURN_DONE;
     }
 

From c920334e9f55f5769f5457342e4b30fc60af6037 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Wed, 2 Nov 2022 18:25:18 -0400
Subject: [PATCH 040/259] sketch out resuming suspended task in the parsec
 backend

---
 ttg/ttg/parsec/ttg.h | 87 +++++++++++++++++++++++++++++---------------
 1 file changed, 57 insertions(+), 30 deletions(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 826831096..46139ecc1 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -595,6 +595,7 @@ namespace ttg_parsec {
       TT* tt;
       key_type key;
       size_goal_t stream[num_streams] = {};
+      void* suspended_task_state = nullptr;  // if not null the function already started
 
       parsec_ttg_task_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class)
           : parsec_ttg_task_base_t(mempool, task_class, num_streams) {
@@ -641,6 +642,7 @@ namespace ttg_parsec {
       static constexpr size_t num_streams = TT::numins;
       TT* tt;
       size_goal_t stream[num_streams] = {};
+      void* suspended_task_state = nullptr;  // if not null the function already started
 
       parsec_ttg_task_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class)
           : parsec_ttg_task_base_t(mempool, task_class, num_streams) {
@@ -1216,6 +1218,7 @@ namespace ttg_parsec {
     /// dispatches a call to derivedT::op if Space == Host, otherwise to derivedT::op_cuda if Space == CUDA
     /// @return void*; if called a synchronous function or a coroutine that completed will return nullptr,
     ///         else points to the state (promise) of the suspended coroutine
+    /// @internal the only reason to return dummy nullptr is to reduce the code duplication in the caller (static_op)
     template <ttg::ExecutionSpace Space, typename... Args>
     void* op(Args &&...args) {
       derivedT *derived = static_cast<derivedT *>(this);
@@ -1287,38 +1290,62 @@ namespace ttg_parsec {
     template <ttg::ExecutionSpace Space>
     static parsec_hook_return_t static_op(parsec_task_t *parsec_task) {
       task_t *task = (task_t*)parsec_task;
-      ttT *baseobj = task->tt;
-      derivedT *obj = static_cast<derivedT *>(baseobj);
-      assert(parsec_ttg_caller == NULL);
-      parsec_ttg_caller = static_cast<detail::parsec_ttg_task_base_t*>(task);
-      if (obj->tracing()) {
-        if constexpr (!ttg::meta::is_void_v<keyT>)
-          ttg::trace(obj->get_world().rank(), ":", obj->get_name(), " : ", task->key, ": executing");
-        else
-          ttg::trace(obj->get_world().rank(), ":", obj->get_name(), " : executing");
-      }
-
-      void* suspended_task_state = nullptr;  // if non-null, must resubmit the task
-      if constexpr (!ttg::meta::is_void_v<keyT> && !ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
-        auto input = make_tuple_of_ref_from_array(task, std::make_index_sequence<numinvals>{});
-        suspended_task_state = baseobj->template op<Space>(task->key, std::move(input), obj->output_terminals);
-      } else if constexpr (!ttg::meta::is_void_v<keyT> && ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
-        suspended_task_state = baseobj->template op<Space>(task->key, obj->output_terminals);
-      } else if constexpr (ttg::meta::is_void_v<keyT> && !ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
-        auto input = make_tuple_of_ref_from_array(task, std::make_index_sequence<numinvals>{});
-        suspended_task_state = baseobj->template op<Space>(std::move(input), obj->output_terminals);
-      } else if constexpr (ttg::meta::is_void_v<keyT> && ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
-        suspended_task_state = baseobj->template op<Space>(obj->output_terminals);
-      } else {
-        abort();
+      void* suspended_task_state = task->suspended_task_state;  // if non-null, must resubmit the task
+      if (suspended_task_state == nullptr) {  // task has not started
+        ttT *baseobj = task->tt;
+        derivedT *obj = static_cast<derivedT *>(baseobj);
+        assert(parsec_ttg_caller == NULL);
+        parsec_ttg_caller = static_cast<detail::parsec_ttg_task_base_t*>(task);
+        if (obj->tracing()) {
+          if constexpr (!ttg::meta::is_void_v<keyT>)
+            ttg::trace(obj->get_world().rank(), ":", obj->get_name(), " : ", task->key, ": executing");
+          else
+            ttg::trace(obj->get_world().rank(), ":", obj->get_name(), " : executing");
+        }
+
+        if constexpr (!ttg::meta::is_void_v<keyT> && !ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
+          auto input = make_tuple_of_ref_from_array(task, std::make_index_sequence<numinvals>{});
+          suspended_task_state = baseobj->template op<Space>(task->key, std::move(input), obj->output_terminals);
+        } else if constexpr (!ttg::meta::is_void_v<keyT> && ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
+          suspended_task_state = baseobj->template op<Space>(task->key, obj->output_terminals);
+        } else if constexpr (ttg::meta::is_void_v<keyT> && !ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
+          auto input = make_tuple_of_ref_from_array(task, std::make_index_sequence<numinvals>{});
+          suspended_task_state = baseobj->template op<Space>(std::move(input), obj->output_terminals);
+        } else if constexpr (ttg::meta::is_void_v<keyT> && ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
+          suspended_task_state = baseobj->template op<Space>(obj->output_terminals);
+        } else {
+          abort();
+        }
+        parsec_ttg_caller = NULL;
       }
-      parsec_ttg_caller = NULL;
+      else {
+#ifdef TTG_HAS_COROUTINE
+        auto* coro_state = static_cast<ttg::resumable_task_state*>(suspended_task_state);
+        auto ret = coro_state->get_return_object();
+        assert(ret.ready());
+        ret.resume();
+        if (ret.completed()) {
+          ret.destroy();
+          suspended_task_state = nullptr;
+        }
+        else { // not yet completed
+          // leave suspended_task_state as is
+        }
+#else
+        abort();  // should not happen
+#endif
+      }
+      task->suspended_task_state = suspended_task_state;
 
-      if (obj->tracing()) {
-        if constexpr (!ttg::meta::is_void_v<keyT>)
-          ttg::trace(obj->get_world().rank(), ":", obj->get_name(), " : ", task->key, ": done executing");
-        else
-          ttg::trace(obj->get_world().rank(), ":", obj->get_name(), " : done executing");
+      if (suspended_task_state == nullptr) {
+        ttT *baseobj = task->tt;
+        derivedT *obj = static_cast<derivedT *>(baseobj);
+        if (obj->tracing()) {
+          if constexpr (!ttg::meta::is_void_v<keyT>)
+            ttg::trace(obj->get_world().rank(), ":", obj->get_name(), " : ", task->key, ": done executing");
+          else
+            ttg::trace(obj->get_world().rank(), ":", obj->get_name(), " : done executing");
+        }
       }
 
       // if op is coro-based this will be something else

From 02c4e3e9b57dddfb09361b5e7589b1cbe26d8d28 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Thu, 3 Nov 2022 08:40:52 -0400
Subject: [PATCH 041/259] parsec reruns suspended tasks successfully

the only missing piece now is event monitoring and resubmission by the worker threads
---
 ttg/ttg/make_tt.h        | 15 ++-------------
 ttg/ttg/parsec/ttg.h     | 17 ++++++++++++-----
 ttg/ttg/util/coroutine.h |  4 +++-
 3 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/ttg/ttg/make_tt.h b/ttg/ttg/make_tt.h
index 861222ed2..85c3bed0e 100644
--- a/ttg/ttg/make_tt.h
+++ b/ttg/ttg/make_tt.h
@@ -162,22 +162,11 @@ class CallableWrapTTArgs
 #ifdef TTG_HAS_COROUTINE
         if constexpr (std::is_same_v<returnT, ttg::resumable_task>) {
           ttg::resumable_task_state *result_value = nullptr;
+          // if task completed destroy it
           if (ret.completed()) {
             ret.destroy();
-          } else {
-            // in general will return the coroutine promise ptr
+          } else {  // if task is suspended return the coroutine promise ptr
             result_value = &ret.promise();
-            // BUT ... now just mark events finished
-            auto events = ret.events();
-            for (auto &event_ptr : events) {
-              event_ptr->finish();
-            }
-            // in general will resume somewhere else ... in our test this proceeds to the end
-            assert(ret.ready());
-            ret.resume();
-            assert(ret.completed());
-            ret.destroy();
-            result_value = nullptr;
           }
           return result_value;
         } else
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 46139ecc1..d7134febb 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1290,7 +1290,8 @@ namespace ttg_parsec {
     template <ttg::ExecutionSpace Space>
     static parsec_hook_return_t static_op(parsec_task_t *parsec_task) {
       task_t *task = (task_t*)parsec_task;
-      void* suspended_task_state = task->suspended_task_state;  // if non-null, must resubmit the task
+
+      void* suspended_task_state = task->suspended_task_state;  // non-null = need to resume the task
       if (suspended_task_state == nullptr) {  // task has not started
         ttT *baseobj = task->tt;
         derivedT *obj = static_cast<derivedT *>(baseobj);
@@ -1348,10 +1349,16 @@ namespace ttg_parsec {
         }
       }
 
-      // if op is coro-based this will be something else
-      // for now cannot handle suspended tasks
-      if (suspended_task_state) std::abort();
-      return PARSEC_HOOK_RETURN_DONE;
+      // TODO PARSEC_HOOK_RETURN_AGAIN -> PARSEC_HOOK_RETURN_ASYNC when event tracking and task resumption (by this thread) is ready
+      // right now can events are not properly implemented, we are only testing the workflow with dummy events
+      // so mark the events finished manually, parsec will rerun this task again and it should complete the second time
+      if (suspended_task_state) {
+        auto events = static_cast<ttg::resumable_task_state*>(suspended_task_state)->events();
+        for (auto &event_ptr : events) {
+          event_ptr->finish();
+        }
+      }
+      return suspended_task_state ? PARSEC_HOOK_RETURN_AGAIN : PARSEC_HOOK_RETURN_DONE;
     }
 
     template <ttg::ExecutionSpace Space>
diff --git a/ttg/ttg/util/coroutine.h b/ttg/ttg/util/coroutine.h
index d05b62aae..d709b62c8 100644
--- a/ttg/ttg/util/coroutine.h
+++ b/ttg/ttg/util/coroutine.h
@@ -51,9 +51,11 @@ namespace ttg {
   /// @note this is the `promise_type` for resumable_task coroutine
   struct resumable_task_state {
     resumable_task_state() noexcept = default;
-    // noncopyiable
+    // these only live on coroutine frames so make noncopyable and nonmovable
     resumable_task_state(const resumable_task_state&) = delete;
     resumable_task_state& operator=(const resumable_task_state&) = delete;
+    resumable_task_state(resumable_task_state&&) = delete;
+    resumable_task_state& operator=(resumable_task_state&&) = delete;
 
     constexpr static inline std::size_t MaxNumEvents = 20;
     using handle_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<resumable_task_state>;

From 943f664982ab093239755d746618602ee5408b5e Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Thu, 3 Nov 2022 13:32:11 -0400
Subject: [PATCH 042/259] cleanup + updated design notes

---
 doc/dox/contrib/Design-Device.md | 127 ++++++++++++++++++++++++++-----
 ttg/ttg/parsec/ttg.h             |  16 +++-
 2 files changed, 122 insertions(+), 21 deletions(-)

diff --git a/doc/dox/contrib/Design-Device.md b/doc/dox/contrib/Design-Device.md
index cf2e7f1b8..2c40f64fd 100644
--- a/doc/dox/contrib/Design-Device.md
+++ b/doc/dox/contrib/Design-Device.md
@@ -32,30 +32,126 @@ Additional memory-related concerns common to both models:
     - hence it makes sense to make representation of object's partial state (`View`) a first-class concept in both models.
 
 ### Asynchrony
-- *Stages*: decompose tasks into stages, with runtime-managed periods between stages for managing asynchrony of the actions scheduled by each stage
-    - pro: most explicit, easier to reason about
+- *Continuations/stages*: decompose tasks into _continuations_ (stages), with runtime-managed scheduling of continutations for managing the asynchrony of the actions initiated by each continuation
+    - pro: most explicit, easier to reason about, fewest performance implications
     - con: most verbose; device-capable tasks look very different from host tasks
-- *"Threads"*: use conventional mechanisms (threads for current backends, coroutines/fibers usable).
+    - con: limited composability
+        - difficult to support general computation patterns (e.g. generator continuation, etc.,)
+- *"Threads"*: use threads to deal with the asynchrony (in principle could use user-space threads = fibers)
     - pro: least host/device dichotomy
-        - threads: tasks are ordinary functions
-        - fibers/coroutines: task functions "look" like ordinary functions (and can be made almost like normal functions using macros)
-    - con: potentially lower performance
-        - threads: due to the need to context switch to "yield" to other tasks
-        - fibers/coroutines: implementation details are more complex and usually involve heap allocation (C++20 coro)
-    - con: runtime is still responsible for managing the executor space heterogeneity (control where to launch a task) and asynchrony (events/host callbacks).
+        - tasks are ordinary (synchronous) functions
+        - fully composable
+    - con: performance implications
+        - due to the need to context switch to "yield" to other tasks
+        - thus even fully synchronous computations will suffer
+    - con: asynchrony artifacts still appear
+        - asynchronous calls must be in general annotated (to force synchronous execution and/or to provide hints to the thread scheduler)
+- *"Coroutines"*: use C++20 coroutines
+    - pro: less host/device dichotomy compared to continuations
+        - task functions "look" like ordinary functions (and can be made almost like normal functions using macros) but returning a custom return object (containing return status + handle to the coroutine) instead of void
+        - fully composable
+    - performance implications
+        - pro: no impact on synchronous tasks
+        - con: coroutine implementation details are complex and usually involve heap allocation
+        - pro: custom allocators can be introduced to elide heap allocation (at the cost of limited generality)
+    - con: asynchrony artifacts still appear
+        - co_await annotate the spots where execution may need to be suspended
+    - con: less mature due to the need for C++20
+        - GCC (10+), LLVM (8+) support coroutines
+        - TTG and all of its dependencies will be impacted by the raised standard requirement
 
 ### other considerations
 
 - it's not possible to manage memory from the device code, hence all program logic, including _device-capable_ tasks, must execute on host executors. In principle if we restricted ourselves to a single-source language (SYLC-extended C++) we could write device capable tasks directly as device code, but current language limitations mandate wrapping everything into host code.
+- runtime is still responsible for managing the executor space heterogeneity (control where to launch a task) and asynchrony (events/host callbacks).
 
 ## Current designs
 - *UM+threads*: use UM for memory management + threads for asynchrony
 - *DM+stages*: use Parsec's device memory pool manager + stage-decomposed tasks
 - *?M+coroutines*: UM/DM for memory + C++20 coroutines for handling the asynchrony
 
-### Coroutine-based design
-C++ coroutines were introduced in C++20. The design is fairly convoluted, with many customization points, and our understanding of the best practices is evolving.
-The basic idea is to have user compose device tasks as a single coroutine that gets decomposed by the compiler into the stages that in the *DM+stages* design user had to express as individual lambdas:
+### Example code: threads vs continuations vs coroutines
+
+How should we map the following host task onto the device?
+```cpp
+make_tt([](auto& key, auto& data1, auto& data2) -> void {
+    double data3 = blap::dot(data1.data(), data2.data());
+    if (data3 >= 0.)
+        send<0>(data1);
+    else
+        send<0>(data2);
+}
+```
+
+Ideally the task will receive `data1` and `data2` already transferred to the memory space(s) accessible from the device execution space:
+```cpp
+make_device_tt([](auto& key, auto& data1, auto& data2) -> void {
+    double data3 = blap::device_dot(data1.data(), data2.data());
+    if (data3 >= 0.)
+        send<0>(data1);
+    else
+        send<0>(data2);
+}
+```
+But now `data3` lives in the host memory so in general we must manage its transfer from the device. Hence either:
+- all intermediate data must be managed explicitly within the task, or
+- except for the cases where user types are aware of multiple memory spaces (but this makes the state of such types asynchronous).
+
+Here are the tentative device versions of this task in each of the 3 approaches (the memory details are omitted).
+
+#### Threads
+```cpp
+make_tt([](auto& key, auto& data1, auto& data2) -> void {
+    // stage 1
+    ConstView view1(data1);
+    ConstView view2(data2);
+    double data3;
+    View view3(data3, NewView | SyncView_D2H);
+    // depending on the memory model may need to wait here for the transfers to complete
+    // could build the waits into View ctors, or need an explicit await()
+    
+    // stage 2
+    cublasDdot(view1.device_ptr(), view2.device_ptr(), view3.device_ptr());
+    // if called an async function need explicit await() here
+    // also: who/how will view3 be synchronized
+
+    if (data3 >= 0.)
+        send<0>(data1);
+    else
+        send<0>(data2);
+}
+```
+N.B. `make_tt`: this is a regular task.
+
+#### Continuations
+```cpp
+make_device_tt(
+  // stage 1
+  [](auto& key, auto& data1, auto& data2) {
+    ConstView view1(data1);
+    ConstView view2(data2);
+    double data3;
+    View view3(data3, NewView | SyncView_D2H);
+    return {view1, view2, view3};
+    }, 
+  // stage 2
+  [](auto& key, auto& views) {
+    auto& [view1, view2, view3] = views;
+    cublasDdot(view1.device_ptr(), view2.device_ptr(), view3.device_ptr());
+  },
+  // stage 3
+  [](auto& key, auto& views) {
+    auto& [view1, view2, view3] = views;
+    if (*view3.host_ptr() >= 0.)
+        send<0>(data1);
+    else
+        send<0>(data2);
+    }
+}
+```
+N.B. `make_device_tt` vs `make_tt`: this is a special task.
+
+#### Coroutines
 ```cpp
 make_tt([](auto& key, auto& data1, auto& data2) -> ttg::resumable_task {
     // stage 1
@@ -63,7 +159,7 @@ make_tt([](auto& key, auto& data1, auto& data2) -> ttg::resumable_task {
     ConstView view2(data2);
     double data3;
     View view3(data3, NewView | SyncView_D2H);
-    co_await sync_views(data1, data2, data3);  // creates list of transfers to be fulfilled by the runtime
+    co_await sync_views(view1, view2, view3);  // creates list of transfers to be fulfilled by the runtime
     
     // stage 2
     cublasDdot(view1.device_ptr(), view2.device_ptr(), view3.device_ptr());
@@ -76,7 +172,4 @@ make_tt([](auto& key, auto& data1, auto& data2) -> ttg::resumable_task {
     co_return;  // processes sends and destroys coroutine
 }, ...);
 ```
-
-The advantages of this design over the *DM+stages* design are:
-- the number and type of stages is completely arbitrary, and
-- it is possible to make the device code for simple cases as similar to the host case as possible.
+N.B. `make_tt` and `ttg::resumable_task`: this is a regular task but with special return type.
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index d7134febb..ccc6d7ca2 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1349,16 +1349,24 @@ namespace ttg_parsec {
         }
       }
 
-      // TODO PARSEC_HOOK_RETURN_AGAIN -> PARSEC_HOOK_RETURN_ASYNC when event tracking and task resumption (by this thread) is ready
-      // right now can events are not properly implemented, we are only testing the workflow with dummy events
-      // so mark the events finished manually, parsec will rerun this task again and it should complete the second time
       if (suspended_task_state) {
+        // right now can events are not properly implemented, we are only testing the workflow with dummy events
+        // so mark the events finished manually, parsec will rerun this task again and it should complete the second time
         auto events = static_cast<ttg::resumable_task_state*>(suspended_task_state)->events();
         for (auto &event_ptr : events) {
           event_ptr->finish();
         }
+
+        // TODO: shove {ptr to parsec_task, ptr to this function} to the list of tasks suspended by this thread (hence stored in TLS)
+        // thread will loop over its list (after running every task? periodically? need a dedicated queue of ready tasks?)
+        // and resume the suspended tasks whose events are ready (N.B. ptr to parsec_task is enough to get the list of pending events)
+        // event clearance will in device case handled by host callbacks run by the dedicated device runtime thread
+
+        // TODO PARSEC_HOOK_RETURN_AGAIN -> PARSEC_HOOK_RETURN_ASYNC when event tracking and task resumption (by this thread) is ready
+        return PARSEC_HOOK_RETURN_AGAIN;
       }
-      return suspended_task_state ? PARSEC_HOOK_RETURN_AGAIN : PARSEC_HOOK_RETURN_DONE;
+      else
+        return PARSEC_HOOK_RETURN_DONE;
     }
 
     template <ttg::ExecutionSpace Space>

From 63195c4b733746e47cf38000daf1598f88e3f313 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Fri, 4 Nov 2022 09:06:39 -0400
Subject: [PATCH 043/259] cleanup + annotation of of the coroutine classes +
 cleanup of the PaRSEC implementation

- introduced ttg::coroutine_handle<> = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<resumable_task_state>
- tested custom new/delete for coroutines, works but commented out; added notes sketching out the interaction with the runtime that will allow heap-free operation
- parsec impl is back to optimal for ordinary function ops; coroutine-based TT::op() returns coroutine_handle instead of ptr to resumable_task_state
---
 ttg/ttg/make_tt.h        | 10 +++----
 ttg/ttg/parsec/ttg.h     | 63 +++++++++++++++++++++++-----------------
 ttg/ttg/tt.h             | 61 ++++++++++++++++++++++++++------------
 ttg/ttg/util/coroutine.h | 53 +++++++++++++++++++++++++++++----
 4 files changed, 131 insertions(+), 56 deletions(-)

diff --git a/ttg/ttg/make_tt.h b/ttg/ttg/make_tt.h
index 85c3bed0e..50d307a98 100644
--- a/ttg/ttg/make_tt.h
+++ b/ttg/ttg/make_tt.h
@@ -145,12 +145,12 @@ class CallableWrapTTArgs
 
   using op_return_type =
 #ifdef TTG_HAS_COROUTINE
-      std::conditional_t<std::is_same_v<returnT, ttg::resumable_task>, ttg::resumable_task_state *, void>;
+      std::conditional_t<std::is_same_v<returnT, ttg::resumable_task>, ttg::coroutine_handle<>, void>;
 #else   // TTG_HAS_COROUTINE
       void;
 #endif  // TTG_HAS_COROUTINE
 
-  /// @return pointer to the coroutine state (promise) if funcT is a coroutine and it did not complete, else void
+  /// @return coroutine handle<> (if funcT is a coroutine), else void
   template <typename Key, typename Tuple, std::size_t... S>
   auto call_func(Key &&key, Tuple &&args_tuple, output_terminalsT &out, std::index_sequence<S...>) {
     using func_args_t = ttg::meta::tuple_concat_t<std::tuple<const Key &>, input_refs_tuple_type, output_edges_type>;
@@ -161,14 +161,14 @@ class CallableWrapTTArgs
       if constexpr (!std::is_void_v<returnT>) {  // protect from compiling for void returnT
 #ifdef TTG_HAS_COROUTINE
         if constexpr (std::is_same_v<returnT, ttg::resumable_task>) {
-          ttg::resumable_task_state *result_value = nullptr;
+          ttg::coroutine_handle<> coro_handle;
           // if task completed destroy it
           if (ret.completed()) {
             ret.destroy();
           } else {  // if task is suspended return the coroutine promise ptr
-            result_value = &ret.promise();
+            coro_handle = ret;
           }
-          return result_value;
+          return coro_handle;
         } else
 #endif
         {
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index ccc6d7ca2..eb855bf77 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -28,9 +28,6 @@
 #include "ttg/util/print.h"
 #include "ttg/util/trace.h"
 #include "ttg/util/typelist.h"
-#ifdef TTG_HAS_COROUTINE
-# include "ttg/util/coroutine.h"
-#endif
 
 #include "ttg/serialization/data_descriptor.h"
 
@@ -595,7 +592,9 @@ namespace ttg_parsec {
       TT* tt;
       key_type key;
       size_goal_t stream[num_streams] = {};
-      void* suspended_task_state = nullptr;  // if not null the function already started
+#ifdef TTG_HAS_COROUTINE
+      void* suspended_task_address = nullptr;  // if not null the function is suspended
+#endif
 
       parsec_ttg_task_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class)
           : parsec_ttg_task_base_t(mempool, task_class, num_streams) {
@@ -642,7 +641,9 @@ namespace ttg_parsec {
       static constexpr size_t num_streams = TT::numins;
       TT* tt;
       size_goal_t stream[num_streams] = {};
-      void* suspended_task_state = nullptr;  // if not null the function already started
+#ifdef TTG_HAS_COROUTINE
+      void* suspended_task_address = nullptr;  // if not null the function is suspended
+#endif
 
       parsec_ttg_task_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class)
           : parsec_ttg_task_base_t(mempool, task_class, num_streams) {
@@ -1216,27 +1217,30 @@ namespace ttg_parsec {
 
    private:
     /// dispatches a call to derivedT::op if Space == Host, otherwise to derivedT::op_cuda if Space == CUDA
-    /// @return void*; if called a synchronous function or a coroutine that completed will return nullptr,
-    ///         else points to the state (promise) of the suspended coroutine
-    /// @internal the only reason to return dummy nullptr is to reduce the code duplication in the caller (static_op)
+    /// @return void if called a synchronous function, or ttg::coroutine_handle<> if called a coroutine (if non-null,
+    ///    points to the suspended coroutine)
     template <ttg::ExecutionSpace Space, typename... Args>
-    void* op(Args &&...args) {
+    auto op(Args &&...args) {
       derivedT *derived = static_cast<derivedT *>(this);
       if constexpr (Space == ttg::ExecutionSpace::Host) {
         using return_type = decltype(derived->op(std::forward<Args>(args)...));
         if constexpr (std::is_same_v<return_type,void>) {
           derived->op(std::forward<Args>(args)...);
-          return nullptr;
+          return;
+        }
+        else {
+          return derived->op(std::forward<Args>(args)...);
         }
-        else return derived->op(std::forward<Args>(args)...);
       }
       else if constexpr (Space == ttg::ExecutionSpace::CUDA) {
         using return_type = decltype(derived->op_cuda(std::forward<Args>(args)...));
         if constexpr (std::is_same_v<return_type,void>) {
           derived->op_cuda(std::forward<Args>(args)...);
-          return nullptr;
+          return;
+        }
+        else {
+          return derived->op_cuda(std::forward<Args>(args)...);
         }
-        else return derived->op_cuda(std::forward<Args>(args)...);
       }
       else
         abort();
@@ -1291,8 +1295,13 @@ namespace ttg_parsec {
     static parsec_hook_return_t static_op(parsec_task_t *parsec_task) {
       task_t *task = (task_t*)parsec_task;
 
-      void* suspended_task_state = task->suspended_task_state;  // non-null = need to resume the task
-      if (suspended_task_state == nullptr) {  // task has not started
+      void* suspended_task_address =
+#ifdef TTG_HAS_COROUTINE
+        task->suspended_task_address;  // non-null = need to resume the task
+#else
+        nullptr;
+#endif
+      if (suspended_task_address == nullptr) {  // task is a coroutine that has not started or an ordinary function
         ttT *baseobj = task->tt;
         derivedT *obj = static_cast<derivedT *>(baseobj);
         assert(parsec_ttg_caller == NULL);
@@ -1306,14 +1315,14 @@ namespace ttg_parsec {
 
         if constexpr (!ttg::meta::is_void_v<keyT> && !ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
           auto input = make_tuple_of_ref_from_array(task, std::make_index_sequence<numinvals>{});
-          suspended_task_state = baseobj->template op<Space>(task->key, std::move(input), obj->output_terminals);
+          TTG_PROCESS_TT_OP_RETURN(suspended_task_address, baseobj->template op<Space>(task->key, std::move(input), obj->output_terminals));
         } else if constexpr (!ttg::meta::is_void_v<keyT> && ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
-          suspended_task_state = baseobj->template op<Space>(task->key, obj->output_terminals);
+          TTG_PROCESS_TT_OP_RETURN(suspended_task_address, baseobj->template op<Space>(task->key, obj->output_terminals));
         } else if constexpr (ttg::meta::is_void_v<keyT> && !ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
           auto input = make_tuple_of_ref_from_array(task, std::make_index_sequence<numinvals>{});
-          suspended_task_state = baseobj->template op<Space>(std::move(input), obj->output_terminals);
+          TTG_PROCESS_TT_OP_RETURN(suspended_task_address, baseobj->template op<Space>(std::move(input), obj->output_terminals));
         } else if constexpr (ttg::meta::is_void_v<keyT> && ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
-          suspended_task_state = baseobj->template op<Space>(obj->output_terminals);
+          TTG_PROCESS_TT_OP_RETURN(suspended_task_address, baseobj->template op<Space>(obj->output_terminals));
         } else {
           abort();
         }
@@ -1321,24 +1330,23 @@ namespace ttg_parsec {
       }
       else {
 #ifdef TTG_HAS_COROUTINE
-        auto* coro_state = static_cast<ttg::resumable_task_state*>(suspended_task_state);
-        auto ret = coro_state->get_return_object();
+        auto ret = static_cast<ttg::resumable_task>(ttg::coroutine_handle<>::from_address(suspended_task_address));
         assert(ret.ready());
         ret.resume();
         if (ret.completed()) {
           ret.destroy();
-          suspended_task_state = nullptr;
+          suspended_task_address = nullptr;
         }
         else { // not yet completed
-          // leave suspended_task_state as is
+          // leave suspended_task_address as is
         }
 #else
         abort();  // should not happen
 #endif
       }
-      task->suspended_task_state = suspended_task_state;
+      task->suspended_task_address = suspended_task_address;
 
-      if (suspended_task_state == nullptr) {
+      if (suspended_task_address == nullptr) {
         ttT *baseobj = task->tt;
         derivedT *obj = static_cast<derivedT *>(baseobj);
         if (obj->tracing()) {
@@ -1349,13 +1357,14 @@ namespace ttg_parsec {
         }
       }
 
-      if (suspended_task_state) {
+      if (suspended_task_address) {
         // right now can events are not properly implemented, we are only testing the workflow with dummy events
         // so mark the events finished manually, parsec will rerun this task again and it should complete the second time
-        auto events = static_cast<ttg::resumable_task_state*>(suspended_task_state)->events();
+        auto events = static_cast<ttg::resumable_task>(ttg::coroutine_handle<>::from_address(suspended_task_address)).events();
         for (auto &event_ptr : events) {
           event_ptr->finish();
         }
+        assert(ttg::coroutine_handle<>::from_address(suspended_task_address).promise().ready());
 
         // TODO: shove {ptr to parsec_task, ptr to this function} to the list of tasks suspended by this thread (hence stored in TLS)
         // thread will loop over its list (after running every task? periodically? need a dedicated queue of ready tasks?)
diff --git a/ttg/ttg/tt.h b/ttg/ttg/tt.h
index 66048906a..b7dd88b82 100644
--- a/ttg/ttg/tt.h
+++ b/ttg/ttg/tt.h
@@ -1,16 +1,24 @@
 #ifndef TTG_TT_H
 #define TTG_TT_H
 
-#include <vector>
 #include <memory>
+#include <vector>
 
 #include "ttg/fwd.h"
 
 #include "ttg/base/tt.h"
 #include "ttg/edge.h"
 
+#ifdef TTG_HAS_COROUTINE
+#include "ttg/util/coroutine.h"
+#endif
+
 namespace ttg {
 
+  // TODO describe TT concept (preferably as a C++20 concept)
+  // N.B. TT::op returns void or ttg::coroutine_handle<>
+  // see TTG_PROCESS_TT_OP_RETURN below
+
   /// @brief a template task graph implementation
 
   /// It contains (owns) one or more TT objects. Since it can also be viewed as a TT object itself,
@@ -35,12 +43,16 @@ namespace ttg {
     TTG(const TTG &) = delete;
     TTG &operator=(const TTG &) = delete;
     // movable
-    TTG(TTG && other) : TTBase(static_cast<TTBase&&>(other)), tts(other.tts), ins(std::move(other.ins)), outs(std::move(other.outs)) {
-        is_ttg_ = true;
-        own_my_tts();
+    TTG(TTG &&other)
+        : TTBase(static_cast<TTBase &&>(other))
+        , tts(other.tts)
+        , ins(std::move(other.ins))
+        , outs(std::move(other.outs)) {
+      is_ttg_ = true;
+      own_my_tts();
     }
-    TTG& operator=(TTG &&other) {
-      static_cast<TTBase&>(*this) = static_cast<TTBase&&>(other);
+    TTG &operator=(TTG &&other) {
+      static_cast<TTBase &>(*this) = static_cast<TTBase &&>(other);
       is_ttg_ = true;
       tts = std::move(other.tts);
       ins = std::move(other.ins);
@@ -91,17 +103,14 @@ namespace ttg {
     void own_my_tts() const {
       for (auto &op : tts) op->owning_ttg = this;
     }
-
   };
 
   template <typename ttseqT, typename input_terminalsT, typename output_terminalsT>
-  auto make_ttg(ttseqT &&tts, const input_terminalsT &ins,
-                const output_terminalsT &outs,
+  auto make_ttg(ttseqT &&tts, const input_terminalsT &ins, const output_terminalsT &outs,
                 const std::string &name = "ttg") {
     return std::make_unique<TTG<input_terminalsT, output_terminalsT>>(std::forward<ttseqT>(tts), ins, outs, name);
   }
 
-
   /// A data sink for one input
   template <typename keyT, typename input_valueT>
   class SinkTT : public TTBase {
@@ -126,20 +135,20 @@ namespace ttg {
       using valueT = std::decay_t<typename terminalT::value_type>;
       auto move_callback = [](const keyT &key, valueT &&value) {};
       auto send_callback = [](const keyT &key, const valueT &value) {};
-      auto broadcast_callback = [](const ttg::span<const keyT>& key, const valueT &value) {};
+      auto broadcast_callback = [](const ttg::span<const keyT> &key, const valueT &value) {};
       auto setsize_callback = [](const keyT &key, std::size_t size) {};
       auto finalize_callback = [](const keyT &key) {};
 
       input.set_callback(send_callback, move_callback, broadcast_callback, setsize_callback, finalize_callback);
     }
 
-  public:
-   SinkTT(const std::string& inname="junk") : TTBase("sink", numins, numouts) {
+   public:
+    SinkTT(const std::string &inname = "junk") : TTBase("sink", numins, numouts) {
       register_input_terminals(input_terminals, std::vector<std::string>{inname});
       register_input_callback(std::get<0>(input_terminals));
     }
 
-    SinkTT(const input_edges_type &inedges, const std::string& inname="junk") : TTBase("sink", numins, numouts) {
+    SinkTT(const input_edges_type &inedges, const std::string &inname = "junk") : TTBase("sink", numins, numouts) {
       register_input_terminals(input_terminals, std::vector<std::string>{inname});
       register_input_callback(std::get<0>(input_terminals));
       std::get<0>(inedges).set_out(&std::get<0>(input_terminals));
@@ -154,12 +163,28 @@ namespace ttg {
     /// Returns pointer to input terminal i to facilitate connection --- terminal cannot be copied, moved or assigned
     template <std::size_t i>
     std::tuple_element_t<i, input_terminals_type> *in() {
-      static_assert(i==0);
+      static_assert(i == 0);
       return &std::get<i>(input_terminals);
     }
   };
 
+}  // namespace ttg
+
+#ifndef TTG_PROCESS_TT_OP_RETURN
+#ifdef TTG_HAS_COROUTINE
+#define TTG_PROCESS_TT_OP_RETURN(result, invoke)     \
+  {                                                  \
+    using return_type = decltype(invoke);            \
+    if constexpr (std::is_same_v<return_type, void>) \
+      invoke;                                        \
+    else {                                           \
+      auto coro_return = invoke;                     \
+      result = coro_return.address();                \
+    }                                                \
+  }
+#else
+#define TTG_PROCESS_OP_RETURN(result, invoke) invoke
+#endif
+#endif  // !defined(TTG_PROCESS_TT_OP_RETURN)
 
-} // namespace ttg
-
-#endif // TTG_TT_H
+#endif  // TTG_TT_H
diff --git a/ttg/ttg/util/coroutine.h b/ttg/ttg/util/coroutine.h
index d709b62c8..3577032f4 100644
--- a/ttg/ttg/util/coroutine.h
+++ b/ttg/ttg/util/coroutine.h
@@ -15,6 +15,10 @@ namespace ttg {
 
   struct resumable_task_state;
 
+  // import coroutine_handle, with default promise redefined to resumable_task_state
+  template <typename Promise = resumable_task_state>
+  using coroutine_handle = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<Promise>;
+
   template <std::size_t N>
   struct resumable_task_events;
 
@@ -30,11 +34,18 @@ namespace ttg {
   };
 
   /// task that can be resumed after some events occur
-  struct resumable_task : TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<resumable_task_state> {
-    using base_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<resumable_task_state>;
+  struct resumable_task : public ttg::coroutine_handle<> {
+    using base_type = ttg::coroutine_handle<>;
+
+    /// these are members mandated by the promise_type concept
+    ///@{
 
     using promise_type = struct resumable_task_state;
 
+    ///@}
+
+    resumable_task(base_type base) : base_type(std::move(base)) {}
+
     base_type handle() { return *this; }
 
     /// @return true if ready to resume
@@ -58,7 +69,10 @@ namespace ttg {
     resumable_task_state& operator=(resumable_task_state&&) = delete;
 
     constexpr static inline std::size_t MaxNumEvents = 20;
-    using handle_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<resumable_task_state>;
+    using handle_type = coroutine_handle<>;
+
+    /// these are members mandated by the promise_type concept
+    ///@{
 
     resumable_task get_return_object() { return resumable_task{handle_type::from_promise(*this)}; }
 
@@ -73,6 +87,26 @@ namespace ttg {
     void return_void() {}
     void unhandled_exception() {}
 
+    ///@}
+
+    /// these are optional members of the promise_type concept
+    ///@{
+
+    // these can be used to use optional storage provided by the runtime (e.g. part of the runtime's task data struct)
+    // N.B. the existing buffer must be passed to operator new via TLS
+    //    void* operator new(std::size_t size)
+    //    {
+    //      return ::operator new(size);
+    //    }
+
+    // N.B. whether the external buffer was used by operator new must be passed via TLS
+    //    void operator delete(void* ptr, std::size_t size)
+    //    {
+    //      ::operator delete(ptr, size);
+    //    }
+
+    ///@}
+
     /// @return true if ready to resume
     constexpr bool ready() const {
       for (std::size_t e = 0; e != nevents_; ++e)
@@ -111,7 +145,9 @@ namespace ttg {
   ttg::span<event*> resumable_task::events() { return base_type::promise().events(); }
 
   /// statically-sized sequence of events on whose completion progress of a given task depends on
-  /// @note this is the `Awaitable` for resumable_task coroutine
+  /// @note this is the `Awaiter` for resumable_task coroutine
+  ///       (the concept is not defined in the standard, see
+  ///       https://lewissbaker.github.io/2017/11/17/understanding-operator-co-await instead )
   template <std::size_t N>
   struct resumable_task_events {
    private:
@@ -124,9 +160,12 @@ namespace ttg {
     template <typename... Events>
     constexpr resumable_task_events(Events&&... events) : events_{(&events)...} {}
 
+    /// these are members mandated by the Awaiter concept
+    ///@{
+
     constexpr bool await_ready() const { return await_ready(std::make_index_sequence<N>{}); }
 
-    void await_suspend(TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<resumable_task_state> pending_task) {
+    void await_suspend(coroutine_handle<> pending_task) {
       pending_task_ = pending_task;
       pending_task_.promise().set_events(events_);
     }
@@ -138,9 +177,11 @@ namespace ttg {
       }
     }
 
+    ///@}
+
    private:
     std::array<event*, N> events_;
-    TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<resumable_task_state> pending_task_;
+    coroutine_handle<> pending_task_;
   };  // resumable_task_events
 
   // deduce the number of events properly

From e2e3b2b3aaf5e6a10e8e11d8a17fddfbd58627be Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Fri, 4 Nov 2022 09:15:13 -0400
Subject: [PATCH 044/259] parsec's TT::static_op_noarg now is also coroutine
 aware

---
 ttg/ttg/parsec/ttg.h | 57 ++++++++++++++++++++++++++++++++++----------
 1 file changed, 44 insertions(+), 13 deletions(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index eb855bf77..64d81dfb2 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1381,20 +1381,51 @@ namespace ttg_parsec {
     template <ttg::ExecutionSpace Space>
     static parsec_hook_return_t static_op_noarg(parsec_task_t *parsec_task) {
       task_t *task = static_cast<task_t*>(parsec_task);
-      ttT *baseobj = (ttT *)task->object_ptr;
-      derivedT *obj = (derivedT *)task->object_ptr;
-      assert(parsec_ttg_caller == NULL);
-      parsec_ttg_caller = task;
-      if constexpr (!ttg::meta::is_void_v<keyT>) {
-        baseobj->template op<Space>(task->key, obj->output_terminals);
-      } else if constexpr (ttg::meta::is_void_v<keyT>) {
-        baseobj->template op<Space>(obj->output_terminals);
-      } else
-        abort();
-      parsec_ttg_caller = NULL;
 
-      // if op is coro-based this will be something else
-      return PARSEC_HOOK_RETURN_DONE;
+      void* suspended_task_address =
+#ifdef TTG_HAS_COROUTINE
+        task->suspended_task_address;  // non-null = need to resume the task
+#else
+        nullptr;
+#endif
+      if (suspended_task_address == nullptr) {  // task is a coroutine that has not started or an ordinary function
+        ttT *baseobj = (ttT *)task->object_ptr;
+        derivedT *obj = (derivedT *)task->object_ptr;
+        assert(parsec_ttg_caller == NULL);
+        parsec_ttg_caller = task;
+        if constexpr (!ttg::meta::is_void_v<keyT>) {
+          TTG_PROCESS_TT_OP_RETURN(suspended_task_address, baseobj->template op<Space>(task->key, obj->output_terminals));
+        } else if constexpr (ttg::meta::is_void_v<keyT>) {
+          TTG_PROCESS_TT_OP_RETURN(suspended_task_address, baseobj->template op<Space>(obj->output_terminals));
+        } else
+          abort();
+        parsec_ttg_caller = NULL;
+      }
+      else {
+#ifdef TTG_HAS_COROUTINE
+        auto ret = static_cast<ttg::resumable_task>(ttg::coroutine_handle<>::from_address(suspended_task_address));
+        assert(ret.ready());
+        ret.resume();
+        if (ret.completed()) {
+          ret.destroy();
+          suspended_task_address = nullptr;
+        }
+        else { // not yet completed
+          // leave suspended_task_address as is
+        }
+#else
+        abort();  // should not happen
+#endif
+      }
+      task->suspended_task_address = suspended_task_address;
+
+      if (suspended_task_address) {
+        abort();  // not yet implemented
+        // see comments in static_op()
+        return PARSEC_HOOK_RETURN_AGAIN;
+      }
+      else
+        return PARSEC_HOOK_RETURN_DONE;
     }
 
    protected:

From 5c6b31602dcfe1915d50c33cfe3345d45c9e0a0f Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Fri, 4 Nov 2022 09:26:19 -0400
Subject: [PATCH 045/259] adapted madness backend to do same (incomplete)
 handling of coroutines as in parsec. enough for proof of concept.

- fibonacci-coro is the only working example, with dummy events that get cleared manually, etc. ...
---
 ttg/ttg/madness/ttg.h | 88 +++++++++++++++++++++++++++++++++++--------
 ttg/ttg/parsec/ttg.h  |  4 +-
 2 files changed, 76 insertions(+), 16 deletions(-)

diff --git a/ttg/ttg/madness/ttg.h b/ttg/ttg/madness/ttg.h
index 8f2929860..cc8b4718d 100644
--- a/ttg/ttg/madness/ttg.h
+++ b/ttg/ttg/madness/ttg.h
@@ -280,6 +280,9 @@ namespace ttg_madness {
       derivedT *derived;                            // Pointer to derived class instance
       bool pull_terminals_invoked = false;
       std::conditional_t<ttg::meta::is_void_v<keyT>, ttg::Void, keyT> key;  // Task key
+#ifdef TTG_HAS_COROUTINE
+      void *suspended_task_address = nullptr;  // if not null the function is suspended
+#endif
 
       /// makes a tuple of references out of tuple of
       template <typename Tuple, std::size_t... Is>
@@ -304,28 +307,83 @@ namespace ttg_madness {
       }
 
       virtual void run(::madness::World &world) override {
-        // ttg::print("starting task");
-
         using ttg::hash;
         ttT::threaddata.key_hash = hash<decltype(key)>{}(key);
         ttT::threaddata.call_depth++;
 
-        if constexpr (!ttg::meta::is_void_v<keyT> && !ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
-          derived->op(key, this->make_input_refs(),
-                      derived->output_terminals);  // !!! NOTE converting input values to refs
-        } else if constexpr (!ttg::meta::is_void_v<keyT> && ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
-          derived->op(key, derived->output_terminals);
-        } else if constexpr (ttg::meta::is_void_v<keyT> && !ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
-          derived->op(this->make_input_refs(),
-                      derived->output_terminals);  // !!! NOTE converting input values to refs
-        } else if constexpr (ttg::meta::is_void_v<keyT> && ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
-          derived->op(derived->output_terminals);
-        } else
-          abort();
+        void *suspended_task_address =
+#ifdef TTG_HAS_COROUTINE
+            this->suspended_task_address;  // non-null = need to resume the task
+#else
+            nullptr;
+#endif
+        if (suspended_task_address == nullptr) {  // task is a coroutine that has not started or an ordinary function
+          // ttg::print("starting task");
+          if constexpr (!ttg::meta::is_void_v<keyT> && !ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
+            TTG_PROCESS_TT_OP_RETURN(
+                suspended_task_address,
+                derived->op(key, this->make_input_refs(),
+                            derived->output_terminals));  // !!! NOTE converting input values to refs
+          } else if constexpr (!ttg::meta::is_void_v<keyT> && ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
+            TTG_PROCESS_TT_OP_RETURN(suspended_task_address, derived->op(key, derived->output_terminals));
+          } else if constexpr (ttg::meta::is_void_v<keyT> && !ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
+            TTG_PROCESS_TT_OP_RETURN(
+                suspended_task_address,
+                derived->op(this->make_input_refs(),
+                            derived->output_terminals));  // !!! NOTE converting input values to refs
+          } else if constexpr (ttg::meta::is_void_v<keyT> && ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
+            TTG_PROCESS_TT_OP_RETURN(suspended_task_address, derived->op(derived->output_terminals));
+          } else
+            abort();
+        } else {  // resume suspended coroutine
+#ifdef TTG_HAS_COROUTINE
+          auto ret = static_cast<ttg::resumable_task>(ttg::coroutine_handle<>::from_address(suspended_task_address));
+          assert(ret.ready());
+          ret.resume();
+          if (ret.completed()) {
+            ret.destroy();
+            suspended_task_address = nullptr;
+          } else {  // not yet completed
+            // leave suspended_task_address as is
+          }
+#else
+          abort();  // should not happen
+#endif
+        }
+        this->suspended_task_address = suspended_task_address;
 
         ttT::threaddata.call_depth--;
 
-        // ttg::print("finishing task",ttT::threaddata.call_depth);
+        // if (suspended_task_address == nullptr) {
+        //   ttg::print("finishing task",ttT::threaddata.call_depth);
+        // }
+
+#ifdef TTG_HAS_COROUTINE
+        if (suspended_task_address) {
+          // TODO implement handling of suspended coroutines properly
+
+          // right now can events are not properly implemented, we are only testing the workflow with dummy events
+          // so mark the events finished manually, parsec will rerun this task again and it should complete the second
+          // time
+          auto events =
+              static_cast<ttg::resumable_task>(ttg::coroutine_handle<>::from_address(suspended_task_address)).events();
+          for (auto &event_ptr : events) {
+            event_ptr->finish();
+          }
+          assert(ttg::coroutine_handle<>::from_address(suspended_task_address).promise().ready());
+
+          // resume the coroutine
+          auto ret = static_cast<ttg::resumable_task>(ttg::coroutine_handle<>::from_address(suspended_task_address));
+          assert(ret.ready());
+          ret.resume();
+          if (ret.completed()) {
+            ret.destroy();
+            suspended_task_address = nullptr;
+          } else {  // not yet completed
+            abort();
+          }
+        }
+#endif  // TTG_HAS_COROUTINE
       }
 
       virtual ~TTArgs() {}  // Will be deleted via TaskInterface*
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 64d81dfb2..ad51aeb0e 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1328,7 +1328,7 @@ namespace ttg_parsec {
         }
         parsec_ttg_caller = NULL;
       }
-      else {
+      else {  // resume the suspended coroutine
 #ifdef TTG_HAS_COROUTINE
         auto ret = static_cast<ttg::resumable_task>(ttg::coroutine_handle<>::from_address(suspended_task_address));
         assert(ret.ready());
@@ -1357,6 +1357,7 @@ namespace ttg_parsec {
         }
       }
 
+#ifdef TTG_HAS_COROUTINE
       if (suspended_task_address) {
         // right now can events are not properly implemented, we are only testing the workflow with dummy events
         // so mark the events finished manually, parsec will rerun this task again and it should complete the second time
@@ -1375,6 +1376,7 @@ namespace ttg_parsec {
         return PARSEC_HOOK_RETURN_AGAIN;
       }
       else
+#endif  // TTG_HAS_COROUTINE
         return PARSEC_HOOK_RETURN_DONE;
     }
 

From 16f45bce809e3c680e6004c7fdf359a3b233dbdf Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Fri, 4 Nov 2022 10:29:45 -0400
Subject: [PATCH 046/259] fixups to make build with C++17 again

---
 ttg/ttg/madness/ttg.h | 2 +-
 ttg/ttg/parsec/ttg.h  | 2 +-
 ttg/ttg/tt.h          | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/ttg/ttg/madness/ttg.h b/ttg/ttg/madness/ttg.h
index cc8b4718d..4acdf7ac1 100644
--- a/ttg/ttg/madness/ttg.h
+++ b/ttg/ttg/madness/ttg.h
@@ -346,11 +346,11 @@ namespace ttg_madness {
           } else {  // not yet completed
             // leave suspended_task_address as is
           }
+          this->suspended_task_address = suspended_task_address;
 #else
           abort();  // should not happen
 #endif
         }
-        this->suspended_task_address = suspended_task_address;
 
         ttT::threaddata.call_depth--;
 
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index ad51aeb0e..d5676d033 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1340,11 +1340,11 @@ namespace ttg_parsec {
         else { // not yet completed
           // leave suspended_task_address as is
         }
+        task->suspended_task_address = suspended_task_address;
 #else
         abort();  // should not happen
 #endif
       }
-      task->suspended_task_address = suspended_task_address;
 
       if (suspended_task_address == nullptr) {
         ttT *baseobj = task->tt;
diff --git a/ttg/ttg/tt.h b/ttg/ttg/tt.h
index b7dd88b82..2c545ce25 100644
--- a/ttg/ttg/tt.h
+++ b/ttg/ttg/tt.h
@@ -183,7 +183,7 @@ namespace ttg {
     }                                                \
   }
 #else
-#define TTG_PROCESS_OP_RETURN(result, invoke) invoke
+#define TTG_PROCESS_TT_OP_RETURN(result, invoke) invoke
 #endif
 #endif  // !defined(TTG_PROCESS_TT_OP_RETURN)
 

From 7adf4287d910c5a1076ec85c7bbe9299dc0f29a7 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Fri, 4 Nov 2022 11:15:26 -0400
Subject: [PATCH 047/259] [cmake] fixup

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 045a74d90..faa2d5a3b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -29,7 +29,7 @@ project(ttg
     LANGUAGES CXX
     HOMEPAGE_URL "https://tesseorg.github.io/ttg/")
 
-if (NOT CMAKE_CXX_STANDARD)
+if (NOT DEFINED CMAKE_CXX_STANDARD)
     set(CMAKE_CXX_STANDARD "17" CACHE STRING "The C++ standard")
 endif()
 if (NOT CMAKE_CXX_EXTENSIONS)

From 59e19bfc52280ece12378d4fd6081325dfea0e83 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Mon, 7 Nov 2022 17:07:54 -0800
Subject: [PATCH 048/259] replaced more std::abort instances with ttg::abort

---
 tests/unit/tt.cc      |  2 +-
 ttg/ttg/base/tt.h     |  4 ++--
 ttg/ttg/madness/ttg.h | 14 +++++++-------
 ttg/ttg/parsec/ttg.h  | 24 ++++++++++++------------
 4 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/tests/unit/tt.cc b/tests/unit/tt.cc
index 3952d84fd..def14684a 100644
--- a/tests/unit/tt.cc
+++ b/tests/unit/tt.cc
@@ -146,7 +146,7 @@ namespace tt_i_iv {
 
   template <typename K, typename D1, typename D2>
   void func0(K &key, D1 &datum1, D2 &&datum2) {
-    abort();
+    ttg::abort();
   }
 }  // namespace tt_i_iv
 
diff --git a/ttg/ttg/base/tt.h b/ttg/ttg/base/tt.h
index e59db15e6..4fa975b40 100644
--- a/ttg/ttg/base/tt.h
+++ b/ttg/ttg/base/tt.h
@@ -144,11 +144,11 @@ namespace ttg {
     virtual ~TTBase() = default;
 
     /// Use this to create a task that takes no data "manually"
-    /// @warning calls std::abort() if the derived class TT did not override this;
+    /// @warning calls ttg::abort() if the derived class TT did not override this;
     ///          only makes sense to override this if the derived TT uses void for key or data
     virtual void invoke() {
       std::cerr << "TTBase::invoke() invoked on a TT that did not override it" << std::endl;
-      abort();
+      ttg::abort();
     }
 
     /// Sets trace for all operations to value and returns previous setting.
diff --git a/ttg/ttg/madness/ttg.h b/ttg/ttg/madness/ttg.h
index 4acdf7ac1..d283d5541 100644
--- a/ttg/ttg/madness/ttg.h
+++ b/ttg/ttg/madness/ttg.h
@@ -333,8 +333,8 @@ namespace ttg_madness {
                             derived->output_terminals));  // !!! NOTE converting input values to refs
           } else if constexpr (ttg::meta::is_void_v<keyT> && ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
             TTG_PROCESS_TT_OP_RETURN(suspended_task_address, derived->op(derived->output_terminals));
-          } else
-            abort();
+          } else  // unreachable
+            ttg::abort();
         } else {  // resume suspended coroutine
 #ifdef TTG_HAS_COROUTINE
           auto ret = static_cast<ttg::resumable_task>(ttg::coroutine_handle<>::from_address(suspended_task_address));
@@ -348,7 +348,7 @@ namespace ttg_madness {
           }
           this->suspended_task_address = suspended_task_address;
 #else
-          abort();  // should not happen
+          ttg::abort();  // should not happen
 #endif
         }
 
@@ -380,7 +380,7 @@ namespace ttg_madness {
             ret.destroy();
             suspended_task_address = nullptr;
           } else {  // not yet completed
-            abort();
+            ttg::abort();
           }
         }
 #endif  // TTG_HAS_COROUTINE
@@ -629,7 +629,7 @@ namespace ttg_madness {
             } else if constexpr (ttg::meta::is_void_v<keyT> && ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
               static_cast<derivedT *>(this)->op(output_terminals);  // Runs immediately
             } else
-              abort();
+              ttg::abort();
             ttT::threaddata.call_depth--;
 
           } else {
@@ -1009,7 +1009,7 @@ namespace ttg_madness {
         auto finalize_callback = [this]() { finalize_argstream<i>(); };
         input.set_callback(send_callback, send_callback, {}, setsize_callback, finalize_callback);
       } else
-        abort();
+        ttg::abort();
     }
 
     template <std::size_t... IS>
@@ -1135,7 +1135,7 @@ namespace ttg_madness {
           for (std::size_t i = 0; i < numins; i++) std::cerr << (item.second->nargs[i] == 0 ? "T" : "F") << " ";
           std::cerr << ")" << std::endl;
         }
-        abort();
+        ttg::abort();
       }
     }
 
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index d5676d033..07e201218 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1243,7 +1243,7 @@ namespace ttg_parsec {
         }
       }
       else
-        abort();
+        ttg::abort();
     }
 
     template <std::size_t i, typename terminalT, typename Key>
@@ -1324,7 +1324,7 @@ namespace ttg_parsec {
         } else if constexpr (ttg::meta::is_void_v<keyT> && ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
           TTG_PROCESS_TT_OP_RETURN(suspended_task_address, baseobj->template op<Space>(obj->output_terminals));
         } else {
-          abort();
+          ttg::abort();
         }
         parsec_ttg_caller = NULL;
       }
@@ -1342,7 +1342,7 @@ namespace ttg_parsec {
         }
         task->suspended_task_address = suspended_task_address;
 #else
-        abort();  // should not happen
+        ttg::abort();  // should not happen
 #endif
       }
 
@@ -1399,8 +1399,8 @@ namespace ttg_parsec {
           TTG_PROCESS_TT_OP_RETURN(suspended_task_address, baseobj->template op<Space>(task->key, obj->output_terminals));
         } else if constexpr (ttg::meta::is_void_v<keyT>) {
           TTG_PROCESS_TT_OP_RETURN(suspended_task_address, baseobj->template op<Space>(obj->output_terminals));
-        } else
-          abort();
+        } else  // unreachable
+          ttg:: abort();
         parsec_ttg_caller = NULL;
       }
       else {
@@ -1416,13 +1416,13 @@ namespace ttg_parsec {
           // leave suspended_task_address as is
         }
 #else
-        abort();  // should not happen
+        ttg::abort();  // should not happen
 #endif
       }
       task->suspended_task_address = suspended_task_address;
 
       if (suspended_task_address) {
-        abort();  // not yet implemented
+        ttg::abort();  // not yet implemented
         // see comments in static_op()
         return PARSEC_HOOK_RETURN_AGAIN;
       }
@@ -1473,7 +1473,7 @@ namespace ttg_parsec {
             (obj->*member)(data, size);
           } else {
             // there is no good reason to have negative param ids
-            abort();
+            ttg::abort();
           }
           break;
         }
@@ -1499,7 +1499,7 @@ namespace ttg_parsec {
           break;
         }
         default:
-          abort();
+          ttg::abort();
       }
     }
 
@@ -1680,8 +1680,8 @@ namespace ttg_parsec {
         // case 5 and 6
       } else if constexpr (ttg::meta::is_void_v<keyT> && std::is_void_v<valueT>) {
         set_arg<i, keyT, ttg::Void>(ttg::Void{});
-      } else {
-        abort();
+      } else {  // unreachable
+        ttg:: abort();
       }
     }
 
@@ -2747,7 +2747,7 @@ namespace ttg_parsec {
       // NOTE: subsumed in case 5 above, kept for historical reasons
       //////////////////////////////////////////////////////////////////
       else
-        abort();
+        ttg::abort();
     }
 
     template <std::size_t... IS>

From 325dcab1490cc526bdccb89ab52aeab29060c843 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Tue, 8 Nov 2022 05:25:31 -0800
Subject: [PATCH 049/259] make MultiIndex mad/boost-serializable

---
 ttg/ttg/util/multiindex.h | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/ttg/ttg/util/multiindex.h b/ttg/ttg/util/multiindex.h
index 7141b7954..5c04494e2 100644
--- a/ttg/ttg/util/multiindex.h
+++ b/ttg/ttg/util/multiindex.h
@@ -5,6 +5,8 @@
 #ifndef TTG_UTIL_MULTIINDEX_H
 #define TTG_UTIL_MULTIINDEX_H
 
+#include "ttg/serialization/std/array.h"
+
 namespace ttg {
 
   template <std::size_t Rank, typename Int = int>
@@ -53,6 +55,11 @@ namespace ttg {
       return data_[idx];
     }
 
+    template <typename Archive>
+    void serialize(Archive &ar, const unsigned int version = 0) {
+      ar &data_;
+    }
+
    private:
     bool valid() {
       bool result = true;

From 779d3975c2c991c5833964cac2df6e598c20840c Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Tue, 8 Nov 2022 06:05:01 -0800
Subject: [PATCH 050/259] [ci] C compiler matches C++ compiler

---
 .github/workflows/cmake.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index b25f773ed..d0b3d9604 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -13,8 +13,10 @@ jobs:
         os : [ macos-latest, ubuntu-20.04 ]
         include:
           - os: ubuntu-20.04
+            cc: /usr/bin/gcc-10
             cxx: /usr/bin/g++-10
           - os: macos-latest
+            cc: clang
             cxx: clang++
 
     name: "${{ matrix.os }}: ${{ matrix.cxx }} ${{ matrix.build_type }}"

From 7206c613100e3989cf57c197399dcb5ac49ec993 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Tue, 8 Nov 2022 06:31:48 -0800
Subject: [PATCH 051/259] [cmake] discover std::coroutine is needed

---
 CMakeLists.txt                          |  4 ++--
 cmake/modules/FindCXXStdCoroutine.cmake |  9 ++++++---
 cmake/ttg-config.cmake.in               | 16 +++++++++++-----
 3 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index faa2d5a3b..2f6cfb95b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -84,8 +84,7 @@ include(FindOrFetchBoost)
 # Cereal
 #include(FindOrFetchCereal)
 # C++ coroutines
-find_package(CXXStdCoroutine COMPONENTS Final Experimental)
-
+find_package(CXXStdCoroutine MODULE REQUIRED COMPONENTS Final Experimental)
 
 ##########################
 #### prerequisite runtimes
@@ -162,6 +161,7 @@ configure_package_config_file(cmake/ttg-config.cmake.in
 install(FILES
         "${PROJECT_SOURCE_DIR}/cmake/modules/AddTTGExecutable.cmake"
         "${PROJECT_SOURCE_DIR}/cmake/modules/AddTTGTestExecutable.cmake"
+        "${PROJECT_SOURCE_DIR}/cmake/modules/FindCXXStdCoroutine.cmake"
         DESTINATION "${CMAKE_INSTALL_CMAKEDIR}/modules"
         COMPONENT ttg-config)
 
diff --git a/cmake/modules/FindCXXStdCoroutine.cmake b/cmake/modules/FindCXXStdCoroutine.cmake
index 8bf8facb9..4562ceb73 100644
--- a/cmake/modules/FindCXXStdCoroutine.cmake
+++ b/cmake/modules/FindCXXStdCoroutine.cmake
@@ -62,10 +62,12 @@ Imported Targets
 Variables
 *********
 
-.. variable:: CXX_COROUTINE_IS_EXPERIMENTAL
+.. variable:: CXX_COROUTINE_COMPONENT
 
-    Set to ``TRUE`` when the :find-component:`coro.Experimental` version of C++
-    coroutine library was found, otherwise ``FALSE``.
+    Set to ``Final`` when the :find-component:`coro.Final` version of C++
+    coroutine library was found, ``Experimental`` when
+    the :find-component:`coro.Experimental` version of C++
+    coroutine library was found, otherwise not defined.
 
 .. variable:: CXX_COROUTINE_HAVE_CORO
 
@@ -149,6 +151,7 @@ foreach(component IN LISTS CXXStdCoroutines_want_components_ordered)
     if(_CXX_COROUTINE_HAVE_HEADER)
       set(CXXStdCoroutines_option "${option}" CACHE STRING "The compiler option needed to use <coroutine>")
       mark_as_advanced(CXXStdCoroutines_option)
+      set(CXX_COROUTINE_COMPONENT "${component}" CACHE STRING "The component of CXXStdCoroutine package found")
       # We found a header, skip rest of tests
       break()
     else()
diff --git a/cmake/ttg-config.cmake.in b/cmake/ttg-config.cmake.in
index b00a9c692..41663c806 100644
--- a/cmake/ttg-config.cmake.in
+++ b/cmake/ttg-config.cmake.in
@@ -7,11 +7,16 @@ set(TTG_EXT_VERSION "@TTG_EXT_VERSION@")
 set(PaRSEC_CONFIG "@PaRSEC_CONFIG@")
 set(MADNESS_CONFIG "@MADNESS_CONFIG@")
 set(Boost_CONFIG "@Boost_CONFIG@")
+set(CXX_COROUTINE_COMPONENT "@CXX_COROUTINE_COMPONENT@")
 
 set(TTG_TRACKED_BOOST_VERSION "@TTG_TRACKED_BOOST_VERSION@")
 
 set(TTG_IGNORE_BUNDLED_EXTERNALS @TTG_IGNORE_BUNDLED_EXTERNALS@)
 
+# make TTG CMake modules discoverable + load AddTTGExecutable by default
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/modules")
+include(AddTTGExecutable)
+
 @PACKAGE_INIT@
 
 if (NOT TARGET MADworld AND MADNESS_CONFIG)
@@ -24,12 +29,17 @@ if (NOT TARGET PaRSEC::parsec AND PaRSEC_CONFIG)
   find_package(PaRSEC CONFIG QUIET REQUIRED COMPONENTS parsec PATHS "${PaRSEC_CONFIG_DIR}" NO_DEFAULT_PATH)
 endif()
 
-# N.B. load Boost
+# if Boost was discovered and used at TTG configure time discover Boost at the same path
 if (NOT TARGET Boost::boost AND Boost_CONFIG)
   get_filename_component(Boost_CONFIG_DIR "${Boost_CONFIG}" DIRECTORY)
   find_package(Boost ${TTG_TRACKED_BOOST_VERSION} CONFIG QUIET REQUIRED OPTIONAL_COMPONENTS serialization PATHS "${Boost_CONFIG_DIR}" NO_DEFAULT_PATH)
 endif()
 
+# if C++ coroutines were used discover same version of them
+if (NOT TARGET std::coroutine AND CXX_COROUTINE_COMPONENT)
+  find_package(CXXStdCoroutine MODULE QUIET REQUIRED COMPONENTS "${CXX_COROUTINE_COMPONENT}")
+endif()
+
 # Include library IMPORT targets
 if(NOT TARGET ttg)
   include("${CMAKE_CURRENT_LIST_DIR}/ttg-targets.cmake")
@@ -47,8 +57,4 @@ if (NOT TARGET Boost::boost)
   endif()
 endif()
 
-# load CMake modules
-list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/modules")
-include(AddTTGExecutable)
-
 set(TTG_FOUND TRUE)

From 72d29191e8faea4d66bfbd906c4df5a0bbfca091 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Tue, 8 Nov 2022 06:38:57 -0800
Subject: [PATCH 052/259] [cmake] install public headers generated in the build
 tree correctly

---
 cmake/modules/AddTTGLibrary.cmake | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/cmake/modules/AddTTGLibrary.cmake b/cmake/modules/AddTTGLibrary.cmake
index 29ce0f8df..5c8c2d560 100644
--- a/cmake/modules/AddTTGLibrary.cmake
+++ b/cmake/modules/AddTTGLibrary.cmake
@@ -106,7 +106,14 @@ macro(add_ttg_library)
     # set_target_properties(${_library} PROPERTIES PUBLIC_HEADER "${ADD_TTG_LIBRARY_PUBLIC_HEADER}")
     # install manually
     foreach ( file ${ADD_TTG_LIBRARY_PUBLIC_HEADER} )
-        file(RELATIVE_PATH _rel_file_path "${PROJECT_SOURCE_DIR}/ttg" "${file}")
+        # N.B. some files are in the build tree
+        if ("${file}" MATCHES "^${PROJECT_SOURCE_DIR}/ttg")
+            file(RELATIVE_PATH _rel_file_path "${PROJECT_SOURCE_DIR}/ttg" "${file}")
+        elseif("${file}" MATCHES "^${PROJECT_BINARY_DIR}/ttg")
+            file(RELATIVE_PATH _rel_file_path "${PROJECT_BINARY_DIR}/ttg" "${file}")
+        else()
+            message(FATAL_ERROR "AddTTGLibrary: could not deduce install location for public header ${file} of component ${_library}")
+        endif()
         get_filename_component( dir "${_rel_file_path}" DIRECTORY )
         install( FILES ${file} DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/${dir}" COMPONENT ${_library})
     endforeach()

From b3111c27aedc7b8940339d344b66c7ee6c1587aa Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Tue, 8 Nov 2022 12:06:07 -0800
Subject: [PATCH 053/259] [ci] dump CMake{Output,Error}.log of install tree
 tests to CI log

---
 .github/workflows/cmake.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index d0b3d9604..eaecd6930 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -128,7 +128,7 @@ jobs:
       working-directory: ${{github.workspace}}/build
       shell: bash
       run: |
-        cmake -S $GITHUB_WORKSPACE/doc/dox/dev/devsamp/main -B test_install_devsamp -DCMAKE_PREFIX_PATH=${{github.workspace}}/install
+        cmake -S $GITHUB_WORKSPACE/doc/dox/dev/devsamp/main -B test_install_devsamp -DCMAKE_PREFIX_PATH=${{github.workspace}}/install || (cat test_install_devsamp/CMakeFiles/CMakeOutput.log && cat test_install_devsamp/CMakeFiles/CMakeError.log)
         cmake --build test_install_devsamp
         cmake -E make_directory test_install_userexamples
         cat > test_install_userexamples/CMakeLists.txt <<EOF
@@ -140,7 +140,7 @@ jobs:
         add_ttg_executable(iterative $GITHUB_WORKSPACE/doc/dox/user/examples/iterative.cc NOT_EXCLUDE_FROM_ALL)
         add_ttg_executable(distributed $GITHUB_WORKSPACE/doc/dox/user/examples/distributed.cc NOT_EXCLUDE_FROM_ALL)
         EOF
-        cmake -S test_install_userexamples -B test_install_userexamples/build -DCMAKE_PREFIX_PATH=${{github.workspace}}/install
+        cmake -S test_install_userexamples -B test_install_userexamples/build -DCMAKE_PREFIX_PATH=${{github.workspace}}/install || (cat test_install_userexamples/CMakeFiles/CMakeOutput.log && cat test_install_userexamples/CMakeFiles/CMakeError.log)
         cmake --build test_install_userexamples/build
 
     - name: Build+Deploy Dox

From adaf4e5c51ad8a981006ab63d83abe231cae7287 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Tue, 8 Nov 2022 12:37:52 -0800
Subject: [PATCH 054/259] [cmake] when checking for std coroutine header try
 without and with -std flags to try to enable c++20

---
 cmake/modules/FindCXXStdCoroutine.cmake | 33 ++++++++++++++-----------
 1 file changed, 18 insertions(+), 15 deletions(-)

diff --git a/cmake/modules/FindCXXStdCoroutine.cmake b/cmake/modules/FindCXXStdCoroutine.cmake
index 4562ceb73..1bf3e88fc 100644
--- a/cmake/modules/FindCXXStdCoroutine.cmake
+++ b/cmake/modules/FindCXXStdCoroutine.cmake
@@ -131,6 +131,7 @@ endforeach()
 # clang may need to use -stdlib=c++ to have coroutines
 # gcc/libstdc++ needs -fcoroutines
 set(CXXStdCoroutines_find_options "" "-stdlib=libc++" "-fcoroutines")
+set(CXXStdCoroutines_std_options "" "-std=c++20" "-std=c++2a")
 set(CXXStdCoroutines_want_components_ordered "${CXXStdCoroutines_want_components}")
 list(SORT CXXStdCoroutines_want_components_ordered ORDER DESCENDING)  # Final before Experimental
 
@@ -143,21 +144,23 @@ foreach(component IN LISTS CXXStdCoroutines_want_components_ordered)
     set(_coro_namespace std::experimental)
   endif()
   foreach(option IN LISTS CXXStdCoroutines_find_options)
-    cmake_push_check_state()
-    list(APPEND CMAKE_REQUIRED_FLAGS "${option}")
-    check_include_file_cxx("${_coro_header}" _CXX_COROUTINE_HAVE_HEADER)
-    mark_as_advanced(_CXX_COROUTINE_HAVE_HEADER)
-    cmake_pop_check_state()
-    if(_CXX_COROUTINE_HAVE_HEADER)
-      set(CXXStdCoroutines_option "${option}" CACHE STRING "The compiler option needed to use <coroutine>")
-      mark_as_advanced(CXXStdCoroutines_option)
-      set(CXX_COROUTINE_COMPONENT "${component}" CACHE STRING "The component of CXXStdCoroutine package found")
-      # We found a header, skip rest of tests
-      break()
-    else()
-      unset(_CXX_COROUTINE_HAVE_HEADER CACHE)
-    endif()
-  endforeach()  # options
+    foreach(stdoption IN LISTS CXXStdCoroutines_std_options)
+      cmake_push_check_state()
+      list(APPEND CMAKE_REQUIRED_FLAGS "${option} ${stdoption}")
+      check_include_file_cxx("${_coro_header}" _CXX_COROUTINE_HAVE_HEADER)
+      mark_as_advanced(_CXX_COROUTINE_HAVE_HEADER)
+      cmake_pop_check_state()
+      if(_CXX_COROUTINE_HAVE_HEADER)
+        set(CXXStdCoroutines_option "${option}" CACHE STRING "The compiler option needed to use <coroutine>")
+        mark_as_advanced(CXXStdCoroutines_option)
+        set(CXX_COROUTINE_COMPONENT "${component}" CACHE STRING "The component of CXXStdCoroutine package found")
+        # We found a header, skip rest of tests
+        break()
+      else()
+        unset(_CXX_COROUTINE_HAVE_HEADER CACHE)
+      endif()
+    endforeach()  # stdoption
+  endforeach()  # option
   if (_CXX_COROUTINE_HAVE_HEADER)
     break()
   endif()

From 419192cb2e4af891eed5b2f535ab0cc13d399be6 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Tue, 8 Nov 2022 16:10:38 -0800
Subject: [PATCH 055/259] [cmake] when compiling std coroutine example try
 without and with -std flags to try to enable c++20

---
 cmake/modules/FindCXXStdCoroutine.cmake | 43 ++++++++++++++++---------
 1 file changed, 27 insertions(+), 16 deletions(-)

diff --git a/cmake/modules/FindCXXStdCoroutine.cmake b/cmake/modules/FindCXXStdCoroutine.cmake
index 1bf3e88fc..7dfc048d8 100644
--- a/cmake/modules/FindCXXStdCoroutine.cmake
+++ b/cmake/modules/FindCXXStdCoroutine.cmake
@@ -146,12 +146,14 @@ foreach(component IN LISTS CXXStdCoroutines_want_components_ordered)
   foreach(option IN LISTS CXXStdCoroutines_find_options)
     foreach(stdoption IN LISTS CXXStdCoroutines_std_options)
       cmake_push_check_state()
-      list(APPEND CMAKE_REQUIRED_FLAGS "${option} ${stdoption}")
+      list(APPEND CMAKE_REQUIRED_FLAGS "${option};${stdoption}")
       check_include_file_cxx("${_coro_header}" _CXX_COROUTINE_HAVE_HEADER)
       mark_as_advanced(_CXX_COROUTINE_HAVE_HEADER)
       cmake_pop_check_state()
       if(_CXX_COROUTINE_HAVE_HEADER)
-        set(CXXStdCoroutines_option "${option}" CACHE STRING "The compiler option needed to use <coroutine>")
+        if (option)
+          set(CXXStdCoroutines_option "${option}" CACHE STRING "The compiler option needed to use <coroutine>")
+        endif()
         mark_as_advanced(CXXStdCoroutines_option)
         set(CXX_COROUTINE_COMPONENT "${component}" CACHE STRING "The component of CXXStdCoroutine package found")
         # We found a header, skip rest of tests
@@ -183,21 +185,30 @@ if(CXX_COROUTINE_HAVE_CORO)
         }
     ]] code @ONLY)
 
-  cmake_push_check_state()
-  if (DEFINED CXXStdCoroutines_option)
-    list(APPEND CMAKE_REQUIRED_FLAGS "${CXXStdCoroutines_option}")
-  endif()
-  check_cxx_source_compiles("${code}" CXX_COROUTINE_NO_LINK_NEEDED)
-  cmake_pop_check_state()
-
-  if(CXX_COROUTINE_NO_LINK_NEEDED)
-    add_library(std::coroutine INTERFACE IMPORTED GLOBAL)
-    target_compile_features(std::coroutine INTERFACE cxx_std_20)
+  # argh ... check_include_file_cxx will discover coroutine header but it is not usable without -std flag
+  foreach(stdoption IN LISTS CXXStdCoroutines_std_options)
+    cmake_push_check_state()
     if (DEFINED CXXStdCoroutines_option)
-      target_compile_options(std::coroutine INTERFACE "${CXXStdCoroutines_option}")
+      set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${CXXStdCoroutines_option}")
     endif()
-    set(_found TRUE)
-  endif()
+    if (stdoption)
+      set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${stdoption}")
+    endif()
+    check_cxx_source_compiles("${code}" CXX_COROUTINE_NO_LINK_NEEDED)
+
+    if(CXX_COROUTINE_NO_LINK_NEEDED)
+      add_library(std::coroutine INTERFACE IMPORTED GLOBAL)
+      target_compile_features(std::coroutine INTERFACE cxx_std_20)
+      if (DEFINED CXXStdCoroutines_option)
+        target_compile_options(std::coroutine INTERFACE "${CXXStdCoroutines_option}")
+      endif()
+      set(_found TRUE)
+      break()
+    else()
+      unset(CXX_COROUTINE_NO_LINK_NEEDED CACHE)
+    endif()
+    cmake_pop_check_state()
+  endforeach()  # stdoption
 endif()
 
 cmake_pop_check_state()
@@ -205,5 +216,5 @@ cmake_pop_check_state()
 set(CXXStdCoroutine_FOUND ${_found} CACHE BOOL "TRUE if we can compile and link a program using std::coroutine" FORCE)
 
 if(CXXStdCoroutine_FIND_REQUIRED AND NOT TARGET std::coroutine)
-  message(FATAL_ERROR "Cannot Compile simple program using std::coroutine")
+  message(FATAL_ERROR "Cannot discover std::coroutine headers and/or compile simple program using std::coroutine")
 endif()

From 8e9a16fcee18bc439afa07cda2e77e37e81e2754 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Tue, 8 Nov 2022 16:55:11 -0800
Subject: [PATCH 056/259] [cmake] do not do separate header and compile checks
 for coroutine does not work (headers may be present but not usable, etc.)

---
 cmake/modules/FindCXXStdCoroutine.cmake | 75 +++++++++----------------
 1 file changed, 25 insertions(+), 50 deletions(-)

diff --git a/cmake/modules/FindCXXStdCoroutine.cmake b/cmake/modules/FindCXXStdCoroutine.cmake
index 7dfc048d8..3a86d7a4d 100644
--- a/cmake/modules/FindCXXStdCoroutine.cmake
+++ b/cmake/modules/FindCXXStdCoroutine.cmake
@@ -146,74 +146,49 @@ foreach(component IN LISTS CXXStdCoroutines_want_components_ordered)
   foreach(option IN LISTS CXXStdCoroutines_find_options)
     foreach(stdoption IN LISTS CXXStdCoroutines_std_options)
       cmake_push_check_state()
-      list(APPEND CMAKE_REQUIRED_FLAGS "${option};${stdoption}")
-      check_include_file_cxx("${_coro_header}" _CXX_COROUTINE_HAVE_HEADER)
-      mark_as_advanced(_CXX_COROUTINE_HAVE_HEADER)
+      set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${option} ${stdoption}")
+
+      string(CONFIGURE [[
+        #include <@_coro_header@>
+
+        int main() {
+            auto x = @_coro_namespace@::suspend_always{};
+            return 0;
+        }
+      ]] code @ONLY)
+
+      check_cxx_source_compiles("${code}" HAVE_USABLE_${_coro_header})
+      mark_as_advanced(HAVE_USABLE_${_coro_header})
       cmake_pop_check_state()
-      if(_CXX_COROUTINE_HAVE_HEADER)
+      if(HAVE_USABLE_${_coro_header})
+        add_library(std::coroutine INTERFACE IMPORTED GLOBAL)
+        target_compile_features(std::coroutine INTERFACE cxx_std_20)
         if (option)
-          set(CXXStdCoroutines_option "${option}" CACHE STRING "The compiler option needed to use <coroutine>")
+          target_compile_options(std::coroutine INTERFACE "${option}")
         endif()
-        mark_as_advanced(CXXStdCoroutines_option)
         set(CXX_COROUTINE_COMPONENT "${component}" CACHE STRING "The component of CXXStdCoroutine package found")
-        # We found a header, skip rest of tests
+        # break out of this loop
         break()
       else()
-        unset(_CXX_COROUTINE_HAVE_HEADER CACHE)
+        unset(HAVE_USABLE_${_coro_header} CACHE)
       endif()
     endforeach()  # stdoption
+    if (TARGET std::coroutine)
+      break()
+    endif()
   endforeach()  # option
-  if (_CXX_COROUTINE_HAVE_HEADER)
+  if (TARGET std::coroutine)
     break()
   endif()
 endforeach() # components
 
-set(CXX_COROUTINE_HAVE_CORO ${_CXX_COROUTINE_HAVE_HEADER} CACHE BOOL "TRUE if we have the C++ coroutine headers")
+set(CXX_COROUTINE_HAVE_CORO ${HAVE_USABLE_${_coro_header}} CACHE BOOL "TRUE if we have usable C++ coroutine headers")
 set(CXX_COROUTINE_HEADER ${_coro_header} CACHE STRING "The header that should be included to obtain the coroutine APIs")
 set(CXX_COROUTINE_NAMESPACE ${_coro_namespace} CACHE STRING "The C++ namespace that contains the coroutine APIs")
 
-set(_found FALSE)
-
-if(CXX_COROUTINE_HAVE_CORO)
-  # We have some coroutine library available. Do compile checks
-  string(CONFIGURE [[
-        #include <@CXX_COROUTINE_HEADER@>
-
-        int main() {
-            auto x = @CXX_COROUTINE_NAMESPACE@::suspend_always{};
-            return 0;
-        }
-    ]] code @ONLY)
-
-  # argh ... check_include_file_cxx will discover coroutine header but it is not usable without -std flag
-  foreach(stdoption IN LISTS CXXStdCoroutines_std_options)
-    cmake_push_check_state()
-    if (DEFINED CXXStdCoroutines_option)
-      set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${CXXStdCoroutines_option}")
-    endif()
-    if (stdoption)
-      set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${stdoption}")
-    endif()
-    check_cxx_source_compiles("${code}" CXX_COROUTINE_NO_LINK_NEEDED)
-
-    if(CXX_COROUTINE_NO_LINK_NEEDED)
-      add_library(std::coroutine INTERFACE IMPORTED GLOBAL)
-      target_compile_features(std::coroutine INTERFACE cxx_std_20)
-      if (DEFINED CXXStdCoroutines_option)
-        target_compile_options(std::coroutine INTERFACE "${CXXStdCoroutines_option}")
-      endif()
-      set(_found TRUE)
-      break()
-    else()
-      unset(CXX_COROUTINE_NO_LINK_NEEDED CACHE)
-    endif()
-    cmake_pop_check_state()
-  endforeach()  # stdoption
-endif()
-
 cmake_pop_check_state()
 
-set(CXXStdCoroutine_FOUND ${_found} CACHE BOOL "TRUE if we can compile and link a program using std::coroutine" FORCE)
+set(CXXStdCoroutine_FOUND ${HAVE_USABLE_${_coro_header}} CACHE BOOL "TRUE if we have usable C++ coroutine headers" FORCE)
 
 if(CXXStdCoroutine_FIND_REQUIRED AND NOT TARGET std::coroutine)
   message(FATAL_ERROR "Cannot discover std::coroutine headers and/or compile simple program using std::coroutine")

From 9fd5b875d2b32d69b1fa5cef635cb55d2e015ca6 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Tue, 24 Jan 2023 15:38:21 -0500
Subject: [PATCH 057/259] Initial implementation of coroutine-based device
 support with PaRSEC

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/ttg.h           | 261 +++++++++++++++++++++++++++++----
 ttg/ttg/parsec/ttg_data_copy.h |  56 +++++++
 ttg/ttg/tt.h                   |  18 +--
 ttg/ttg/view.h                 | 241 ++++++++++++++++++++++++++++--
 4 files changed, 519 insertions(+), 57 deletions(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 07e201218..010bebed2 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -520,6 +520,7 @@ namespace ttg_parsec {
       parsec_task_t parsec_task;
       int32_t in_data_count = 0;  //< number of satisfied inputs
       int32_t data_count = 0;     //< number of data elements in parsec_task.data
+      ttg_data_copy_t *copies;    //< pointer to the fixed copies array of the derived task
       parsec_hash_table_item_t tt_ht_item = {};
       parsec_static_op_t function_template_class_ptr[ttg::runtime_traits<ttg::Runtime::PaRSEC>::num_execution_spaces] =
           {nullptr};
@@ -554,9 +555,10 @@ namespace ttg_parsec {
        * but always be use through parsec_ttg_task_t.
        */
 
-      parsec_ttg_task_base_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class, int data_count,
+      parsec_ttg_task_base_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class,
+                             int data_count, ttg_data_copy_t *copies,
                              bool defer_writer = TTG_PARSEC_DEFER_WRITER)
-          : data_count(data_count), defer_writer(defer_writer) {
+          : data_count(data_count), copies(copies) defer_writer(defer_writer) {
         PARSEC_LIST_ITEM_SINGLETON(&parsec_task.super);
         parsec_task.mempool_owner = mempool;
         parsec_task.task_class = task_class;
@@ -564,10 +566,12 @@ namespace ttg_parsec {
       }
 
       parsec_ttg_task_base_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class,
-                             parsec_taskpool_t *taskpool, int32_t priority, int data_count,
+                             parsec_taskpool_t *taskpool, int32_t priority,
+                             int data_count, ttg_data_copy_t *copies,
                              release_task_fn *release_fn,
                              bool defer_writer = TTG_PARSEC_DEFER_WRITER)
           : data_count(data_count)
+          , copies(copies)
           , defer_writer(defer_writer)
           , release_task_cb(release_fn) {
             int32_t p = priority;
@@ -595,15 +599,13 @@ namespace ttg_parsec {
 #ifdef TTG_HAS_COROUTINE
       void* suspended_task_address = nullptr;  // if not null the function is suspended
 #endif
+      ttg_data_copy_t *copies[num_streams+1] = { nullptr };  // the data copies tracked by this task
+                                                             // +1 for the copy needed during send/bcas
 
       parsec_ttg_task_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class)
           : parsec_ttg_task_base_t(mempool, task_class, num_streams) {
         tt_ht_item.key = pkey();
 
-        for (int i = 0; i < num_streams; ++i) {
-          parsec_task.data[i].data_in = nullptr;
-        }
-
         // We store the hash of the key and the address where it can be found in locals considered as a scratchpad
         *(uintptr_t*)&(parsec_task.locals[0]) = 0; //there is no key
         *(uintptr_t*)&(parsec_task.locals[2]) = 0; //there is no key
@@ -613,14 +615,10 @@ namespace ttg_parsec {
                         parsec_task_class_t *task_class, parsec_taskpool_t *taskpool,
                         TT *tt_ptr, int32_t priority)
           : parsec_ttg_task_base_t(mempool, task_class, taskpool, priority,
-                                   num_streams, &release_task, tt_ptr->m_defer_writer)
+                                   num_streams, copies, &release_task, tt_ptr->m_defer_writer)
           , tt(tt_ptr), key(key) {
         tt_ht_item.key = pkey();
 
-        for (int i = 0; i < num_streams; ++i) {
-          parsec_task.data[i].data_in = nullptr;
-        }
-
         // We store the hash of the key and the address where it can be found in locals considered as a scratchpad
         uint64_t hv = ttg::hash<std::decay_t<decltype(key)>>{}(key);
         *(uintptr_t*)&(parsec_task.locals[0]) = hv;
@@ -644,26 +642,20 @@ namespace ttg_parsec {
 #ifdef TTG_HAS_COROUTINE
       void* suspended_task_address = nullptr;  // if not null the function is suspended
 #endif
+      ttg_data_copy_t *copies[num_streams+1] = { nullptr };  // the data copies tracked by this task
+                                                             // +1 for the copy needed during send/bcas
 
       parsec_ttg_task_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class)
-          : parsec_ttg_task_base_t(mempool, task_class, num_streams) {
+          : parsec_ttg_task_base_t(mempool, task_class, num_streams, copies) {
         tt_ht_item.key = pkey();
-
-        for (int i = 0; i < num_streams; ++i) {
-          parsec_task.data[i].data_in = nullptr;
-        }
       }
 
       parsec_ttg_task_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class,
                         parsec_taskpool_t *taskpool, TT *tt_ptr, int32_t priority)
           : parsec_ttg_task_base_t(mempool, task_class, taskpool, priority,
-                                   num_streams, &release_task, tt_ptr->m_defer_writer)
+                                   num_streams, copies, &release_task, tt_ptr->m_defer_writer)
           , tt(tt_ptr) {
         tt_ht_item.key = pkey();
-
-        for (int i = 0; i < num_streams; ++i) {
-          parsec_task.data[i].data_in = nullptr;
-        }
       }
 
       static void release_task(parsec_ttg_task_base_t* task_base) {
@@ -681,7 +673,7 @@ namespace ttg_parsec {
         return res;
       }
       for (int i = 0; i < task->data_count; ++i) {
-        auto copy = static_cast<ttg_data_copy_t *>(task->parsec_task.data[i].data_in);
+        auto copy = static_cast<ttg_data_copy_t *>(task->copies[i]);
         if (NULL != copy && copy->device_private == ptr) {
           res = copy;
           break;
@@ -696,7 +688,7 @@ namespace ttg_parsec {
         return i;
       }
       for (i = 0; i < task->data_count; ++i) {
-        auto copy = static_cast<ttg_data_copy_t *>(task->parsec_task.data[i].data_in);
+        auto copy = static_cast<ttg_data_copy_t *>(task->copies[i]);
         if (NULL != copy && copy->device_private == ptr) {
           return i;
         }
@@ -713,7 +705,7 @@ namespace ttg_parsec {
         throw std::logic_error("Too many data copies, check MAX_PARAM_COUNT!");
       }
 
-      task->parsec_task.data[task->data_count].data_in = copy;
+      task->copies[task->data_count] = copy;
       task->data_count++;
       return true;
     }
@@ -722,17 +714,17 @@ namespace ttg_parsec {
       int i;
       /* find and remove entry; copies are usually appended and removed, so start from back */
       for (i = task->data_count-1; i >= 0; --i) {
-        if (copy == task->parsec_task.data[i].data_in) {
+        if (copy == task->copies[i]) {
           break;
         }
       }
       if (i < 0) return;
       /* move all following elements one up */
       for (; i < task->data_count - 1; ++i) {
-        task->parsec_task.data[i].data_in = task->parsec_task.data[i + 1].data_in;
+        task->copies[i] = task->copies[i + 1];
       }
       /* null last element */
-      task->parsec_task.data[i].data_in = nullptr;
+      task->copies[i] = nullptr;
       task->data_count--;
     }
 
@@ -957,8 +949,8 @@ namespace ttg_parsec {
           new_copy->mark_mutable();
           /* replace the copy in the deferred task */
           for (int i = 0; i < deferred_op->data_count; ++i) {
-            if (deferred_op->parsec_task.data[i].data_in == copy_in) {
-              deferred_op->parsec_task.data[i].data_in = new_copy;
+            if (deferred_op->copies[i] == copy_in) {
+              deferred_op->copies[i] = new_copy;
               break;
             }
           }
@@ -1201,6 +1193,13 @@ namespace ttg_parsec {
     constexpr static std::array<void (TT::*)(void *, std::size_t), numinedges> get_from_pull_msg_fcts =
         make_get_from_pull_fcts(std::make_index_sequence<numinedges>{});
 
+    template<std::size_t... IS>
+    constexpr auto make_input_is_const(std::index_sequence<IS...>) {
+      using resultT = decltype(input_is_const);
+      return resultT{{std::is_const_v<std::tuple_element_t<IS, input_args_type>>...}};
+    }
+    constexpr static std::array<bool, numins> input_is_const = make_input_is_const(std::make_index_sequence<numins>{});
+
     ttg::World world;
     ttg::meta::detail::keymap_t<keyT> keymap;
     ttg::meta::detail::keymap_t<keyT> priomap;
@@ -1288,7 +1287,199 @@ namespace ttg_parsec {
     static input_refs_tuple_type make_tuple_of_ref_from_array(task_t *task, std::index_sequence<IS...>) {
       return input_refs_tuple_type{static_cast<std::tuple_element_t<IS, input_refs_tuple_type>>(
           *reinterpret_cast<std::remove_reference_t<std::tuple_element_t<IS, input_refs_tuple_type>> *>(
-              task->parsec_task.data[IS].data_in->device_private))...};
+              task->copies[IS]->device_private))...};
+    }
+
+    /**
+     * Submit callback called by PaRSEC once all input transfers have completed.
+     */
+    template <ttg::ExecutionSpace Space>
+    static parsec_hook_return_t device_static_submit(parsec_task_t* parsec_task) {
+
+      task_t *task = static_cast<task_t*>(parsec_task);
+      // get the device task from the coroutine handle
+      device_task* dev_task = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<device_task_promise_type>::from_address(task->suspended_task_address);
+
+      // get the promise which contains the views
+      auto dev_data = dev_task->promise();
+
+      /* we should still be waiting for the transfer to complete */
+      assert(dev_data.state() == TTG_DEVICE_CORO_WAIT_TRANSFER);
+
+      /* update the device pointers in the device views */
+      int i = 0;
+      for (auto& view : dev_data) {
+        /* iterate over all viewspans of this view (i.e., all memory ranges in this view) */
+        for (auto& view_span : view) {
+          view_span.set_data(task->parsec_task.data[i].copy_out);
+          ++i;
+        }
+      }
+
+      /* Here we call back into the coroutine again after the transfers have completed */
+      static_op(parsec_task);
+
+      /* for now make sure we're waiting for the kernel to complete and the coro hasn't skipped this step */
+      assert(dev_data.state() == TTG_DEVICE_CORO_WAIT_KERNEL);
+
+      /* next time we will come back into device_static_stage_complete */
+      return PARSEC_HOOK_RETURN_DONE;
+    }
+
+    /**
+     * Completion callback called by PaRSEC once the kernel submitted in the submit callback above
+     * have completed.
+     */
+    template <ttg::ExecutionSpace Space>
+    static parsec_hook_return_t device_static_stage_complete(parsec_task_t* parsec_task) {
+
+      // get the device task from the coroutine handle
+      device_task* dev_task = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<device_task_promise_type>::from_address(task->suspended_task_address);
+
+      // get the promise which contains the views
+      auto dev_data = dev_task->promise();
+
+      /* for now make sure we're waiting for the kernel to complete and the coro hasn't skipped this step */
+      assert(dev_data.state() == TTG_DEVICE_CORO_WAIT_KERNEL);
+
+      /* the kernel has completed, resume the coroutine once again to get to the send stage */
+      static_op(parsec_task);
+
+      /* the coroutine should have completed and we cannot access the promise anymore */
+      return PARSEC_HOOK_RETURN_DONE;
+    }
+
+    template <ttg::ExecutionSpace Space>
+    static parsec_hook_return_t device_static_op(parsec_task_t* parsec_task) {
+      static_assert(derived_has_cuda_op);
+      task_t *task = static_cast<task_t*>(parsec_task);
+
+      int dev_index;
+      dev_index = parsec_get_best_device(this_task, ratio);
+      assert(dev_index >= 0);
+      if (dev_index < 2) {
+          return PARSEC_HOOK_RETURN_NEXT; /* Fall back */
+      }
+
+      /* set up a device task */
+      /* TODO: take them from a free-list */
+      parsec_gpu_task_t *gpu_task;
+      gpu_task = (parsec_gpu_task_t *) calloc(1, sizeof(parsec_gpu_task_t));
+      PARSEC_OBJ_CONSTRUCT(gpu_task, parsec_list_item_t);
+
+      gpu_task->ec = parsec_task;
+      /* come back into this function
+        * TODO: have a specialized function that checks
+        */
+      gpu_task->submit = device_static_submit<Space>;
+      gpu_task->complete_stage = device_static_stage_complete<Space>;
+      gpu_task->task_type = 0; // user task
+      gpu_task->load = 1.0;    // TODO: can we do better?
+      gpu_task->last_data_check_epoch = -1; // used internally
+      gpu_task->pushout = 0;
+
+      // first invocation of the coroutine to get the coroutine handle
+      static_op(parsec_task);
+
+      // get the device task from the coroutine handle
+      device_task* dev_task = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<device_task_promise_type>::from_address(task->suspended_task_address);
+
+      // get the promise which contains the views
+      auto dev_data = dev_task->promise();
+
+      /* for now make sure we're waiting for transfers and the coro hasn't skipped this step */
+      assert(dev_data.state() == TTG_DEVICE_CORO_WAIT_TRANSFER);
+
+      // manage the gpu_task flows
+
+      // set the input flows
+      int i = 0;
+      /* TODO: need to free flows at the end of the task lifetime */
+      parsec_flow_t* flows[] = new parsec_flow_t[MAX_PARAM_COUNT];
+      for (auto& view : dev_data) {
+        void *host_obj = view.host_obj();
+
+        /* iterate over all viewspans of this view (i.e., all memory ranges in this view) */
+        for (auto& view_span : view) {
+          gpu_task->flow_nb_elts[i] = view_span.size(); // size in bytes
+          gpu_task->flow[i] = &flows[i];
+          parsec_flow_t flow = {.name = nullptr,
+                                .sym_type = PARSEC_SYM_INOUT,
+                                .flow_flags = view->is_out() ? PARSEC_FLOW_ACCESS_RW : PARSEC_FLOW_ACCESS_READ,
+                                .flow_index = i,
+                                .flow_datatype_mask = ~0 };
+          *gpu_task->flow[i] = flow;
+
+          parsec_data_copy_t* copy = nullptr;
+          ttg_data_copy_t* obj_copy = nullptr;
+          int input_obj_idx = -1;
+          /* try to find the view in the task and allocate a new copy if needed */
+          for (int i = 0; nullptr == copy && i < numins; ++i) {
+            ttg_data_copy_t* obj_copy = task->copies[i];
+            if (obj_copy->device_private == host_obj) {
+              for (auto dev_copy : obj_copy) {
+                if (view_span.data() == dev_copy->device_private) {
+                  copy = dev_copy;
+                  input_obj_idx = i;
+                  break;
+                }
+              }
+            }
+          }
+
+          /* push all data back out, EXCEPT if the host object is a const input for the task
+           * TODO [JS]: if PaRSEC lets us send from the device we can avoid pushing out here
+           */
+          if (input_obj_idx == -1 || !input_is_const[input_obj_idx]) {
+            gpu_task->pushout |= 1<<i;
+          }
+
+          /* no copy found, create a new copy */
+          if (nullptr == copy) {
+            parsec_data_t *data = parsec_data_new();
+            copy = parsec_data_copy_new(data, 0, parsec_datatype_int8_t, 0);
+            copy->device_private = view_span.data();
+            parsec_data_copy_attach(data, copy, 0); // TODO: device 0 is the CPU?
+          }
+
+          if (obj_copy != nullptr) {
+            /* add the */
+            obj_copy->add_device_copy(copy);
+          }
+          /* register the copy with the task */
+          task->parsec_task.data[i].copy_in = copy;
+
+          ++i;
+        }
+      }
+
+      /* mark all other flows as ignored */
+      for (; i < MAX_PARAM_COUNT; ++i) {
+        gpu_task->flow_nb_elts[i] = 0;
+        gpu_task->flow[i] = &flows[i];
+        gpu_task->flow[i]->flow_flags = PARSEC_FLOW_ACCESS_NONE;
+      }
+
+      parsec_device_module_t *device = parsec_mca_device_get(dev_index);
+      assert(NULL != device);
+      switch(device->type) {
+
+#if defined(PARSEC_HAVE_CUDA)
+        case PARSEC_DEV_CUDA:
+          if constexpr (Space == ttg::ExecutionSpace::CUDA) {
+            /* TODO: we need custom staging functions because PaRSEC looks at the
+             *       task-class to determine the number of flows. */
+            gpu_task->stage_in  = parsec_default_cuda_stage_in;
+            gpu_task->stage_out = parsec_default_cuda_stage_out;
+            return parsec_cuda_kernel_scheduler(es, gpu_task, dev_index);
+          }
+          break;
+#endif
+        default:
+          break;
+      }
+      ttg::print_error(get_name(), " : received mismatching device type ", device->type, " from PaRSEC");
+      ttg::abort();
     }
 
     template <ttg::ExecutionSpace Space>
@@ -1302,6 +1493,7 @@ namespace ttg_parsec {
         nullptr;
 #endif
       if (suspended_task_address == nullptr) {  // task is a coroutine that has not started or an ordinary function
+
         ttT *baseobj = task->tt;
         derivedT *obj = static_cast<derivedT *>(baseobj);
         assert(parsec_ttg_caller == NULL);
@@ -1357,6 +1549,8 @@ namespace ttg_parsec {
         }
       }
 
+// XXX the below code is not needed, should be removed once the fib test has been changed
+#if 0
 #ifdef TTG_HAS_COROUTINE
       if (suspended_task_address) {
         // right now can events are not properly implemented, we are only testing the workflow with dummy events
@@ -1377,6 +1571,7 @@ namespace ttg_parsec {
       }
       else
 #endif  // TTG_HAS_COROUTINE
+#endif // 0
         return PARSEC_HOOK_RETURN_DONE;
     }
 
@@ -2924,7 +3119,9 @@ namespace ttg_parsec {
       self.task_class_id = get_instance_id();
       self.nb_parameters = 0;
       self.nb_locals = 0;
-      self.nb_flows = numflows;
+      //self.nb_flows = numflows;
+      self.nb_flows = MAX_PARAM_COUNT; // we're not using all flows but have to
+                                       // trick the device handler into looking at all of them
 
       if( world_impl.profiling() ) {
         // first two ints are used to store the hash of the key.
diff --git a/ttg/ttg/parsec/ttg_data_copy.h b/ttg/ttg/parsec/ttg_data_copy.h
index 461984e3d..bed82db10 100644
--- a/ttg/ttg/parsec/ttg_data_copy.h
+++ b/ttg/ttg/parsec/ttg_data_copy.h
@@ -3,6 +3,8 @@
 
 #include <utility>
 #include <limits>
+#include <vector>
+#include <iterator>
 
 #include <parsec.h>
 
@@ -82,6 +84,60 @@ namespace ttg_parsec {
 
       /* mark destructor as virtual */
       virtual ~ttg_data_copy_t() = default;
+
+
+
+      /* manage device copies owned by this object
+       * we only touch the vector if we have more than one copies to track
+       * and otherwise use the single-element member.
+       */
+      using device_iterator = parsec_data_copy_t**;
+
+      void add_device_copy(parsec_data_copy_t* copy) {
+        switch (m_num_dev_copies) {
+          case 0:
+            m_single_dev_copy = copy;
+            break;
+          case 1:
+            /* move single copy into vector and add new copy below */
+            m_dev_copies.push_back(m_single_dev_copy);
+            /* fall-through */
+          default:
+            /* store in multi-copy vector */
+            m_dev_copies.push_back(copy);
+            break;
+        }
+        m_num_dev_copies++;
+      }
+
+      int num_dev_copies() const {
+        return m_num_dev_copies;
+      }
+
+      device_iterator begin() {
+        switch(m_num_dev_copies) {
+          // no device copies
+          case 0: return end();
+          case 1: return &m_single_dev_copy;
+          default: return m_dev_copies.data();
+        }
+      }
+
+      device_iterator end() {
+        switch(m_num_dev_copies) {
+          case 0:
+          case 1:
+            return &(m_single_dev_copy) + 1;
+          default:
+            return m_dev_copies.data() + m_dev_copies.size();
+        }
+      }
+
+    private:
+      std::vector<parsec_data_copy_t*> m_dev_copies;   //< used if there are multiple device copies
+                                                       //  that belong to this object
+      parsec_data_copy_t* m_single_dev_copy;           //< used if there is a single device copy
+      int m_num_dev_copies = 0;                        //< number of device copies
     };
 
 
diff --git a/ttg/ttg/tt.h b/ttg/ttg/tt.h
index 2c545ce25..a6f3ef9d4 100644
--- a/ttg/ttg/tt.h
+++ b/ttg/ttg/tt.h
@@ -172,15 +172,15 @@ namespace ttg {
 
 #ifndef TTG_PROCESS_TT_OP_RETURN
 #ifdef TTG_HAS_COROUTINE
-#define TTG_PROCESS_TT_OP_RETURN(result, invoke)     \
-  {                                                  \
-    using return_type = decltype(invoke);            \
-    if constexpr (std::is_same_v<return_type, void>) \
-      invoke;                                        \
-    else {                                           \
-      auto coro_return = invoke;                     \
-      result = coro_return.address();                \
-    }                                                \
+#define TTG_PROCESS_TT_OP_RETURN(result, invoke)       \
+  {                                                    \
+    using return_type = decltype(invoke);              \
+    if constexpr (std::is_same_v<return_type, void>) { \
+      invoke;                                          \
+    } else {                                           \
+      auto coro_return = invoke;                       \
+      result = coro_return.address();                  \
+    }                                                  \
   }
 #else
 #define TTG_PROCESS_TT_OP_RETURN(result, invoke) invoke
diff --git a/ttg/ttg/view.h b/ttg/ttg/view.h
index 842081f40..5183b7a50 100644
--- a/ttg/ttg/view.h
+++ b/ttg/ttg/view.h
@@ -3,6 +3,7 @@
 
 #include <array>
 #include "ttg/util/iovec.h"
+#include "ttg/util/coroutine.h"
 
 namespace ttg {
 
@@ -12,15 +13,23 @@ namespace ttg {
     SyncOut   = 0x2
   };
 
+  /**
+   * A view span that can be type-punned.
+   * We use it instead of a std::span to be able
+   * to remove the type and convert to void pointers instead.
+   */
   template<typename T>
-  struct ViewSpan {
+  struct ViewSpan;
 
-    using element_type = T;
-    using value_type   = std::remove_cv_t<T>;
+  struct ViewSpan<void> {
+
+    using element_type  = void;
+    using value_type    = void;
+    using viewspan_type = ViewSpan<value_type>;
 
     constexpr ViewSpan() = default;
-    constexpr ViewSpan(T* ptr, std::size_t size, ViewScope scope = ViewScope::SyncIn)
-    : m_ptr(ptr)
+    constexpr ViewSpan(void* ptr, std::size_t size, ViewScope scope = ViewScope::SyncIn)
+    : m_data_ptr(ptr)
     , m_size(size)
     , m_scope(scope)
     { }
@@ -29,8 +38,12 @@ namespace ttg {
       return m_size;
     }
 
-    constexpr T* data() const {
-      return m_ptr;
+    constexpr void* data() const {
+      return m_data_ptr;
+    }
+
+    void set_data(void *ptr) {
+      m_data_ptr = ptr;
     }
 
     constexpr bool is_allocate() const {
@@ -50,24 +63,64 @@ namespace ttg {
     }
 
   private:
-    T *m_ptr = nullptr;
+    void *m_data_ptr = nullptr;
     std::size_t m_size = 0;
     ViewScope m_scope = ViewScope::Allocate;
   };
 
+
+  template<typename T>
+  struct ViewSpan<T> : public ViewSpan<void> {
+
+    using element_type  = T;
+    using value_type    = std::remove_cv_t<T>;
+    using viewspan_type = ViewSpan<value_type>;
+
+    constexpr ViewSpan() = default;
+    constexpr ViewSpan(T* ptr, std::size_t size, ViewScope scope = ViewScope::SyncIn)
+    : m_ptr(ptr)
+    , m_size(size)
+    , m_scope(scope)
+    { }
+
+    constexpr T* data() const {
+      return static_cast<T*>(m_ptr);
+    }
+
+  }
+
   template<typename HostT, typename... DevTypeTs>
-  struct View {
+  struct View;
+
+  template<>
+  struct View<void> {
+    void *m_obj = nullptr;
+    std::array<
+  }
+
+  template<typename HostT, typename... DevTypeTs>
+  struct View<HostT, DevTypeTs...> {
 
     using span_tuple_type  = std::tuple<ttg::ViewSpan<DevTypeTs>...>;
     using host_type = HostT;
 
     using view_type = View<HostT, DevTypeTs...>;
 
+    constexpr static std::size_t num_spans = std::tuple_size_v<span_tuple_type>;
+
+  private:
+    template<std::size_t... Is>
+    View(HostT& obj, span_tuple_type& spans, std::index_sequence<Is...>)
+    : m_obj(&obj)
+    , m_spans({std::get<Is>(m_spans)...})
+    { }
+
+  public:
+
     constexpr View() = default;
 
     View(HostT& obj, span_tuple_type spans)
-    : m_obj(&obj)
-    , m_spans(std::move(spans))
+    : View(obj, spans, std::index_sequence_for<DevTypeTs...>{})
     { }
 
     View(view_type&&) = default;
@@ -79,12 +132,12 @@ namespace ttg {
 
     template<std::size_t i>
     auto get_device_ptr() {
-      return std::get<i>(m_spans).data();
+      return static_cast<std::tuple_element<i, span_tuple_type>>(std::get<i>(m_spans).data());
     }
 
     template<std::size_t i>
     const auto get_device_ptr() const {
-      return std::get<i>(m_spans).data();
+      return static_cast<std::tuple_element<i, span_tuple_type>>(std::get<i>(m_spans).data());
     }
 
     template<std::size_t i>
@@ -93,7 +146,7 @@ namespace ttg {
     }
 
     template<std::size_t i>
-    auto get_span() const {
+    auto& get_viewspan() const {
       return std::get<i>(m_spans);
     }
 
@@ -111,12 +164,19 @@ namespace ttg {
     }
 
     constexpr static std::size_t size() {
-      return std::tuple_size_v<decltype(m_spans)>;
+      return num_spans;
+    }
+
+    /* return a std::span of type-punned ViewSpans */
+    std::span<ViewSpan<void>> view_spans() {
+      return {m_spans.begin(), m_spans.end()};
     }
 
   private:
     HostT* m_obj = nullptr;
-    span_tuple_type m_spans{};
+    /* type-punned storage, cast to actual types in get_device_ptr */
+    std::array<ViewSpan<void>, num_spans> m_spans;
+    //span_tuple_type m_spans{};
   };
 
   template<typename HostT, typename... ViewSpanTs>
@@ -124,6 +184,155 @@ namespace ttg {
     return View(obj, std::make_tuple(std::move(spans)...));
   }
 
+  /* yielded when waiting on a kernel to complete */
+  struct device_op_wait_kernel
+  { };
+
+  enum ttg_device_coro_state {
+    TTG_DEVICE_CORO_STATE_NONE,
+    TTG_DEVICE_CORO_INIT,
+    TTG_DEVICE_CORO_WAIT_TRANSFER,
+    TTG_DEVICE_CORO_WAIT_KERNEL,
+    TTG_DEVICE_CORO_COMPLETE
+  };
+
+  /* type-punned version of the View, providing access to the object
+   * pointer and a std::span over the views of that object */
+  struct device_obj_view {
+    using span_type = std::span<ViewSpan<void>>;
+    using iterator = typename span_type::iterator;
+
+    device_obj_span(void *obj, span_type&& span)
+    : m_obj(obj)
+    , m_span(std::forward<span_type>(span))
+    { }
+
+    void *host_obj() {
+      return m_obj;
+    }
+
+    auto begin() {
+      return m_span.begin();
+    }
+
+    auto end() {
+      return m_span.end();
+    }
+
+  private:
+    void *m_obj;
+    span_type m_span;
+  };
+
+  struct device_task_promise_type;
+
+  /// task that can be resumed after some events occur
+  struct device_task : public TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<device_task_promise_type> {
+    using base_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<device_task_promise_type>;
+
+    /// these are members mandated by the promise_type concept
+    ///@{
+
+    using promise_type = device_op_promise_type;
+
+    ///@}
+
+    device_task(base_type base) : base_type(std::move(base)) {}
+
+    base_type handle() { return *this; }
+
+    /// @return true if ready to resume
+    inline bool ready() {
+      return true;
+    }
+
+    /// @return true if task completed and can be destroyed
+    inline bool completed();
+  };
+
+  /* The promise type that stores the views provided by the
+   * application task coroutine on the first co_yield. It subsequently
+   * tracks the state of the task when it moves from waiting for transfers
+   * to waiting for the submitted kernel to complete. */
+  struct device_task_promise_type {
+
+    /* do not suspend the coroutine on first invocation, we want to run
+     * the coroutine immediately and suspend when we get the device transfers.
+     */
+    std::suspend_never initial_suspend() {
+      m_state = TTG_DEVICE_CORO_INIT;
+      return {};
+    }
+
+    /* suspend the coroutine at the end of the execution
+     * so we can access the promise.
+     * TODO: necessary? maybe we can save one suspend here
+     */
+    std::suspend_never final_suspend() noexcept {
+      m_state = TTG_DEVICE_CORO_COMPLETE;
+      return {};
+    }
+
+    /* waiting for transfers to complete should always suspend
+     * TODO: as an optimization, we could check here if all data
+     *       is already available and avoid suspending...
+     */
+    template<typename... Views>
+    std::suspend_always yield_value(std:tuple<Views...> &views) {
+      /* gather all the views (host object + view spans, type-punned) into a vector */
+      constexpr std::size_t view_count = std::tuple_size_v<std:tuple<Views...>>;
+      m_spans.clear(); // in case we ever come back here
+      m_spans.reserve(view_count);
+      if constexpr(view_count > 0) {
+        auto unpack_lambda = [&](Views&... view){
+                                ((m_spans.push_back(device_obj_view(view.host_obj(), view.view_spans()))),...);
+                              };
+        std::apply(unpack_lambda, views);
+      }
+      m_state = TTG_DEVICE_CORO_WAIT_TRANSFER;
+      return {};
+    }
+
+    /* waiting for the kernel to complete should always suspend */
+    std::suspend_always yield_value(device_op_wait_kernel) {
+      m_state = TTG_DEVICE_CORO_WAIT_KERNEL;
+      return {};
+    }
+
+    void return_void() {
+      m_state = TTG_DEVICE_CORO_COMPLETE;
+    }
+
+    bool complete const {
+      return m_state == TTG_DEVICE_CORO_COMPLETE;
+    }
+
+    using handle_type = device_task;
+    device_task get_return_object() { return device_task{handle_type::from_promise(*this)}; }
+
+    using iterator = std::vector<device_obj_span>::iterator;
+
+    auto begin() {
+      return m_spans.begin();
+    }
+
+    auto end() {
+      return m_spans.end();
+    }
+
+    auto state() {
+      return m_state;
+    }
+
+  private:
+
+    std::vector<device_obj_view> m_spans;
+    ttg_device_coro_state m_state = TTG_DEVICE_CORO_STATE_NONE;
+
+  };
+
+  bool device_task::completed() { return base_type::promise().state() == TTG_DEVICE_CORO_COMPLETE; }
+
   /// std::span mirrored between host and device memory
   struct HDSpan {
     HDSpan() = default;

From 3fe08a80f404b5d424d8520a7057c79e1d1076e0 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 25 Jan 2023 18:25:27 -0500
Subject: [PATCH 058/259] Compiler fixes

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 CMakeLists.txt            |  2 +-
 tests/unit/CMakeLists.txt |  1 +
 ttg/ttg/parsec/ttg.h      | 41 +++++++++++++++++---------
 ttg/ttg/view.h            | 62 ++++++++++++++++++++-------------------
 4 files changed, 61 insertions(+), 45 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2f6cfb95b..b6381948f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -30,7 +30,7 @@ project(ttg
     HOMEPAGE_URL "https://tesseorg.github.io/ttg/")
 
 if (NOT DEFINED CMAKE_CXX_STANDARD)
-    set(CMAKE_CXX_STANDARD "17" CACHE STRING "The C++ standard")
+    set(CMAKE_CXX_STANDARD "20" CACHE STRING "The C++ standard")
 endif()
 if (NOT CMAKE_CXX_EXTENSIONS)
   set(CMAKE_CXX_EXTENSIONS OFF CACHE BOOL "The C++ standard extensions allowed?")
diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt
index bb0df6255..47dc543e6 100644
--- a/tests/unit/CMakeLists.txt
+++ b/tests/unit/CMakeLists.txt
@@ -3,6 +3,7 @@ include(AddTTGExecutable)
 # TT unit test: core TTG ops
 set(ut_src
         fibonacci.cc
+        device_coro.cc
         ranges.cc
         tt.cc
         unit_main.cpp)
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 010bebed2..dfac43ca9 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -50,6 +50,11 @@
 #include <tuple>
 #include <vector>
 
+/* TODO: remove once we use PaRSEC master */
+#ifndef PARSEC_HAVE_CUDA
+#define PARSEC_HAVE_CUDA 1
+#endif // PARSEC_HAVE_CUDA
+
 #include <parsec.h>
 #include <parsec/class/parsec_hash_table.h>
 #include <parsec/data_internal.h>
@@ -59,6 +64,9 @@
 #include <parsec/parsec_comm_engine.h>
 #include <parsec/parsec_internal.h>
 #include <parsec/scheduling.h>
+/* TODO: once we use parsec master we need to switch this include */
+#include <parsec/mca/device/cuda/device_cuda.h>
+//#include <parsec/mca/device/device_gpu.h>
 #if defined(PARSEC_PROF_TRACE)
 #include <parsec/profiling.h>
 #undef PARSEC_TTG_PROFILE_BACKEND
@@ -520,7 +528,7 @@ namespace ttg_parsec {
       parsec_task_t parsec_task;
       int32_t in_data_count = 0;  //< number of satisfied inputs
       int32_t data_count = 0;     //< number of data elements in parsec_task.data
-      ttg_data_copy_t *copies;    //< pointer to the fixed copies array of the derived task
+      ttg_data_copy_t **copies;    //< pointer to the fixed copies array of the derived task
       parsec_hash_table_item_t tt_ht_item = {};
       parsec_static_op_t function_template_class_ptr[ttg::runtime_traits<ttg::Runtime::PaRSEC>::num_execution_spaces] =
           {nullptr};
@@ -556,9 +564,11 @@ namespace ttg_parsec {
        */
 
       parsec_ttg_task_base_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class,
-                             int data_count, ttg_data_copy_t *copies,
+                             int data_count, ttg_data_copy_t **copies,
                              bool defer_writer = TTG_PARSEC_DEFER_WRITER)
-          : data_count(data_count), copies(copies) defer_writer(defer_writer) {
+          : data_count(data_count)
+          , copies(copies)
+          , defer_writer(defer_writer) {
         PARSEC_LIST_ITEM_SINGLETON(&parsec_task.super);
         parsec_task.mempool_owner = mempool;
         parsec_task.task_class = task_class;
@@ -567,7 +577,7 @@ namespace ttg_parsec {
 
       parsec_ttg_task_base_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class,
                              parsec_taskpool_t *taskpool, int32_t priority,
-                             int data_count, ttg_data_copy_t *copies,
+                             int data_count, ttg_data_copy_t **copies,
                              release_task_fn *release_fn,
                              bool defer_writer = TTG_PARSEC_DEFER_WRITER)
           : data_count(data_count)
@@ -603,7 +613,7 @@ namespace ttg_parsec {
                                                              // +1 for the copy needed during send/bcas
 
       parsec_ttg_task_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class)
-          : parsec_ttg_task_base_t(mempool, task_class, num_streams) {
+          : parsec_ttg_task_base_t(mempool, task_class, num_streams, copies) {
         tt_ht_item.key = pkey();
 
         // We store the hash of the key and the address where it can be found in locals considered as a scratchpad
@@ -1194,7 +1204,7 @@ namespace ttg_parsec {
         make_get_from_pull_fcts(std::make_index_sequence<numinedges>{});
 
     template<std::size_t... IS>
-    constexpr auto make_input_is_const(std::index_sequence<IS...>) {
+    constexpr static auto make_input_is_const(std::index_sequence<IS...>) {
       using resultT = decltype(input_is_const);
       return resultT{{std::is_const_v<std::tuple_element_t<IS, input_args_type>>...}};
     }
@@ -1298,13 +1308,13 @@ namespace ttg_parsec {
 
       task_t *task = static_cast<task_t*>(parsec_task);
       // get the device task from the coroutine handle
-      device_task* dev_task = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<device_task_promise_type>::from_address(task->suspended_task_address);
+      ttg::device_task* dev_task = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<ttg::device_task_promise_type>::from_address(task->suspended_task_address);
 
       // get the promise which contains the views
       auto dev_data = dev_task->promise();
 
       /* we should still be waiting for the transfer to complete */
-      assert(dev_data.state() == TTG_DEVICE_CORO_WAIT_TRANSFER);
+      assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_TRANSFER);
 
       /* update the device pointers in the device views */
       int i = 0;
@@ -1320,7 +1330,7 @@ namespace ttg_parsec {
       static_op(parsec_task);
 
       /* for now make sure we're waiting for the kernel to complete and the coro hasn't skipped this step */
-      assert(dev_data.state() == TTG_DEVICE_CORO_WAIT_KERNEL);
+      assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_KERNEL);
 
       /* next time we will come back into device_static_stage_complete */
       return PARSEC_HOOK_RETURN_DONE;
@@ -1333,14 +1343,16 @@ namespace ttg_parsec {
     template <ttg::ExecutionSpace Space>
     static parsec_hook_return_t device_static_stage_complete(parsec_task_t* parsec_task) {
 
+      task_t *task = static_cast<task_t*>(parsec_task);
+
       // get the device task from the coroutine handle
-      device_task* dev_task = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<device_task_promise_type>::from_address(task->suspended_task_address);
+      ttg::device_task* dev_task = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<ttg::device_task_promise_type>::from_address(task->suspended_task_address);
 
       // get the promise which contains the views
       auto dev_data = dev_task->promise();
 
       /* for now make sure we're waiting for the kernel to complete and the coro hasn't skipped this step */
-      assert(dev_data.state() == TTG_DEVICE_CORO_WAIT_KERNEL);
+      assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_KERNEL);
 
       /* the kernel has completed, resume the coroutine once again to get to the send stage */
       static_op(parsec_task);
@@ -1355,7 +1367,8 @@ namespace ttg_parsec {
       task_t *task = static_cast<task_t*>(parsec_task);
 
       int dev_index;
-      dev_index = parsec_get_best_device(this_task, ratio);
+      double ratio = 1.0;
+      dev_index = parsec_get_best_device(parsec_task, ratio);
       assert(dev_index >= 0);
       if (dev_index < 2) {
           return PARSEC_HOOK_RETURN_NEXT; /* Fall back */
@@ -1382,13 +1395,13 @@ namespace ttg_parsec {
       static_op(parsec_task);
 
       // get the device task from the coroutine handle
-      device_task* dev_task = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<device_task_promise_type>::from_address(task->suspended_task_address);
+      ttg::device_task* dev_task = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<ttg::device_task_promise_type>::from_address(task->suspended_task_address);
 
       // get the promise which contains the views
       auto dev_data = dev_task->promise();
 
       /* for now make sure we're waiting for transfers and the coro hasn't skipped this step */
-      assert(dev_data.state() == TTG_DEVICE_CORO_WAIT_TRANSFER);
+      assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_TRANSFER);
 
       // manage the gpu_task flows
 
diff --git a/ttg/ttg/view.h b/ttg/ttg/view.h
index 5183b7a50..a8a71ff9c 100644
--- a/ttg/ttg/view.h
+++ b/ttg/ttg/view.h
@@ -2,8 +2,8 @@
 #define TTG_VIEW_H
 
 #include <array>
-#include "ttg/util/iovec.h"
-#include "ttg/util/coroutine.h"
+#include <type_traits>
+#include <span>
 
 namespace ttg {
 
@@ -18,10 +18,11 @@ namespace ttg {
    * We use it instead of a std::span to be able
    * to remove the type and convert to void pointers instead.
    */
-  template<typename T>
+  template<typename T, typename = void>
   struct ViewSpan;
 
-  struct ViewSpan<void> {
+  template<>
+  struct ViewSpan<void, void> {
 
     using element_type  = void;
     using value_type    = void;
@@ -70,7 +71,7 @@ namespace ttg {
 
 
   template<typename T>
-  struct ViewSpan<T> : public ViewSpan<void> {
+  struct ViewSpan<T, void> : public ViewSpan<void, void> {
 
     using element_type  = T;
     using value_type    = std::remove_cv_t<T>;
@@ -78,28 +79,17 @@ namespace ttg {
 
     constexpr ViewSpan() = default;
     constexpr ViewSpan(T* ptr, std::size_t size, ViewScope scope = ViewScope::SyncIn)
-    : m_ptr(ptr)
-    , m_size(size)
-    , m_scope(scope)
+    : ViewSpan<void, void>(ptr, size, scope)
     { }
 
     constexpr T* data() const {
-      return static_cast<T*>(m_ptr);
+      return static_cast<T*>(m_data_ptr);
     }
 
-  }
-
-  template<typename HostT, typename... DevTypeTs>
-  struct View;
-
-  template<>
-  struct View<void> {
-    void *m_obj = nullptr;
-    std::array<
-  }
+  };
 
   template<typename HostT, typename... DevTypeTs>
-  struct View<HostT, DevTypeTs...> {
+  struct View {
 
     using span_tuple_type  = std::tuple<ttg::ViewSpan<DevTypeTs>...>;
     using host_type = HostT;
@@ -184,6 +174,12 @@ namespace ttg {
     return View(obj, std::make_tuple(std::move(spans)...));
   }
 
+  /* overload for trivially-copyable host objects */
+  template<typename HostT, typename = std::enable_if_t<std::is_trivially_copyable_v<HostT>>>
+  auto make_view(HostT& obj) {
+    return View(obj, std::make_tuple(ViewSpan{&obj, sizeof(HostT)}));
+  }
+
   /* yielded when waiting on a kernel to complete */
   struct device_op_wait_kernel
   { };
@@ -202,7 +198,7 @@ namespace ttg {
     using span_type = std::span<ViewSpan<void>>;
     using iterator = typename span_type::iterator;
 
-    device_obj_span(void *obj, span_type&& span)
+    device_obj_view(void *obj, span_type&& span)
     : m_obj(obj)
     , m_span(std::forward<span_type>(span))
     { }
@@ -233,7 +229,7 @@ namespace ttg {
     /// these are members mandated by the promise_type concept
     ///@{
 
-    using promise_type = device_op_promise_type;
+    using promise_type = device_task_promise_type;
 
     ///@}
 
@@ -259,7 +255,7 @@ namespace ttg {
     /* do not suspend the coroutine on first invocation, we want to run
      * the coroutine immediately and suspend when we get the device transfers.
      */
-    std::suspend_never initial_suspend() {
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_never initial_suspend() {
       m_state = TTG_DEVICE_CORO_INIT;
       return {};
     }
@@ -268,7 +264,7 @@ namespace ttg {
      * so we can access the promise.
      * TODO: necessary? maybe we can save one suspend here
      */
-    std::suspend_never final_suspend() noexcept {
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_never final_suspend() noexcept {
       m_state = TTG_DEVICE_CORO_COMPLETE;
       return {};
     }
@@ -278,9 +274,9 @@ namespace ttg {
      *       is already available and avoid suspending...
      */
     template<typename... Views>
-    std::suspend_always yield_value(std:tuple<Views...> &views) {
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_always yield_value(std::tuple<Views&...> &views) {
       /* gather all the views (host object + view spans, type-punned) into a vector */
-      constexpr std::size_t view_count = std::tuple_size_v<std:tuple<Views...>>;
+      constexpr static std::size_t view_count = std::tuple_size_v<std::tuple<Views...>>;
       m_spans.clear(); // in case we ever come back here
       m_spans.reserve(view_count);
       if constexpr(view_count > 0) {
@@ -293,8 +289,14 @@ namespace ttg {
       return {};
     }
 
+    /* convenience-function to yield a single view */
+    template<typename HostT, typename... DeviceViewTs>
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_always yield_value(View<HostT, DeviceViewTs...> &view) {
+      return yield_value(std::tie(view));
+    }
+
     /* waiting for the kernel to complete should always suspend */
-    std::suspend_always yield_value(device_op_wait_kernel) {
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_always yield_value(device_op_wait_kernel) {
       m_state = TTG_DEVICE_CORO_WAIT_KERNEL;
       return {};
     }
@@ -303,14 +305,14 @@ namespace ttg {
       m_state = TTG_DEVICE_CORO_COMPLETE;
     }
 
-    bool complete const {
+    bool complete() const {
       return m_state == TTG_DEVICE_CORO_COMPLETE;
     }
 
-    using handle_type = device_task;
+    using handle_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<device_task_promise_type>;
     device_task get_return_object() { return device_task{handle_type::from_promise(*this)}; }
 
-    using iterator = std::vector<device_obj_span>::iterator;
+    using iterator = std::vector<device_obj_view>::iterator;
 
     auto begin() {
       return m_spans.begin();

From 78785a1726a28664bdf7b7e691cf218f8979fb86 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 25 Jan 2023 18:28:36 -0500
Subject: [PATCH 059/259] Add device_coro.cc test file

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 tests/unit/device_coro.cc | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 tests/unit/device_coro.cc

diff --git a/tests/unit/device_coro.cc b/tests/unit/device_coro.cc
new file mode 100644
index 000000000..48f22bb2a
--- /dev/null
+++ b/tests/unit/device_coro.cc
@@ -0,0 +1,33 @@
+#include <catch2/catch.hpp>
+
+#include "ttg.h"
+#include "ttg/ttg/view.h"
+
+
+TEST_CASE("Device", "coro") {
+  SECTION("device_task") {
+    ttg::Edge<int, double> edge;
+    auto fn = [](const int& key, double&& val) -> ttg::device_task {
+      ttg::View<double, double> view = ttg::make_view(val);
+      /* wait for the view to be available on the device */
+      co_yield view;
+      /* once we're back here the data has been transferred */
+      CHECK(view.get_device_ptr() != nullptr);
+      CHECK(view.size() == sizeof(val));
+      CHECK(&view.get_host_object() == &val);
+
+      /* here we suspend to wait for a kernel to complete */
+      co_yield device_op_wait_kernel{};
+
+      /* we're back, the kernel executed and we can send */
+      if (key < 10) {
+        ttg::send<0>(key+1, val);
+      }
+    };
+    auto tt = ttg::make_tt(fn, ttg::edges(edge), ttg::edges(edge),
+                           "device_task", {"edge_in"}, {"edge_out"});
+    make_graph_executable(tt);
+    if (ttg::default_execution_context().rank() == 0) tt->invoke(0, 0.0);
+    ttg::ttg_fence();
+  }
+}

From d7893df3d8ad524b8660f66eddf6d214d018b113 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 27 Jan 2023 14:45:56 -0500
Subject: [PATCH 060/259] Some more device coroutine examples

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 tests/unit/device_coro.cc | 85 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 84 insertions(+), 1 deletion(-)

diff --git a/tests/unit/device_coro.cc b/tests/unit/device_coro.cc
index 48f22bb2a..693b7a3bf 100644
--- a/tests/unit/device_coro.cc
+++ b/tests/unit/device_coro.cc
@@ -11,13 +11,16 @@ TEST_CASE("Device", "coro") {
       ttg::View<double, double> view = ttg::make_view(val);
       /* wait for the view to be available on the device */
       co_yield view;
+      // co_yield std::tie(view1, view2);
+      // TTG_WAIT_VIEW(view);
       /* once we're back here the data has been transferred */
       CHECK(view.get_device_ptr() != nullptr);
       CHECK(view.size() == sizeof(val));
       CHECK(&view.get_host_object() == &val);
 
       /* here we suspend to wait for a kernel to complete */
-      co_yield device_op_wait_kernel{};
+      co_yield ttg::device_op_wait_kernel{};
+      // TTG_WAIT_KERNEL();
 
       /* we're back, the kernel executed and we can send */
       if (key < 10) {
@@ -30,4 +33,84 @@ TEST_CASE("Device", "coro") {
     if (ttg::default_execution_context().rank() == 0) tt->invoke(0, 0.0);
     ttg::ttg_fence();
   }
+
+  struct A {
+    int a[10];
+    double b[10];
+  };
+
+
+  SECTION("device_task_struct") {
+    ttg::Edge<int, double> edge;
+    auto fn = [](const int& key, A&& val) -> ttg::device_task {
+      auto view = ttg::make_view(val, {val.a, 10, ttg::ViewScope::SyncIn},
+                                      {val.b, 10, ttg::ViewScope::SyncIn|ttg::ViewScope::SyncOut});
+      /* wait for the view to be available on the device */
+      co_yield view;
+      // co_yield std::tie(view1, view2);
+      // TTG_WAIT_VIEW(view);
+      /* once we're back here the data has been transferred */
+      CHECK(view.get_device_ptr<0>() != nullptr);
+      CHECK(view.size<0>() == 10*sizeof(int));
+      CHECK(&view.get_host_object() == &val);
+
+      // <submit kernel here>
+
+      /* here we suspend to wait for a kernel to complete */
+      co_yield ttg::device_op_wait_kernel{};
+      // TTG_WAIT_KERNEL();
+
+      /* we're back, the kernel executed and we can send */
+      if (key < 10) {
+        ttg::send<0>(key+1, val);
+      }
+    };
+    auto tt = ttg::make_tt(fn, ttg::edges(edge), ttg::edges(edge),
+                           "device_task", {"edge_in"}, {"edge_out"});
+    make_graph_executable(tt);
+    if (ttg::default_execution_context().rank() == 0) tt->invoke(0, 0.0);
+    ttg::ttg_fence();
+  }
+
+
+
+  SECTION("device_task_struct_multi") {
+    ttg::Edge<int, double> edge;
+    auto fn = [](const int& key, A&& val) -> ttg::device_task {
+      auto view = ttg::make_view(val, {val.a, 10},
+                                      {val.b, 10, ttg::ViewScope::SyncOut});
+      /* wait for the view to be available on the device */
+      co_yield view;
+      // co_yield std::tie(view1, view2);
+      // TTG_WAIT_VIEW(view);
+      /* once we're back here the data has been transferred */
+      CHECK(view.get_device_ptr<0>() != nullptr);
+      CHECK(view.size<0>() == 10*sizeof(int));
+      CHECK(&view.get_host_object() == &val);
+
+      // <submit kernel here>
+
+      /* here we suspend to wait for a kernel to complete */
+      co_yield ttg::device_op_wait_kernel{};
+      // TTG_WAIT_KERNEL();
+
+      if (val.b[0] < 100.0) {
+        // <submit another kernel here>
+        view.sync_out<0>();
+        // anything submitted before this point must finish before the data is returned
+        co_yield view;
+      }
+
+      /* we're back, the kernel executed and we can send */
+      if (key < 10) {
+        ttg::send<0>(key+1, val);
+      }
+    };
+    auto tt = ttg::make_tt(fn, ttg::edges(edge), ttg::edges(edge),
+                           "device_task", {"edge_in"}, {"edge_out"});
+    make_graph_executable(tt);
+    if (ttg::default_execution_context().rank() == 0) tt->invoke(0, 0.0);
+    ttg::ttg_fence();
+  }
+
 }

From 7a17baf54ff061548feca218f289c5803ba7e55f Mon Sep 17 00:00:00 2001
From: Thomas Herault <herault@icl.utk.edu>
Date: Wed, 16 Nov 2022 17:53:10 -0600
Subject: [PATCH 061/259] Base of the beginning to log the appropriate info
 from the profiling

---
 ttg/ttg/parsec/ttg.h | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index dfac43ca9..0faa06e73 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -439,7 +439,11 @@ namespace ttg_parsec {
       // TODO PROFILING: 0 and NULL should be replaced with something that depends on the key human-readable serialization...
       // Typically, we would put something like 3*sizeof(int32_t), "m{int32_t};n{int32_t};k{int32_t}" to say
       // there are three fields, named m, n and k, stored in this order, and each of size int32_t
+<<<<<<< HEAD
       parsec_profiling_add_dictionary_keyword(name, "fill:000000", 64, "key{char[64]}",
+=======
+      parsec_profiling_add_dictionary_keyword(name, "fill:000000", 0, NULL,
+>>>>>>> f3544fb3 (Base of the beginning to log the appropriate info from the profiling)
                                               (int*)&tpool->profiling_array[2*position],
                                               (int*)&tpool->profiling_array[2*position+1]);
     }
@@ -3156,7 +3160,11 @@ namespace ttg_parsec {
       self.task_snprintf = parsec_ttg_task_snprintf;
 
 #if defined(PARSEC_PROF_TRACE)
+<<<<<<< HEAD
       self.profile_info = &parsec_ttg_task_info;
+=======
+      self.profile_info = parsec_ttg_task_info;
+>>>>>>> f3544fb3 (Base of the beginning to log the appropriate info from the profiling)
 #endif
 
       world_impl.taskpool()->nb_task_classes = std::max(world_impl.taskpool()->nb_task_classes, static_cast<decltype(world_impl.taskpool()->nb_task_classes)>(self.task_class_id+1));

From cbb30b6c6350d80526ed433d684ccd5b261c744b Mon Sep 17 00:00:00 2001
From: Thomas Herault <herault@icl.utk.edu>
Date: Sat, 19 Nov 2022 16:21:00 -0500
Subject: [PATCH 062/259] Update to appropriate version of parsec; use arrays
 of chars to store the key in the profiling info under the parsec driver

---
 ttg/ttg/parsec/ttg.h | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 0faa06e73..dfac43ca9 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -439,11 +439,7 @@ namespace ttg_parsec {
       // TODO PROFILING: 0 and NULL should be replaced with something that depends on the key human-readable serialization...
       // Typically, we would put something like 3*sizeof(int32_t), "m{int32_t};n{int32_t};k{int32_t}" to say
       // there are three fields, named m, n and k, stored in this order, and each of size int32_t
-<<<<<<< HEAD
       parsec_profiling_add_dictionary_keyword(name, "fill:000000", 64, "key{char[64]}",
-=======
-      parsec_profiling_add_dictionary_keyword(name, "fill:000000", 0, NULL,
->>>>>>> f3544fb3 (Base of the beginning to log the appropriate info from the profiling)
                                               (int*)&tpool->profiling_array[2*position],
                                               (int*)&tpool->profiling_array[2*position+1]);
     }
@@ -3160,11 +3156,7 @@ namespace ttg_parsec {
       self.task_snprintf = parsec_ttg_task_snprintf;
 
 #if defined(PARSEC_PROF_TRACE)
-<<<<<<< HEAD
       self.profile_info = &parsec_ttg_task_info;
-=======
-      self.profile_info = parsec_ttg_task_info;
->>>>>>> f3544fb3 (Base of the beginning to log the appropriate info from the profiling)
 #endif
 
       world_impl.taskpool()->nb_task_classes = std::max(world_impl.taskpool()->nb_task_classes, static_cast<decltype(world_impl.taskpool()->nb_task_classes)>(self.task_class_id+1));

From cc12ba23031e6a0be5ae70992bb542b29f7af026 Mon Sep 17 00:00:00 2001
From: Thomas Herault <herault@icl.utk.edu>
Date: Tue, 3 Jan 2023 17:27:25 -0500
Subject: [PATCH 063/259] Port deprecated types and functions to parsec master

---
 examples/potrf/testing_dlauum.cc | 32 ++++++++++++++++++++++++++++++++
 examples/potrf/testing_dpotrf.cc | 32 ++++++++++++++++++++++++++++++++
 examples/potrf/testing_dtrtri.cc | 32 ++++++++++++++++++++++++++++++++
 3 files changed, 96 insertions(+)

diff --git a/examples/potrf/testing_dlauum.cc b/examples/potrf/testing_dlauum.cc
index c25e36763..51dc7f81a 100644
--- a/examples/potrf/testing_dlauum.cc
+++ b/examples/potrf/testing_dlauum.cc
@@ -142,3 +142,35 @@ int main(int argc, char **argv)
   ttg::finalize();
   return ret;
 }
+
+static void
+dplasma_dprint_tile( int m, int n,
+                     const parsec_tiled_matrix_t* descA,
+                     const double *M )
+{
+    int tempmm = ( m == descA->mt-1 ) ? descA->m - m*descA->mb : descA->mb;
+    int tempnn = ( n == descA->nt-1 ) ? descA->n - n*descA->nb : descA->nb;
+    int ldam = BLKLDD( descA, m );
+
+    int ii, jj;
+
+    fflush(stdout);
+    for(ii=0; ii<tempmm; ii++) {
+        if ( ii == 0 )
+            fprintf(stdout, "(%2d, %2d) :", m, n);
+        else
+            fprintf(stdout, "          ");
+        for(jj=0; jj<tempnn; jj++) {
+#if defined(PRECISION_z) || defined(PRECISION_c)
+            fprintf(stdout, " (% e, % e)",
+                    creal( M[jj*ldam + ii] ),
+                    cimag( M[jj*ldam + ii] ));
+#else
+            fprintf(stdout, " % e", M[jj*ldam + ii]);
+#endif
+        }
+        fprintf(stdout, "\n");
+    }
+    fflush(stdout);
+    usleep(1000);
+}
diff --git a/examples/potrf/testing_dpotrf.cc b/examples/potrf/testing_dpotrf.cc
index 36ea59b16..5cef0bd20 100644
--- a/examples/potrf/testing_dpotrf.cc
+++ b/examples/potrf/testing_dpotrf.cc
@@ -210,6 +210,38 @@ int main(int argc, char **argv)
   return ret;
 }
 
+static void
+dplasma_dprint_tile( int m, int n,
+                     const parsec_tiled_matrix_t* descA,
+                     const double *M )
+{
+    int tempmm = ( m == descA->mt-1 ) ? descA->m - m*descA->mb : descA->mb;
+    int tempnn = ( n == descA->nt-1 ) ? descA->n - n*descA->nb : descA->nb;
+    int ldam = BLKLDD( descA, m );
+
+    int ii, jj;
+
+    fflush(stdout);
+    for(ii=0; ii<tempmm; ii++) {
+        if ( ii == 0 )
+            fprintf(stdout, "(%2d, %2d) :", m, n);
+        else
+            fprintf(stdout, "          ");
+        for(jj=0; jj<tempnn; jj++) {
+#if defined(PRECISION_z) || defined(PRECISION_c)
+            fprintf(stdout, " (% e, % e)",
+                    creal( M[jj*ldam + ii] ),
+                    cimag( M[jj*ldam + ii] ));
+#else
+            fprintf(stdout, " % e", M[jj*ldam + ii]);
+#endif
+        }
+        fprintf(stdout, "\n");
+    }
+    fflush(stdout);
+    usleep(1000);
+}
+
 int check_dpotrf( double *A, double *A0, int N )
 {
     int ret;
diff --git a/examples/potrf/testing_dtrtri.cc b/examples/potrf/testing_dtrtri.cc
index 370e2a08c..4a24dec4a 100644
--- a/examples/potrf/testing_dtrtri.cc
+++ b/examples/potrf/testing_dtrtri.cc
@@ -231,6 +231,38 @@ int main(int argc, char **argv)
   return ret;
 }
 
+static void
+dplasma_dprint_tile( int m, int n,
+                     const parsec_tiled_matrix_t* descA,
+                     const double *M )
+{
+    int tempmm = ( m == descA->mt-1 ) ? descA->m - m*descA->mb : descA->mb;
+    int tempnn = ( n == descA->nt-1 ) ? descA->n - n*descA->nb : descA->nb;
+    int ldam = BLKLDD( descA, m );
+
+    int ii, jj;
+
+    fflush(stdout);
+    for(ii=0; ii<tempmm; ii++) {
+        if ( ii == 0 )
+            fprintf(stdout, "(%2d, %2d) :", m, n);
+        else
+            fprintf(stdout, "          ");
+        for(jj=0; jj<tempnn; jj++) {
+#if defined(PRECISION_z) || defined(PRECISION_c)
+            fprintf(stdout, " (% e, % e)",
+                    creal( M[jj*ldam + ii] ),
+                    cimag( M[jj*ldam + ii] ));
+#else
+            fprintf(stdout, " % e", M[jj*ldam + ii]);
+#endif
+        }
+        fprintf(stdout, "\n");
+    }
+    fflush(stdout);
+    usleep(1000);
+}
+
 int check_dtrtri( lapack::Diag diag, lapack::Uplo uplo, double *A, double *Ainv, int N )
 {
     int ret;

From 078ad464c13798ad5655696fdccb4a9eb71e7066 Mon Sep 17 00:00:00 2001
From: Thomas Herault <herault@icl.utk.edu>
Date: Wed, 4 Jan 2023 19:45:21 -0500
Subject: [PATCH 064/259] Port the PaRSEC backend over a candidate PaRSEC
 master

---
 ttg/ttg/parsec/ttg.h | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index dfac43ca9..17ff9e416 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -183,6 +183,18 @@ namespace ttg_parsec {
       return comm_rank;
     }
 
+    static void ttg_parsec_ce_up(parsec_comm_engine_t *comm_engine, void *user_data)
+    {
+      parsec_ce.tag_register(WorldImpl::parsec_ttg_tag(), &detail::static_unpack_msg, user_data, PARSEC_TTG_MAX_AM_SIZE);
+      parsec_ce.tag_register(WorldImpl::parsec_ttg_rma_tag(), &detail::get_remote_complete_cb, user_data, 128);
+    }
+
+    static void ttg_parsec_ce_down(parsec_comm_engine_t *comm_engine, void *user_data)
+    {
+      parsec_ce.tag_unregister(WorldImpl::parsec_ttg_tag());
+      parsec_ce.tag_unregister(WorldImpl::parsec_ttg_rma_tag());
+    }
+
    public:
 #if defined(PARSEC_PROF_TRACE) && defined(PARSEC_TTG_PROFILE_BACKEND)
     int parsec_ttg_profile_backend_set_arg_start, parsec_ttg_profile_backend_set_arg_end;
@@ -330,9 +342,9 @@ namespace ttg_parsec {
         ttg::detail::deregister_world(*this);
         destroy_tpool();
         if (own_ctx) {
-          unregister_parsec_tags(nullptr);
+          unregister_parsec_tags(&parsec_comm_engine_cb_idx);
         } else {
-          parsec_context_at_fini(unregister_parsec_tags, nullptr);
+          parsec_context_at_fini(unregister_parsec_tags, &parsec_comm_engine_cb_idx);
         }
 #if defined(PARSEC_PROF_TRACE)
         if(nullptr != profiling_array) {
@@ -479,7 +491,7 @@ namespace ttg_parsec {
 #endif
   };
 
-  inline void unregister_parsec_tags(void *_)
+  static void unregister_parsec_tags(void *_pidx)
   {
     if(NULL != parsec_ce.tag_unregister) {
       parsec_ce.tag_unregister(WorldImpl::parsec_ttg_tag());

From a2caac6d82dc7a8982482324b1e2689a6043c865 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 27 Jan 2023 18:53:26 -0500
Subject: [PATCH 065/259] Make device coro tasks actually compile

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 tests/unit/device_coro.cc      | 17 +++++++++--------
 ttg/ttg/make_tt.h              | 12 ++++++++++--
 ttg/ttg/parsec/ttg.h           | 24 +++++++++++++-----------
 ttg/ttg/parsec/ttg_data_copy.h |  6 +++---
 ttg/ttg/view.h                 | 19 +++++++++++++------
 5 files changed, 48 insertions(+), 30 deletions(-)

diff --git a/tests/unit/device_coro.cc b/tests/unit/device_coro.cc
index 693b7a3bf..eaa33cb1a 100644
--- a/tests/unit/device_coro.cc
+++ b/tests/unit/device_coro.cc
@@ -1,21 +1,21 @@
 #include <catch2/catch.hpp>
 
 #include "ttg.h"
-#include "ttg/ttg/view.h"
+#include "ttg/view.h"
 
 
 TEST_CASE("Device", "coro") {
   SECTION("device_task") {
     ttg::Edge<int, double> edge;
-    auto fn = [](const int& key, double&& val) -> ttg::device_task {
+    auto fn = [&](const int& key, double&& val) -> ttg::device_task {
       ttg::View<double, double> view = ttg::make_view(val);
       /* wait for the view to be available on the device */
       co_yield view;
       // co_yield std::tie(view1, view2);
       // TTG_WAIT_VIEW(view);
       /* once we're back here the data has been transferred */
-      CHECK(view.get_device_ptr() != nullptr);
-      CHECK(view.size() == sizeof(val));
+      CHECK(view.get_device_ptr<0>()  != nullptr);
+      CHECK(view.get_device_size<0>() == sizeof(val));
       CHECK(&view.get_host_object() == &val);
 
       /* here we suspend to wait for a kernel to complete */
@@ -31,9 +31,10 @@ TEST_CASE("Device", "coro") {
                            "device_task", {"edge_in"}, {"edge_out"});
     make_graph_executable(tt);
     if (ttg::default_execution_context().rank() == 0) tt->invoke(0, 0.0);
-    ttg::ttg_fence();
+    ttg::ttg_fence(ttg::default_execution_context());
   }
 
+#if 0
   struct A {
     int a[10];
     double b[10];
@@ -69,7 +70,7 @@ TEST_CASE("Device", "coro") {
                            "device_task", {"edge_in"}, {"edge_out"});
     make_graph_executable(tt);
     if (ttg::default_execution_context().rank() == 0) tt->invoke(0, 0.0);
-    ttg::ttg_fence();
+    ttg::ttg_fence(ttg::default_execution_context());
   }
 
 
@@ -110,7 +111,7 @@ TEST_CASE("Device", "coro") {
                            "device_task", {"edge_in"}, {"edge_out"});
     make_graph_executable(tt);
     if (ttg::default_execution_context().rank() == 0) tt->invoke(0, 0.0);
-    ttg::ttg_fence();
+    ttg::ttg_fence(ttg::default_execution_context());
   }
-
+#endif // 0
 }
diff --git a/ttg/ttg/make_tt.h b/ttg/ttg/make_tt.h
index 50d307a98..e731ee960 100644
--- a/ttg/ttg/make_tt.h
+++ b/ttg/ttg/make_tt.h
@@ -145,7 +145,11 @@ class CallableWrapTTArgs
 
   using op_return_type =
 #ifdef TTG_HAS_COROUTINE
-      std::conditional_t<std::is_same_v<returnT, ttg::resumable_task>, ttg::coroutine_handle<>, void>;
+      std::conditional_t<std::is_same_v<returnT, ttg::resumable_task>,
+                         ttg::coroutine_handle<>,
+                         std::conditional_t<std::is_same_v<returnT, ttg::device_task>,
+                                            ttg::device_task::base_type,
+                                            void>>;
 #else   // TTG_HAS_COROUTINE
       void;
 #endif  // TTG_HAS_COROUTINE
@@ -169,7 +173,11 @@ class CallableWrapTTArgs
             coro_handle = ret;
           }
           return coro_handle;
-        } else
+        } else if constexpr (std::is_same_v<returnT, ttg::device_task>) {
+          ttg::device_task::base_type coro_handle = ret;
+          return coro_handle;
+        }
+        if constexpr (!(std::is_same_v<returnT, ttg::resumable_task> || std::is_same_v<returnT, ttg::device_task>))
 #endif
         {
           static_assert(std::tuple_size_v<std::remove_reference_t<decltype(out)>> == 1,
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 17ff9e416..7561b0946 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -64,9 +64,10 @@
 #include <parsec/parsec_comm_engine.h>
 #include <parsec/parsec_internal.h>
 #include <parsec/scheduling.h>
+#include <parsec/remote_dep.h>
 /* TODO: once we use parsec master we need to switch this include */
 #include <parsec/mca/device/cuda/device_cuda.h>
-//#include <parsec/mca/device/device_gpu.h>
+#include <parsec/mca/device/device_gpu.h>
 #if defined(PARSEC_PROF_TRACE)
 #include <parsec/profiling.h>
 #undef PARSEC_TTG_PROFILE_BACKEND
@@ -1339,7 +1340,7 @@ namespace ttg_parsec {
       }
 
       /* Here we call back into the coroutine again after the transfers have completed */
-      static_op(parsec_task);
+      static_op<Space>(parsec_task);
 
       /* for now make sure we're waiting for the kernel to complete and the coro hasn't skipped this step */
       assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_KERNEL);
@@ -1367,7 +1368,7 @@ namespace ttg_parsec {
       assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_KERNEL);
 
       /* the kernel has completed, resume the coroutine once again to get to the send stage */
-      static_op(parsec_task);
+      static_op<Space>(parsec_task);
 
       /* the coroutine should have completed and we cannot access the promise anymore */
       return PARSEC_HOOK_RETURN_DONE;
@@ -1377,6 +1378,7 @@ namespace ttg_parsec {
     static parsec_hook_return_t device_static_op(parsec_task_t* parsec_task) {
       static_assert(derived_has_cuda_op);
       task_t *task = static_cast<task_t*>(parsec_task);
+      parsec_execution_stream_s *es = task->world().get_impl().execution_stream();
 
       int dev_index;
       double ratio = 1.0;
@@ -1404,13 +1406,13 @@ namespace ttg_parsec {
       gpu_task->pushout = 0;
 
       // first invocation of the coroutine to get the coroutine handle
-      static_op(parsec_task);
+      static_op<Space>(parsec_task);
 
       // get the device task from the coroutine handle
       ttg::device_task* dev_task = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<ttg::device_task_promise_type>::from_address(task->suspended_task_address);
 
       // get the promise which contains the views
-      auto dev_data = dev_task->promise();
+      ttg::device_task_promise_type& dev_data = dev_task->promise();
 
       /* for now make sure we're waiting for transfers and the coro hasn't skipped this step */
       assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_TRANSFER);
@@ -1420,7 +1422,7 @@ namespace ttg_parsec {
       // set the input flows
       int i = 0;
       /* TODO: need to free flows at the end of the task lifetime */
-      parsec_flow_t* flows[] = new parsec_flow_t[MAX_PARAM_COUNT];
+      parsec_flow_t* flows = new parsec_flow_t[MAX_PARAM_COUNT];
       for (auto& view : dev_data) {
         void *host_obj = view.host_obj();
 
@@ -1430,19 +1432,19 @@ namespace ttg_parsec {
           gpu_task->flow[i] = &flows[i];
           parsec_flow_t flow = {.name = nullptr,
                                 .sym_type = PARSEC_SYM_INOUT,
-                                .flow_flags = view->is_out() ? PARSEC_FLOW_ACCESS_RW : PARSEC_FLOW_ACCESS_READ,
+                                .flow_flags = view_span.is_sync_out() ? PARSEC_FLOW_ACCESS_RW : PARSEC_FLOW_ACCESS_READ,
                                 .flow_index = i,
                                 .flow_datatype_mask = ~0 };
           *gpu_task->flow[i] = flow;
 
           parsec_data_copy_t* copy = nullptr;
-          ttg_data_copy_t* obj_copy = nullptr;
+          ttg_parsec::detail::ttg_data_copy_t* obj_copy = nullptr;
           int input_obj_idx = -1;
           /* try to find the view in the task and allocate a new copy if needed */
           for (int i = 0; nullptr == copy && i < numins; ++i) {
-            ttg_data_copy_t* obj_copy = task->copies[i];
+            ttg_parsec::detail::ttg_data_copy_t* obj_copy = task->copies[i];
             if (obj_copy->device_private == host_obj) {
-              for (auto dev_copy : obj_copy) {
+              for (auto& dev_copy : *obj_copy) {
                 if (view_span.data() == dev_copy->device_private) {
                   copy = dev_copy;
                   input_obj_idx = i;
@@ -1503,7 +1505,7 @@ namespace ttg_parsec {
         default:
           break;
       }
-      ttg::print_error(get_name(), " : received mismatching device type ", device->type, " from PaRSEC");
+      ttg::print_error(task->get_name(), " : received mismatching device type ", device->type, " from PaRSEC");
       ttg::abort();
     }
 
diff --git a/ttg/ttg/parsec/ttg_data_copy.h b/ttg/ttg/parsec/ttg_data_copy.h
index bed82db10..0d5b88f49 100644
--- a/ttg/ttg/parsec/ttg_data_copy.h
+++ b/ttg/ttg/parsec/ttg_data_copy.h
@@ -91,7 +91,7 @@ namespace ttg_parsec {
        * we only touch the vector if we have more than one copies to track
        * and otherwise use the single-element member.
        */
-      using device_iterator = parsec_data_copy_t**;
+      using iterator = parsec_data_copy_t**;
 
       void add_device_copy(parsec_data_copy_t* copy) {
         switch (m_num_dev_copies) {
@@ -114,7 +114,7 @@ namespace ttg_parsec {
         return m_num_dev_copies;
       }
 
-      device_iterator begin() {
+      iterator begin() {
         switch(m_num_dev_copies) {
           // no device copies
           case 0: return end();
@@ -123,7 +123,7 @@ namespace ttg_parsec {
         }
       }
 
-      device_iterator end() {
+      iterator end() {
         switch(m_num_dev_copies) {
           case 0:
           case 1:
diff --git a/ttg/ttg/view.h b/ttg/ttg/view.h
index a8a71ff9c..ad209d601 100644
--- a/ttg/ttg/view.h
+++ b/ttg/ttg/view.h
@@ -122,12 +122,12 @@ namespace ttg {
 
     template<std::size_t i>
     auto get_device_ptr() {
-      return static_cast<std::tuple_element<i, span_tuple_type>>(std::get<i>(m_spans).data());
+      return static_cast<std::tuple_element_t<i, span_tuple_type>::value_type*>(std::get<i>(m_spans).data());
     }
 
     template<std::size_t i>
     const auto get_device_ptr() const {
-      return static_cast<std::tuple_element<i, span_tuple_type>>(std::get<i>(m_spans).data());
+      return static_cast<std::tuple_element_t<i, span_tuple_type>::value_type>(std::get<i>(m_spans).data());
     }
 
     template<std::size_t i>
@@ -177,7 +177,7 @@ namespace ttg {
   /* overload for trivially-copyable host objects */
   template<typename HostT, typename = std::enable_if_t<std::is_trivially_copyable_v<HostT>>>
   auto make_view(HostT& obj) {
-    return View(obj, std::make_tuple(ViewSpan{&obj, sizeof(HostT)}));
+    return View<HostT, HostT>(obj, std::make_tuple(ViewSpan<HostT>(&obj, sizeof(HostT))));
   }
 
   /* yielded when waiting on a kernel to complete */
@@ -235,7 +235,7 @@ namespace ttg {
 
     device_task(base_type base) : base_type(std::move(base)) {}
 
-    base_type handle() { return *this; }
+    base_type& handle() { return *this; }
 
     /// @return true if ready to resume
     inline bool ready() {
@@ -281,7 +281,9 @@ namespace ttg {
       m_spans.reserve(view_count);
       if constexpr(view_count > 0) {
         auto unpack_lambda = [&](Views&... view){
-                                ((m_spans.push_back(device_obj_view(view.host_obj(), view.view_spans()))),...);
+                                ((m_spans.push_back(device_obj_view(&view.get_host_object(),
+                                                                    view.view_spans()))),
+                                  ...);
                               };
         std::apply(unpack_lambda, views);
       }
@@ -292,7 +294,8 @@ namespace ttg {
     /* convenience-function to yield a single view */
     template<typename HostT, typename... DeviceViewTs>
     TTG_CXX_COROUTINE_NAMESPACE::suspend_always yield_value(View<HostT, DeviceViewTs...> &view) {
-      return yield_value(std::tie(view));
+      auto tmp_tuple = std::tie(view);
+      return yield_value(tmp_tuple);
     }
 
     /* waiting for the kernel to complete should always suspend */
@@ -312,6 +315,10 @@ namespace ttg {
     using handle_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<device_task_promise_type>;
     device_task get_return_object() { return device_task{handle_type::from_promise(*this)}; }
 
+    void unhandled_exception() {
+
+    }
+
     using iterator = std::vector<device_obj_view>::iterator;
 
     auto begin() {

From 8cc11a6dbbab7f9d430d91a0b53c2ed2316ae9b6 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 1 Feb 2023 18:47:52 -0500
Subject: [PATCH 066/259] Many fixes to coro-based device tasks

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 tests/unit/device_coro.cc      |   9 +-
 ttg/ttg/make_tt.h              |  63 ++++++---
 ttg/ttg/parsec/fwd.h           |   1 +
 ttg/ttg/parsec/ttg.h           | 252 +++++++++++++++++++++++----------
 ttg/ttg/parsec/ttg_data_copy.h |   2 +
 ttg/ttg/view.h                 |  22 +--
 6 files changed, 243 insertions(+), 106 deletions(-)

diff --git a/tests/unit/device_coro.cc b/tests/unit/device_coro.cc
index eaa33cb1a..1e35c4586 100644
--- a/tests/unit/device_coro.cc
+++ b/tests/unit/device_coro.cc
@@ -8,7 +8,8 @@ TEST_CASE("Device", "coro") {
   SECTION("device_task") {
     ttg::Edge<int, double> edge;
     auto fn = [&](const int& key, double&& val) -> ttg::device_task {
-      ttg::View<double, double> view = ttg::make_view(val);
+      ttg::View<double, double> view = ttg::make_view(val, ttg::ViewScope::SyncInOut);
+      ttg::print("device_task key ", key, ", value ", val);
       /* wait for the view to be available on the device */
       co_yield view;
       // co_yield std::tie(view1, view2);
@@ -18,6 +19,8 @@ TEST_CASE("Device", "coro") {
       CHECK(view.get_device_size<0>() == sizeof(val));
       CHECK(&view.get_host_object() == &val);
 
+      ttg::print("device_task key ", key, ", device pointer ", view.get_device_ptr<0>());
+
       /* here we suspend to wait for a kernel to complete */
       co_yield ttg::device_op_wait_kernel{};
       // TTG_WAIT_KERNEL();
@@ -27,8 +30,8 @@ TEST_CASE("Device", "coro") {
         ttg::send<0>(key+1, val);
       }
     };
-    auto tt = ttg::make_tt(fn, ttg::edges(edge), ttg::edges(edge),
-                           "device_task", {"edge_in"}, {"edge_out"});
+    auto tt = ttg::make_tt<ttg::ExecutionSpace::CUDA>(fn, ttg::edges(edge), ttg::edges(edge),
+                                                      "device_task", {"edge_in"}, {"edge_out"});
     make_graph_executable(tt);
     if (ttg::default_execution_context().rank() == 0) tt->invoke(0, 0.0);
     ttg::ttg_fence(ttg::default_execution_context());
diff --git a/ttg/ttg/make_tt.h b/ttg/ttg/make_tt.h
index e731ee960..1c01a2554 100644
--- a/ttg/ttg/make_tt.h
+++ b/ttg/ttg/make_tt.h
@@ -8,7 +8,8 @@
 // case 1 (keyT != void): void op(auto&& key, std::tuple<input_valuesT...>&&, std::tuple<output_terminalsT...>&)
 // case 2 (keyT == void): void op(std::tuple<input_valuesT...>&&, std::tuple<output_terminalsT...>&)
 //
-template <typename funcT, bool funcT_receives_outterm_tuple, typename keyT, typename output_terminalsT,
+template <typename funcT, bool funcT_receives_outterm_tuple,
+          typename keyT, typename output_terminalsT,
           typename... input_valuesT>
 class CallableWrapTT
     : public TT<keyT, output_terminalsT,
@@ -100,15 +101,19 @@ class CallableWrapTT
   }
 };
 
-template <typename funcT, bool funcT_receives_outterm_tuple, typename keyT, typename output_terminalsT,
+template <typename funcT, bool funcT_receives_outterm_tuple,
+          typename keyT, typename output_terminalsT,
           typename input_values_tupleT>
 struct CallableWrapTTUnwrapTypelist;
 
-template <typename funcT, bool funcT_receives_outterm_tuple, typename keyT, typename output_terminalsT,
+template <typename funcT, bool funcT_receives_outterm_tuple,
+          typename keyT, typename output_terminalsT,
           typename... input_valuesT>
-struct CallableWrapTTUnwrapTypelist<funcT, funcT_receives_outterm_tuple, keyT, output_terminalsT,
+struct CallableWrapTTUnwrapTypelist<funcT, funcT_receives_outterm_tuple,
+                                    keyT, output_terminalsT,
                                     std::tuple<input_valuesT...>> {
-  using type = CallableWrapTT<funcT, funcT_receives_outterm_tuple, keyT, output_terminalsT,
+  using type = CallableWrapTT<funcT, funcT_receives_outterm_tuple,
+                              keyT, output_terminalsT,
                               std::remove_reference_t<input_valuesT>...>;
 };
 
@@ -126,12 +131,12 @@ struct CallableWrapTTUnwrapTypelist<funcT, funcT_receives_outterm_tuple, keyT, o
 // case 2 (keyT == void): returnT op(input_valuesT&&..., std::tuple<output_terminalsT...>&)
 //
 // returnT is void for funcT = synchronous (ordinary) function and the appropriate return type for funcT=coroutine
-template <typename funcT, typename returnT, bool funcT_receives_outterm_tuple, typename keyT,
-          typename output_terminalsT, typename... input_valuesT>
+template <typename funcT, typename returnT, bool funcT_receives_outterm_tuple, ttg::ExecutionSpace space,
+          typename keyT, typename output_terminalsT, typename... input_valuesT>
 class CallableWrapTTArgs
     : public TT<
           keyT, output_terminalsT,
-          CallableWrapTTArgs<funcT, returnT, funcT_receives_outterm_tuple, keyT, output_terminalsT, input_valuesT...>,
+          CallableWrapTTArgs<funcT, returnT, funcT_receives_outterm_tuple, space, keyT, output_terminalsT, input_valuesT...>,
           ttg::typelist<input_valuesT...>> {
   using baseT = typename CallableWrapTTArgs::ttT;
 
@@ -154,6 +159,11 @@ class CallableWrapTTArgs
       void;
 #endif  // TTG_HAS_COROUTINE
 
+public:
+  static constexpr bool have_cuda_op = (space == ttg::ExecutionSpace::CUDA);
+
+protected:
+
   /// @return coroutine handle<> (if funcT is a coroutine), else void
   template <typename Key, typename Tuple, std::size_t... S>
   auto call_func(Key &&key, Tuple &&args_tuple, output_terminalsT &out, std::index_sequence<S...>) {
@@ -318,23 +328,23 @@ class CallableWrapTTArgs
   };
 };
 
-template <typename funcT, typename returnT, bool funcT_receives_outterm_tuple, typename keyT,
-          typename output_terminalsT, typename input_values_typelistT>
+template <typename funcT, typename returnT, bool funcT_receives_outterm_tuple, ttg::ExecutionSpace space,
+          typename keyT, typename output_terminalsT, typename input_values_typelistT>
 struct CallableWrapTTArgsAsTypelist;
 
-template <typename funcT, typename returnT, bool funcT_receives_outterm_tuple, typename keyT,
-          typename output_terminalsT, typename... input_valuesT>
-struct CallableWrapTTArgsAsTypelist<funcT, returnT, funcT_receives_outterm_tuple, keyT, output_terminalsT,
+template <typename funcT, typename returnT, bool funcT_receives_outterm_tuple, ttg::ExecutionSpace space,
+          typename keyT, typename output_terminalsT, typename... input_valuesT>
+struct CallableWrapTTArgsAsTypelist<funcT, returnT, funcT_receives_outterm_tuple, space, keyT, output_terminalsT,
                                     std::tuple<input_valuesT...>> {
-  using type = CallableWrapTTArgs<funcT, returnT, funcT_receives_outterm_tuple, keyT, output_terminalsT,
+  using type = CallableWrapTTArgs<funcT, returnT, funcT_receives_outterm_tuple, space, keyT, output_terminalsT,
                                   std::remove_reference_t<input_valuesT>...>;
 };
 
-template <typename funcT, typename returnT, bool funcT_receives_outterm_tuple, typename keyT,
-          typename output_terminalsT, typename... input_valuesT>
-struct CallableWrapTTArgsAsTypelist<funcT, returnT, funcT_receives_outterm_tuple, keyT, output_terminalsT,
+template <typename funcT, typename returnT, bool funcT_receives_outterm_tuple, ttg::ExecutionSpace space,
+          typename keyT, typename output_terminalsT, typename... input_valuesT>
+struct CallableWrapTTArgsAsTypelist<funcT, returnT, funcT_receives_outterm_tuple, space, keyT, output_terminalsT,
                                     ttg::meta::typelist<input_valuesT...>> {
-  using type = CallableWrapTTArgs<funcT, returnT, funcT_receives_outterm_tuple, keyT, output_terminalsT,
+  using type = CallableWrapTTArgs<funcT, returnT, funcT_receives_outterm_tuple, space, keyT, output_terminalsT,
                                   std::remove_reference_t<input_valuesT>...>;
 };
 
@@ -494,7 +504,9 @@ auto make_tt_tpl(funcT &&func, const std::tuple<ttg::Edge<keyT, input_edge_value
 /// @warning Although generic arguments annotated by `const auto&` are also permitted, their use is discouraged to avoid confusion;
 ///          namely, `const auto&` denotes a _consumable_ argument, NOT read-only, despite the `const`.
 // clang-format on
-template <typename keyT = void, typename funcT, typename... input_edge_valuesT, typename... output_edgesT>
+template <ttg::ExecutionSpace space,
+          typename keyT = void, typename funcT,
+          typename... input_edge_valuesT, typename... output_edgesT>
 auto make_tt(funcT &&func, const std::tuple<ttg::Edge<keyT, input_edge_valuesT>...> &inedges = std::tuple<>{},
              const std::tuple<output_edgesT...> &outedges = std::tuple<>{}, const std::string &name = "wrapper",
              const std::vector<std::string> &innames = std::vector<std::string>(sizeof...(input_edge_valuesT), "input"),
@@ -569,12 +581,21 @@ auto make_tt(funcT &&func, const std::tuple<ttg::Edge<keyT, input_edge_valuesT>.
   using decayed_input_args_t = ttg::meta::decayed_typelist_t<input_args_t>;
   // 3. full_input_args_t = edge-types with non-void types replaced by input_args_t
   using full_input_args_t = ttg::meta::replace_nonvoid_t<input_edge_value_types, input_args_t>;
-  using wrapT = typename CallableWrapTTArgsAsTypelist<funcT, func_return_t, have_outterm_tuple, keyT,
+  using wrapT = typename CallableWrapTTArgsAsTypelist<funcT, func_return_t, have_outterm_tuple, space, keyT,
                                                       output_terminals_type, full_input_args_t>::type;
 
   return std::make_unique<wrapT>(std::forward<funcT>(func), inedges, outedges, name, innames, outnames);
 }
 
+template <typename keyT = void, typename funcT,
+          typename... input_edge_valuesT, typename... output_edgesT>
+auto make_tt(funcT &&func, const std::tuple<ttg::Edge<keyT, input_edge_valuesT>...> &inedges = std::tuple<>{},
+             const std::tuple<output_edgesT...> &outedges = std::tuple<>{}, const std::string &name = "wrapper",
+             const std::vector<std::string> &innames = std::vector<std::string>(sizeof...(input_edge_valuesT), "input"),
+             const std::vector<std::string> &outnames = std::vector<std::string>(sizeof...(output_edgesT), "output")) {
+  return make_tt<ttg::ExecutionSpace::Host>(std::forward<funcT>(func), inedges, outedges, name, innames, outnames);
+}
+
 template <typename keyT, typename funcT, typename... input_valuesT, typename... output_edgesT>
 [[deprecated("use make_tt_tpl instead")]] inline auto wrapt(
     funcT &&func, const std::tuple<ttg::Edge<keyT, input_valuesT>...> &inedges,
@@ -593,6 +614,6 @@ template <typename keyT, typename funcT, typename... input_edge_valuesT, typenam
   return make_tt<keyT>(std::forward<funcT>(func), inedges, outedges, name, innames, outnames);
 }
 
-#include "ttg/make_device_tt.h"
+//#include "ttg/make_device_tt.h"
 
 #endif  // TTG_MAKE_TT_H
diff --git a/ttg/ttg/parsec/fwd.h b/ttg/ttg/parsec/fwd.h
index 4511a6231..cc38f68d8 100644
--- a/ttg/ttg/parsec/fwd.h
+++ b/ttg/ttg/parsec/fwd.h
@@ -28,6 +28,7 @@ namespace ttg_parsec {
 
   inline void ttg_finalize();
 
+  [[noreturn]]
   static inline void ttg_abort();
 
   inline ttg::World ttg_default_execution_context();
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 7561b0946..5f0474bde 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -103,6 +103,42 @@ int parsec_add_fetch_runtime_task(parsec_taskpool_t *tp, int tasks);
 
 #include "ttg/view.h"
 
+namespace test::meta {
+
+    template <class...>
+    using void_t = void;
+  namespace detail {
+      template <class Default, class Enabler, template <class...> class TT, class... Args>
+      struct detector {
+        using value_t = std::false_type;
+        using type = Default;
+      };
+
+      template <class Default, template <class...> class TT, class... Args>
+      struct detector<Default, void_t<TT<Args...>>, TT, Args...> {
+        using value_t = std::true_type;
+        using type = TT<Args...>;
+      };
+
+  }
+
+      template <template <class...> class TT, class... Args>
+      using is_detected = typename detail::detector<ttg::meta::nonesuch, void, TT, Args...>::value_t;
+
+    template <template <class...> class TT, class... Args>
+    using is_detected = typename detail::detector<ttg::meta::nonesuch, void, TT, Args...>::value_t;
+
+    template <template <class...> class TT, class... Args>
+    using detected_t = typename detail::detector<ttg::meta::nonesuch, void, TT, Args...>::type;
+
+    template <class Default, template <class...> class TT, class... Args>
+    using detected_or = detail::detector<Default, void, TT, Args...>;
+
+    template <template <class...> class TT, class... Args>
+    constexpr bool is_detected_v = is_detected<TT, Args...>::value;
+
+}
+
 namespace ttg_parsec {
   typedef void (*static_set_arg_fct_type)(void *, size_t, ttg::TTBase *);
   typedef std::pair<static_set_arg_fct_type, ttg::TTBase *> static_set_arg_fct_call_t;
@@ -258,6 +294,15 @@ namespace ttg_parsec {
       tpool->taskpool_name = strdup("TTG Taskpool");
       parsec_taskpool_reserve_id(tpool);
 
+      tpool->devices_index_mask = 0;
+      // TODO DEBUG: reenable CPU incarnations
+      for(int i = 2; i < (int)parsec_nb_devices; i++) {
+          parsec_device_module_t *device = parsec_mca_device_get(i);
+          if( NULL == device ) continue;
+          tpool->devices_index_mask |= (1 << device->device_index);
+      }
+
+
 #ifdef TTG_USE_USER_TERMDET
       parsec_termdet_open_module(tpool, "user_trigger");
 #else   // TTG_USE_USER_TERMDET
@@ -623,7 +668,7 @@ namespace ttg_parsec {
       void* suspended_task_address = nullptr;  // if not null the function is suspended
 #endif
       ttg_data_copy_t *copies[num_streams+1] = { nullptr };  // the data copies tracked by this task
-                                                             // +1 for the copy needed during send/bcas
+                                                             // +1 for the copy needed during send/bcast
 
       parsec_ttg_task_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class)
           : parsec_ttg_task_base_t(mempool, task_class, num_streams, copies) {
@@ -793,8 +838,8 @@ namespace ttg_parsec {
 
     inline parsec_hook_return_t hook_cuda(struct parsec_execution_stream_s *es, parsec_task_t *parsec_task) {
       parsec_ttg_task_base_t *me = (parsec_ttg_task_base_t *)parsec_task;
-      me->function_template_class_ptr[static_cast<std::size_t>(ttg::ExecutionSpace::CUDA)](parsec_task);
-      return PARSEC_HOOK_RETURN_DONE;
+      auto ret = me->function_template_class_ptr[static_cast<std::size_t>(ttg::ExecutionSpace::CUDA)](parsec_task);
+      return ret;
     }
 
     static parsec_key_fn_t parsec_tasks_hash_fcts = {.key_equal = parsec_hash_table_generic_64bits_key_equal,
@@ -1027,7 +1072,8 @@ namespace ttg_parsec {
     if (detail::initialized_mpi()) MPI_Finalize();
   }
   inline ttg::World ttg_default_execution_context() { return ttg::get_default_world(); }
-  inline void ttg_abort() { MPI_Abort(ttg_default_execution_context().impl().comm(), 1); }
+  [[noreturn]]
+  inline void ttg_abort() { MPI_Abort(ttg_default_execution_context().impl().comm(), 1); std::abort(); }
   inline void ttg_execute(ttg::World world) { world.impl().execute(); }
   inline void ttg_fence(ttg::World world) { world.impl().fence(); }
 
@@ -1121,7 +1167,7 @@ namespace ttg_parsec {
 
     // check for a non-type member named have_cuda_op
     template <typename T>
-    using have_cuda_op_non_type_t = decltype(&T::have_cuda_op);
+    using have_cuda_op_non_type_t = decltype(T::have_cuda_op);
 
     bool alive = true;
 
@@ -1132,7 +1178,8 @@ namespace ttg_parsec {
 
     /// @return true if derivedT::have_cuda_op exists and is defined to true
     static constexpr bool derived_has_cuda_op() {
-      if constexpr (ttg::meta::is_detected_v<have_cuda_op_non_type_t, derivedT>) {
+      //if constexpr (test::meta::is_detected_v<have_cuda_op_non_type_t, derivedT>) {
+      if constexpr (derivedT::have_cuda_op) {
         return derivedT::have_cuda_op;
       } else {
         return false;
@@ -1244,7 +1291,8 @@ namespace ttg_parsec {
     template <ttg::ExecutionSpace Space, typename... Args>
     auto op(Args &&...args) {
       derivedT *derived = static_cast<derivedT *>(this);
-      if constexpr (Space == ttg::ExecutionSpace::Host) {
+      // TODO: do we still distinguish op and op_cuda? How do we handle support for multiple devices?
+      //if constexpr (Space == ttg::ExecutionSpace::Host) {
         using return_type = decltype(derived->op(std::forward<Args>(args)...));
         if constexpr (std::is_same_v<return_type,void>) {
           derived->op(std::forward<Args>(args)...);
@@ -1253,6 +1301,7 @@ namespace ttg_parsec {
         else {
           return derived->op(std::forward<Args>(args)...);
         }
+#if 0
       }
       else if constexpr (Space == ttg::ExecutionSpace::CUDA) {
         using return_type = decltype(derived->op_cuda(std::forward<Args>(args)...));
@@ -1266,6 +1315,7 @@ namespace ttg_parsec {
       }
       else
         ttg::abort();
+#endif // 0
     }
 
     template <std::size_t i, typename terminalT, typename Key>
@@ -1317,14 +1367,18 @@ namespace ttg_parsec {
      * Submit callback called by PaRSEC once all input transfers have completed.
      */
     template <ttg::ExecutionSpace Space>
-    static parsec_hook_return_t device_static_submit(parsec_task_t* parsec_task) {
+    static int device_static_submit(parsec_device_gpu_module_t  *gpu_device,
+                                    parsec_gpu_task_t           *gpu_task,
+                                    parsec_gpu_exec_stream_t    *gpu_stream) {
 
-      task_t *task = static_cast<task_t*>(parsec_task);
+      task_t *task = (task_t*)gpu_task->ec;
       // get the device task from the coroutine handle
-      ttg::device_task* dev_task = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<ttg::device_task_promise_type>::from_address(task->suspended_task_address);
+      ttg::device_task dev_task = ttg::device_task_handle_type::from_address(task->suspended_task_address);
+
+      std::cout << "device_static_submit task " << task << std::endl;
 
       // get the promise which contains the views
-      auto dev_data = dev_task->promise();
+      auto dev_data = dev_task.promise();
 
       /* we should still be waiting for the transfer to complete */
       assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_TRANSFER);
@@ -1334,59 +1388,33 @@ namespace ttg_parsec {
       for (auto& view : dev_data) {
         /* iterate over all viewspans of this view (i.e., all memory ranges in this view) */
         for (auto& view_span : view) {
-          view_span.set_data(task->parsec_task.data[i].copy_out);
+          view_span.set_data(task->parsec_task.data[i].data_out);
           ++i;
         }
       }
 
       /* Here we call back into the coroutine again after the transfers have completed */
-      static_op<Space>(parsec_task);
+      static_op<Space>(&task->parsec_task);
 
+      dev_task = ttg::device_task_handle_type::from_address(task->suspended_task_address);
+      dev_data = dev_task.promise();
       /* for now make sure we're waiting for the kernel to complete and the coro hasn't skipped this step */
       assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_KERNEL);
 
-      /* next time we will come back into device_static_stage_complete */
-      return PARSEC_HOOK_RETURN_DONE;
-    }
-
-    /**
-     * Completion callback called by PaRSEC once the kernel submitted in the submit callback above
-     * have completed.
-     */
-    template <ttg::ExecutionSpace Space>
-    static parsec_hook_return_t device_static_stage_complete(parsec_task_t* parsec_task) {
-
-      task_t *task = static_cast<task_t*>(parsec_task);
-
-      // get the device task from the coroutine handle
-      ttg::device_task* dev_task = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<ttg::device_task_promise_type>::from_address(task->suspended_task_address);
-
-      // get the promise which contains the views
-      auto dev_data = dev_task->promise();
-
-      /* for now make sure we're waiting for the kernel to complete and the coro hasn't skipped this step */
-      assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_KERNEL);
-
-      /* the kernel has completed, resume the coroutine once again to get to the send stage */
-      static_op<Space>(parsec_task);
-
-      /* the coroutine should have completed and we cannot access the promise anymore */
+      /* next time we will come back into complete_task_and_release */
       return PARSEC_HOOK_RETURN_DONE;
     }
 
     template <ttg::ExecutionSpace Space>
     static parsec_hook_return_t device_static_op(parsec_task_t* parsec_task) {
-      static_assert(derived_has_cuda_op);
-      task_t *task = static_cast<task_t*>(parsec_task);
-      parsec_execution_stream_s *es = task->world().get_impl().execution_stream();
+      static_assert(derived_has_cuda_op());
+      task_t *task = (task_t*)parsec_task;
+      parsec_execution_stream_s *es = task->tt->world.impl().execution_stream();
+
+      std::cout << "device_static_op: task " << parsec_task << std::endl;
 
       int dev_index;
       double ratio = 1.0;
-      dev_index = parsec_get_best_device(parsec_task, ratio);
-      assert(dev_index >= 0);
-      if (dev_index < 2) {
-          return PARSEC_HOOK_RETURN_NEXT; /* Fall back */
-      }
 
       /* set up a device task */
       /* TODO: take them from a free-list */
@@ -1398,8 +1426,7 @@ namespace ttg_parsec {
       /* come back into this function
         * TODO: have a specialized function that checks
         */
-      gpu_task->submit = device_static_submit<Space>;
-      gpu_task->complete_stage = device_static_stage_complete<Space>;
+      gpu_task->submit = &TT::device_static_submit<Space>;
       gpu_task->task_type = 0; // user task
       gpu_task->load = 1.0;    // TODO: can we do better?
       gpu_task->last_data_check_epoch = -1; // used internally
@@ -1409,10 +1436,10 @@ namespace ttg_parsec {
       static_op<Space>(parsec_task);
 
       // get the device task from the coroutine handle
-      ttg::device_task* dev_task = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<ttg::device_task_promise_type>::from_address(task->suspended_task_address);
+      auto dev_task = ttg::device_task_handle_type::from_address(task->suspended_task_address);
 
       // get the promise which contains the views
-      ttg::device_task_promise_type& dev_data = dev_task->promise();
+      ttg::device_task_promise_type& dev_data = dev_task.promise();
 
       /* for now make sure we're waiting for transfers and the coro hasn't skipped this step */
       assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_TRANSFER);
@@ -1420,7 +1447,7 @@ namespace ttg_parsec {
       // manage the gpu_task flows
 
       // set the input flows
-      int i = 0;
+      uint8_t i = 0;
       /* TODO: need to free flows at the end of the task lifetime */
       parsec_flow_t* flows = new parsec_flow_t[MAX_PARAM_COUNT];
       for (auto& view : dev_data) {
@@ -1435,7 +1462,8 @@ namespace ttg_parsec {
                                 .flow_flags = view_span.is_sync_out() ? PARSEC_FLOW_ACCESS_RW : PARSEC_FLOW_ACCESS_READ,
                                 .flow_index = i,
                                 .flow_datatype_mask = ~0 };
-          *gpu_task->flow[i] = flow;
+          std::cout << "view_span.is_sync_out() " << view_span.is_sync_out() << std::endl;
+          *(parsec_flow_t*)gpu_task->flow[i] = flow; // why are flows constant?!
 
           parsec_data_copy_t* copy = nullptr;
           ttg_parsec::detail::ttg_data_copy_t* obj_copy = nullptr;
@@ -1464,9 +1492,13 @@ namespace ttg_parsec {
           /* no copy found, create a new copy */
           if (nullptr == copy) {
             parsec_data_t *data = parsec_data_new();
-            copy = parsec_data_copy_new(data, 0, parsec_datatype_int8_t, 0);
+            copy = parsec_data_copy_new(data, 0, parsec_datatype_int8_t, PARSEC_DATA_FLAG_PARSEC_MANAGED);
             copy->device_private = view_span.data();
-            parsec_data_copy_attach(data, copy, 0); // TODO: device 0 is the CPU?
+            copy->version = 1; // this version is valid
+            data->nb_elts = view_span.size();
+            data->owner_device = 0;
+            copy->coherency_state = PARSEC_DATA_COHERENCY_SHARED;
+            std::cout << "copy " << copy << " device_private " << copy->device_private << std::endl;
           }
 
           if (obj_copy != nullptr) {
@@ -1474,7 +1506,8 @@ namespace ttg_parsec {
             obj_copy->add_device_copy(copy);
           }
           /* register the copy with the task */
-          task->parsec_task.data[i].copy_in = copy;
+          task->parsec_task.data[i].data_in = copy;
+          task->parsec_task.data[i].source_repo_entry = NULL;
 
           ++i;
         }
@@ -1483,8 +1516,17 @@ namespace ttg_parsec {
       /* mark all other flows as ignored */
       for (; i < MAX_PARAM_COUNT; ++i) {
         gpu_task->flow_nb_elts[i] = 0;
+        flows[i].flow_flags = PARSEC_FLOW_ACCESS_NONE;
+        flows[i].flow_index = i;
         gpu_task->flow[i] = &flows[i];
-        gpu_task->flow[i]->flow_flags = PARSEC_FLOW_ACCESS_NONE;
+        task->parsec_task.data[i].data_in = nullptr;
+        task->parsec_task.data[i].source_repo_entry = NULL;
+      }
+
+      dev_index = parsec_get_best_device(parsec_task, ratio);
+      assert(dev_index >= 0);
+      if (dev_index < 2) {
+          return PARSEC_HOOK_RETURN_NEXT; /* Fall back */
       }
 
       parsec_device_module_t *device = parsec_mca_device_get(dev_index);
@@ -1505,25 +1547,47 @@ namespace ttg_parsec {
         default:
           break;
       }
-      ttg::print_error(task->get_name(), " : received mismatching device type ", device->type, " from PaRSEC");
+      ttg::print_error(task->tt->get_name(), " : received mismatching device type ", device->type, " from PaRSEC");
       ttg::abort();
+      return PARSEC_HOOK_RETURN_DONE; // will not be reacehed
     }
 
+    class A {
+    public:
+      static constexpr bool have_cuda_op = true;
+    };
+
     template <ttg::ExecutionSpace Space>
     static parsec_hook_return_t static_op(parsec_task_t *parsec_task) {
-      task_t *task = (task_t*)parsec_task;
 
+      //static_assert(ttg::meta::is_detected_v<have_cuda_op_non_type_t, A>);
+      //static_assert(ttg::meta::is_detected_v<have_cuda_op_non_type_t, derivedT>);
+      //static_assert(ttg::meta::is_detected<have_cuda_op_non_type_t, derivedT>::value);
+      // TODO: this does not compile
+      //static_assert(ttg::meta::detail::detector<ttg::meta::nonesuch, void, have_cuda_op_non_type_t, derivedT>::value_t::value);
+      // TODO: this does compile. WTF?!
+      //static_assert(test::meta::detail::detector<ttg::meta::nonesuch, void, have_cuda_op_non_type_t, derivedT>::value_t::value);
+      //static_assert(test::meta::is_detected<have_cuda_op_non_type_t, derivedT>::value);
+      //static_assert(derivedT::have_cuda_op);
+      //static_assert(detector<ttg::meta::nonesuch, void, have_cuda_op_non_type_t, derivedT>::value_t::value);
+      using t = ttg::meta::void_t<have_cuda_op_non_type_t<derivedT>>;
+
+      task_t *task = (task_t*)parsec_task;
+      have_cuda_op_non_type_t<derivedT> tmp = 0;
+      std::cout << "static_op derived_has_cuda_op " << derived_has_cuda_op() << " detected " << test::meta::is_detected_v<have_cuda_op_non_type_t, derivedT>
+                << " have_cuda_op " << derivedT::have_cuda_op << " detected " << ttg::meta::detail::detector<ttg::meta::nonesuch, void, have_cuda_op_non_type_t, derivedT>::value_t::value << std::endl;
       void* suspended_task_address =
 #ifdef TTG_HAS_COROUTINE
         task->suspended_task_address;  // non-null = need to resume the task
 #else
         nullptr;
 #endif
+      std::cout << "static_op: suspended_task_address " << suspended_task_address << std::endl;
       if (suspended_task_address == nullptr) {  // task is a coroutine that has not started or an ordinary function
 
         ttT *baseobj = task->tt;
         derivedT *obj = static_cast<derivedT *>(baseobj);
-        assert(parsec_ttg_caller == NULL);
+        assert(parsec_ttg_caller == nullptr);
         parsec_ttg_caller = static_cast<detail::parsec_ttg_task_base_t*>(task);
         if (obj->tracing()) {
           if constexpr (!ttg::meta::is_void_v<keyT>)
@@ -1545,9 +1609,23 @@ namespace ttg_parsec {
         } else {
           ttg::abort();
         }
-        parsec_ttg_caller = NULL;
+        parsec_ttg_caller = nullptr;
       }
       else {  // resume the suspended coroutine
+        auto coro = static_cast<ttg::device_task>(ttg::device_task_handle_type::from_address(suspended_task_address));
+        assert(parsec_ttg_caller == nullptr);
+        parsec_ttg_caller = static_cast<detail::parsec_ttg_task_base_t*>(task);
+        // TODO: unify the outputs tls handling
+        auto old_output_tls_ptr = task->tt->outputs_tls_ptr_accessor();
+        task->tt->set_outputs_tls_ptr();
+        coro.resume();
+        if (coro.completed()) {
+          coro.destroy();
+          suspended_task_address = nullptr;
+        }
+        task->tt->set_outputs_tls_ptr(old_output_tls_ptr);
+        parsec_ttg_caller = nullptr;
+#if 0
 #ifdef TTG_HAS_COROUTINE
         auto ret = static_cast<ttg::resumable_task>(ttg::coroutine_handle<>::from_address(suspended_task_address));
         assert(ret.ready());
@@ -1561,9 +1639,11 @@ namespace ttg_parsec {
         }
         task->suspended_task_address = suspended_task_address;
 #else
+#endif // 0
         ttg::abort();  // should not happen
 #endif
       }
+      task->suspended_task_address = suspended_task_address;
 
       if (suspended_task_address == nullptr) {
         ttT *baseobj = task->tt;
@@ -1744,7 +1824,7 @@ namespace ttg_parsec {
       // TODO: do we need to copy static_stream_goal in dummy?
 
       /* set the received value as the dummy's only data */
-      dummy->parsec_task.data[0].data_in = copy;
+      dummy->copies[0] = copy;
 
       /* We received the task on this world, so it's using the same taskpool */
       dummy->parsec_task.taskpool = world.impl().taskpool();
@@ -1903,7 +1983,7 @@ namespace ttg_parsec {
       } else if constexpr (ttg::meta::is_void_v<keyT> && std::is_void_v<valueT>) {
         set_arg<i, keyT, ttg::Void>(ttg::Void{});
       } else {  // unreachable
-        ttg:: abort();
+        ttg::abort();
       }
     }
 
@@ -2015,7 +2095,7 @@ namespace ttg_parsec {
           reinterpret_cast<detail::parsec_static_op_t>(&TT::static_op<ttg::ExecutionSpace::Host>);
       if constexpr (derived_has_cuda_op())
         newtask->function_template_class_ptr[static_cast<std::size_t>(ttg::ExecutionSpace::CUDA)] =
-            reinterpret_cast<detail::parsec_static_op_t>(&TT::static_op<ttg::ExecutionSpace::CUDA>);
+            reinterpret_cast<detail::parsec_static_op_t>(&TT::device_static_op<ttg::ExecutionSpace::CUDA>);
 
       for (int i = 0; i < static_stream_goal.size(); ++i) {
         newtask->stream[i].goal = static_stream_goal[i];
@@ -2115,12 +2195,12 @@ namespace ttg_parsec {
         if constexpr (!ttg::meta::is_void_v<valueT>) {  // for data values
           // have a value already? if not, set, otherwise reduce
           detail::ttg_data_copy_t *copy = nullptr;
-          if (nullptr == (copy = static_cast<detail::ttg_data_copy_t *>(task->parsec_task.data[i].data_in))) {
+          if (nullptr == (copy = task->copies[i])) {
             using decay_valueT = std::decay_t<valueT>;
             /* For now, we always create a copy because we cannot rely on the task_release
              * mechanism (it would release the task, not the reduction value). */
             copy = detail::create_new_datacopy(std::forward<Value>(value));
-            task->parsec_task.data[i].data_in = copy;
+            task->copies[i] = copy;
           } else {
             reducer(*reinterpret_cast<std::decay_t<valueT> *>(copy->device_private), value);
           }
@@ -2137,7 +2217,7 @@ namespace ttg_parsec {
       } else {
         /* whether the task needs to be deferred or not */
         if constexpr (!valueT_is_Void) {
-          if (nullptr != task->parsec_task.data[i].data_in) {
+          if (nullptr != task->copies[i]) {
             ttg::print_error(get_name(), " : ", key, ": error argument is already set : ", i);
             throw std::logic_error("bad set arg");
           }
@@ -2157,7 +2237,7 @@ namespace ttg_parsec {
            * we need to defer the release of this task to give other tasks a chance to
            * make a copy of the original data */
           release = (copy->push_task != &task->parsec_task);
-          task->parsec_task.data[i].data_in = copy;
+          task->copies[i] = copy;
         }
       }
       task->remove_from_hash = remove_from_hash;
@@ -3098,13 +3178,37 @@ namespace ttg_parsec {
 
     parsec_key_fn_t tasks_hash_fcts = {key_equal, key_print, key_hash};
 
-    static parsec_hook_return_t complete_task_and_release(parsec_execution_stream_t *es, parsec_task_t *t) {
-      auto *task = (detail::parsec_ttg_task_base_t *)t;
+    static parsec_hook_return_t complete_task_and_release(parsec_execution_stream_t *es, parsec_task_t *parsec_task) {
+
+      std::cout << "complete_task_and_release: task " << parsec_task << std::endl;
+
+      task_t *task = (task_t*)parsec_task;
+
+      /* if we still have a coroutine handle we invoke it one more time to get the sends/broadcasts */
+      if (task->suspended_task_address) {
+        // get the device task from the coroutine handle
+        auto dev_task = ttg::device_task_handle_type::from_address(task->suspended_task_address);
+
+        // get the promise which contains the views
+        auto dev_data = dev_task.promise();
+
+        /* for now make sure we're waiting for the kernel to complete and the coro hasn't skipped this step */
+        assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_KERNEL);
+
+        /* the kernel has completed, resume the coroutine once again to get to the send stage */
+        /* TODO: how can we get the execution space here? */
+        static_op<ttg::ExecutionSpace::CUDA>(parsec_task);
+
+        /* the coroutine should have completed and we cannot access the promise anymore */
+        task->suspended_task_address = nullptr;
+      }
+
+      /* release our data copies */
       for (int i = 0; i < task->data_count; i++) {
-        detail::ttg_data_copy_t *copy = static_cast<detail::ttg_data_copy_t *>(task->parsec_task.data[i].data_in);
+        detail::ttg_data_copy_t *copy = task->copies[i];
         if (nullptr == copy) continue;
         detail::release_data_copy(copy);
-        task->parsec_task.data[i].data_in = nullptr;
+        task->copies[i] = nullptr;
       }
       return PARSEC_HOOK_RETURN_DONE;
     }
@@ -3175,7 +3279,8 @@ namespace ttg_parsec {
 
       world_impl.taskpool()->nb_task_classes = std::max(world_impl.taskpool()->nb_task_classes, static_cast<decltype(world_impl.taskpool()->nb_task_classes)>(self.task_class_id+1));
       //    function_id_to_instance[self.task_class_id] = this;
-
+      //self.incarnations = incarnations_array.data();
+//#if 0
       if constexpr (derived_has_cuda_op()) {
         self.incarnations = (__parsec_chore_t *)malloc(3 * sizeof(__parsec_chore_t));
         ((__parsec_chore_t *)self.incarnations)[0].type = PARSEC_DEV_CUDA;
@@ -3196,6 +3301,7 @@ namespace ttg_parsec {
         ((__parsec_chore_t *)self.incarnations)[1].evaluate = NULL;
         ((__parsec_chore_t *)self.incarnations)[1].hook = NULL;
       }
+//#endif // 0
 
       self.release_task = &parsec_release_task_to_mempool_update_nbtasks;
       self.complete_execution = complete_task_and_release;
diff --git a/ttg/ttg/parsec/ttg_data_copy.h b/ttg/ttg/parsec/ttg_data_copy.h
index 0d5b88f49..88d5531f3 100644
--- a/ttg/ttg/parsec/ttg_data_copy.h
+++ b/ttg/ttg/parsec/ttg_data_copy.h
@@ -94,6 +94,8 @@ namespace ttg_parsec {
       using iterator = parsec_data_copy_t**;
 
       void add_device_copy(parsec_data_copy_t* copy) {
+        // TODO: properly release again!
+        PARSEC_OBJ_RETAIN(copy);
         switch (m_num_dev_copies) {
           case 0:
             m_single_dev_copy = copy;
diff --git a/ttg/ttg/view.h b/ttg/ttg/view.h
index ad209d601..11cb3dcde 100644
--- a/ttg/ttg/view.h
+++ b/ttg/ttg/view.h
@@ -10,7 +10,8 @@ namespace ttg {
   enum class ViewScope {
     Allocate  = 0x0,
     SyncIn    = 0x1,
-    SyncOut   = 0x2
+    SyncOut   = 0x2,
+    SyncInOut = 0x3
   };
 
   /**
@@ -102,7 +103,7 @@ namespace ttg {
     template<std::size_t... Is>
     View(HostT& obj, span_tuple_type& spans, std::index_sequence<Is...>)
     : m_obj(&obj)
-    , m_spans({std::get<Is>(m_spans)...})
+    , m_spans({std::get<Is>(spans)...})
     { }
 
   public:
@@ -176,8 +177,8 @@ namespace ttg {
 
   /* overload for trivially-copyable host objects */
   template<typename HostT, typename = std::enable_if_t<std::is_trivially_copyable_v<HostT>>>
-  auto make_view(HostT& obj) {
-    return View<HostT, HostT>(obj, std::make_tuple(ViewSpan<HostT>(&obj, sizeof(HostT))));
+  auto make_view(HostT& obj, ViewScope scope = ViewScope::SyncIn) {
+    return View<HostT, HostT>(obj, std::make_tuple(ViewSpan<HostT>(&obj, sizeof(HostT), scope)));
   }
 
   /* yielded when waiting on a kernel to complete */
@@ -222,9 +223,11 @@ namespace ttg {
 
   struct device_task_promise_type;
 
+  using device_task_handle_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<device_task_promise_type>;
+
   /// task that can be resumed after some events occur
-  struct device_task : public TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<device_task_promise_type> {
-    using base_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<device_task_promise_type>;
+  struct device_task : public device_task_handle_type {
+    using base_type = device_task_handle_type;
 
     /// these are members mandated by the promise_type concept
     ///@{
@@ -264,7 +267,7 @@ namespace ttg {
      * so we can access the promise.
      * TODO: necessary? maybe we can save one suspend here
      */
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_never final_suspend() noexcept {
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_always final_suspend() noexcept {
       m_state = TTG_DEVICE_CORO_COMPLETE;
       return {};
     }
@@ -277,6 +280,7 @@ namespace ttg {
     TTG_CXX_COROUTINE_NAMESPACE::suspend_always yield_value(std::tuple<Views&...> &views) {
       /* gather all the views (host object + view spans, type-punned) into a vector */
       constexpr static std::size_t view_count = std::tuple_size_v<std::tuple<Views...>>;
+      std::cout << "yield_value: views" << std::endl;
       m_spans.clear(); // in case we ever come back here
       m_spans.reserve(view_count);
       if constexpr(view_count > 0) {
@@ -300,6 +304,7 @@ namespace ttg {
 
     /* waiting for the kernel to complete should always suspend */
     TTG_CXX_COROUTINE_NAMESPACE::suspend_always yield_value(device_op_wait_kernel) {
+      std::cout << "yield_value: device_op_wait_kernel" << std::endl;
       m_state = TTG_DEVICE_CORO_WAIT_KERNEL;
       return {};
     }
@@ -312,8 +317,7 @@ namespace ttg {
       return m_state == TTG_DEVICE_CORO_COMPLETE;
     }
 
-    using handle_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<device_task_promise_type>;
-    device_task get_return_object() { return device_task{handle_type::from_promise(*this)}; }
+    device_task get_return_object() { return device_task{device_task_handle_type::from_promise(*this)}; }
 
     void unhandled_exception() {
 

From 82c2a8995461bd4aa9d4946d6b77c0b8cc73913e Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 3 Feb 2023 10:44:53 -0500
Subject: [PATCH 067/259] Clean up code from ttg::meta::is_detected debugging

---
 ttg/ttg/parsec/ttg.h | 59 +-------------------------------------------
 1 file changed, 1 insertion(+), 58 deletions(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 5f0474bde..c593d623f 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -103,42 +103,6 @@ int parsec_add_fetch_runtime_task(parsec_taskpool_t *tp, int tasks);
 
 #include "ttg/view.h"
 
-namespace test::meta {
-
-    template <class...>
-    using void_t = void;
-  namespace detail {
-      template <class Default, class Enabler, template <class...> class TT, class... Args>
-      struct detector {
-        using value_t = std::false_type;
-        using type = Default;
-      };
-
-      template <class Default, template <class...> class TT, class... Args>
-      struct detector<Default, void_t<TT<Args...>>, TT, Args...> {
-        using value_t = std::true_type;
-        using type = TT<Args...>;
-      };
-
-  }
-
-      template <template <class...> class TT, class... Args>
-      using is_detected = typename detail::detector<ttg::meta::nonesuch, void, TT, Args...>::value_t;
-
-    template <template <class...> class TT, class... Args>
-    using is_detected = typename detail::detector<ttg::meta::nonesuch, void, TT, Args...>::value_t;
-
-    template <template <class...> class TT, class... Args>
-    using detected_t = typename detail::detector<ttg::meta::nonesuch, void, TT, Args...>::type;
-
-    template <class Default, template <class...> class TT, class... Args>
-    using detected_or = detail::detector<Default, void, TT, Args...>;
-
-    template <template <class...> class TT, class... Args>
-    constexpr bool is_detected_v = is_detected<TT, Args...>::value;
-
-}
-
 namespace ttg_parsec {
   typedef void (*static_set_arg_fct_type)(void *, size_t, ttg::TTBase *);
   typedef std::pair<static_set_arg_fct_type, ttg::TTBase *> static_set_arg_fct_call_t;
@@ -1178,8 +1142,7 @@ namespace ttg_parsec {
 
     /// @return true if derivedT::have_cuda_op exists and is defined to true
     static constexpr bool derived_has_cuda_op() {
-      //if constexpr (test::meta::is_detected_v<have_cuda_op_non_type_t, derivedT>) {
-      if constexpr (derivedT::have_cuda_op) {
+      if constexpr (ttg::meta::is_detected_v<have_cuda_op_non_type_t, derivedT>) {
         return derivedT::have_cuda_op;
       } else {
         return false;
@@ -1552,30 +1515,10 @@ namespace ttg_parsec {
       return PARSEC_HOOK_RETURN_DONE; // will not be reacehed
     }
 
-    class A {
-    public:
-      static constexpr bool have_cuda_op = true;
-    };
-
     template <ttg::ExecutionSpace Space>
     static parsec_hook_return_t static_op(parsec_task_t *parsec_task) {
 
-      //static_assert(ttg::meta::is_detected_v<have_cuda_op_non_type_t, A>);
-      //static_assert(ttg::meta::is_detected_v<have_cuda_op_non_type_t, derivedT>);
-      //static_assert(ttg::meta::is_detected<have_cuda_op_non_type_t, derivedT>::value);
-      // TODO: this does not compile
-      //static_assert(ttg::meta::detail::detector<ttg::meta::nonesuch, void, have_cuda_op_non_type_t, derivedT>::value_t::value);
-      // TODO: this does compile. WTF?!
-      //static_assert(test::meta::detail::detector<ttg::meta::nonesuch, void, have_cuda_op_non_type_t, derivedT>::value_t::value);
-      //static_assert(test::meta::is_detected<have_cuda_op_non_type_t, derivedT>::value);
-      //static_assert(derivedT::have_cuda_op);
-      //static_assert(detector<ttg::meta::nonesuch, void, have_cuda_op_non_type_t, derivedT>::value_t::value);
-      using t = ttg::meta::void_t<have_cuda_op_non_type_t<derivedT>>;
-
       task_t *task = (task_t*)parsec_task;
-      have_cuda_op_non_type_t<derivedT> tmp = 0;
-      std::cout << "static_op derived_has_cuda_op " << derived_has_cuda_op() << " detected " << test::meta::is_detected_v<have_cuda_op_non_type_t, derivedT>
-                << " have_cuda_op " << derivedT::have_cuda_op << " detected " << ttg::meta::detail::detector<ttg::meta::nonesuch, void, have_cuda_op_non_type_t, derivedT>::value_t::value << std::endl;
       void* suspended_task_address =
 #ifdef TTG_HAS_COROUTINE
         task->suspended_task_address;  // non-null = need to resume the task

From 5df835078fff9f3eb34bf15d1381c705cb85565b Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 3 Feb 2023 10:45:07 -0500
Subject: [PATCH 068/259] Clean up code from ttg::meta::is_detected debugging

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 tests/unit/CMakeLists.txt | 17 +++----
 tests/unit/device_coro.cc | 96 ++++++++++++++++++++++++++++++++++++++-
 ttg/ttg/fwd.h             |  1 +
 ttg/ttg/run.h             |  1 +
 4 files changed, 106 insertions(+), 9 deletions(-)

diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt
index 47dc543e6..faff08780 100644
--- a/tests/unit/CMakeLists.txt
+++ b/tests/unit/CMakeLists.txt
@@ -2,16 +2,17 @@ include(AddTTGExecutable)
 
 # TT unit test: core TTG ops
 set(ut_src
-        fibonacci.cc
-        device_coro.cc
-        ranges.cc
-        tt.cc
-        unit_main.cpp)
+        #fibonacci.cc
+        #ranges.cc
+        #tt.cc
+        unit_main.cpp
+    )
 set(ut_libs Catch2::Catch2)
-if (TARGET std::coroutine)
-    list(APPEND ut_src fibonacci-coro.cc)
+#if (TARGET std::coroutine)
+    #list(APPEND ut_src fibonacci-coro.cc)
+    list(APPEND ut_src device_coro.cc)
     list(APPEND ut_libs std::coroutine)
-endif()
+#endif()
 add_ttg_executable(core-unittests-ttg "${ut_src}" LINK_LIBRARIES "${ut_libs}")
 
 
diff --git a/tests/unit/device_coro.cc b/tests/unit/device_coro.cc
index 1e35c4586..86a0cf1dd 100644
--- a/tests/unit/device_coro.cc
+++ b/tests/unit/device_coro.cc
@@ -38,6 +38,100 @@ TEST_CASE("Device", "coro") {
   }
 
 #if 0
+  SECTION("device_task") {
+    ttg::Edge<int, double> edge;
+    auto fn = [&](const int& key, double&& val) -> ttg::device_task {
+      // will schedule the view for transfer to and from the device
+      ttg::View<double> view = ttg::make_view(val, ttg::ViewScope::SyncInOut);
+      ttg::print("device_task key ", key, ", value ", val);
+      /* wait for the view to be available on the device */
+      co_await ttg::device_task_wait_views{};
+
+      /* once we're back here the data has been transferred */
+      CHECK(view.get_device_ptr<0>()  != nullptr);
+      CHECK(view.get_device_size<0>() == sizeof(val));
+      CHECK(&view.get_host_object() == &val);
+
+      ttg::print("device_task key ", key, ", device pointer ", view.get_device_ptr<0>());
+
+      while (val < 10.0) {
+
+        view.set_scope(ttg::ViewScope::SyncOut);
+
+        /* <submit kernel here> */
+
+        /* here we suspend to wait for a kernel to complete */
+        co_await ttg::device_task_wait_kernel{};
+
+        // TTG_WAIT_KERNEL();
+      }
+
+      /* we're back, the kernel executed and we can send */
+      if (key < 10) {
+        ttg::send<0>(key+1, val);
+      }
+    };
+    auto tt = ttg::make_tt<ttg::ExecutionSpace::CUDA>(fn, ttg::edges(edge), ttg::edges(edge),
+                                                      "device_task", {"edge_in"}, {"edge_out"});
+    make_graph_executable(tt);
+    if (ttg::default_execution_context().rank() == 0) tt->invoke(0, 0.0);
+    ttg::ttg_fence(ttg::default_execution_context());
+  }
+
+  struct A {
+    double norm;
+    std::vector<double> d;
+  };
+
+  SECTION("device_task") {
+    ttg::Edge<int, double> edge;
+    auto fn = [&](const int& key, ttg::ptr<A>&& a) -> ttg::device_task {
+      // will schedule the view for transfer to and from the device
+      View<double> norm_view = a.to_host(&A::norm);
+
+      View<A::norm> norm_view{a};
+      co_await ttg::device::wait_transfer{};
+
+
+
+
+      if (val)
+      val += 1.0;
+      ptr.sync_to_device();
+      /* wait for the view to be available on the device */
+      co_await ttg::device_task_wait_views{};
+
+      /* once we're back here the data has been transferred */
+      CHECK(view.get_device_ptr<0>()  != nullptr);
+      CHECK(view.get_device_size<0>() == sizeof(val));
+      CHECK(&view.get_host_object() == &val);
+
+      ttg::print("device_task key ", key, ", device pointer ", view.get_device_ptr<0>());
+
+      while (val < 10.0) {
+
+        view.set_scope(ttg::ViewScope::SyncOut);
+
+        /* <submit kernel here> */
+
+        /* here we suspend to wait for a kernel to complete */
+        co_await ttg::device_task_wait_kernel{};
+
+        // TTG_WAIT_KERNEL();
+      }
+
+      /* we're back, the kernel executed and we can send */
+      if (key < 10) {
+        ttg::send<0>(key+1, val);
+      }
+    };
+    auto tt = ttg::make_tt<ttg::ExecutionSpace::CUDA>(fn, ttg::edges(edge), ttg::edges(edge),
+                                                      "device_task", {"edge_in"}, {"edge_out"});
+    make_graph_executable(tt);
+    if (ttg::default_execution_context().rank() == 0) tt->invoke(0, 0.0);
+    ttg::ttg_fence(ttg::default_execution_context());
+  }
+
   struct A {
     int a[10];
     double b[10];
@@ -48,7 +142,7 @@ TEST_CASE("Device", "coro") {
     ttg::Edge<int, double> edge;
     auto fn = [](const int& key, A&& val) -> ttg::device_task {
       auto view = ttg::make_view(val, {val.a, 10, ttg::ViewScope::SyncIn},
-                                      {val.b, 10, ttg::ViewScope::SyncIn|ttg::ViewScope::SyncOut});
+                                      {val.b, 10, ttg::ViewScope::SyncIn});
       /* wait for the view to be available on the device */
       co_yield view;
       // co_yield std::tie(view1, view2);
diff --git a/ttg/ttg/fwd.h b/ttg/ttg/fwd.h
index df32505d0..f9b8d1c0f 100644
--- a/ttg/ttg/fwd.h
+++ b/ttg/ttg/fwd.h
@@ -47,6 +47,7 @@ namespace ttg {
   template <typename... RestOfArgs>
   void initialize(int argc, char **argv, int num_threads = -1, RestOfArgs &&...);
   void finalize();
+  [[noreturn]]
   void abort();
   World default_execution_context();
   void execute(ttg::World world);
diff --git a/ttg/ttg/run.h b/ttg/ttg/run.h
index 21ec337e8..b4dc593de 100644
--- a/ttg/ttg/run.h
+++ b/ttg/ttg/run.h
@@ -57,6 +57,7 @@ namespace ttg {
   inline void finalize() { TTG_IMPL_NS::ttg_finalize(); }
 
   /// Aborts the TTG program using the default backend's `ttg_abort` method
+  [[noreturn]]
   inline void abort() { TTG_IMPL_NS::ttg_abort(); }
 
   /// Accesses the default backend's default execution context

From e0c0b7850f116c4a9e206f33351fa2774691947e Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 27 Feb 2023 10:21:51 -0500
Subject: [PATCH 069/259] Implement ttg::buffer and ttg::devicescratch in the
 PaRSEC backend

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 tests/unit/device_coro.cc      | 147 ++++++++-
 ttg/CMakeLists.txt             |   9 +
 ttg/ttg.h                      |   5 +
 ttg/ttg/buffer.h               |  58 ++++
 ttg/ttg/devicescope.h          |  15 +
 ttg/ttg/devicescratch.h        |  19 ++
 ttg/ttg/parsec/buffer.h        | 341 +++++++++++++++++++
 ttg/ttg/parsec/devicescratch.h | 129 ++++++++
 ttg/ttg/parsec/fwd.h           |  18 +
 ttg/ttg/parsec/ptr.h           | 214 ++++++++++++
 ttg/ttg/parsec/task.h          | 207 ++++++++++++
 ttg/ttg/parsec/thread_local.h  |  22 ++
 ttg/ttg/parsec/ttg.h           | 578 ++++++++++++++++++---------------
 ttg/ttg/parsec/ttg_data_copy.h | 249 +++++++++++---
 ttg/ttg/ptr.h                  |  23 ++
 ttg/ttg/view.h                 | 356 +++++++++++++++-----
 16 files changed, 2010 insertions(+), 380 deletions(-)
 create mode 100644 ttg/ttg/buffer.h
 create mode 100644 ttg/ttg/devicescope.h
 create mode 100644 ttg/ttg/devicescratch.h
 create mode 100644 ttg/ttg/parsec/buffer.h
 create mode 100644 ttg/ttg/parsec/devicescratch.h
 create mode 100644 ttg/ttg/parsec/ptr.h
 create mode 100644 ttg/ttg/parsec/task.h
 create mode 100644 ttg/ttg/parsec/thread_local.h
 create mode 100644 ttg/ttg/ptr.h

diff --git a/tests/unit/device_coro.cc b/tests/unit/device_coro.cc
index 86a0cf1dd..a2cc3f7a2 100644
--- a/tests/unit/device_coro.cc
+++ b/tests/unit/device_coro.cc
@@ -3,12 +3,102 @@
 #include "ttg.h"
 #include "ttg/view.h"
 
+struct value_t {
+  ttg::buffer<double> db; // TODO: rename
+  int quark;
+
+  template<typename Archive>
+  void ttg_serialize(Archive& ar) {
+    ar& quark;
+    ar& db; // input:
+  }
+};
+
+/* devicebuf is non-POD so provide serialization
+ * information for members not a devicebuf */
+namespace madness::archive {
+  template <class Archive>
+  struct ArchiveSerializeImpl<Archive, value_t> {
+    static inline void serialize(const Archive& ar, value_t& obj) { ar& obj.quark; };
+  };
+}  // namespace madness::archive
+
+/* announce that this type contains a device buffer */
+template<>
+struct ttg::container_trait<value_t> {
+  static auto devicebuf_members(value_t&& v) {
+    return std::tie(v.db);
+  }
+};
+
+
 
 TEST_CASE("Device", "coro") {
+  SECTION("devicebuf") {
+
+    ttg::Edge<int, value_t> edge;
+    ttg::ptr<value_t> ptr;
+    auto fn = [&](const int& key, value_t&& val) -> ttg::device_task {
+      double scratch = 1.0;
+      ttg::devicescratch<double> scratch_view = ttg::make_scratch(&scratch, ttg::scope::SyncOut);
+      ttg::print("device_task key ", key);
+      /* wait for the view to be available on the device */
+      co_await ttg::to_device(scratch_view, val.db);
+      // co_yield std::tie(view1, view2);
+      // TTG_WAIT_VIEW(view);
+      /* once we're back here the data has been transferred */
+      //CHECK(view.get_rw_device_ptr<0>()  != nullptr);
+      CHECK(scratch_view.device_ptr()  != nullptr);
+
+      ttg::print("device_task key ", key, ", device pointer ", scratch_view.device_ptr());
+
+      /* here we suspend to wait for a kernel to complete */
+      co_await ttg::device_op_wait_kernel{};
+      // TTG_WAIT_KERNEL();
+
+      /* force the data back to the host
+       * useful if the buffer tracks a non-owned pointer
+       * and we want to send on that pointer
+       * TODO: needs implementing, how does this interact with scratch?
+       */
+      //co_await ttg::to_host(val.db);
+
+      /* we're back, the kernel executed and we can send */
+      if (key < 1 || scratch < 0.0) {
+        /* TODO: should we move the view in here if we want to get the device side data */
+        ttg::send<0>(key+1, std::move(val));
+      } else {
+        /* exfiltrate the value */
+        ptr = ttg::get_ptr(val);
+      }
+    };
+
+    //ptr.get_view<ttg::ExecutionSpace::CUDA>(device_id);
+
+    auto tt = ttg::make_tt<ttg::ExecutionSpace::CUDA>(fn, ttg::edges(edge), ttg::edges(edge),
+                                                      "device_task", {"edge_in"}, {"edge_out"});
+    make_graph_executable(tt);
+    if (ttg::default_execution_context().rank() == 0) tt->invoke(0, value_t{});
+    ttg::ttg_fence(ttg::default_execution_context());
+    CHECK(ptr.is_valid());
+    ptr.reset();
+  }
+
+
+
+
+
+
+
+
+
+
+#if 0
+
   SECTION("device_task") {
     ttg::Edge<int, double> edge;
     auto fn = [&](const int& key, double&& val) -> ttg::device_task {
-      ttg::View<double, double> view = ttg::make_view(val, ttg::ViewScope::SyncInOut);
+      ttg::View<double> view = ttg::make_view(val, ttg::ViewScope::SyncInOut);
       ttg::print("device_task key ", key, ", value ", val);
       /* wait for the view to be available on the device */
       co_yield view;
@@ -27,7 +117,7 @@ TEST_CASE("Device", "coro") {
 
       /* we're back, the kernel executed and we can send */
       if (key < 10) {
-        ttg::send<0>(key+1, val);
+        ttg::send<0>(key+1, std::move(val));
       }
     };
     auto tt = ttg::make_tt<ttg::ExecutionSpace::CUDA>(fn, ttg::edges(edge), ttg::edges(edge),
@@ -37,7 +127,60 @@ TEST_CASE("Device", "coro") {
     ttg::ttg_fence(ttg::default_execution_context());
   }
 
+  SECTION("get_ptr") {
+    ttg::Edge<int, double> edge;
+    ttg::ptr<double> ptr;
+    auto fn = [&](const int& key, double&& val) -> ttg::device_task {
+      ttg::View<double> view = ttg::make_view(val, ttg::ViewScope::SyncInOut);
+      ttg::print("device_task key ", key, ", value ", val);
+      /* wait for the view to be available on the device */
+      co_yield view;
+      // co_yield std::tie(view1, view2);
+      // TTG_WAIT_VIEW(view);
+      /* once we're back here the data has been transferred */
+      //CHECK(view.get_rw_device_ptr<0>()  != nullptr);
+      CHECK(view.get_device_ptr<0>()  != nullptr);
+      CHECK(view.get_device_size<0>() == sizeof(val));
+      CHECK(&view.get_host_object() == &val);
+
+      ttg::print("device_task key ", key, ", device pointer ", view.get_device_ptr<0>());
+
+      /* here we suspend to wait for a kernel to complete */
+      co_yield ttg::device_op_wait_kernel{};
+      // TTG_WAIT_KERNEL();
+
+      /* we're back, the kernel executed and we can send */
+      if (key < 10) {
+        /* TODO: should we move the view in here if we want to get the device side data */
+        ttg::send<0>(key+1, std::move(val));
+      } else {
+        /* exfiltrate the value */
+        ptr = ttg::get_ptr(val);
+      }
+    };
+
+    //ptr.get_view<ttg::ExecutionSpace::CUDA>(device_id);
+
+    auto tt = ttg::make_tt<ttg::ExecutionSpace::CUDA>(fn, ttg::edges(edge), ttg::edges(edge),
+                                                      "device_task", {"edge_in"}, {"edge_out"});
+    make_graph_executable(tt);
+    if (ttg::default_execution_context().rank() == 0) tt->invoke(0, 0.0);
+    ttg::ttg_fence(ttg::default_execution_context());
+
 #if 0
+    /* feed the host-side value back into the graph */
+    if (ttg::default_execution_context().rank() == 0) tt->invoke(0, pview.get_ptr());
+    ttg::ttg_fence(ttg::default_execution_context());
+
+    /* feed the device-side value back into the graph */
+    if (ttg::default_execution_context().rank() == 0) tt->invoke(0, pview);
+    ttg::ttg_fence(ttg::default_execution_context());
+#endif // 0
+  }
+
+
+
+
   SECTION("device_task") {
     ttg::Edge<int, double> edge;
     auto fn = [&](const int& key, double&& val) -> ttg::device_task {
diff --git a/ttg/CMakeLists.txt b/ttg/CMakeLists.txt
index 8945bc581..ef868f556 100644
--- a/ttg/CMakeLists.txt
+++ b/ttg/CMakeLists.txt
@@ -44,12 +44,16 @@ configure_file(
 )
 set(ttg-impl-headers
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/broadcast.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/ttg/buffer.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/ttg/devicescope.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/ttg/devicescratch.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/edge.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/execution.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/func.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/fwd.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/impl_selector.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/tt.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/ttg/ptr.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/reduce.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/run.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/runtimes.h
@@ -203,8 +207,13 @@ endif(TARGET MADworld)
 ########################
 if (TARGET PaRSEC::parsec)
   set(ttg-parsec-headers
+          ${CMAKE_CURRENT_SOURCE_DIR}/ttg/parsec/buffer.h
+          ${CMAKE_CURRENT_SOURCE_DIR}/ttg/parsec/devicescratch.h
           ${CMAKE_CURRENT_SOURCE_DIR}/ttg/parsec/fwd.h
           ${CMAKE_CURRENT_SOURCE_DIR}/ttg/parsec/import.h
+          ${CMAKE_CURRENT_SOURCE_DIR}/ttg/parsec/ptr.h
+          ${CMAKE_CURRENT_SOURCE_DIR}/ttg/parsec/task.h
+          ${CMAKE_CURRENT_SOURCE_DIR}/ttg/parsec/thread_local.h
           ${CMAKE_CURRENT_SOURCE_DIR}/ttg/parsec/ttg.h
           ${CMAKE_CURRENT_SOURCE_DIR}/ttg/parsec/ttg_data_copy.h
           )
diff --git a/ttg/ttg.h b/ttg/ttg.h
index e0fa9a702..09ccfaf90 100644
--- a/ttg/ttg.h
+++ b/ttg/ttg.h
@@ -27,6 +27,11 @@
 
 #include "ttg/edge.h"
 
+#include "ttg/ptr.h"
+#include "ttg/buffer.h"
+#include "ttg/devicescratch.h"
+#include "ttg/devicescope.h"
+
 #if defined(TTG_USE_PARSEC)
 #include "ttg/parsec/ttg.h"
 #elif defined(TTG_USE_MADNESS)
diff --git a/ttg/ttg/buffer.h b/ttg/ttg/buffer.h
new file mode 100644
index 000000000..7261404a2
--- /dev/null
+++ b/ttg/ttg/buffer.h
@@ -0,0 +1,58 @@
+#ifndef TTG_BUFFER_H
+#define TTG_BUFFER_H
+
+#include <memory>
+
+namespace ttg {
+
+template<typename T>
+using buffer = TTG_IMPL_NS::buffer<T>;
+
+template<typename T>
+struct hostbuf {
+private:
+  T* m_ptr = nullptr;
+  std::size_t m_count = 0;
+
+public:
+  hostbuf() = default;
+
+  hostbuf(T* ptr, std::size_t count)
+  : m_ptr(ptr)
+  , m_count(count)
+  { }
+
+  T* ptr() const {
+    return m_ptr;
+  }
+
+  std::size_t count() const {
+    return m_count;
+  }
+
+  std::size_t size() const {
+    return m_count * sizeof(T);
+  }
+};
+
+/* Applications may override this trait to
+ * expose device and host buffer members of
+ * data structures. Example:
+ * struct data_t {
+ *   ttg::buffer<double> db;
+ *   double *more_host_data;
+ *   int size;
+ * };
+ * template<>
+ * struct container_trait<data_t> {
+ *   static auto buffers() {
+ *     return std::make_tuple(db);
+ *   }
+ * };
+ */
+template<typename T>
+struct container_trait;
+
+} // namespace ttg
+
+#endif // TTG_buffer_H
\ No newline at end of file
diff --git a/ttg/ttg/devicescope.h b/ttg/ttg/devicescope.h
new file mode 100644
index 000000000..a8427fb56
--- /dev/null
+++ b/ttg/ttg/devicescope.h
@@ -0,0 +1,15 @@
+#ifndef TTG_DEVICESCOPE_H
+#define TTG_DEVICESCOPE_H
+
+namespace ttg {
+  enum class scope {
+    Allocate     = 0x0,  //< memory allocated as scratch, but not moved in or out
+    SyncIn       = 0x2,  //< data will be allocated on and transferred to device
+                         //< if latest version resides on the device (no previous sync-out) the data will
+                         //< not be transferred again
+    SyncOut      = 0x4,  //< value will be transferred from device to host after kernel completes
+    SyncInOut    = 0x8,  //< data will be moved in and synchronized back out after the kernel completes
+  };
+} // namespace ttg
+
+#endif // TTG_DEVICESCOPE_H
\ No newline at end of file
diff --git a/ttg/ttg/devicescratch.h b/ttg/ttg/devicescratch.h
new file mode 100644
index 000000000..425b3b676
--- /dev/null
+++ b/ttg/ttg/devicescratch.h
@@ -0,0 +1,19 @@
+#ifndef TTG_DEVICESCRATCH_H
+#define TTG_DEVICESCRATCH_H
+
+#include <ttg/devicescope.h>
+#include <ttg/parsec/devicescratch.h>
+
+namespace ttg {
+
+template<typename T>
+using devicescratch = TTG_IMPL_NS::devicescratch<T>;
+
+template<typename T>
+auto make_scratch(T* val, ttg::scope scope, std::size_t count = 1) {
+  return devicescratch<T>(val, scope, 1);
+}
+
+} // namespace ttg
+
+#endif // TTG_DEVICESCRATCH_H
\ No newline at end of file
diff --git a/ttg/ttg/parsec/buffer.h b/ttg/ttg/parsec/buffer.h
new file mode 100644
index 000000000..3d2d3a90c
--- /dev/null
+++ b/ttg/ttg/parsec/buffer.h
@@ -0,0 +1,341 @@
+#ifndef TTG_PARSEC_BUFFER_H
+#define TTG_PARSEC_BUFFER_H
+
+// TODO: replace with short vector
+#define TTG_PARSEC_MAX_NUM_DEVICES 4
+
+#include <array>
+#include <parsec.h>
+#include <parsec/data_internal.h>
+#include <parsec/mca/device/device.h>
+#include "ttg/parsec/ttg_data_copy.h"
+
+namespace ttg_parsec {
+
+
+namespace detail {
+  // fwd decl
+  template<typename T>
+  parsec_data_t* get_parsec_data(const ttg_parsec::buffer<T>& db);
+} // namespace detail
+
+/**
+ * A buffer that is mirrored between host memory
+ * and different devices. The runtime is free to
+ * move data between device and host memory based
+ * on where the tasks are executing.
+ *
+ * Note that a buffer is movable and should not
+ * be shared between two objects (e.g., through a pointer)
+ * in order for TTG to properly facilitate ownership
+ * tracking of the containing object.
+ */
+template<typename T>
+struct buffer {
+
+  using element_type = std::decay_t<T>;
+
+  static_assert(std::is_trivially_copyable_v<element_type>,
+                "Only trivially copyable types are supported for devices.");
+  static_assert(std::is_default_constructible_v<element_type>,
+                "Only default constructible types are supported for devices.");
+
+private:
+  using delete_fn_t = std::add_pointer_t<void(element_type*)>;
+
+  std::unique_ptr<parsec_data_t, decltype(&parsec_data_destroy)> m_data;
+  std::unique_ptr<element_type[], delete_fn_t> m_host_data;
+  std::size_t m_count = 0;
+  detail::ttg_data_copy_t *m_ttg_copy = nullptr;
+
+  static void delete_owned(element_type *ptr) {
+    delete[] ptr;
+  }
+
+  static void delete_non_owned(element_type *ptr) {
+    // nothing to be done, we don't own the memory
+  }
+
+  static void delete_parsec_data(parsec_data_t *data) {
+    std::cout << "delete parsec_data " << data << std::endl;
+    parsec_data_destroy(data);
+  }
+
+  static void delete_null_parsec_data(parsec_data_t *) {
+    // nothing to be done, only used for nullptr
+  }
+
+  void create_host_copy() {
+    /* create a new copy for the host object */
+    parsec_data_copy_t* copy;
+    copy = parsec_data_copy_new(m_data.get(), 0, parsec_datatype_int8_t, PARSEC_DATA_FLAG_PARSEC_MANAGED);
+    copy->device_private = m_host_data.get();
+    copy->coherency_state = PARSEC_DATA_COHERENCY_SHARED;
+    copy->version = 1; // this version is valid
+    m_data->nb_elts = sizeof(element_type)*m_count;
+    m_data->owner_device = 0;
+    /* register the new data with the host copy */
+    if (nullptr != m_ttg_copy) {
+      m_ttg_copy->add_device_data(m_data.get());
+    }
+  }
+
+  void reset() {
+    if (m_data) {
+      if (nullptr != m_ttg_copy) {
+        m_ttg_copy->remove_device_data(m_data.get());
+      }
+      m_data.reset();
+      m_count = 0;
+    }
+  }
+
+  friend parsec_data_t* detail::get_parsec_data<T>(const ttg_parsec::buffer<T>&);
+
+public:
+
+  /* The device ID of the CPU. */
+  static constexpr int cpu_device = 0;
+
+  buffer() : buffer(1)
+  { }
+
+  buffer(std::size_t count)
+  : m_data(parsec_data_new(), &delete_parsec_data)
+  , m_host_data(new element_type[count](), &delete_owned)
+  , m_count(count)
+  , m_ttg_copy(detail::ttg_data_copy_container())
+  {
+    create_host_copy();
+  }
+
+  /* Constructing a buffer using application-managed memory.
+   * The memory pointed to by ptr must be accessible during
+   * the life-time of the buffer. */
+  buffer(element_type* ptr, std::size_t count = 1)
+  : m_data(parsec_data_new(), &parsec_data_destroy)
+  , m_host_data(ptr, &delete_non_owned)
+  , m_count(count)
+  , m_ttg_copy(detail::ttg_data_copy_container())
+  {
+    create_host_copy();
+  }
+
+  ~buffer() {
+    unpin(); // make sure the copies are not pinned
+    /* remove the tracked copy */
+    if (nullptr != m_ttg_copy && m_data) {
+      m_ttg_copy->remove_device_data(m_data.get());
+    }
+  }
+
+  /* allow moving device buffers */
+  buffer(buffer&& db)
+  : m_data(std::move(db.m_data))
+  , m_host_data(std::move(db.m_host_data))
+  , m_count(db.m_count)
+  {
+    db.m_count = 0;
+    /* don't update the ttg_copy, we keep the connection */
+  }
+
+  /* copy the host data but leave the devices untouched */
+  buffer(const buffer& db)
+  : m_data(db.m_count ? parsec_data_new() : nullptr,
+           db.m_count ? &parsec_data_destroy : &delete_null_parsec_data)
+  , m_host_data(db.m_count ? new element_type[db.m_count] : nullptr,
+                db.m_count ? &delete_owned : delete_non_owned)
+  , m_count(db.m_count)
+  , m_ttg_copy(detail::ttg_data_copy_container())
+  {
+    /* copy host data */
+    std::copy(db.m_host_data.get(),
+              db.m_host_data.get() + m_count,
+              m_host_data.get());
+    /* create the host copy with the allocated memory */
+    create_host_copy();
+  }
+
+  /* allow moving device buffers */
+  buffer& operator=(buffer&& db) {
+    m_data = std::move(db.m_data);
+    m_host_data = std::move(db.m_host_data);
+    m_count = db.m_count;
+    db.m_count = 0;
+    /* don't update the ttg_copy, we keep the connection */
+  }
+
+  /* copy the host buffer content but leave the devices untouched */
+  buffer& operator=(const buffer& db) {
+    if (db.m_count == 0) {
+      m_data = std::unique_ptr<parsec_data_t>(nullptr, &delete_null_parsec_data);
+      m_host_data = std::unique_ptr<element_type>(nullptr, &delete_non_owned);
+    } else {
+      m_data = std::unique_ptr(parsec_data_new(), &parsec_data_destroy);
+      m_host_data = std::unique_ptr(new element_type[db.m_count], &delete_owned);
+      /* copy host data */
+      std::copy(db.m_host_data.get(),
+                db.m_host_data.get() + db.m_count,
+                m_host_data.get());
+      /* create the host copy with the allocated memory */
+      create_host_copy();
+    }
+    m_count = db.m_count;
+  }
+
+  /* set the current device, useful when a device
+   * buffer was modified outside of a TTG */
+  void set_current_device(int device_id) {
+    assert(is_valid());
+    /* make sure it's a valid device */
+    assert(parsec_nb_devices > device_id);
+    /* make sure it's a valid copy */
+    assert(m_data->device_copies[device_id] != nullptr);
+    m_data->owner_device = device_id;
+  }
+
+  /* get the current device ID, i.e., the last updated
+   * device buffer.  */
+  int get_current_device() const {
+    assert(is_valid());
+    return m_data->owner_device;
+  }
+
+  /* get the current device pointer */
+  element_type* current_device_ptr() {
+    assert(is_valid());
+    return static_cast<element_type*>(m_data->device_copies[m_data->owner_device]->device_private);
+  }
+
+  /* get the current device pointer */
+  const element_type* current_device_ptr() const {
+    assert(is_valid());
+    return static_cast<element_type*>(m_data->device_copies[m_data->owner_device]->device_private);
+  }
+
+  /* get the device pointer at the given device
+   * \sa cpu_device
+   */
+  element_type* device_ptr_on(int device_id) {
+    assert(is_valid());
+    return static_cast<element_type*>(parsec_data_get_ptr(m_data.get(), device_id));
+  }
+
+  /* get the device pointer at the given device
+   * \sa cpu_device
+   */
+  const element_type* device_ptr_on(int device_id) const {
+    assert(is_valid());
+    return static_cast<element_type*>(parsec_data_get_ptr(m_data.get(), device_id));
+  }
+
+  bool is_valid_on(int device_id) const {
+    assert(is_valid());
+    return (parsec_data_get_ptr(m_data.get(), device_id) != nullptr);
+  }
+
+  void allocate_on(int device_id) {
+    /* TODO: need exposed PaRSEC memory allocator */
+  }
+
+  /* TODO: can we do this automatically?
+   * Pin the memory on all devices we currently track.
+   * Pinned memory won't be released by PaRSEC and can be used
+   * at any time.
+   */
+  void pin() {
+    for (int i = 1; i < parsec_nb_devices; ++i) {
+      pin_on(i);
+    }
+  }
+
+  /* Unpin the memory on all devices we currently track. */
+  void unpin() {
+    if (!is_valid()) return;
+    for (int i = 1; i < parsec_nb_devices; ++i) {
+      unpin_on(i);
+    }
+  }
+
+  /* Pin the memory on a given device */
+  void pin_on(int device_id) {
+    /* TODO: how can we pin memory on a device? */
+  }
+
+  /* Pin the memory on a given device */
+  void unpin_on(int device_id) {
+    /* TODO: how can we unpin memory on a device? */
+  }
+
+  bool is_valid() const {
+    return !!m_data;
+  }
+
+  operator bool() const {
+    return is_valid();
+  }
+
+  std::size_t size() const {
+    return m_count;
+  }
+
+  /* Reallocate the buffer with count elements */
+  void reset(std::size_t count) {
+    /* TODO: can we resize if count is smaller than m_count? */
+    /* drop the current data and reallocate */
+    reset();
+    if (count == 0) {
+      m_data = std::unique_ptr<parsec_data_t>(nullptr, &delete_null_parsec_data);
+      m_host_data = std::unique_ptr<element_type>(nullptr, &delete_non_owned);
+    } else {
+      m_data = std::unique_ptr(parsec_data_new(), &parsec_data_destroy);
+      m_host_data = std::unique_ptr(new element_type[count], &delete_owned);
+      /* create the host copy with the allocated memory */
+      create_host_copy();
+    }
+    m_count = count;
+    /* don't touch the ttg_copy, we still belong to the same container */
+  }
+
+  /* Reset the buffer to use the ptr to count elements */
+  void reset(T* ptr, std::size_t count = 1) {
+    /* TODO: can we resize if count is smaller than m_count? */
+    /* drop the current data and reallocate */
+    reset();
+    if (nullptr == ptr) {
+      m_data = std::unique_ptr<parsec_data_t>(nullptr, &delete_null_parsec_data);
+      m_host_data = std::unique_ptr<element_type>(nullptr, &delete_non_owned);
+      m_count = 0;
+    } else {
+      m_data = std::unique_ptr(parsec_data_new(), &parsec_data_destroy);
+      m_host_data = std::unique_ptr(ptr, &delete_non_owned);
+      /* create the host copy with the allocated memory */
+      create_host_copy();
+      m_count = count;
+    }
+    /* don't touch the ttg_copy, we still belong to the same container */
+  }
+
+};
+
+template<typename T>
+struct is_buffer : std::false_type
+{ };
+
+template<typename T>
+struct is_buffer<buffer<T>> : std::true_type
+{ };
+
+template<typename T>
+constexpr static const bool is_buffer_v = is_buffer<T>::value;
+
+namespace detail {
+  template<typename T>
+  parsec_data_t* get_parsec_data(const ttg_parsec::buffer<T>& db) {
+    return const_cast<parsec_data_t*>(db.m_data.get());
+  }
+} // namespace detail
+
+} // namespace ttg_parsec
+
+#endif // TTG_PARSEC_BUFFER_H
\ No newline at end of file
diff --git a/ttg/ttg/parsec/devicescratch.h b/ttg/ttg/parsec/devicescratch.h
new file mode 100644
index 000000000..2a2e928d4
--- /dev/null
+++ b/ttg/ttg/parsec/devicescratch.h
@@ -0,0 +1,129 @@
+#ifndef TTG_PARSEC_DEVICESCRATCH_H
+#define TTG_PARSEC_DEVICESCRATCH_H
+
+// TODO: replace with short vector
+#define TTG_PARSEC_MAX_NUM_DEVICES 4
+
+#include <array>
+#include <parsec.h>
+#include <parsec/data_internal.h>
+#include <parsec/mca/device/device.h>
+#include <ttg/devicescope.h>
+
+namespace ttg_parsec {
+
+namespace detail {
+  // fwd decl
+  template<typename T>
+  parsec_data_t* get_parsec_data(const ttg_parsec::devicescratch<T>&);
+} // namespace detail
+
+/**
+ * Scratch-space for task-local variables.
+ * TTG will allocate memory on the device
+ * and transfer data in and out based on the scope.
+ */
+template<typename T>
+struct devicescratch {
+
+  using element_type = std::decay_t<T>;
+
+  static_assert(std::is_trivially_copyable_v<element_type>,
+                "Only trivially copyable types are supported for devices.");
+  static_assert(std::is_default_constructible_v<element_type>,
+                "Only default constructible types are supported for devices.");
+
+private:
+
+  parsec_data_t m_data;
+  parsec_data_copy_t m_data_copy;
+  ttg::scope m_scope;
+
+  void create_host_copy(element_type *ptr, std::size_t count) {
+    /* TODO: is the construction call necessary? */
+    /* TODO: handle the scope */
+    PARSEC_OBJ_CONSTRUCT(&m_data_copy, parsec_data_copy_t);
+    m_data_copy.device_index    = 0;
+    m_data_copy.original        = &m_data;
+    m_data_copy.older           = NULL;
+    m_data_copy.flags           = PARSEC_DATA_FLAG_PARSEC_MANAGED;
+    m_data_copy.dtt             = parsec_datatype_int8_t;
+    m_data_copy.version         = 1;
+    m_data_copy.device_private  = ptr;
+    m_data_copy.coherency_state = PARSEC_DATA_COHERENCY_SHARED;
+
+    PARSEC_OBJ_CONSTRUCT(&m_data, parsec_data_t);
+    m_data.device_copies[0]     = &m_data_copy;
+    m_data.nb_elts              = count * sizeof(element_type);
+    m_data.owner_device         = 0;
+  }
+
+  friend parsec_data_t* detail::get_parsec_data<T>(const ttg_parsec::devicescratch<T>&);
+
+public:
+
+  /* Constructing a devicescratch using application-managed memory.
+   * The memory pointed to by ptr must be accessible during
+   * the life-time of the devicescratch. */
+  devicescratch(element_type* ptr, ttg::scope scope = ttg::scope::SyncIn, std::size_t count = 1)
+  : m_scope(scope) {
+    create_host_copy(ptr, count);
+  }
+
+  /* don't allow moving */
+  devicescratch(devicescratch&&) = delete;
+
+  /* don't allow copying */
+  devicescratch(const devicescratch& db) = delete;
+
+  /* don't allow moving */
+  devicescratch& operator=(devicescratch&&) = delete;
+
+  /* don't allow copying */
+  devicescratch& operator=(const devicescratch& db) = delete;
+
+  /* get the current device pointer */
+  element_type* device_ptr() {
+    assert(is_valid());
+    return static_cast<element_type*>(m_data.device_copies[m_data.owner_device]->device_private);
+  }
+
+  /* get the current device pointer */
+  const element_type* device_ptr() const {
+    assert(is_valid());
+    return static_cast<element_type*>(m_data.device_copies[m_data.owner_device]->device_private);
+  }
+
+  bool is_valid() const {
+    // TODO: how to get the current device
+    // return (m_data.owner_device == parsec_current_device);
+    return true;
+  }
+
+  ttg::scope scope() const {
+    return m_scope;
+  }
+
+};
+
+template<typename T>
+struct is_devicescratch : std::false_type
+{ };
+
+template<typename T>
+struct is_devicescratch<devicescratch<T>> : std::true_type
+{ };
+
+template<typename T>
+constexpr static const bool is_devicescratch_v = is_devicescratch<T>::value;
+
+namespace detail {
+  template<typename T>
+  parsec_data_t* get_parsec_data(const ttg_parsec::devicescratch<T>& scratch) {
+    return const_cast<parsec_data_t*>(&scratch.m_data);
+  }
+} // namespace detail
+
+} // namespace ttg_parsec
+
+#endif // TTG_PARSEC_DEVICESCRATCH_H
\ No newline at end of file
diff --git a/ttg/ttg/parsec/fwd.h b/ttg/ttg/parsec/fwd.h
index cc38f68d8..690b4b7ad 100644
--- a/ttg/ttg/parsec/fwd.h
+++ b/ttg/ttg/parsec/fwd.h
@@ -13,6 +13,17 @@ namespace ttg_parsec {
   template <typename keyT, typename output_terminalsT, typename derivedT, typename input_valueTs = ttg::typelist<>>
   class TT;
 
+  template<typename T>
+  struct ptr;
+
+  template<typename T>
+  struct buffer;
+  template<typename T>
+  struct devicescratch;
+
+  template<typename... Views>
+  bool register_device_memory(std::tuple<Views&...> &views);
+
   /// \internal the OG name
   template <typename keyT, typename output_terminalsT, typename derivedT, typename... input_valueTs>
   using Op [[deprecated("use TT instead")]] = TT<keyT, output_terminalsT, derivedT, ttg::typelist<input_valueTs...>>;
@@ -54,6 +65,13 @@ namespace ttg_parsec {
   template <typename T>
   static void ttg_broadcast(ttg::World world, T &data, int source_rank);
 
+  template<typename T>
+  ptr<T> get_ptr(const T& obj);
+
+  template<typename T, typename... Args>
+  ptr<T> make_ptr(Args&&... args);
+
+
 }  // namespace ttg_parsec
 
 #endif  // TTG_PARSEC_FWD_H
diff --git a/ttg/ttg/parsec/ptr.h b/ttg/ttg/parsec/ptr.h
new file mode 100644
index 000000000..5c8c39524
--- /dev/null
+++ b/ttg/ttg/parsec/ptr.h
@@ -0,0 +1,214 @@
+#ifndef TTG_PARSEC_PTR_H
+#define TTG_PARSEC_PTR_H
+
+#include <unordered_map>
+#include <mutex>
+
+#include "ttg/parsec/ttg_data_copy.h"
+#include "ttg/parsec/thread_local.h"
+#include "ttg/parsec/task.h"
+
+namespace ttg_parsec {
+
+  namespace detail {
+    /* fwd decl */
+    template <typename Value>
+    inline ttg_data_copy_t *create_new_datacopy(Value &&value);
+
+    struct ptr {
+      using copy_type = detail::ttg_data_copy_t;
+
+    private:
+      static inline std::unordered_map<ptr*, bool> m_ptr_map;
+      static inline std::mutex m_ptr_map_mtx;
+
+      copy_type *m_copy = nullptr;
+
+      void drop_copy() {
+        std::cout << "ptr drop_copy " << m_copy << " ref " << m_copy->num_ref() << std::endl;
+        if (nullptr != m_copy && 1 == m_copy->drop_ref()) {
+          delete m_copy;
+        }
+        m_copy = nullptr;
+      }
+
+      void register_self() {
+        /* insert ourselves from the list of ptr */
+        std::lock_guard {m_ptr_map_mtx};
+        m_ptr_map.insert(std::pair{this, true});
+      }
+
+      void deregister_self() {
+        /* remove ourselves from the list of ptr */
+        std::lock_guard _{m_ptr_map_mtx};
+        if (m_ptr_map.contains(this)) {
+          m_ptr_map.erase(this);
+        }
+      }
+
+    public:
+      ptr(copy_type *copy)
+      : m_copy(copy)
+      {
+        register_self();
+        m_copy->add_ref();
+        std::cout << "ptr copy_obj ref " << m_copy->num_ref() << std::endl;
+      }
+
+      copy_type* get_copy() const {
+        return m_copy;
+      }
+
+      ptr(const ptr& p)
+      : m_copy(p.m_copy)
+      {
+        register_self();
+        m_copy->add_ref();
+        std::cout << "ptr cpy " << m_copy << " ref " << m_copy->num_ref() << std::endl;
+      }
+
+      ptr(ptr&& p)
+      : m_copy(p.m_copy)
+      {
+        register_self();
+        p.m_copy = nullptr;
+        std::cout << "ptr mov " << m_copy << " ref " << m_copy->num_ref() << std::endl;
+      }
+
+      ~ptr() {
+        deregister_self();
+        drop_copy();
+      }
+
+      ptr& operator=(const ptr& p)
+      {
+        drop_copy();
+        m_copy = p.m_copy;
+        m_copy->add_ref();
+        std::cout << "ptr cpy " << m_copy << " ref " << m_copy->num_ref() << std::endl;
+        return *this;
+      }
+
+      ptr& operator=(ptr&& p) {
+        drop_copy();
+        m_copy = p.m_copy;
+        p.m_copy = nullptr;
+        std::cout << "ptr mov " << m_copy << " ref " << m_copy->num_ref() << std::endl;
+        return *this;
+      }
+
+      bool is_valid() const {
+        return (nullptr != m_copy);
+      }
+
+      void reset() {
+        drop_copy();
+      }
+
+      /* drop all currently registered ptr
+       * \note this function is not thread-safe
+       *       and should only be called at the
+       *       end of the execution, e.g., during finalize.
+       */
+      static void drop_all_ptr() {
+        for(auto it : m_ptr_map) {
+          it.first->drop_copy();
+        }
+      }
+    };
+  } // namespace detail
+
+  template<typename T, typename... Args>
+  ptr<T> make_ptr(Args&&... args);
+
+  template<typename T>
+  ptr<T> get_ptr(const T& obj);
+
+  template<typename T>
+  struct ptr {
+
+    using value_type = std::decay_t<T>;
+
+  private:
+    using copy_type = detail::ttg_data_value_copy_t<value_type>;
+
+    std::unique_ptr<detail::ptr> m_ptr;
+
+    /* only PaRSEC backend functions are allowed to touch our private parts */
+    template<typename... Args>
+    friend ptr<T> make_ptr(Args&&... args);
+    template<typename S>
+    friend ptr<S> get_ptr(const S& obj);
+    friend ttg::detail::value_copy_handler<ttg::Runtime::PaRSEC>;
+
+    /* only accessible by get_ptr and make_ptr */
+    ptr(detail::ptr::copy_type *copy)
+    : m_ptr(new detail::ptr(copy))
+    { }
+
+    copy_type* get_copy() const {
+      return static_cast<copy_type*>(m_ptr->get_copy());
+    }
+
+  public:
+
+    ptr() = default;
+
+    ptr(const ptr& p)
+    : ptr(p.get_copy())
+    { }
+
+    ptr(ptr&& p) = default;
+
+    ~ptr() = default;
+
+    ptr& operator=(const ptr& p) {
+      m_ptr.reset(new detail::ptr(p.get_copy()));
+      return *this;
+    }
+
+    ptr& operator=(ptr&& p) = default;
+
+    value_type& operator*() const {
+      return **static_cast<copy_type*>(m_ptr->get_copy());
+    }
+
+    value_type& operator->() const {
+      return **static_cast<copy_type*>(m_ptr->get_copy());
+    }
+
+    bool is_valid() const {
+      return m_ptr && m_ptr->is_valid();
+    }
+
+    void reset() {
+      m_ptr.reset();
+    }
+  };
+
+  template<typename T>
+  ptr<T> get_ptr(const T& obj) {
+    if (nullptr != detail::parsec_ttg_caller) {
+      for (int i = 0; i < detail::parsec_ttg_caller->data_count; ++i) {
+        detail::ttg_data_copy_t *copy = detail::parsec_ttg_caller->copies[i];
+        if (nullptr != copy) {
+          if (copy->get_ptr() == &obj) {
+            return ptr<T>(copy);
+          }
+        }
+      }
+    }
+    /* object not tracked, make a new ptr that is now tracked */
+    detail::ttg_data_copy_t *copy = detail::create_new_datacopy(obj);
+    return ptr<T>(copy);
+  }
+
+  template<typename T, typename... Args>
+  ptr<T> make_ptr(Args&&... args) {
+    detail::ttg_data_copy_t *copy = detail::create_new_datacopy(T(std::forward<Args>(args)...));
+    return ptr<T>(copy);
+  }
+
+} // namespace ttg_parsec
+
+#endif // TTG_PARSEC_PTR_H
\ No newline at end of file
diff --git a/ttg/ttg/parsec/task.h b/ttg/ttg/parsec/task.h
new file mode 100644
index 000000000..c8465b81c
--- /dev/null
+++ b/ttg/ttg/parsec/task.h
@@ -0,0 +1,207 @@
+#ifndef TTG_PARSEC_TASK_H
+#define TTG_PARSEC_TASK_H
+
+#include "ttg/parsec/ttg_data_copy.h"
+
+#include <parsec/parsec_internal.h>
+#include <parsec/mca/device/device_gpu.h>
+
+namespace ttg_parsec {
+
+  namespace detail {
+
+    struct device_ptr_t {
+      parsec_gpu_task_t* gpu_task = nullptr;
+      parsec_flow_t* flows = nullptr;
+    };
+
+    template<bool SupportDevice>
+    struct device_state_t
+    {
+      static constexpr bool support_device = false;
+      static constexpr size_t num_flows = 0;
+      device_state_t(parsec_task_t *parsec_task)
+      { }
+      static constexpr device_ptr_t* dev_ptr() {
+        return nullptr;
+      }
+    };
+
+    template<>
+    struct device_state_t<true> {
+      static constexpr bool support_device = false;
+      static constexpr size_t num_flows = MAX_PARAM_COUNT;
+      parsec_flow_t m_flows[num_flows];
+      device_ptr_t m_dev_ptr = {nullptr, &m_flows[0]}; // gpu_task will be allocated in each task
+      device_ptr_t* dev_ptr() {
+        return &m_dev_ptr;
+      }
+    };
+
+    typedef parsec_hook_return_t (*parsec_static_op_t)(void *);  // static_op will be cast to this type
+
+    struct parsec_ttg_task_base_t {
+      parsec_task_t parsec_task;
+      int32_t in_data_count = 0;   //< number of satisfied inputs
+      int32_t data_count = 0;      //< number of data elements in the copies array
+      ttg_data_copy_t **copies;    //< pointer to the fixed copies array of the derived task
+      parsec_hash_table_item_t tt_ht_item = {};
+      parsec_static_op_t function_template_class_ptr[ttg::runtime_traits<ttg::Runtime::PaRSEC>::num_execution_spaces] =
+          {nullptr};
+
+      typedef struct {
+        std::size_t goal;
+        std::size_t size;
+      } size_goal_t;
+
+      typedef void (release_task_fn)(parsec_ttg_task_base_t*);
+      /* Poor-mans virtual function
+       * We cannot use virtual inheritance or private visibility because we
+       * need offsetof for the mempool and scheduling.
+       */
+      release_task_fn* release_task_cb = nullptr;
+      device_ptr_t* dev_ptr;
+      bool remove_from_hash = true;
+      bool is_dummy = false;
+      bool defer_writer = TTG_PARSEC_DEFER_WRITER; // whether to defer writer instead of creating a new copy
+
+
+      /*
+      virtual void release_task() = 0;
+      */
+    //public:
+      void release_task() {
+        release_task_cb(this);
+      }
+
+     protected:
+      /**
+       * Protected constructors: this class should not be instantiated directly
+       * but always be use through parsec_ttg_task_t.
+       */
+
+      parsec_ttg_task_base_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class,
+                             int data_count, ttg_data_copy_t **copies, device_ptr_t *dev_ptr,
+                             bool defer_writer = TTG_PARSEC_DEFER_WRITER)
+          : data_count(data_count)
+          , copies(copies)
+          , dev_ptr(dev_ptr)
+          , defer_writer(defer_writer) {
+        PARSEC_LIST_ITEM_SINGLETON(&parsec_task.super);
+        parsec_task.mempool_owner = mempool;
+        parsec_task.task_class = task_class;
+        parsec_task.priority = 0;
+      }
+
+      parsec_ttg_task_base_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class,
+                             parsec_taskpool_t *taskpool, int32_t priority,
+                             int data_count, ttg_data_copy_t **copies, device_ptr_t *dev_ptr,
+                             release_task_fn *release_fn,
+                             bool defer_writer = TTG_PARSEC_DEFER_WRITER)
+          : data_count(data_count)
+          , copies(copies)
+          , release_task_cb(release_fn)
+          , dev_ptr(dev_ptr)
+          , defer_writer(defer_writer) {
+        PARSEC_LIST_ITEM_SINGLETON(&parsec_task.super);
+        parsec_task.mempool_owner = mempool;
+        parsec_task.task_class = task_class;
+        parsec_task.status = PARSEC_TASK_STATUS_HOOK;
+        parsec_task.taskpool = taskpool;
+        parsec_task.priority = priority;
+        parsec_task.chore_mask = 1<<0;
+      }
+
+    public:
+      void set_dummy(bool d) { is_dummy = d; }
+      bool dummy() { return is_dummy; }
+    };
+
+    template <typename TT, bool KeyIsVoid = ttg::meta::is_void_v<typename TT::key_type>>
+    struct parsec_ttg_task_t : public parsec_ttg_task_base_t {
+      using key_type = typename TT::key_type;
+      static constexpr size_t num_streams = TT::numins;
+      TT* tt;
+      key_type key;
+      size_goal_t stream[num_streams] = {};
+#ifdef TTG_HAS_COROUTINE
+      void* suspended_task_address = nullptr;  // if not null the function is suspended
+#endif
+      ttg_data_copy_t *copies[num_streams+1] = { nullptr };  // the data copies tracked by this task
+                                                             // +1 for the copy needed during send/bcast
+      device_state_t<TT::derived_has_cuda_op()> dev_state;
+
+      parsec_ttg_task_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class)
+          : parsec_ttg_task_base_t(mempool, task_class, num_streams, copies, dev_state.dev_ptr()) {
+        tt_ht_item.key = pkey();
+
+        // We store the hash of the key and the address where it can be found in locals considered as a scratchpad
+        *(uintptr_t*)&(parsec_task.locals[0]) = 0; //there is no key
+        *(uintptr_t*)&(parsec_task.locals[2]) = 0; //there is no key
+      }
+
+      parsec_ttg_task_t(const key_type& key, parsec_thread_mempool_t *mempool,
+                        parsec_task_class_t *task_class, parsec_taskpool_t *taskpool,
+                        TT *tt_ptr, int32_t priority)
+          : parsec_ttg_task_base_t(mempool, task_class, taskpool, priority,
+                                   num_streams, copies, dev_state.dev_ptr(),
+                                   &release_task, tt_ptr->m_defer_writer)
+          , tt(tt_ptr), key(key) {
+        tt_ht_item.key = pkey();
+
+        // We store the hash of the key and the address where it can be found in locals considered as a scratchpad
+        uint64_t hv = ttg::hash<std::decay_t<decltype(key)>>{}(key);
+        *(uintptr_t*)&(parsec_task.locals[0]) = hv;
+        *(uintptr_t*)&(parsec_task.locals[2]) = reinterpret_cast<uintptr_t>(&this->key);
+      }
+
+      static void release_task(parsec_ttg_task_base_t* task_base) {
+        parsec_ttg_task_t *task = static_cast<parsec_ttg_task_t*>(task_base);
+        TT *tt = task->tt;
+        tt->release_task(task);
+      }
+
+      parsec_key_t pkey() { return reinterpret_cast<parsec_key_t>(&key); }
+    };
+
+    template <typename TT>
+    struct parsec_ttg_task_t<TT, true> : public parsec_ttg_task_base_t {
+      static constexpr size_t num_streams = TT::numins;
+      TT* tt;
+      size_goal_t stream[num_streams] = {};
+#ifdef TTG_HAS_COROUTINE
+      void* suspended_task_address = nullptr;  // if not null the function is suspended
+#endif
+      ttg_data_copy_t *copies[num_streams+1] = { nullptr };  // the data copies tracked by this task
+                                                             // +1 for the copy needed during send/bcast
+      device_state_t<TT::derived_has_cuda_op()> dev_state;
+
+      parsec_ttg_task_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class)
+          : parsec_ttg_task_base_t(mempool, task_class, num_streams, copies, dev_state.dev_ptr()) {
+        tt_ht_item.key = pkey();
+      }
+
+      parsec_ttg_task_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class,
+                        parsec_taskpool_t *taskpool, TT *tt_ptr, int32_t priority)
+          : parsec_ttg_task_base_t(mempool, task_class, taskpool, priority,
+                                   num_streams, copies, dev_state.dev_ptr(),
+                                   &release_task, tt_ptr->m_defer_writer)
+          , tt(tt_ptr) {
+        tt_ht_item.key = pkey();
+      }
+
+      static void release_task(parsec_ttg_task_base_t* task_base) {
+        parsec_ttg_task_t *task = static_cast<parsec_ttg_task_t*>(task_base);
+        TT *tt = task->tt;
+        tt->release_task(task);
+      }
+
+      parsec_key_t pkey() { return 0; }
+    };
+
+
+  } // namespace detail
+
+} // namespace ttg_parsec
+
+#endif // TTG_PARSEC_TASK_H
\ No newline at end of file
diff --git a/ttg/ttg/parsec/thread_local.h b/ttg/ttg/parsec/thread_local.h
new file mode 100644
index 000000000..54b98885e
--- /dev/null
+++ b/ttg/ttg/parsec/thread_local.h
@@ -0,0 +1,22 @@
+#ifndef TTG_PARSEC_THREAD_LOCAL_H
+#define TTG_PARSEC_THREAD_LOCAL_H
+
+namespace ttg_parsec {
+
+namespace detail {
+
+  // fwd decls
+  struct parsec_ttg_task_base_t;
+  struct ttg_data_copy_t;
+
+  inline thread_local parsec_ttg_task_base_t *parsec_ttg_caller = nullptr;
+
+  inline ttg_data_copy_t*& ttg_data_copy_container() {
+    static thread_local ttg_data_copy_t *ptr = nullptr;
+    return ptr;
+  }
+
+} // namespace detail
+} // namespace ttg_parsec
+
+#endif // TTG_PARSEC_THREAD_LOCAL_H
\ No newline at end of file
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index c593d623f..64c66b4e6 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -7,6 +7,11 @@
 #define TTG_USE_PARSEC 1
 #endif  // !defined(TTG_IMPL_NAME)
 
+/* Whether to defer a potential writer if there are readers.
+ * This may avoid extra copies in exchange for concurrency.
+ * This may cause deadlocks, so use with caution. */
+#define TTG_PARSEC_DEFER_WRITER false
+
 #include "ttg/impl_selector.h"
 
 /* include ttg header to make symbols available in case this header is included directly */
@@ -31,8 +36,14 @@
 
 #include "ttg/serialization/data_descriptor.h"
 
+#include "ttg/view.h"
+
 #include "ttg/parsec/fwd.h"
 
+#include "ttg/parsec/buffer.h"
+#include "ttg/parsec/devicescratch.h"
+#include "ttg/parsec/thread_local.h"
+
 #include <algorithm>
 #include <array>
 #include <cassert>
@@ -79,6 +90,9 @@
 #include <cstring>
 
 #include "ttg/parsec/ttg_data_copy.h"
+#include "ttg/parsec/thread_local.h"
+#include "ttg/parsec/ptr.h"
+#include "ttg/parsec/task.h"
 
 /* This is missing in the parsec_comm_engine.h interface... But we are discussing
    if this execution stream should be exposed. Workaround this for now. */
@@ -90,11 +104,6 @@ extern "C" parsec_execution_stream_t parsec_comm_es;
 #include <unordered_set>
 #endif
 
-/* Whether to defer a potential writer if there are readers.
- * This may avoid extra copies in exchange for concurrency.
- * This may cause deadlocks, so use with caution. */
-#define TTG_PARSEC_DEFER_WRITER false
-
 /* PaRSEC function declarations */
 extern "C" {
 void parsec_taskpool_termination_detected(parsec_taskpool_t *tp);
@@ -511,8 +520,6 @@ namespace ttg_parsec {
 
   namespace detail {
 
-    typedef parsec_hook_return_t (*parsec_static_op_t)(void *);  // static_op will be cast to this type
-
     const parsec_symbol_t parsec_taskclass_param0 = {
       .flags = PARSEC_SYMBOL_IS_STANDALONE|PARSEC_SYMBOL_IS_GLOBAL,
       .name = "HASH0",
@@ -546,159 +553,6 @@ namespace ttg_parsec {
       .expr_inc = nullptr,
       .cst_inc = 0 };
 
-    struct parsec_ttg_task_base_t {
-      parsec_task_t parsec_task;
-      int32_t in_data_count = 0;  //< number of satisfied inputs
-      int32_t data_count = 0;     //< number of data elements in parsec_task.data
-      ttg_data_copy_t **copies;    //< pointer to the fixed copies array of the derived task
-      parsec_hash_table_item_t tt_ht_item = {};
-      parsec_static_op_t function_template_class_ptr[ttg::runtime_traits<ttg::Runtime::PaRSEC>::num_execution_spaces] =
-          {nullptr};
-      bool is_dummy = false;
-      bool defer_writer = TTG_PARSEC_DEFER_WRITER; // whether to defer writer instead of creating a new copy
-
-      typedef void (release_task_fn)(parsec_ttg_task_base_t*);
-
-      typedef struct {
-        std::size_t goal;
-        std::size_t size;
-      } size_goal_t;
-
-      /* Poor-mans virtual function
-       * We cannot use virtual inheritance or private visibility because we
-       * need offsetof for the mempool and scheduling.
-       */
-      release_task_fn* release_task_cb = nullptr;
-      bool remove_from_hash = true;
-
-      /*
-      virtual void release_task() = 0;
-      */
-    //public:
-      void release_task() {
-        release_task_cb(this);
-      }
-
-     protected:
-      /**
-       * Protected constructors: this class should not be instantiated directly
-       * but always be use through parsec_ttg_task_t.
-       */
-
-      parsec_ttg_task_base_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class,
-                             int data_count, ttg_data_copy_t **copies,
-                             bool defer_writer = TTG_PARSEC_DEFER_WRITER)
-          : data_count(data_count)
-          , copies(copies)
-          , defer_writer(defer_writer) {
-        PARSEC_LIST_ITEM_SINGLETON(&parsec_task.super);
-        parsec_task.mempool_owner = mempool;
-        parsec_task.task_class = task_class;
-        parsec_task.priority = 0;
-      }
-
-      parsec_ttg_task_base_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class,
-                             parsec_taskpool_t *taskpool, int32_t priority,
-                             int data_count, ttg_data_copy_t **copies,
-                             release_task_fn *release_fn,
-                             bool defer_writer = TTG_PARSEC_DEFER_WRITER)
-          : data_count(data_count)
-          , copies(copies)
-          , defer_writer(defer_writer)
-          , release_task_cb(release_fn) {
-            int32_t p = priority;
-        PARSEC_LIST_ITEM_SINGLETON(&parsec_task.super);
-        parsec_task.mempool_owner = mempool;
-        parsec_task.task_class = task_class;
-        parsec_task.status = PARSEC_TASK_STATUS_HOOK;
-        parsec_task.taskpool = taskpool;
-        parsec_task.priority = priority;
-        parsec_task.chore_mask = 1<<0;
-      }
-
-    public:
-      void set_dummy(bool d) { is_dummy = d; }
-      bool dummy() { return is_dummy; }
-    };
-
-    template <typename TT, bool KeyIsVoid = ttg::meta::is_void_v<typename TT::key_type>>
-    struct parsec_ttg_task_t : public parsec_ttg_task_base_t {
-      using key_type = typename TT::key_type;
-      static constexpr size_t num_streams = TT::numins;
-      TT* tt;
-      key_type key;
-      size_goal_t stream[num_streams] = {};
-#ifdef TTG_HAS_COROUTINE
-      void* suspended_task_address = nullptr;  // if not null the function is suspended
-#endif
-      ttg_data_copy_t *copies[num_streams+1] = { nullptr };  // the data copies tracked by this task
-                                                             // +1 for the copy needed during send/bcast
-
-      parsec_ttg_task_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class)
-          : parsec_ttg_task_base_t(mempool, task_class, num_streams, copies) {
-        tt_ht_item.key = pkey();
-
-        // We store the hash of the key and the address where it can be found in locals considered as a scratchpad
-        *(uintptr_t*)&(parsec_task.locals[0]) = 0; //there is no key
-        *(uintptr_t*)&(parsec_task.locals[2]) = 0; //there is no key
-      }
-
-      parsec_ttg_task_t(const key_type& key, parsec_thread_mempool_t *mempool,
-                        parsec_task_class_t *task_class, parsec_taskpool_t *taskpool,
-                        TT *tt_ptr, int32_t priority)
-          : parsec_ttg_task_base_t(mempool, task_class, taskpool, priority,
-                                   num_streams, copies, &release_task, tt_ptr->m_defer_writer)
-          , tt(tt_ptr), key(key) {
-        tt_ht_item.key = pkey();
-
-        // We store the hash of the key and the address where it can be found in locals considered as a scratchpad
-        uint64_t hv = ttg::hash<std::decay_t<decltype(key)>>{}(key);
-        *(uintptr_t*)&(parsec_task.locals[0]) = hv;
-        *(uintptr_t*)&(parsec_task.locals[2]) = reinterpret_cast<uintptr_t>(&this->key);
-      }
-
-      static void release_task(parsec_ttg_task_base_t* task_base) {
-        parsec_ttg_task_t *task = static_cast<parsec_ttg_task_t*>(task_base);
-        TT *tt = task->tt;
-        tt->release_task(task);
-      }
-
-      parsec_key_t pkey() { return reinterpret_cast<parsec_key_t>(&key); }
-    };
-
-    template <typename TT>
-    struct parsec_ttg_task_t<TT, true> : public parsec_ttg_task_base_t {
-      static constexpr size_t num_streams = TT::numins;
-      TT* tt;
-      size_goal_t stream[num_streams] = {};
-#ifdef TTG_HAS_COROUTINE
-      void* suspended_task_address = nullptr;  // if not null the function is suspended
-#endif
-      ttg_data_copy_t *copies[num_streams+1] = { nullptr };  // the data copies tracked by this task
-                                                             // +1 for the copy needed during send/bcas
-
-      parsec_ttg_task_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class)
-          : parsec_ttg_task_base_t(mempool, task_class, num_streams, copies) {
-        tt_ht_item.key = pkey();
-      }
-
-      parsec_ttg_task_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class,
-                        parsec_taskpool_t *taskpool, TT *tt_ptr, int32_t priority)
-          : parsec_ttg_task_base_t(mempool, task_class, taskpool, priority,
-                                   num_streams, copies, &release_task, tt_ptr->m_defer_writer)
-          , tt(tt_ptr) {
-        tt_ht_item.key = pkey();
-      }
-
-      static void release_task(parsec_ttg_task_base_t* task_base) {
-        parsec_ttg_task_t *task = static_cast<parsec_ttg_task_t*>(task_base);
-        TT *tt = task->tt;
-        tt->release_task(task);
-      }
-
-      parsec_key_t pkey() { return 0; }
-    };
-
     inline ttg_data_copy_t *find_copy_in_task(parsec_ttg_task_base_t *task, const void *ptr) {
       ttg_data_copy_t *res = nullptr;
       if (task == nullptr || ptr == nullptr) {
@@ -706,7 +560,7 @@ namespace ttg_parsec {
       }
       for (int i = 0; i < task->data_count; ++i) {
         auto copy = static_cast<ttg_data_copy_t *>(task->copies[i]);
-        if (NULL != copy && copy->device_private == ptr) {
+        if (NULL != copy && copy->get_ptr() == ptr) {
           res = copy;
           break;
         }
@@ -721,7 +575,7 @@ namespace ttg_parsec {
       }
       for (i = 0; i < task->data_count; ++i) {
         auto copy = static_cast<ttg_data_copy_t *>(task->copies[i]);
-        if (NULL != copy && copy->device_private == ptr) {
+        if (NULL != copy && copy->get_ptr() == ptr) {
           return i;
         }
       }
@@ -863,7 +717,7 @@ namespace ttg_parsec {
     }
 
     inline void release_data_copy(ttg_data_copy_t *copy) {
-      if (copy->is_mutable()) {
+      if (copy->is_mutable() && nullptr == copy->get_next_task()) {
         /* current task mutated the data but there are no consumers so prepare
         * the copy to be freed below */
         copy->reset_readers();
@@ -873,19 +727,24 @@ namespace ttg_parsec {
       if (readers > 1) {
         /* potentially more than one reader, decrement atomically */
         readers = copy->decrement_readers();
-      }
-      /* if there was only one reader (the current task) we release the copy */
-      if (1 == readers) {
-        if (nullptr != copy->push_task) {
+      } else if (readers == 1) {
+        /* make sure readers drop to zero */
+        readers = copy->decrement_readers<false>();
+      }
+      /* if there was only one reader (the current task) or
+       * a mutable copy and a successor, we release the copy */
+      if (1 == readers || copy->is_mutable()) {
+        if (nullptr != copy->get_next_task()) {
           /* Release the deferred task.
-          * The copy was mutable and will be mutated by the released task,
-          * so simply transfer ownership.
-          */
-          parsec_task_t *push_task = copy->push_task;
-          copy->push_task = nullptr;
-          parsec_ttg_task_base_t *deferred_op = (parsec_ttg_task_base_t *)push_task;
+           * The copy was mutable and will be mutated by the released task,
+           * so simply transfer ownership.
+           */
+          parsec_task_t *next_task = copy->get_next_task();
+          copy->set_next_task(nullptr);
+          parsec_ttg_task_base_t *deferred_op = (parsec_ttg_task_base_t *)next_task;
           deferred_op->release_task();
-        } else {
+        } else if ((1 == copy->num_ref()) || (1 == copy->drop_ref())) {
+          /* we are the last reference, delete the copy */
 #if defined(TTG_PARSEC_DEBUG_TRACK_DATA_COPIES)
           {
             const std::lock_guard<std::mutex> lock(pending_copies_mutex);
@@ -921,10 +780,10 @@ namespace ttg_parsec {
       }
 
       if (readers == copy_in->mutable_tag) {
-        if (copy_res->push_task != nullptr) {
+        if (copy_res->get_next_task() != nullptr) {
           if (readonly) {
-            parsec_ttg_task_base_t *push_task = reinterpret_cast<parsec_ttg_task_base_t *>(copy_res->push_task);
-            if (push_task->defer_writer) {
+            parsec_ttg_task_base_t *next_task = reinterpret_cast<parsec_ttg_task_base_t *>(copy_res->get_next_task());
+            if (next_task->defer_writer) {
               /* there is a writer but it signalled that it wants to wait for readers to complete */
               return copy_res;
             }
@@ -959,13 +818,13 @@ namespace ttg_parsec {
            * of the task
            */
           copy_in->mark_mutable();
-          assert(nullptr == copy_in->push_task);
+          assert(nullptr == copy_in->get_next_task());
           assert(nullptr != task);
-          copy_in->push_task = &task->parsec_task;
+          copy_in->set_next_task(&task->parsec_task);
         } else {
-          if (task->defer_writer && nullptr == copy_in->push_task) {
+          if (task->defer_writer && nullptr == copy_in->get_next_task()) {
             /* we're the first writer and want to wait for all readers to complete */
-            copy_res->push_task = &task->parsec_task;
+            copy_res->set_next_task(&task->parsec_task);
           } else {
             /* there are writers and/or waiting already of this copy already, make a copy that we can mutate */
             copy_res = NULL;
@@ -974,10 +833,10 @@ namespace ttg_parsec {
       }
 
       if (NULL == copy_res) {
-        ttg_data_copy_t *new_copy = detail::create_new_datacopy(*static_cast<Value *>(copy_in->device_private));
-        if (replace && nullptr != copy_in->push_task) {
+        ttg_data_copy_t *new_copy = detail::create_new_datacopy(*static_cast<Value *>(copy_in->get_ptr()));
+        if (replace && nullptr != copy_in->get_next_task()) {
           /* replace the task that was deferred */
-          parsec_ttg_task_base_t *deferred_op = (parsec_ttg_task_base_t *)copy_in->push_task;
+          parsec_ttg_task_base_t *deferred_op = (parsec_ttg_task_base_t *)copy_in->get_next_task();
           new_copy->mark_mutable();
           /* replace the copy in the deferred task */
           for (int i = 0; i < deferred_op->data_count; ++i) {
@@ -986,7 +845,7 @@ namespace ttg_parsec {
               break;
             }
           }
-          copy_in->push_task = nullptr;
+          copy_in->set_next_task(nullptr);
           deferred_op->release_task();
           copy_in->reset_readers();            // set the copy back to being read-only
           copy_in->increment_readers<false>(); // register as reader
@@ -1003,8 +862,6 @@ namespace ttg_parsec {
 
   }  // namespace detail
 
-  inline thread_local detail::parsec_ttg_task_base_t *parsec_ttg_caller;
-
   inline void ttg_initialize(int argc, char **argv, int num_threads, parsec_context_t *ctx) {
     if (detail::initialized_mpi()) throw std::runtime_error("ttg_parsec::ttg_initialize: can only be called once");
 
@@ -1032,6 +889,7 @@ namespace ttg_parsec {
     if(0 == ttg::default_execution_context().rank())
       ttg::default_execution_context().impl().final_task();
     ttg::detail::set_default_world(ttg::World{});  // reset the default world
+    detail::ptr::drop_all_ptr();
     ttg::detail::destroy_worlds<ttg_parsec::WorldImpl>();
     if (detail::initialized_mpi()) MPI_Finalize();
   }
@@ -1140,6 +998,7 @@ namespace ttg_parsec {
     static constexpr int numouts = std::tuple_size_v<output_terminalsT>;       // number of outputs
     static constexpr int numflows = std::max(numins, numouts);                 // max number of flows
 
+   public:
     /// @return true if derivedT::have_cuda_op exists and is defined to true
     static constexpr bool derived_has_cuda_op() {
       if constexpr (ttg::meta::is_detected_v<have_cuda_op_non_type_t, derivedT>) {
@@ -1149,7 +1008,6 @@ namespace ttg_parsec {
       }
     }
 
-   public:
     using ttT = TT;
     using key_type = keyT;
     using input_terminals_type = ttg::detail::input_terminals_tuple_t<keyT, input_tuple_type>;
@@ -1323,7 +1181,7 @@ namespace ttg_parsec {
     static input_refs_tuple_type make_tuple_of_ref_from_array(task_t *task, std::index_sequence<IS...>) {
       return input_refs_tuple_type{static_cast<std::tuple_element_t<IS, input_refs_tuple_type>>(
           *reinterpret_cast<std::remove_reference_t<std::tuple_element_t<IS, input_refs_tuple_type>> *>(
-              task->copies[IS]->device_private))...};
+              task->copies[IS]->get_ptr()))...};
     }
 
     /**
@@ -1371,33 +1229,43 @@ namespace ttg_parsec {
     template <ttg::ExecutionSpace Space>
     static parsec_hook_return_t device_static_op(parsec_task_t* parsec_task) {
       static_assert(derived_has_cuda_op());
+
+      int dev_index;
+      double ratio = 1.0;
+
+      /* get a device and come back if we need another one */
+      dev_index = parsec_get_best_device(parsec_task, ratio);
+      assert(dev_index >= 0);
+      if (dev_index < 2) {
+          return PARSEC_HOOK_RETURN_NEXT; /* Fall back */
+      }
+
       task_t *task = (task_t*)parsec_task;
       parsec_execution_stream_s *es = task->tt->world.impl().execution_stream();
 
       std::cout << "device_static_op: task " << parsec_task << std::endl;
 
-      int dev_index;
-      double ratio = 1.0;
 
       /* set up a device task */
-      /* TODO: take them from a free-list */
       parsec_gpu_task_t *gpu_task;
-      gpu_task = (parsec_gpu_task_t *) calloc(1, sizeof(parsec_gpu_task_t));
+      /* PaRSEC wants to free the gpu_task, because F***K ownerships */
+      gpu_task = static_cast<parsec_gpu_task_t*>(std::calloc(1, sizeof(*gpu_task)));
       PARSEC_OBJ_CONSTRUCT(gpu_task, parsec_list_item_t);
-
       gpu_task->ec = parsec_task;
-      /* come back into this function
-        * TODO: have a specialized function that checks
-        */
-      gpu_task->submit = &TT::device_static_submit<Space>;
       gpu_task->task_type = 0; // user task
       gpu_task->load = 1.0;    // TODO: can we do better?
       gpu_task->last_data_check_epoch = -1; // used internally
       gpu_task->pushout = 0;
+      gpu_task->submit = &TT::device_static_submit<Space>;
+
+      /* set the gpu_task so it's available in register_device_memory */
+      task->dev_ptr->gpu_task = gpu_task;
 
       // first invocation of the coroutine to get the coroutine handle
       static_op<Space>(parsec_task);
 
+      /* when we come back here, the flows in gpu_task are set (see register_device_memory) */
+
       // get the device task from the coroutine handle
       auto dev_task = ttg::device_task_handle_type::from_address(task->suspended_task_address);
 
@@ -1407,6 +1275,7 @@ namespace ttg_parsec {
       /* for now make sure we're waiting for transfers and the coro hasn't skipped this step */
       assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_TRANSFER);
 
+#if 0
       // manage the gpu_task flows
 
       // set the input flows
@@ -1434,7 +1303,7 @@ namespace ttg_parsec {
           /* try to find the view in the task and allocate a new copy if needed */
           for (int i = 0; nullptr == copy && i < numins; ++i) {
             ttg_parsec::detail::ttg_data_copy_t* obj_copy = task->copies[i];
-            if (obj_copy->device_private == host_obj) {
+            if (obj_copy->get_ptr() == host_obj) {
               for (auto& dev_copy : *obj_copy) {
                 if (view_span.data() == dev_copy->device_private) {
                   copy = dev_copy;
@@ -1485,12 +1354,7 @@ namespace ttg_parsec {
         task->parsec_task.data[i].data_in = nullptr;
         task->parsec_task.data[i].source_repo_entry = NULL;
       }
-
-      dev_index = parsec_get_best_device(parsec_task, ratio);
-      assert(dev_index >= 0);
-      if (dev_index < 2) {
-          return PARSEC_HOOK_RETURN_NEXT; /* Fall back */
-      }
+#endif // 0
 
       parsec_device_module_t *device = parsec_mca_device_get(dev_index);
       assert(NULL != device);
@@ -1530,8 +1394,8 @@ namespace ttg_parsec {
 
         ttT *baseobj = task->tt;
         derivedT *obj = static_cast<derivedT *>(baseobj);
-        assert(parsec_ttg_caller == nullptr);
-        parsec_ttg_caller = static_cast<detail::parsec_ttg_task_base_t*>(task);
+        assert(detail::parsec_ttg_caller == nullptr);
+        detail::parsec_ttg_caller = static_cast<detail::parsec_ttg_task_base_t*>(task);
         if (obj->tracing()) {
           if constexpr (!ttg::meta::is_void_v<keyT>)
             ttg::trace(obj->get_world().rank(), ":", obj->get_name(), " : ", task->key, ": executing");
@@ -1552,12 +1416,12 @@ namespace ttg_parsec {
         } else {
           ttg::abort();
         }
-        parsec_ttg_caller = nullptr;
+        detail::parsec_ttg_caller = nullptr;
       }
       else {  // resume the suspended coroutine
         auto coro = static_cast<ttg::device_task>(ttg::device_task_handle_type::from_address(suspended_task_address));
-        assert(parsec_ttg_caller == nullptr);
-        parsec_ttg_caller = static_cast<detail::parsec_ttg_task_base_t*>(task);
+        assert(detail::parsec_ttg_caller == nullptr);
+        detail::parsec_ttg_caller = static_cast<detail::parsec_ttg_task_base_t*>(task);
         // TODO: unify the outputs tls handling
         auto old_output_tls_ptr = task->tt->outputs_tls_ptr_accessor();
         task->tt->set_outputs_tls_ptr();
@@ -1567,7 +1431,7 @@ namespace ttg_parsec {
           suspended_task_address = nullptr;
         }
         task->tt->set_outputs_tls_ptr(old_output_tls_ptr);
-        parsec_ttg_caller = nullptr;
+        detail::parsec_ttg_caller = nullptr;
 #if 0
 #ifdef TTG_HAS_COROUTINE
         auto ret = static_cast<ttg::resumable_task>(ttg::coroutine_handle<>::from_address(suspended_task_address));
@@ -1638,15 +1502,15 @@ namespace ttg_parsec {
       if (suspended_task_address == nullptr) {  // task is a coroutine that has not started or an ordinary function
         ttT *baseobj = (ttT *)task->object_ptr;
         derivedT *obj = (derivedT *)task->object_ptr;
-        assert(parsec_ttg_caller == NULL);
-        parsec_ttg_caller = task;
+        assert(detail::parsec_ttg_caller == NULL);
+        detail::parsec_ttg_caller = task;
         if constexpr (!ttg::meta::is_void_v<keyT>) {
           TTG_PROCESS_TT_OP_RETURN(suspended_task_address, baseobj->template op<Space>(task->key, obj->output_terminals));
         } else if constexpr (ttg::meta::is_void_v<keyT>) {
           TTG_PROCESS_TT_OP_RETURN(suspended_task_address, baseobj->template op<Space>(obj->output_terminals));
         } else  // unreachable
           ttg:: abort();
-        parsec_ttg_caller = NULL;
+        detail::parsec_ttg_caller = NULL;
       }
       else {
 #ifdef TTG_HAS_COROUTINE
@@ -1773,13 +1637,13 @@ namespace ttg_parsec {
       dummy->parsec_task.taskpool = world.impl().taskpool();
 
       /* save the current task and set the dummy task */
-      auto parsec_ttg_caller_save = parsec_ttg_caller;
-      parsec_ttg_caller = dummy;
+      auto parsec_ttg_caller_save = detail::parsec_ttg_caller;
+      detail::parsec_ttg_caller = dummy;
 
       /* iterate over the keys and have them use the copy we made */
       parsec_task_t *task_ring = nullptr;
       for (auto &&key : keylist) {
-        set_arg_local_impl<i>(key, *reinterpret_cast<valueT *>(copy->device_private), copy, &task_ring);
+        set_arg_local_impl<i>(key, *reinterpret_cast<valueT *>(copy->get_ptr()), copy, &task_ring);
       }
 
       if (nullptr != task_ring) {
@@ -1789,7 +1653,7 @@ namespace ttg_parsec {
       }
 
       /* restore the previous task */
-      parsec_ttg_caller = parsec_ttg_caller_save;
+      detail::parsec_ttg_caller = parsec_ttg_caller_save;
 
       /* release the dummy task */
       complete_task_and_release(es, &dummy->parsec_task);
@@ -1828,7 +1692,7 @@ namespace ttg_parsec {
           using decvalueT = std::decay_t<valueT>;
           if constexpr (!ttg::has_split_metadata<decvalueT>::value) {
             detail::ttg_data_copy_t *copy = detail::create_new_datacopy(decvalueT{});
-            unpack(*static_cast<decvalueT *>(copy->device_private), msg->bytes, pos);
+            unpack(*static_cast<decvalueT *>(copy->get_ptr()), msg->bytes, pos);
 
             set_arg_from_msg_keylist<i, decvalueT>(ttg::span<keyT>(&keylist[0], num_keys), copy);
           } else {
@@ -1870,7 +1734,7 @@ namespace ttg_parsec {
                     set_arg_from_msg_keylist<i, decvalueT>(keylist, copy);
                     this->world.impl().decrement_inflight_msg();
                   });
-              auto &val = *static_cast<decvalueT *>(copy->device_private);
+              auto &val = *static_cast<decvalueT *>(copy->get_ptr());
 
               using ActivationT = std::decay_t<decltype(*activation)>;
 
@@ -2111,8 +1975,8 @@ namespace ttg_parsec {
 
       if( world_impl.dag_profiling() ) {
 #if defined(PARSEC_PROF_GRAPHER)
-        if(NULL != parsec_ttg_caller && !parsec_ttg_caller->dummy()) {
-          int orig_index = detail::find_index_of_copy_in_task(parsec_ttg_caller, &value);
+        if(NULL != detail::parsec_ttg_caller && !detail::parsec_ttg_caller->dummy()) {
+          int orig_index = detail::find_index_of_copy_in_task(detail::parsec_ttg_caller, &value);
           char orig_str[32];
           char dest_str[32];
           if(orig_index >= 0) {
@@ -2125,7 +1989,7 @@ namespace ttg_parsec {
                               .flow_index = 0, .flow_datatype_mask = ~0 };
           parsec_flow_t dest{ .name = dest_str, .sym_type = PARSEC_SYM_INOUT, .flow_flags = PARSEC_FLOW_ACCESS_RW,
                               .flow_index = 0, .flow_datatype_mask = ~0 };
-          parsec_prof_grapher_dep(&parsec_ttg_caller->parsec_task, &task->parsec_task, discover_task ? 1 : 0, &orig, &dest);
+          parsec_prof_grapher_dep(&detail::parsec_ttg_caller->parsec_task, &task->parsec_task, discover_task ? 1 : 0, &orig, &dest);
         }
 #endif
       }
@@ -2145,7 +2009,7 @@ namespace ttg_parsec {
             copy = detail::create_new_datacopy(std::forward<Value>(value));
             task->copies[i] = copy;
           } else {
-            reducer(*reinterpret_cast<std::decay_t<valueT> *>(copy->device_private), value);
+            reducer(*reinterpret_cast<std::decay_t<valueT> *>(copy->get_ptr()), value);
           }
         } else {
           reducer();  // even if this was a control input, must execute the reducer for possible side effects
@@ -2166,8 +2030,8 @@ namespace ttg_parsec {
           }
 
           detail::ttg_data_copy_t *copy = copy_in;
-          if (nullptr == copy_in && nullptr != parsec_ttg_caller) {
-            copy = detail::find_copy_in_task(parsec_ttg_caller, &value);
+          if (nullptr == copy_in && nullptr != detail::parsec_ttg_caller) {
+            copy = detail::find_copy_in_task(detail::parsec_ttg_caller, &value);
           }
 
           if (nullptr != copy) {
@@ -2179,7 +2043,7 @@ namespace ttg_parsec {
           /* if we registered as a writer and were the first to register with this copy
            * we need to defer the release of this task to give other tasks a chance to
            * make a copy of the original data */
-          release = (copy->push_task != &task->parsec_task);
+          release = (copy->get_next_task() != &task->parsec_task);
           task->copies[i] = copy;
         }
       }
@@ -2315,7 +2179,7 @@ namespace ttg_parsec {
           pos = pack(value, msg->bytes, pos);
         } else {
           detail::ttg_data_copy_t *copy;
-          copy = detail::find_copy_in_task(parsec_ttg_caller, &value);
+          copy = detail::find_copy_in_task(detail::parsec_ttg_caller, &value);
           if (nullptr == copy) {
             // We need to create a copy for this data, as it does not exist yet.
             copy = detail::create_new_datacopy(std::forward<Value>(value));
@@ -2333,7 +2197,7 @@ namespace ttg_parsec {
           std::memcpy(msg->bytes + pos, &rank, sizeof(rank));
           pos += sizeof(rank);
 
-          auto iovecs = descr.get_data(*static_cast<decvalueT *>(copy->device_private));
+          auto iovecs = descr.get_data(*static_cast<decvalueT *>(copy->get_ptr()));
 
           int32_t num_iovs = std::distance(std::begin(iovecs), std::end(iovecs));
           std::memcpy(msg->bytes + pos, &num_iovs, sizeof(num_iovs));
@@ -2390,8 +2254,8 @@ namespace ttg_parsec {
       }
 #endif
 #if defined(PARSEC_PROF_GRAPHER)
-      if(NULL != parsec_ttg_caller && !parsec_ttg_caller->dummy()) {
-        int orig_index = detail::find_index_of_copy_in_task(parsec_ttg_caller, &value);
+      if(NULL != detail::parsec_ttg_caller && !detail::parsec_ttg_caller->dummy()) {
+        int orig_index = detail::find_index_of_copy_in_task(detail::parsec_ttg_caller, &value);
         char orig_str[32];
         char dest_str[32];
         if(orig_index >= 0) {
@@ -2405,7 +2269,7 @@ namespace ttg_parsec {
         parsec_flow_t dest{ .name = dest_str, .sym_type = PARSEC_SYM_INOUT, .flow_flags = PARSEC_FLOW_ACCESS_RW,
                             .flow_index = 0, .flow_datatype_mask = ~0 };
         task_t *task = create_new_task(key);
-        parsec_prof_grapher_dep(&parsec_ttg_caller->parsec_task, &task->parsec_task, 0, &orig, &dest);
+        parsec_prof_grapher_dep(&detail::parsec_ttg_caller->parsec_task, &task->parsec_task, 0, &orig, &dest);
         delete task;
       }
 #endif
@@ -2420,8 +2284,8 @@ namespace ttg_parsec {
 #endif
       parsec_task_t *task_ring = nullptr;
       detail::ttg_data_copy_t *copy = nullptr;
-      if (nullptr != parsec_ttg_caller) {
-        copy = detail::find_copy_in_task(parsec_ttg_caller, &value);
+      if (nullptr != detail::parsec_ttg_caller) {
+        copy = detail::find_copy_in_task(detail::parsec_ttg_caller, &value);
       }
 
       for (auto it = begin; it != end; ++it) {
@@ -2566,7 +2430,7 @@ namespace ttg_parsec {
         size_t metadata_size = sizeof(metadata);
 
         detail::ttg_data_copy_t *copy;
-        copy = detail::find_copy_in_task(parsec_ttg_caller, &value);
+        copy = detail::find_copy_in_task(detail::parsec_ttg_caller, &value);
         assert(nullptr != copy);
 
         parsec_taskpool_t *tp = world_impl.taskpool();
@@ -3249,7 +3113,7 @@ namespace ttg_parsec {
       self.release_task = &parsec_release_task_to_mempool_update_nbtasks;
       self.complete_execution = complete_task_and_release;
 
-      for (i = 0; i < numins; i++) {
+      for (i = 0; i < MAX_PARAM_COUNT; i++) {
         parsec_flow_t *flow = new parsec_flow_t;
         flow->name = strdup((std::string("flow in") + std::to_string(i)).c_str());
         flow->sym_type = PARSEC_SYM_INOUT;
@@ -3258,13 +3122,13 @@ namespace ttg_parsec {
         flow->dep_in[0] = NULL;
         flow->dep_out[0] = NULL;
         flow->flow_index = i;
-        flow->flow_datatype_mask = (1 << i);
+        flow->flow_datatype_mask = ~0;
         *((parsec_flow_t **)&(self.in[i])) = flow;
       }
-      *((parsec_flow_t **)&(self.in[i])) = NULL;
-      initialize_flows<input_terminals_type>(self.in);
+      //*((parsec_flow_t **)&(self.in[i])) = NULL;
+      //initialize_flows<input_terminals_type>(self.in);
 
-      for (i = 0; i < numouts; i++) {
+      for (i = 0; i < MAX_PARAM_COUNT; i++) {
         parsec_flow_t *flow = new parsec_flow_t;
         flow->name = strdup((std::string("flow out") + std::to_string(i)).c_str());
         flow->sym_type = PARSEC_SYM_INOUT;
@@ -3275,7 +3139,7 @@ namespace ttg_parsec {
         flow->flow_datatype_mask = (1 << i);
         *((parsec_flow_t **)&(self.out[i])) = flow;
       }
-      *((parsec_flow_t **)&(self.out[i])) = NULL;
+      //*((parsec_flow_t **)&(self.out[i])) = NULL;
 
       self.flags = 0;
       self.dependencies_goal = numins; /* (~(uint32_t)0) >> (32 - numins); */
@@ -3469,6 +3333,36 @@ namespace ttg_parsec {
         TTBase::invoke();
     }
 
+#if 0
+    template<typename Key, typename Arg, typename... Args, std::size_t I, std::size_t... Is>
+    void invoke_arglist(const Key& key, Arg&& arg, Args&&... args, std::index_sequence<I, Is...>) {
+      if constexpr (ttg::detail::view_trait<Arg>::is_view) {
+        /* need to unpack the view */
+        if constexpr (std::is_rvalue_reference_v<Arg>) {
+          ttg::detail::value_copy_handler
+        }
+      } else {
+        ttg::detail::value_copy_handler<ttg::Runtime::PaRSEC> vch{arg};
+      }
+      if constexpr (sizeof...(Is) > 0) {
+        /* recursive next argument */
+        invoke_arglist(key, std::forward<Args>(args)..., std::index_sequence<Is...>{});
+      }
+    }
+
+    // Manual injection of a task with all input arguments specified as a tuple
+    template <typename Key = keyT, typename Arg, typename... Args>
+    std::enable_if_t<!ttg::meta::is_void_v<Key> && !ttg::meta::is_empty_tuple_v<input_values_tuple_type>, void> invoke(
+        const Key &key, Arg&& arg, Args&&... args) {
+      TTG_OP_ASSERT_EXECUTABLE();
+      /* trigger non-void inputs */
+      set_args(ttg::meta::nonvoid_index_seq<actual_input_tuple_type>{}, key, args);
+      /* trigger void inputs */
+      using void_index_seq = ttg::meta::void_index_seq<actual_input_tuple_type>;
+      set_args(void_index_seq{}, key, ttg::detail::make_void_tuple<void_index_seq::size()>());
+    }
+#endif // 0
+
     void set_defer_writer(bool value) {
       m_defer_writer = value;
     }
@@ -3548,6 +3442,106 @@ namespace ttg_parsec {
 
 #include "ttg/make_tt.h"
 
+
+  namespace detail {
+    template<typename... Views, std::size_t I, std::size_t... Is>
+    bool register_device_memory(std::tuple<Views&...> &views, std::index_sequence<I, Is...>) {
+      static_assert(I < MAX_PARAM_COUNT,
+                    "PaRSEC only supports MAX_PARAM_COUNT device input/outputs. "
+                    "Increase MAX_PARAM_COUNT and recompile PaRSEC/TTG.");
+      using view_type = std::remove_reference_t<std::tuple_element_t<I, std::tuple<Views&...>>>;
+      auto& view = std::get<I>(views);
+      bool is_current = false;
+      static_assert(ttg::is_buffer_v<view_type> || ttg_parsec::is_devicescratch_v<view_type>);
+      /* get_parsec_data is overloaded for buffer and devicescratch */
+      parsec_data_t* data = detail::get_parsec_data(view);
+      /* TODO: check whether the device is current */
+
+      auto flow_flags = PARSEC_FLOW_ACCESS_RW;
+      bool pushout = false;
+      if constexpr (std::is_const_v<view_type>) {
+        flow_flags = PARSEC_FLOW_ACCESS_READ;
+      } else if constexpr (ttg_parsec::is_devicescratch_v<view_type>) {
+        switch(view.scope()) {
+          case ttg::scope::Allocate:
+            flow_flags = PARSEC_FLOW_ACCESS_NONE;
+            break;
+          case ttg::scope::SyncIn:
+            flow_flags = PARSEC_FLOW_ACCESS_READ;
+            break;
+          case ttg::scope::SyncOut:
+            flow_flags = PARSEC_FLOW_ACCESS_WRITE;
+            pushout = true;
+            break;
+          case ttg::scope::SyncInOut:
+            flow_flags = PARSEC_FLOW_ACCESS_RW;
+            pushout = true;
+            break;
+        }
+      }
+      assert(nullptr != detail::parsec_ttg_caller->dev_ptr);
+      parsec_gpu_task_t *gpu_task = detail::parsec_ttg_caller->dev_ptr->gpu_task;
+      parsec_flow_t *flows = detail::parsec_ttg_caller->dev_ptr->flows;
+
+      std::cout << "register_device_memory task " << detail::parsec_ttg_caller << " data " << I << " "
+                << data << " size " << data->nb_elts << std::endl;
+
+      /* build the flow */
+      /* TODO: reuse the flows of the task class? How can we control the sync direction then? */
+      flows[I] = parsec_flow_t{.name = nullptr,
+                               .sym_type = PARSEC_SYM_INOUT,
+                               .flow_flags = static_cast<uint8_t>(flow_flags),
+                               .flow_index = I,
+                               .flow_datatype_mask = ~0 };
+
+      gpu_task->flow_nb_elts[I] = data->nb_elts; // size in bytes
+      gpu_task->flow[I] = &flows[I];
+
+      if (pushout) {
+        std::cout << "PUSHOUT " << I << std::endl;
+        gpu_task->pushout |= 1<<I;
+      }
+
+      /* set the input data copy, parsec will take care of the transfer
+       * and the buffer will look at the parsec_data_t for the current pointer */
+      //detail::parsec_ttg_caller->parsec_task.data[I].data_in = data->device_copies[data->owner_device];
+      detail::parsec_ttg_caller->parsec_task.data[I].data_in = data->device_copies[0];
+      detail::parsec_ttg_caller->parsec_task.data[I].source_repo_entry = NULL;
+
+      if constexpr (sizeof...(Is) > 0) {
+        is_current |= register_device_memory(views, std::index_sequence<Is...>{});
+      }
+      return is_current;
+    }
+  } // namespace detail
+
+  /* Takes a tuple of ttg::Views or ttg::buffers and register them
+   * with the currently executing task. Returns true if all memory
+   * is current on the target device, false if transfers are required. */
+  template<typename... Views>
+  bool register_device_memory(std::tuple<Views&...> &views) {
+    if (nullptr == detail::parsec_ttg_caller) {
+      throw std::runtime_error("register_device_memory may only be invoked from inside a task!");
+    }
+
+    if (nullptr == detail::parsec_ttg_caller->dev_ptr) {
+      throw std::runtime_error("register_device_memory called inside a non-gpu task!");
+    }
+
+    bool is_current = detail::register_device_memory(views, std::index_sequence_for<Views...>{});
+
+    /* reset all entries in the current task */
+    for (int i = sizeof...(Views); i < MAX_PARAM_COUNT; ++i) {
+      detail::parsec_ttg_caller->parsec_task.data[i].data_in = nullptr;
+      detail::parsec_ttg_caller->dev_ptr->flows[i].flow_flags = PARSEC_FLOW_ACCESS_NONE;
+      detail::parsec_ttg_caller->dev_ptr->flows[i].flow_index = i;
+      detail::parsec_ttg_caller->dev_ptr->gpu_task->flow[i] = &detail::parsec_ttg_caller->dev_ptr->flows[i];
+      detail::parsec_ttg_caller->dev_ptr->gpu_task->flow_nb_elts[i] = 0;
+    }
+
+    return is_current;
+  }
+
 }  // namespace ttg_parsec
 
 /**
@@ -3560,31 +3554,34 @@ struct ttg::detail::value_copy_handler<ttg::Runtime::PaRSEC> {
  private:
   ttg_parsec::detail::ttg_data_copy_t *copy_to_remove = nullptr;
 
+
+
  public:
   ~value_copy_handler() {
     if (nullptr != copy_to_remove) {
-      ttg_parsec::detail::remove_data_copy(copy_to_remove, parsec_ttg_caller);
+      ttg_parsec::detail::remove_data_copy(copy_to_remove, ttg_parsec::detail::parsec_ttg_caller);
       ttg_parsec::detail::release_data_copy(copy_to_remove);
     }
   }
 
   template <typename Value>
   inline Value &&operator()(Value &&value) {
-    if (nullptr == parsec_ttg_caller) {
+    if (nullptr == ttg_parsec::detail::parsec_ttg_caller) {
       ttg::print("ERROR: ttg_send or ttg_broadcast called outside of a task!\n");
     }
+    using value_type = std::remove_reference_t<Value>;
     ttg_parsec::detail::ttg_data_copy_t *copy;
-    copy = ttg_parsec::detail::find_copy_in_task(parsec_ttg_caller, &value);
-    Value *value_ptr = &value;
+    copy = ttg_parsec::detail::find_copy_in_task(ttg_parsec::detail::parsec_ttg_caller, &value);
+    value_type *value_ptr = &value;
     if (nullptr == copy) {
       /**
        * the value is not known, create a copy that we can track
        * depending on Value, this uses either the copy or move constructor
        */
       copy = ttg_parsec::detail::create_new_datacopy(std::forward<Value>(value));
-      bool inserted = ttg_parsec::detail::add_copy_to_task(copy, parsec_ttg_caller);
+      bool inserted = ttg_parsec::detail::add_copy_to_task(copy, ttg_parsec::detail::parsec_ttg_caller);
       assert(inserted);
-      value_ptr = reinterpret_cast<Value *>(copy->device_private);
+      value_ptr = reinterpret_cast<value_type *>(copy->get_ptr());
       copy_to_remove = copy;
     } else {
       /* this copy won't be modified anymore so mark it as read-only */
@@ -3595,11 +3592,11 @@ struct ttg::detail::value_copy_handler<ttg::Runtime::PaRSEC> {
 
   template <typename Value>
   inline const Value &operator()(const Value &value) {
-    if (nullptr == parsec_ttg_caller) {
+    if (nullptr == ttg_parsec::detail::parsec_ttg_caller) {
       ttg::print("ERROR: ttg_send or ttg_broadcast called outside of a task!\n");
     }
     ttg_parsec::detail::ttg_data_copy_t *copy;
-    copy = ttg_parsec::detail::find_copy_in_task(parsec_ttg_caller, &value);
+    copy = ttg_parsec::detail::find_copy_in_task(ttg_parsec::detail::parsec_ttg_caller, &value);
     const Value *value_ptr = &value;
     if (nullptr == copy) {
       /**
@@ -3607,30 +3604,103 @@ struct ttg::detail::value_copy_handler<ttg::Runtime::PaRSEC> {
        * depending on Value, this uses either the copy or move constructor
        */
       copy = ttg_parsec::detail::create_new_datacopy(value);
-      bool inserted = ttg_parsec::detail::add_copy_to_task(copy, parsec_ttg_caller);
+      bool inserted = ttg_parsec::detail::add_copy_to_task(copy, ttg_parsec::detail::parsec_ttg_caller);
       assert(inserted);
-      value_ptr = reinterpret_cast<Value *>(copy->device_private);
+      value_ptr = reinterpret_cast<Value *>(copy->get_ptr());
       copy_to_remove = copy;
     }
     return *value_ptr;
   }
 
+#if 0
   /* we have to make a copy of non-const data as the user may modify it after
    * send/broadcast */
   template <typename Value, typename Enabler = std::enable_if_t<!std::is_const_v<Value>>>
   inline Value &operator()(Value &value) {
-    if (nullptr == parsec_ttg_caller) {
+    if (nullptr == ttg_parsec::detail::parsec_ttg_caller) {
       ttg::print("ERROR: ttg_send or ttg_broadcast called outside of a task!\n");
     }
     /* the value is not known, create a copy that we can track */
     ttg_parsec::detail::ttg_data_copy_t *copy;
     copy = ttg_parsec::detail::create_new_datacopy(value);
-    bool inserted = ttg_parsec::detail::add_copy_to_task(copy, parsec_ttg_caller);
+    bool inserted = ttg_parsec::detail::add_copy_to_task(copy, ttg_parsec::detail::parsec_ttg_caller);
     assert(inserted);
-    Value *value_ptr = reinterpret_cast<Value *>(copy->device_private);
+    Value *value_ptr = reinterpret_cast<Value *>(copy->get_ptr());
     copy_to_remove = copy;
     return *value_ptr;
   }
+
+  /**
+   * Overload for PersistentView objects
+   *
+   * TODO: make sure the device copy is current?!
+   */
+
+  template <typename Value>
+  inline Value &operator()(ttg::PersistentView<Value> &&ptr) {
+    Value& value_ref = *ptr;
+
+    /* register the copy with the task so that the TT can find it */
+    ttg_parsec::detail::ttg_data_copy_t *copy;
+    copy = ptr.get_copy();
+    bool inserted = ttg_parsec::detail::add_copy_to_task(copy, ttg_parsec::detail::parsec_ttg_caller);
+    if (inserted) {
+      copy_to_remove = copy;
+      /* if this is the first time we see this copy again we add the TTG reference */
+      if (0 == copy->num_readers()) {
+        copy->add_ref();
+      }
+      /* steal the copy from the ptr */
+      ptr.reset();
+    }
+
+    return value_ref;
+  }
+
+  template <typename Value>
+  inline Value &operator()(ttg::PersistentView<Value> &ptr) {
+    Value& value_ref = *ptr;
+
+    /* register the copy with the task so that the TT can find it */
+    ttg_parsec::detail::ttg_data_copy_t *copy;
+    copy = ptr.get_copy();
+    bool inserted = ttg_parsec::detail::add_copy_to_task(copy, ttg_parsec::detail::parsec_ttg_caller);
+    if (inserted) {
+      copy_to_remove = copy;
+      /* if this is the first time we see this copy again we add the TTG reference */
+      if (0 == copy->readers) {
+        copy->add_ref();
+      }
+    }
+    return value_ref;
+  }
+
+  template <typename Value>
+  inline const Value &operator()(const ttg::PersistentView<Value> &ptr) {
+    const Value& value_ref = *ptr;
+    /* register the copy with the task so that the TT can find it */
+    ttg_parsec::detail::ttg_data_copy_t *copy;
+    copy = ptr.get_copy();
+    bool inserted = ttg_parsec::detail::add_copy_to_task(copy, ttg_parsec::detail::parsec_ttg_caller);
+    if (inserted) {
+      copy_to_remove = copy;
+      /* if this is the first time we see this copy again we add the TTG reference */
+      if (0 == copy->num_readers()) {
+        copy->add_ref();
+      }
+    }
+    return value_ref;
+  }
+
+  /**
+   * Overload for ttg::ptr objects
+   *
+   * TODO: implement :)
+   *
+   * TODO: make sure the host copies are current?!
+   */
+#endif // 0
+
 };
 
 #endif  // PARSEC_TTG_H_INCLUDED
diff --git a/ttg/ttg/parsec/ttg_data_copy.h b/ttg/ttg/parsec/ttg_data_copy.h
index 88d5531f3..66c907356 100644
--- a/ttg/ttg/parsec/ttg_data_copy.h
+++ b/ttg/ttg/parsec/ttg_data_copy.h
@@ -5,36 +5,102 @@
 #include <limits>
 #include <vector>
 #include <iterator>
+#include <atomic>
+#include <type_traits>
 
 #include <parsec.h>
 
+#include "ttg/parsec/thread_local.h"
+
 
 namespace ttg_parsec {
 
   namespace detail {
 
-    /* Extension of PaRSEC's data copy. Note that we use the readers field
-    * to facilitate the ref-counting of the data copy.
-    * TODO: create abstractions for all fields in parsec_data_copy_t that we access.
-    */
-    struct ttg_data_copy_t : public parsec_data_copy_t {
-#if defined(PARSEC_PROF_TRACE) && defined(PARSEC_TTG_PROFILE_BACKEND)
-      int64_t size;
-      int64_t uid;
-#endif
+    /* Non-owning copy-tracking wrapper, accounting for N readers or 1 writer.
+     * Also counts external references, which are not treated as
+     * readers or writers but merely prevent the object from being
+     * destroyed once no readers/writers exist.
+     */
+    struct ttg_data_copy_t {
 
       /* special value assigned to parsec_data_copy_t::readers to mark the copy as
       * mutable, i.e., a task will modify it */
       static constexpr int mutable_tag = std::numeric_limits<int>::min();
 
+      ttg_data_copy_t()
+      {
+        /* set the container ptr here, will be reset in the the ttg_data_value_copy_t ctor */
+        ttg_data_copy_container() = this;
+      }
+
+      ttg_data_copy_t(const ttg_data_copy_t& c) {
+        /* we allow copying but do not copy any data over from the original
+         * device copies will have to be allocated again
+         * and it's a new object to reference */
+
+        /* set the container ptr here, will be reset in the the ttg_data_value_copy_t ctor */
+        ttg_data_copy_container() = this;
+      }
+
+      ttg_data_copy_t(ttg_data_copy_t&& c)
+      : m_ptr(c.m_ptr)
+      , m_next_task(c.m_next_task)
+      , m_readers(c.m_readers)
+      , m_refs(c.m_refs.load(std::memory_order_relaxed))
+      , m_dev_data(std::move(c.m_dev_data))
+      , m_single_dev_data(c.m_single_dev_data)
+      , m_num_dev_data(c.m_num_dev_data)
+      {
+        c.m_num_dev_data = 0;
+        c.m_readers = 0;
+        c.m_single_dev_data = nullptr;
+
+        /* set the container ptr here, will be reset in the the ttg_data_value_copy_t ctor */
+        ttg_data_copy_container() = this;
+      }
+
+      ttg_data_copy_t& operator=(ttg_data_copy_t&& c)
+      {
+        m_ptr = c.m_ptr;
+        c.m_ptr = nullptr;
+        m_next_task = c.m_next_task;
+        c.m_next_task = nullptr;
+        m_readers = c.m_readers;
+        c.m_readers = 0;
+        m_refs.store(c.m_refs.load(std::memory_order_relaxed), std::memory_order_relaxed);
+        c.m_refs.store(0, std::memory_order_relaxed);
+        m_dev_data = std::move(c.m_dev_data);
+        m_single_dev_data = c.m_single_dev_data;
+        c.m_single_dev_data = nullptr;
+        m_num_dev_data = c.m_num_dev_data;
+        c.m_num_dev_data = 0;
+        /* set the container ptr here, will be reset in the the ttg_data_value_copy_t ctor */
+        ttg_data_copy_container() = this;
+        return *this;
+      }
+
+      ttg_data_copy_t& operator=(const ttg_data_copy_t& c) {
+        /* we allow copying but do not copy any data over from the original
+         * device copies will have to be allocated again
+         * and it's a new object to reference */
+
+        /* set the container ptr here, will be reset in the the ttg_data_value_copy_t ctor */
+        ttg_data_copy_container() = this;
+        return *this;
+      }
+
+      /* mark destructor as virtual */
+      virtual ~ttg_data_copy_t() = default;
+
       /* Returns true if the copy is mutable */
       bool is_mutable() const {
-        return this->readers == mutable_tag;
+        return m_readers == mutable_tag;
       }
 
       /* Mark the copy as mutable */
       void mark_mutable() {
-        this->readers = mutable_tag;
+        m_readers = mutable_tag;
       }
 
       /* Increment the reader counter and return previous value
@@ -43,9 +109,11 @@ namespace ttg_parsec {
       template<bool Atomic = true>
       int increment_readers() {
         if constexpr(Atomic) {
-          return parsec_atomic_fetch_inc_int32(&this->readers);
+          //return parsec_atomic_fetch_inc_int32(&m_readers);
+          std::atomic_ref<int32_t> a{m_readers};
+          return a.fetch_add(1, std::memory_order_relaxed);
         } else {
-          return this->readers++;
+          return m_readers++;
         }
       }
 
@@ -53,7 +121,7 @@ namespace ttg_parsec {
       * Reset the number of readers to read-only with a single reader.
       */
       void reset_readers() {
-        this->readers = 1;
+        m_readers = 1;
       }
 
       /* Decrement the reader counter and return previous value.
@@ -62,84 +130,122 @@ namespace ttg_parsec {
       template<bool Atomic = true>
       int decrement_readers() {
         if constexpr(Atomic) {
-          return parsec_atomic_fetch_dec_int32(&this->readers);
+          //return parsec_atomic_fetch_dec_int32(&m_readers);
+          std::atomic_ref<int32_t> a{m_readers};
+          return a.fetch_sub(1, std::memory_order_relaxed);
         } else {
-          return this->readers--;
+          return m_readers--;
         }
       }
 
       /* Returns the number of readers if the copy is immutable, or \c mutable_tag
       * if the copy is mutable */
       int num_readers() const {
-        return this->readers;
+        return m_readers;
       }
 
-      ttg_data_copy_t()
-      {
-        /* TODO: do we need this construction? */
-        PARSEC_OBJ_CONSTRUCT(this, parsec_data_copy_t);
-        this->readers = 1;
-        this->push_task = nullptr;
+      void *get_ptr() const {
+        return m_ptr;
       }
 
-      /* mark destructor as virtual */
-      virtual ~ttg_data_copy_t() = default;
+      parsec_task_t* get_next_task() const {
+        return m_next_task;
+      }
 
+      void set_next_task(parsec_task_t* task) {
+        m_next_task = task;
+      }
+
+      int32_t add_ref() {
+        return m_refs.fetch_add(1, std::memory_order_relaxed);
+      }
+
+      int32_t drop_ref() {
+        return m_refs.fetch_sub(1, std::memory_order_relaxed);
+      }
+
+      bool has_ref() {
+        return (m_refs.load(std::memory_order_relaxed) != 0);
+      }
 
+      int32_t num_ref() const {
+        return m_refs.load(std::memory_order_relaxed);
+      }
 
       /* manage device copies owned by this object
        * we only touch the vector if we have more than one copies to track
        * and otherwise use the single-element member.
        */
-      using iterator = parsec_data_copy_t**;
+      using iterator = parsec_data_t**;
 
-      void add_device_copy(parsec_data_copy_t* copy) {
+      void add_device_data(parsec_data_t* data) {
         // TODO: properly release again!
-        PARSEC_OBJ_RETAIN(copy);
-        switch (m_num_dev_copies) {
+        PARSEC_OBJ_RETAIN(data);
+        switch (m_num_dev_data) {
           case 0:
-            m_single_dev_copy = copy;
+            m_single_dev_data = data;
             break;
           case 1:
             /* move single copy into vector and add new copy below */
-            m_dev_copies.push_back(m_single_dev_copy);
+            m_dev_data.push_back(m_single_dev_data);
             /* fall-through */
           default:
             /* store in multi-copy vector */
-            m_dev_copies.push_back(copy);
+            m_dev_data.push_back(data);
             break;
         }
-        m_num_dev_copies++;
+        m_num_dev_data++;
       }
 
-      int num_dev_copies() const {
-        return m_num_dev_copies;
+      void remove_device_data(parsec_data_t* data) {
+        if (m_num_dev_data == 1) {
+          m_single_dev_data = nullptr;
+        } else if (m_num_dev_data > 1) {
+          auto it = std::find(m_dev_data.begin(), m_dev_data.end(), data);
+          if (it != m_dev_data.end()) {
+            m_dev_data.erase(it);
+          }
+        }
+        --m_num_dev_data;
+      }
+
+      int num_dev_data() const {
+        return m_num_dev_data;
       }
 
       iterator begin() {
-        switch(m_num_dev_copies) {
+        switch(m_num_dev_data) {
           // no device copies
           case 0: return end();
-          case 1: return &m_single_dev_copy;
-          default: return m_dev_copies.data();
+          case 1: return &m_single_dev_data;
+          default: return m_dev_data.data();
         }
       }
 
       iterator end() {
-        switch(m_num_dev_copies) {
+        switch(m_num_dev_data) {
           case 0:
           case 1:
-            return &(m_single_dev_copy) + 1;
+            return &(m_single_dev_data) + 1;
           default:
-            return m_dev_copies.data() + m_dev_copies.size();
+            return m_dev_data.data() + m_dev_data.size();
         }
       }
 
-    private:
-      std::vector<parsec_data_copy_t*> m_dev_copies;   //< used if there are multiple device copies
-                                                       //  that belong to this object
-      parsec_data_copy_t* m_single_dev_copy;           //< used if there is a single device copy
-      int m_num_dev_copies = 0;                        //< number of device copies
+#if defined(PARSEC_PROF_TRACE) && defined(PARSEC_TTG_PROFILE_BACKEND)
+      int64_t size;
+      int64_t uid;
+#endif
+    protected:
+      void          *m_ptr;
+      parsec_task_t *m_next_task = nullptr;
+      int32_t        m_readers  = 1;
+      std::atomic<int32_t>  m_refs = 1; // number of entities referencing this copy (TTGs, external)
+
+      std::vector<parsec_data_t*> m_dev_data;   //< used if there are multiple device copies
+                                                  //  that belong to this object
+      parsec_data_t *m_single_dev_data;           //< used if there is a single device copy
+      int m_num_dev_data = 0;                   //< number of device copies
     };
 
 
@@ -150,14 +256,57 @@ namespace ttg_parsec {
     */
     template<typename ValueT>
     struct ttg_data_value_copy_t final : public ttg_data_copy_t {
-      using value_type = std::decay_t<ValueT>;
+      using value_type = ValueT;
       value_type m_value;
 
       template<typename T>
       ttg_data_value_copy_t(T&& value)
-      : ttg_data_copy_t(), m_value(std::forward<T>(value))
+      : ttg_data_copy_t()
+      , m_value(std::forward<T>(value))
+      {
+        this->m_ptr = const_cast<value_type*>(&m_value);
+        /* reset the container tracker */
+        ttg_data_copy_container() = nullptr;
+      }
+
+      ttg_data_value_copy_t(ttg_data_value_copy_t&& c)
+        noexcept(std::is_nothrow_move_constructible_v<value_type>)
+      : ttg_data_copy_t(std::move(c))
+      , m_value(std::move(c.m_value))
       {
-        this->device_private = const_cast<value_type*>(&m_value);
+        /* reset the container tracker */
+        ttg_data_copy_container() = nullptr;
+      }
+
+      ttg_data_value_copy_t(const ttg_data_value_copy_t& c)
+        noexcept(std::is_nothrow_copy_constructible_v<value_type>)
+      : ttg_data_copy_t(c)
+      , m_value(c.m_value)
+      {
+        /* reset the container tracker */
+        ttg_data_copy_container() = nullptr;
+      }
+
+      ttg_data_value_copy_t& operator=(ttg_data_value_copy_t&& c)
+        noexcept(std::is_nothrow_move_assignable_v<value_type>)
+      {
+        ttg_data_copy_t::operator=(std::move(c));
+        m_value = std::move(c.m_value);
+        /* reset the container tracker */
+        ttg_data_copy_container() = nullptr;
+      }
+
+      ttg_data_value_copy_t& operator=(const ttg_data_value_copy_t& c)
+        noexcept(std::is_nothrow_copy_assignable_v<value_type>)
+      {
+        ttg_data_copy_t::operator=(c);
+        m_value = c.m_value;
+        /* reset the container tracker */
+        ttg_data_copy_container() = nullptr;
+      }
+
+      value_type& operator*() {
+        return m_value;
       }
 
       /* will destruct the value */
@@ -168,4 +317,4 @@ namespace ttg_parsec {
 
 } // namespace ttg_parsec
 
-#endif // TTG_DATA_COPY_H
+#endif // TTG_DATA_COPY_H
\ No newline at end of file
diff --git a/ttg/ttg/ptr.h b/ttg/ttg/ptr.h
new file mode 100644
index 000000000..f2265f65e
--- /dev/null
+++ b/ttg/ttg/ptr.h
@@ -0,0 +1,23 @@
+#ifndef TTG_PTR_H
+#define TTG_PTR_H
+
+#include "ttg/impl_selector.h"
+
+namespace ttg {
+
+template<typename T>
+using ptr = TTG_IMPL_NS::ptr<T>;
+
+template<typename T>
+ptr<T> get_ptr(const T& obj) {
+  return TTG_IMPL_NS::get_ptr(obj);
+}
+
+template<typename T, typename... Args>
+ptr<T> make_ptr(Args&&... args) {
+  return TTG_IMPL_NS::make_ptr(std::forward<Args>(args)...);
+}
+
+} // namespace ttg
+
+#endif // TTG_PTR_H
\ No newline at end of file
diff --git a/ttg/ttg/view.h b/ttg/ttg/view.h
index 11cb3dcde..c8e48ae4c 100644
--- a/ttg/ttg/view.h
+++ b/ttg/ttg/view.h
@@ -8,10 +8,14 @@
 namespace ttg {
 
   enum class ViewScope {
-    Allocate  = 0x0,
-    SyncIn    = 0x1,
-    SyncOut   = 0x2,
-    SyncInOut = 0x3
+    Allocate     = 0x0,  //< memory allocated as scratch, but not moved in or out
+    Available    = 0x1,  //< data will be reused on device if available, transferred to device otherwise
+    SyncIn       = 0x2,  //< data will be allocated on and transferred to device
+                         //< if latest version resides on the device (no previous sync-out) the data will
+                         //< not be transferred again
+    SyncOut      = 0x4,  //< value will be transferred from device to host after kernel completes
+    SyncInOut    = 0x8,  //< data will be moved in and synchronized back out after the kernel completes
+    AvailableOut = 0x16, //< similar to Available and data is transferred back to device after kernel completes
   };
 
   /**
@@ -20,17 +24,17 @@ namespace ttg {
    * to remove the type and convert to void pointers instead.
    */
   template<typename T, typename = void>
-  struct ViewSpan;
+  struct ViewPart;
 
   template<>
-  struct ViewSpan<void, void> {
+  struct ViewPart<void, void> {
 
     using element_type  = void;
     using value_type    = void;
-    using viewspan_type = ViewSpan<value_type>;
+    using view_part_type = ViewPart<value_type>;
 
-    constexpr ViewSpan() = default;
-    constexpr ViewSpan(void* ptr, std::size_t size, ViewScope scope = ViewScope::SyncIn)
+    constexpr ViewPart() = default;
+    constexpr ViewPart(void* ptr, std::size_t size, ViewScope scope = ViewScope::SyncIn)
     : m_data_ptr(ptr)
     , m_size(size)
     , m_scope(scope)
@@ -72,15 +76,15 @@ namespace ttg {
 
 
   template<typename T>
-  struct ViewSpan<T, void> : public ViewSpan<void, void> {
+  struct ViewPart<T, void> : public ViewPart<void, void> {
 
     using element_type  = T;
     using value_type    = std::remove_cv_t<T>;
-    using viewspan_type = ViewSpan<value_type>;
+    using view_part_type = ViewPart<value_type>;
 
-    constexpr ViewSpan() = default;
-    constexpr ViewSpan(T* ptr, std::size_t size, ViewScope scope = ViewScope::SyncIn)
-    : ViewSpan<void, void>(ptr, size, scope)
+    constexpr ViewPart() = default;
+    constexpr ViewPart(T* ptr, std::size_t size, ViewScope scope = ViewScope::SyncIn)
+    : ViewPart<void, void>(ptr, size, scope)
     { }
 
     constexpr T* data() const {
@@ -89,13 +93,22 @@ namespace ttg {
 
   };
 
-  template<typename HostT, typename... DevTypeTs>
+  namespace detail {
+    template<typename T>
+    struct view_trait
+    {
+      static constexpr bool is_view = false;
+      static constexpr bool is_persistent = false;
+    };
+  } // namespace detail
+
+  template<typename HostT, typename DevT = HostT, typename... DevTypeTs>
   struct View {
 
-    using span_tuple_type  = std::tuple<ttg::ViewSpan<DevTypeTs>...>;
+    using span_tuple_type  = std::tuple<ttg::ViewPart<DevT>, ttg::ViewPart<DevTypeTs>...>;
     using host_type = HostT;
 
-    using view_type = View<HostT, DevTypeTs...>;
+    using view_type = View<HostT, DevT, DevTypeTs...>;
 
     constexpr static std::size_t num_spans = std::tuple_size_v<span_tuple_type>;
 
@@ -106,20 +119,36 @@ namespace ttg {
     , m_spans({std::get<Is>(spans)...})
     { }
 
+    View(HostT& obj, span_tuple_type spans)
+    : View(obj, spans, std::make_index_sequence<sizeof...(DevTypeTs)+1>{})
+    {
+      /* TODO: let the runtime handle the view */
+      //ttg::detail::register_view(*this);
+    }
+
+    /* hidden so that users cannot create views outside of a task */
+    template<typename T, typename... ArgsT>
+    friend auto make_view(T& obj, ViewPart<ArgsT>... spans);
+
   public:
 
-    constexpr View() = default;
+    constexpr View() = delete;
 
-    View(HostT& obj, span_tuple_type spans)
-    : View(obj, spans, std::index_sequence_for<DevTypeTs...>{})
-    { }
+    /* move ctor deleted to prevent moving out of a task */
+    View(view_type&&) = delete;
 
-    View(view_type&&) = default;
+    /* copy ctor deleted to prevent copying out of a task */
+    View(const view_type&) = delete;
 
-    View(const view_type&) = default;
+    ~View() {
+      /* TODO: let the runtime remove the view */
+      //ttg::detail::drop_view(*this);
+    }
 
-    view_type& operator=(view_type&&) = default;
-    view_type& operator=(const view_type&) = default;
+    /* move operator deleted to prevent moving out of a task */
+    view_type& operator=(view_type&&) = delete;
+    /* copy operator deleted to prevent moving out of a task */
+    view_type& operator=(const view_type&) = delete;
 
     template<std::size_t i>
     auto get_device_ptr() {
@@ -137,7 +166,7 @@ namespace ttg {
     }
 
     template<std::size_t i>
-    auto& get_viewspan() const {
+    auto& get_ViewPart() const {
       return std::get<i>(m_spans);
     }
 
@@ -158,29 +187,38 @@ namespace ttg {
       return num_spans;
     }
 
-    /* return a std::span of type-punned ViewSpans */
-    std::span<ViewSpan<void>> view_spans() {
+    /* return a std::span of type-punned ViewParts */
+    std::span<ViewPart<void>> view_spans() {
       return {m_spans.begin(), m_spans.end()};
     }
 
   private:
     HostT* m_obj = nullptr;
     /* type-punned storage, cast to actual types in get_device_ptr */
-    std::array<ViewSpan<void>, num_spans> m_spans;
+    std::array<ViewPart<void>, num_spans> m_spans;
     //span_tuple_type m_spans{};
   };
 
-  template<typename HostT, typename... ViewSpanTs>
-  auto make_view(HostT& obj, ViewSpan<ViewSpanTs>... spans) {
+  template<typename HostT, typename... ViewPartTs>
+  auto make_view(HostT& obj, ViewPart<ViewPartTs>... spans) {
     return View(obj, std::make_tuple(std::move(spans)...));
   }
 
   /* overload for trivially-copyable host objects */
   template<typename HostT, typename = std::enable_if_t<std::is_trivially_copyable_v<HostT>>>
   auto make_view(HostT& obj, ViewScope scope = ViewScope::SyncIn) {
-    return View<HostT, HostT>(obj, std::make_tuple(ViewSpan<HostT>(&obj, sizeof(HostT), scope)));
+    return make_view(obj, ViewPart<HostT>(&obj, sizeof(HostT), scope));
   }
 
+  namespace detail {
+    template<typename HostT, typename... DevTs>
+    struct view_trait<View<HostT, DevTs...>>
+    {
+      static constexpr bool is_view = true;
+      static constexpr bool is_persistent = false;
+    };
+  } // namespace detail
+
   /* yielded when waiting on a kernel to complete */
   struct device_op_wait_kernel
   { };
@@ -196,7 +234,7 @@ namespace ttg {
   /* type-punned version of the View, providing access to the object
    * pointer and a std::span over the views of that object */
   struct device_obj_view {
-    using span_type = std::span<ViewSpan<void>>;
+    using span_type = std::span<ViewPart<void>>;
     using iterator = typename span_type::iterator;
 
     device_obj_view(void *obj, span_type&& span)
@@ -221,6 +259,164 @@ namespace ttg {
     span_type m_span;
   };
 
+
+  /**
+   * A view that is persistent and can be copied in and out of the TTG
+   */
+  template<typename HostT, typename DevT = HostT, typename... DevTypeTs>
+  struct PersistentView {
+
+    using span_tuple_type  = std::tuple<ttg::ViewPart<DevT>, ttg::ViewPart<DevTypeTs>...>;
+    using host_type = HostT;
+
+    using view_type = PersistentView<HostT, DevT, DevTypeTs...>;
+
+    using ptr_type = ttg::ptr<host_type>;
+
+    constexpr static std::size_t num_spans = std::tuple_size_v<span_tuple_type>;
+
+  private:
+    template<std::size_t... Is>
+    PersistentView(ptr_type&& ptr, span_tuple_type& spans, std::index_sequence<Is...>)
+    : m_ptr(std::forward<ptr_type>(ptr))
+    , m_spans({std::get<Is>(spans)...})
+    { }
+
+  public:
+
+    constexpr PersistentView() = default;
+
+    PersistentView(ptr_type ptr, span_tuple_type spans)
+    : PersistentView(std::move(ptr), spans, std::make_index_sequence<sizeof...(DevTypeTs)+1>{})
+    {
+      /* TODO: let the runtime handle the view */
+      //ttg::detail::register_view(*this);
+    }
+
+    PersistentView(view_type&&) = default;
+
+    PersistentView(const view_type&) = default;
+
+    ~PersistentView() {
+      /* TODO: let the runtime remove the view */
+      //ttg::detail::drop_view(*this);
+    }
+
+    view_type& operator=(view_type&&) = default;
+    view_type& operator=(const view_type&) = default;
+
+    template<std::size_t i>
+    auto get_device_ptr() {
+      return static_cast<std::tuple_element_t<i, span_tuple_type>::value_type*>(std::get<i>(m_spans).data());
+    }
+
+    template<std::size_t i>
+    const auto get_device_ptr() const {
+      return static_cast<std::tuple_element_t<i, span_tuple_type>::value_type>(std::get<i>(m_spans).data());
+    }
+
+    template<std::size_t i>
+    std::size_t get_device_size() const {
+      return std::get<i>(m_spans).size();
+    }
+
+    template<std::size_t i>
+    auto& get_ViewPart() const {
+      return std::get<i>(m_spans);
+    }
+
+    HostT& get_host_object() {
+      return *m_ptr;
+    }
+
+    const HostT& get_host_object() const {
+      return *m_ptr;
+    }
+
+    template<std::size_t i>
+    ViewScope get_scope() const {
+      return std::get<i>(m_spans).scope();
+    }
+
+    constexpr static std::size_t size() {
+      return num_spans;
+    }
+
+    ptr_type get_ptr() const {
+      return m_ptr;
+    }
+
+    /* return a std::span of type-punned ViewParts */
+    std::span<ViewPart<void>> view_spans() {
+      return {m_spans.begin(), m_spans.end()};
+    }
+
+  private:
+    ptr_type m_ptr;
+    /* type-punned storage, cast to actual types in get_device_ptr */
+    std::array<ViewPart<void>, num_spans> m_spans;
+    //span_tuple_type m_spans{};
+  };
+
+  template<typename HostT, typename... ViewPartTs>
+  auto make_persistent_view(HostT& obj, ViewPart<ViewPartTs>... spans) {
+    return PersistentView(obj, std::make_tuple(std::move(spans)...));
+  }
+
+  /* overload for trivially-copyable host objects */
+  template<typename HostT, typename = std::enable_if_t<std::is_trivially_copyable_v<HostT>>>
+  auto make_persistent_view(HostT& obj, ViewScope scope = ViewScope::SyncIn) {
+    return PersistentView<HostT, HostT>(obj, std::make_tuple(ViewPart<HostT>(&obj, sizeof(HostT), scope)));
+  }
+
+
+  namespace detail {
+    template<typename HostT, typename... DevTs>
+    struct view_trait<PersistentView<HostT, DevTs...>>
+    {
+      static constexpr bool is_view = true;
+      static constexpr bool is_persistent = true;
+    };
+  } // namespace detail
+
+
+  namespace detail {
+    /* TODO: is this still needed? */
+    template<typename... Ts>
+    struct await_t {
+      std::tuple<Ts&...> ties;
+    };
+  }
+
+  template<typename... Args>
+  auto make_await(Args&&... args) {
+    return detail::await_t{std::tie(std::forward<Args>(args)...)};
+  }
+
+  namespace detail {
+    template<typename... Ts>
+    struct to_device_t {
+      std::tuple<Ts&...> ties;
+    };
+  }
+
+  template<typename... Args>
+  auto to_device(Args&&... args) {
+    return detail::to_device_t{std::tie(std::forward<Args>(args)...)};
+  }
+
+  namespace detail {
+    template<typename... Ts>
+    struct to_host_t {
+      std::tuple<Ts&...> ties;
+    };
+  }
+
+  template<typename... Args>
+  auto to_host(Args&&... args) {
+    return detail::to_host_t{std::tie(std::forward<Args>(args)...)};
+  }
+
   struct device_task_promise_type;
 
   using device_task_handle_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<device_task_promise_type>;
@@ -302,6 +498,13 @@ namespace ttg {
       return yield_value(tmp_tuple);
     }
 
+    /* convenience-function to yield a single view */
+    template<typename HostT, typename... DeviceViewTs>
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_always yield_value(PersistentView<HostT, DeviceViewTs...> &view) {
+      auto tmp_tuple = std::tie(view);
+      return yield_value(tmp_tuple);
+    }
+
     /* waiting for the kernel to complete should always suspend */
     TTG_CXX_COROUTINE_NAMESPACE::suspend_always yield_value(device_op_wait_kernel) {
       std::cout << "yield_value: device_op_wait_kernel" << std::endl;
@@ -309,6 +512,50 @@ namespace ttg {
       return {};
     }
 
+    /* Allow co_await on a tuple */
+    template<typename... Views>
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(std::tuple<Views&...> &views) {
+      return yield_value(views);
+    }
+
+    /* convenience-function to await a single view */
+    template<typename HostT, typename... DeviceViewTs>
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(View<HostT, DeviceViewTs...> &view) {
+      auto tmp_tuple = std::tie(view);
+      return yield_value(tmp_tuple);
+    }
+
+    /* convenience-function to await a single view */
+    template<typename HostT, typename... DeviceViewTs>
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(PersistentView<HostT, DeviceViewTs...> &view) {
+      auto tmp_tuple = std::tie(view);
+      return yield_value(tmp_tuple);
+    }
+
+    /* co_await for the kernel to complete should always suspend */
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(device_op_wait_kernel) {
+      std::cout << "yield_value: device_op_wait_kernel" << std::endl;
+      m_state = TTG_DEVICE_CORO_WAIT_KERNEL;
+      return {};
+    }
+
+    template<typename... Ts>
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(detail::await_t<Ts...>&& a) {
+      bool need_transfer = !(TTG_IMPL_NS::register_device_memory(a.ties));
+      /* TODO: are we allowed to not suspend here and launch the kernel directly? */
+      m_state = TTG_DEVICE_CORO_WAIT_TRANSFER;
+      return {};
+    }
+
+    template<typename... Ts>
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(detail::to_device_t<Ts...>&& a) {
+      bool need_transfer = !(TTG_IMPL_NS::register_device_memory(a.ties));
+      /* TODO: are we allowed to not suspend here and launch the kernel directly? */
+      m_state = TTG_DEVICE_CORO_WAIT_TRANSFER;
+      return {};
+    }
+
+
     void return_void() {
       m_state = TTG_DEVICE_CORO_COMPLETE;
     }
@@ -346,47 +593,8 @@ namespace ttg {
 
   bool device_task::completed() { return base_type::promise().state() == TTG_DEVICE_CORO_COMPLETE; }
 
-  /// std::span mirrored between host and device memory
-  struct HDSpan {
-    HDSpan() = default;
-    HDSpan(std::byte* ptr, std::size_t nbytes) {
-      ptrs_[0] = ptr;
-      nbytes_ = nbytes;
-      last_touched_space_ = 0;
-    }
-
-    std::size_t nbytes() const { return nbytes_; }
-
-    const std::byte* host_data() const { return ptrs_[0]; }
-
-    std::byte* host_data() {
-      last_touched_space_ = 0;
-      return ptrs_[0];
-    }
-
-    const std::byte* device_data() const { return ptrs_[1]; }
-
-    std::byte* device_data() {
-      last_touched_space_ = 1;
-      return ptrs_[1];
-    }
-
-    void mark_synched() { last_touched_space_ = 2; }
-
-   private:
-    std::array<std::byte*, 2> ptrs_ = {nullptr, nullptr};
-    std::size_t nbytes_ = 0;
-    std::size_t last_touched_space_ = 2;
-  };
-
-  /// set of std::span's mirrored between host and device memory
-  template <std::size_t N = 10>
-  struct HDSpans {
-    HDSpans() = default;
-
-   private:
-    std::array<HDSpan, N> spans_;
-  };
+  struct device_wait_kernel
+  { };
 
 }  // namespace ttg
 

From 93a7dc5fa094735dc23534ab4dced495db151bf3 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 3 Mar 2023 09:36:10 -0500
Subject: [PATCH 070/259] Support for moving between device and host tasks,
 much more

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 CMakeLists.txt                 |  11 ++
 examples/madness/mrattg.cc     |  40 +++++++
 tests/unit/CMakeLists.txt      |   1 +
 tests/unit/device_coro.cc      | 118 +++++++++++++++------
 ttg/ttg/buffer.h               |  61 +++--------
 ttg/ttg/devicescratch.h        |   4 +-
 ttg/ttg/parsec/buffer.h        |  75 ++++++++++---
 ttg/ttg/parsec/devicescratch.h |  34 +++---
 ttg/ttg/parsec/fwd.h           |  11 +-
 ttg/ttg/parsec/ptr.h           |  69 +++++++++++-
 ttg/ttg/parsec/ttg.h           | 185 +++++++++++----------------------
 ttg/ttg/ptr.h                  | 101 ++++++++++++++++--
 ttg/ttg/view.h                 |  58 ++++++++++-
 13 files changed, 519 insertions(+), 249 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b6381948f..f16532188 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -100,6 +100,17 @@ if (TARGET MADworld)
   message(STATUS "MADNESS_FOUND=1")
 endif(TARGET MADworld)
 
+
+##########################
+#### CUDA
+##########################
+check_language(CUDA)
+if(CMAKE_CUDA_COMPILER)
+    enable_language(CUDA)
+endif(CMAKE_CUDA_COMPILER)
+set(TTG_HAVE_CUDA ${CMAKE_CUDA_COMPILER} CACHE BOOL "True if TTG supports compiling .cu files")
+
+
 ##########################
 #### Examples
 ##########################
diff --git a/examples/madness/mrattg.cc b/examples/madness/mrattg.cc
index 47bce1ce9..2b2d6c113 100644
--- a/examples/madness/mrattg.cc
+++ b/examples/madness/mrattg.cc
@@ -124,6 +124,46 @@ auto make_project(functorT& f,
   return ttg::make_tt(F, edges(fuse(refine, ctl)), edges(refine, result), name, {"control"}, {"refine", "result"});
 }
 
+
+/// Returns an std::unique_ptr to the object
+template <typename functorT, typename T, size_t K, Dimension NDIM>
+auto make_project_device(functorT& f,
+                         const T thresh,  /// should be scalar value not complex
+                         ctlEdge<NDIM>& ctl, rnodeEdge<T, K, NDIM>& result, const std::string& name = "project") {
+  auto F = [f, thresh](const Key<NDIM>& key, std::tuple<ctlOut<NDIM>, rnodeOut<T, K, NDIM>>& out) {
+    FunctionReconstructedNode<T, K, NDIM> node(key);  // Our eventual result
+    auto& coeffs = node.coeffs;                       // Need to clean up OO design
+    bool is_leaf;
+
+    if (key.level() < initial_level(f)) {
+      for (auto child : children(key)) ttg::sendk<0>(child, out);
+      coeffs = T(1e7);  // set to obviously bad value to detect incorrect use
+      is_leaf = false;
+    } else if (is_negligible<functorT, T, NDIM>(f, Domain<NDIM>::template bounding_box<T>(key),
+                                                truncate_tol(key, thresh))) {
+      coeffs = T(0.0);
+      is_leaf = true;
+    } else {
+      auto node_view  = ttg::make_view(node, ttg::ViewScope::Out); // no need to move node onto the device
+      auto is_leaf_view = ttg::make_view(is_leaf, ttg::ViewScope::Out);
+      co_await ttg::device::wait_views{};
+      fcoeffs<functorT, T, K>(f, key, thresh,
+                              node_view.get_device_ptr<0>(),
+                              is_leaf_view.get_device_ptr<0>());  // cannot deduce K
+      co_await ttg::device::wait_kernel{};
+      if (!is_leaf) {
+        for (auto child : children(key)) ttg::sendk<0>(child, out);  // should be broadcast ?
+      }
+    }
+    node.is_leaf = is_leaf;
+    ttg::send<1>(key, node, out);  // always produce a result
+  };
+  ctlEdge<NDIM> refine("refine");
+  return ttg::make_tt(F, edges(fuse(refine, ctl)), edges(refine, result), name, {"control"}, {"refine", "result"});
+}
+
+
+
 namespace detail {
   template <typename T, size_t K, Dimension NDIM>
   struct tree_types {};
diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt
index faff08780..73d8a152a 100644
--- a/tests/unit/CMakeLists.txt
+++ b/tests/unit/CMakeLists.txt
@@ -11,6 +11,7 @@ set(ut_libs Catch2::Catch2)
 #if (TARGET std::coroutine)
     #list(APPEND ut_src fibonacci-coro.cc)
     list(APPEND ut_src device_coro.cc)
+    list(APPEND ut_src cuda_kernel.cu)
     list(APPEND ut_libs std::coroutine)
 #endif()
 add_ttg_executable(core-unittests-ttg "${ut_src}" LINK_LIBRARIES "${ut_libs}")
diff --git a/tests/unit/device_coro.cc b/tests/unit/device_coro.cc
index a2cc3f7a2..11ed77d91 100644
--- a/tests/unit/device_coro.cc
+++ b/tests/unit/device_coro.cc
@@ -3,6 +3,8 @@
 #include "ttg.h"
 #include "ttg/view.h"
 
+#include "cuda_kernel.h"
+
 struct value_t {
   ttg::buffer<double> db; // TODO: rename
   int quark;
@@ -19,57 +21,33 @@ struct value_t {
 namespace madness::archive {
   template <class Archive>
   struct ArchiveSerializeImpl<Archive, value_t> {
-    static inline void serialize(const Archive& ar, value_t& obj) { ar& obj.quark; };
+    static inline void serialize(const Archive& ar, value_t& obj) { ar& obj.quark & obj.db; };
   };
 }  // namespace madness::archive
 
-/* announce that this type contains a device buffer */
-template<>
-struct ttg::container_trait<value_t> {
-  static auto devicebuf_members(value_t&& v) {
-    return std::tie(v.db);
-  }
-};
-
 
 
 TEST_CASE("Device", "coro") {
+
   SECTION("devicebuf") {
 
     ttg::Edge<int, value_t> edge;
-    ttg::ptr<value_t> ptr;
     auto fn = [&](const int& key, value_t&& val) -> ttg::device_task {
-      double scratch = 1.0;
-      ttg::devicescratch<double> scratch_view = ttg::make_scratch(&scratch, ttg::scope::SyncOut);
       ttg::print("device_task key ", key);
       /* wait for the view to be available on the device */
-      co_await ttg::to_device(scratch_view, val.db);
-      // co_yield std::tie(view1, view2);
-      // TTG_WAIT_VIEW(view);
+      co_await ttg::to_device(val.db);
       /* once we're back here the data has been transferred */
-      //CHECK(view.get_rw_device_ptr<0>()  != nullptr);
-      CHECK(scratch_view.device_ptr()  != nullptr);
+      CHECK(val.db.current_device_ptr() != nullptr);
 
-      ttg::print("device_task key ", key, ", device pointer ", scratch_view.device_ptr());
+      /* NO KERNEL */
 
       /* here we suspend to wait for a kernel to complete */
-      co_await ttg::device_op_wait_kernel{};
-      // TTG_WAIT_KERNEL();
-
-      /* force the data back to the host
-       * useful if the buffer tracks a non-owned pointer
-       * and we want to send on that pointer
-       * TODO: needs implementing, how does this interact with scratch?
-       */
-      //co_await ttg::to_host(val.db);
+      co_await ttg::wait_kernel();
 
       /* we're back, the kernel executed and we can send */
-      if (key < 1 || scratch < 0.0) {
+      if (key < 1) {
         /* TODO: should we move the view in here if we want to get the device side data */
         ttg::send<0>(key+1, std::move(val));
-      } else {
-        /* exfiltrate the value */
-        ptr = ttg::get_ptr(val);
       }
     };
 
@@ -80,17 +58,91 @@ TEST_CASE("Device", "coro") {
     make_graph_executable(tt);
     if (ttg::default_execution_context().rank() == 0) tt->invoke(0, value_t{});
     ttg::ttg_fence(ttg::default_execution_context());
-    CHECK(ptr.is_valid());
-    ptr.reset();
   }
 
+  SECTION("scratch") {
+
+    ttg::Edge<int, value_t> edge;
+    auto fn = [&](const int& key, value_t&& val) -> ttg::device_task {
+      double scratch = 0.0;
+      ttg::devicescratch<double> ds = ttg::make_scratch(&scratch, ttg::scope::SyncOut);
+
+      /* wait for the view to be available on the device */
+      co_await ttg::to_device(ds, val.db);
+      /* once we're back here the data has been transferred */
+      CHECK(ds.device_ptr()  != nullptr);
+
+      /* call a kernel */
+      increment_buffer(val.db.current_device_ptr(), val.db.size(), ds.device_ptr(), ds.size());
+
+      /* here we suspend to wait for a kernel to complete */
+      co_await ttg::wait_kernel();
+
+      /* buffer is increment once per task, so it should be the same as key */
+      CHECK(static_cast<int>(scratch) == key);
+
+      /* we're back, the kernel executed and we can send */
+      if (key < 10) {
+        /* TODO: should we move the view in here if we want to get the device side data */
+        ttg::send<0>(key+1, std::move(val));
+      }
+    };
+
+    auto tt = ttg::make_tt<ttg::ExecutionSpace::CUDA>(fn, ttg::edges(edge), ttg::edges(edge),
+                                                      "device_task", {"edge_in"}, {"edge_out"});
+    make_graph_executable(tt);
+    if (ttg::default_execution_context().rank() == 0) tt->invoke(0, value_t{});
+    ttg::ttg_fence(ttg::default_execution_context());
+  }
+
+  SECTION("ptr") {
 
+    ttg::Edge<int, value_t> edge;
+    ttg::Ptr<value_t> ptr;
+    auto fn = [&](const int& key, value_t&& val) -> ttg::device_task {
+      double scratch = 1.0;
+      ttg::devicescratch<double> ds = ttg::make_scratch(&scratch, ttg::scope::SyncOut);
 
+      /* wait for the view to be available on the device */
+      co_await ttg::to_device(ds, val.db);
+      /* once we're back here the data has been transferred */
+      CHECK(ds.device_ptr()  != nullptr);
 
+      /* KERNEL */
+      increment_buffer(val.db.current_device_ptr(), val.db.size(), ds.device_ptr(), ds.size());
 
+      /* here we suspend to wait for a kernel and the out-transfer to complete */
+      co_await ttg::wait_kernel_out(val.db);
 
+      /* buffer is increment once per task, so it should be the same as key */
+      CHECK(static_cast<int>(scratch) == key);
+      CHECK(static_cast<int>(*val.db.host_ptr()) == key);
 
+      /* we're back, the kernel executed and we can send */
+      if (key < 10 || scratch < 0.0) {
+        ttg::send<0>(key+1, std::move(val));
+      } else {
+        /* exfiltrate the value */
+        /* TODO: what consistency do we expect from get_ptr? */
+        ptr = ttg::get_ptr(val);
+      }
+    };
+
+    //ptr.get_view<ttg::ExecutionSpace::CUDA>(device_id);
+
+    auto tt = ttg::make_tt<ttg::ExecutionSpace::CUDA>(fn, ttg::edges(edge), ttg::edges(edge),
+                                                      "device_task", {"edge_in"}, {"edge_out"});
+    make_graph_executable(tt);
+    if (ttg::default_execution_context().rank() == 0) tt->invoke(0, value_t{});
+    ttg::ttg_fence(ttg::default_execution_context());
+    CHECK(ptr.is_valid());
 
+    /* feed the ptr back into a graph */
+    if (ttg::default_execution_context().rank() == 0) tt->invoke(11, ptr);
+    ttg::ttg_fence(ttg::default_execution_context());
+
+    ptr.reset();
+  }
 
 
 #if 0
diff --git a/ttg/ttg/buffer.h b/ttg/ttg/buffer.h
index 7261404a2..1868b7e0c 100644
--- a/ttg/ttg/buffer.h
+++ b/ttg/ttg/buffer.h
@@ -2,56 +2,29 @@
 #define TTG_BUFFER_H
 
 #include <memory>
+#include "ttg/impl_selector.h"
+
 
 namespace ttg {
 
 template<typename T>
 using buffer = TTG_IMPL_NS::buffer<T>;
 
-template<typename T>
-struct hostbuf {
-private:
-  T* m_ptr = nullptr;
-  std::size_t m_count = 0;
-
-public:
-  hostbuf() = default;
-
-  hostbuf(T* ptr, std::size_t count)
-  : m_ptr(ptr)
-  , m_count(count)
-  { }
-
-  T* ptr() const {
-    return m_ptr;
-  }
-
-  std::size_t count() const {
-    return m_count;
-  }
-
-  std::size_t size() const {
-    return m_count * sizeof(T);
-  }
-};
-
-/* Applications may override this trait to
- * expose device and host buffer members of
- * data structures. Example:
- * struct data_t {
- *   ttg::buffer<double> db;
- *   double *more_host_data;
- *   int size;
- * };
- * template<>
- * struct container_trait<data_t> {
- *   static auto buffers() {
- *     return std::make_tuple(db);
- *   }
- * };
- */
-template<typename T>
-struct container_trait;
+namespace detail {
+  template<typename T>
+  struct is_buffer : std::false_type
+  { };
+
+  template<typename T>
+  struct is_buffer<ttg::buffer<T>> : std::true_type
+  { };
+
+  template<typename T>
+  constexpr bool is_buffer_v = is_buffer<T>::value;
+
+  static_assert(is_buffer_v<ttg::buffer<double>>);
+  static_assert(is_buffer_v<TTG_IMPL_NS::buffer<double>>);
+} // namespace detail
 
 } // namespace ttg
 
diff --git a/ttg/ttg/devicescratch.h b/ttg/ttg/devicescratch.h
index 425b3b676..1478d82cf 100644
--- a/ttg/ttg/devicescratch.h
+++ b/ttg/ttg/devicescratch.h
@@ -1,8 +1,8 @@
 #ifndef TTG_DEVICESCRATCH_H
 #define TTG_DEVICESCRATCH_H
 
-#include <ttg/devicescope.h>
-#include <ttg/parsec/devicescratch.h>
+#include "ttg/devicescope.h"
+#include "ttg/impl_selector.h"
 
 namespace ttg {
 
diff --git a/ttg/ttg/parsec/buffer.h b/ttg/ttg/parsec/buffer.h
index 3d2d3a90c..6d1a0c16f 100644
--- a/ttg/ttg/parsec/buffer.h
+++ b/ttg/ttg/parsec/buffer.h
@@ -43,8 +43,10 @@ struct buffer {
 private:
   using delete_fn_t = std::add_pointer_t<void(element_type*)>;
 
-  std::unique_ptr<parsec_data_t, decltype(&parsec_data_destroy)> m_data;
-  std::unique_ptr<element_type[], delete_fn_t> m_host_data;
+  using parsec_data_ptr = std::unique_ptr<parsec_data_t, decltype(&parsec_data_destroy)>;
+  using host_data_ptr   = std::unique_ptr<element_type[], delete_fn_t>;
+  parsec_data_ptr m_data;
+  host_data_ptr m_host_data;
   std::size_t m_count = 0;
   detail::ttg_data_copy_t *m_ttg_copy = nullptr;
 
@@ -168,11 +170,11 @@ struct buffer {
   /* copy the host buffer content but leave the devices untouched */
   buffer& operator=(const buffer& db) {
     if (db.m_count == 0) {
-      m_data = std::unique_ptr<parsec_data_t>(nullptr, &delete_null_parsec_data);
-      m_host_data = std::unique_ptr<element_type>(nullptr, &delete_non_owned);
+      m_data = parsec_data_ptr(nullptr, &delete_null_parsec_data);
+      m_host_data = host_data_ptr(nullptr, &delete_non_owned);
     } else {
-      m_data = std::unique_ptr(parsec_data_new(), &parsec_data_destroy);
-      m_host_data = std::unique_ptr(new element_type[db.m_count], &delete_owned);
+      m_data = parsec_data_ptr(parsec_data_new(), &parsec_data_destroy);
+      m_host_data = host_data_ptr(new element_type[db.m_count], &delete_owned);
       /* copy host data */
       std::copy(db.m_host_data.get(),
                 db.m_host_data.get() + db.m_count,
@@ -229,6 +231,14 @@ struct buffer {
     return static_cast<element_type*>(parsec_data_get_ptr(m_data.get(), device_id));
   }
 
+  element_type* host_ptr() {
+    return device_ptr_on(cpu_device);
+  }
+
+  const element_type* host_ptr() const {
+    return device_ptr_on(cpu_device);
+  }
+
   bool is_valid_on(int device_id) const {
     assert(is_valid());
     return (parsec_data_get_ptr(m_data.get(), device_id) != nullptr);
@@ -285,11 +295,11 @@ struct buffer {
     /* drop the current data and reallocate */
     reset();
     if (count == 0) {
-      m_data = std::unique_ptr<parsec_data_t>(nullptr, &delete_null_parsec_data);
-      m_host_data = std::unique_ptr<element_type>(nullptr, &delete_non_owned);
+      m_data = parsec_data_ptr(nullptr, &delete_null_parsec_data);
+      m_host_data = host_data_ptr(nullptr, &delete_non_owned);
     } else {
-      m_data = std::unique_ptr(parsec_data_new(), &parsec_data_destroy);
-      m_host_data = std::unique_ptr(new element_type[count], &delete_owned);
+      m_data = parsec_data_ptr(parsec_data_new(), &parsec_data_destroy);
+      m_host_data = host_data_ptr(new element_type[count], &delete_owned);
       /* create the host copy with the allocated memory */
       create_host_copy();
     }
@@ -303,12 +313,12 @@ struct buffer {
     /* drop the current data and reallocate */
     reset();
     if (nullptr == ptr) {
-      m_data = std::unique_ptr<parsec_data_t>(nullptr, &delete_null_parsec_data);
-      m_host_data = std::unique_ptr<element_type>(nullptr, &delete_non_owned);
+      m_data = parsec_data_ptr(nullptr, &delete_null_parsec_data);
+      m_host_data = host_data_ptr(nullptr, &delete_non_owned);
       m_count = 0;
     } else {
-      m_data = std::unique_ptr(parsec_data_new(), &parsec_data_destroy);
-      m_host_data = std::unique_ptr(ptr, &delete_non_owned);
+      m_data = parsec_data_ptr(parsec_data_new(), &parsec_data_destroy);
+      m_host_data = host_data_ptr(ptr, &delete_non_owned);
       /* create the host copy with the allocated memory */
       create_host_copy();
       m_count = count;
@@ -316,6 +326,43 @@ struct buffer {
     /* don't touch the ttg_copy, we still belong to the same container */
   }
 
+  /* serialization support */
+
+#ifdef TTG_SERIALIZATION_SUPPORTS_CEREAL
+  template <class Archive>
+  std::enable_if_t<std::is_base_of_v<cereal::detail::InputArchiveBase, Archive> ||
+                    std::is_base_of_v<cereal::detail::OutputArchiveBase, Archive>>
+  serialize(Archive& ar) {
+    if constexpr (ttg::detail::is_output_archive_v<Archive>)
+      std::size_t s = size();
+      ar(s);
+    else {
+      std::size_t s;
+      ar(s);
+      reset(s);
+    }
+    ar(value);
+  }
+#endif // TTG_SERIALIZATION_SUPPORTS_CEREAL
+
+#ifdef TTG_SERIALIZATION_SUPPORTS_MADNESS
+  template <typename Archive>
+  std::enable_if_t<std::is_base_of_v<madness::archive::BufferInputArchive, Archive> ||
+                   std::is_base_of_v<madness::archive::BufferOutputArchive, Archive>>
+  serialize(Archive& ar) {
+    if constexpr (ttg::detail::is_output_archive_v<Archive>)
+      ar& size();
+    else {
+      std::size_t s;
+      ar & s;
+      /* initialize internal pointers and then reset */
+
+      reset(s);
+    }
+  }
+#endif // TTG_SERIALIZATION_SUPPORTS_MADNESS
+
+
 };
 
 template<typename T>
diff --git a/ttg/ttg/parsec/devicescratch.h b/ttg/ttg/parsec/devicescratch.h
index 2a2e928d4..1c0487cdf 100644
--- a/ttg/ttg/parsec/devicescratch.h
+++ b/ttg/ttg/parsec/devicescratch.h
@@ -35,7 +35,7 @@ struct devicescratch {
 
 private:
 
-  parsec_data_t m_data;
+  parsec_data_t* m_data = nullptr;
   parsec_data_copy_t m_data_copy;
   ttg::scope m_scope;
 
@@ -44,18 +44,17 @@ struct devicescratch {
     /* TODO: handle the scope */
     PARSEC_OBJ_CONSTRUCT(&m_data_copy, parsec_data_copy_t);
     m_data_copy.device_index    = 0;
-    m_data_copy.original        = &m_data;
-    m_data_copy.older           = NULL;
+    //m_data_copy.original        = &m_data;
+    //m_data_copy.older           = NULL;
     m_data_copy.flags           = PARSEC_DATA_FLAG_PARSEC_MANAGED;
     m_data_copy.dtt             = parsec_datatype_int8_t;
     m_data_copy.version         = 1;
     m_data_copy.device_private  = ptr;
     m_data_copy.coherency_state = PARSEC_DATA_COHERENCY_SHARED;
 
-    PARSEC_OBJ_CONSTRUCT(&m_data, parsec_data_t);
-    m_data.device_copies[0]     = &m_data_copy;
-    m_data.nb_elts              = count * sizeof(element_type);
-    m_data.owner_device         = 0;
+    m_data->nb_elts              = count * sizeof(element_type);
+    m_data->owner_device         = 0;
+    parsec_data_copy_attach(m_data, &m_data_copy, 0);
   }
 
   friend parsec_data_t* detail::get_parsec_data<T>(const ttg_parsec::devicescratch<T>&);
@@ -66,7 +65,8 @@ struct devicescratch {
    * The memory pointed to by ptr must be accessible during
    * the life-time of the devicescratch. */
   devicescratch(element_type* ptr, ttg::scope scope = ttg::scope::SyncIn, std::size_t count = 1)
-  : m_scope(scope) {
+  : m_data(parsec_data_new())
+  , m_scope(scope) {
     create_host_copy(ptr, count);
   }
 
@@ -82,21 +82,27 @@ struct devicescratch {
   /* don't allow copying */
   devicescratch& operator=(const devicescratch& db) = delete;
 
+  ~devicescratch() {
+    PARSEC_OBJ_DESTRUCT(&m_data_copy);
+    parsec_data_destroy(m_data);
+    m_data = nullptr;
+  }
+
   /* get the current device pointer */
   element_type* device_ptr() {
     assert(is_valid());
-    return static_cast<element_type*>(m_data.device_copies[m_data.owner_device]->device_private);
+    return static_cast<element_type*>(m_data->device_copies[m_data->owner_device]->device_private);
   }
 
   /* get the current device pointer */
   const element_type* device_ptr() const {
     assert(is_valid());
-    return static_cast<element_type*>(m_data.device_copies[m_data.owner_device]->device_private);
+    return static_cast<element_type*>(m_data->device_copies[m_data->owner_device]->device_private);
   }
 
   bool is_valid() const {
     // TODO: how to get the current device
-    // return (m_data.owner_device == parsec_current_device);
+    // return (m_data->owner_device == parsec_current_device);
     return true;
   }
 
@@ -104,6 +110,10 @@ struct devicescratch {
     return m_scope;
   }
 
+  std::size_t size() const {
+    return (m_data->nb_elts / sizeof(element_type));
+  }
+
 };
 
 template<typename T>
@@ -120,7 +130,7 @@ constexpr static const bool is_devicescratch_v = is_devicescratch<T>::value;
 namespace detail {
   template<typename T>
   parsec_data_t* get_parsec_data(const ttg_parsec::devicescratch<T>& scratch) {
-    return const_cast<parsec_data_t*>(&scratch.m_data);
+    return const_cast<parsec_data_t*>(scratch.m_data);
   }
 } // namespace detail
 
diff --git a/ttg/ttg/parsec/fwd.h b/ttg/ttg/parsec/fwd.h
index 690b4b7ad..f01439a1d 100644
--- a/ttg/ttg/parsec/fwd.h
+++ b/ttg/ttg/parsec/fwd.h
@@ -22,7 +22,10 @@ namespace ttg_parsec {
   struct devicescratch;
 
   template<typename... Views>
-  bool register_device_memory(std::tuple<Views&...> &views);
+  inline bool register_device_memory(std::tuple<Views&...> &views);
+
+  template<typename... Buffer>
+  inline void mark_device_out(std::tuple<Buffer&...> &b);
 
   /// \internal the OG name
   template <typename keyT, typename output_terminalsT, typename derivedT, typename... input_valueTs>
@@ -65,11 +68,11 @@ namespace ttg_parsec {
   template <typename T>
   static void ttg_broadcast(ttg::World world, T &data, int source_rank);
 
-  template<typename T>
-  ptr<T> get_ptr(const T& obj);
+  template<typename... Args>
+  inline std::pair<bool, std::tuple<ptr<std::decay_t<Args>>...>> get_ptr(Args&&... args);
 
   template<typename T, typename... Args>
-  ptr<T> make_ptr(Args&&... args);
+  inline ptr<T> make_ptr(Args&&... args);
 
 
 }  // namespace ttg_parsec
diff --git a/ttg/ttg/parsec/ptr.h b/ttg/ttg/parsec/ptr.h
index 5c8c39524..1c43db418 100644
--- a/ttg/ttg/parsec/ptr.h
+++ b/ttg/ttg/parsec/ptr.h
@@ -10,6 +10,10 @@
 
 namespace ttg_parsec {
 
+  // fwd decl
+  template<typename T>
+  struct ptr;
+
   namespace detail {
     /* fwd decl */
     template <typename Value>
@@ -116,6 +120,10 @@ namespace ttg_parsec {
         }
       }
     };
+
+
+    template<typename T>
+    ttg_parsec::detail::ttg_data_copy_t* get_copy(ttg_parsec::ptr<T>& p);
   } // namespace detail
 
   template<typename T, typename... Args>
@@ -139,6 +147,8 @@ namespace ttg_parsec {
     friend ptr<T> make_ptr(Args&&... args);
     template<typename S>
     friend ptr<S> get_ptr(const S& obj);
+    template<typename S>
+    friend detail::ttg_data_copy_t* detail::get_copy(ptr<S>& p);
     friend ttg::detail::value_copy_handler<ttg::Runtime::PaRSEC>;
 
     /* only accessible by get_ptr and make_ptr */
@@ -186,6 +196,55 @@ namespace ttg_parsec {
     }
   };
 
+  namespace detail {
+    template<typename Arg>
+    inline auto get_ptr(Arg&& obj) {
+
+      for (int i = 0; i < detail::parsec_ttg_caller->data_count; ++i) {
+        detail::ttg_data_copy_t *copy = detail::parsec_ttg_caller->copies[i];
+        if (nullptr != copy) {
+          if (copy->get_ptr() == &obj) {
+            bool is_ready = true;
+            /* TODO: how can we force-sync host and device? Current data could be on either. */
+#if 0
+            /* check all tracked device data for validity */
+            for (auto it : copy) {
+              parsec_data_t *data = *it;
+              for (int i = 0; i < parsec_nb_devices; ++i) {
+                if (nullptr != data->device_copies[i]) {
+
+                } else {
+                  is_ready = false;
+                }
+              }
+            }
+#endif // 0
+            return std::make_pair(is_ready, std::tuple{ttg_parsec::ptr<std::decay_t<Arg>>(copy)});
+          }
+        }
+      }
+
+      throw std::runtime_error("ttg::get_ptr called on an unknown object!");
+    }
+  }
+
+  template<typename... Args>
+  inline std::pair<bool, std::tuple<ptr<std::decay_t<Args>>...>> get_ptr(Args&&... args) {
+    if (nullptr == detail::parsec_ttg_caller) {
+      throw std::runtime_error("ttg::get_ptr called outside of a task!");
+    }
+
+    bool ready = true;
+    auto fn = [&](auto&& arg){
+      auto pair = get_ptr(std::forward<decltype(arg)>(arg));
+      ready &= pair.first;
+      return std::move(pair.second);
+    };
+    std::tuple<ptr<std::decay_t<Args>>...> tpl = {(fn(std::forward<Args>(args)))...};
+    return {ready, std::move(tpl)};
+  }
+
+#if 0
   template<typename T>
   ptr<T> get_ptr(const T& obj) {
     if (nullptr != detail::parsec_ttg_caller) {
@@ -202,13 +261,21 @@ namespace ttg_parsec {
     detail::ttg_data_copy_t *copy = detail::create_new_datacopy(obj);
     return ptr<T>(copy);
   }
+#endif // 0
 
   template<typename T, typename... Args>
-  ptr<T> make_ptr(Args&&... args) {
+  inline ptr<T> make_ptr(Args&&... args) {
     detail::ttg_data_copy_t *copy = detail::create_new_datacopy(T(std::forward<Args>(args)...));
     return ptr<T>(copy);
   }
 
+  namespace detail {
+    template<typename T>
+    detail::ttg_data_copy_t* get_copy(ttg_parsec::ptr<T>& p) {
+      return p.get_copy();
+    }
+  } // namespace detail
+
 } // namespace ttg_parsec
 
 #endif // TTG_PARSEC_PTR_H
\ No newline at end of file
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 64c66b4e6..2a706ac77 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -43,6 +43,7 @@
 #include "ttg/parsec/buffer.h"
 #include "ttg/parsec/devicescratch.h"
 #include "ttg/parsec/thread_local.h"
+#include "ttg/parsec/devicefunc.h"
 
 #include <algorithm>
 #include <array>
@@ -268,14 +269,12 @@ namespace ttg_parsec {
       parsec_taskpool_reserve_id(tpool);
 
       tpool->devices_index_mask = 0;
-      // TODO DEBUG: reenable CPU incarnations
-      for(int i = 2; i < (int)parsec_nb_devices; i++) {
+      for(int i = 0; i < (int)parsec_nb_devices; i++) {
           parsec_device_module_t *device = parsec_mca_device_get(i);
           if( NULL == device ) continue;
           tpool->devices_index_mask |= (1 << device->device_index);
       }
 
-
 #ifdef TTG_USE_USER_TERMDET
       parsec_termdet_open_module(tpool, "user_trigger");
 #else   // TTG_USE_USER_TERMDET
@@ -1204,6 +1203,7 @@ namespace ttg_parsec {
       /* we should still be waiting for the transfer to complete */
       assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_TRANSFER);
 
+#if 0
       /* update the device pointers in the device views */
       int i = 0;
       for (auto& view : dev_data) {
@@ -1213,6 +1213,7 @@ namespace ttg_parsec {
           ++i;
         }
       }
+#endif // 0
 
       /* Here we call back into the coroutine again after the transfers have completed */
       static_op<Space>(&task->parsec_task);
@@ -1233,13 +1234,6 @@ namespace ttg_parsec {
       int dev_index;
       double ratio = 1.0;
 
-      /* get a device and come back if we need another one */
-      dev_index = parsec_get_best_device(parsec_task, ratio);
-      assert(dev_index >= 0);
-      if (dev_index < 2) {
-          return PARSEC_HOOK_RETURN_NEXT; /* Fall back */
-      }
-
       task_t *task = (task_t*)parsec_task;
       parsec_execution_stream_s *es = task->tt->world.impl().execution_stream();
 
@@ -1275,6 +1269,16 @@ namespace ttg_parsec {
       /* for now make sure we're waiting for transfers and the coro hasn't skipped this step */
       assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_TRANSFER);
 
+      /* TODO: is this the right place to set the mask? */
+      task->parsec_task.chore_mask = PARSEC_DEV_ALL;
+      /* get a device and come back if we need another one */
+      dev_index = parsec_get_best_device(parsec_task, ratio);
+      assert(dev_index >= 0);
+      if (dev_index < 2) {
+          return PARSEC_HOOK_RETURN_NEXT; /* Fall back */
+      }
+
+
 #if 0
       // manage the gpu_task flows
 
@@ -2040,6 +2044,19 @@ namespace ttg_parsec {
           } else {
             copy = detail::create_new_datacopy(std::forward<Value>(value));
           }
+
+          /* if this is a host task make sure tracked buffers get copied to the host */
+          if constexpr(!derived_has_cuda_op()) {
+            int c = 0;
+            for (auto it : *copy) {
+              parsec_data_t *data = *it;
+              if (data->owner_device != 0) {
+                task->parsec_task.data[c].data_in = data->device_copies[0];
+                task->parsec_task.data[c].source_repo_entry = NULL;
+                ++c;
+              }
+            }
+          }
           /* if we registered as a writer and were the first to register with this copy
            * we need to defer the release of this task to give other tasks a chance to
            * make a copy of the original data */
@@ -2133,7 +2150,7 @@ namespace ttg_parsec {
 
     // Used to set the i'th argument
     template <std::size_t i, typename Key, typename Value>
-    void set_arg_impl(const Key &key, Value &&value) {
+    void set_arg_impl(const Key &key, Value &&value, detail::ttg_data_copy_t *copy_in = nullptr) {
       int owner;
 
 #if defined(PARSEC_PROF_TRACE) && defined(PARSEC_TTG_PROFILE_BACKEND)
@@ -2148,9 +2165,9 @@ namespace ttg_parsec {
         owner = keymap();
       if (owner == world.rank()) {
         if constexpr (!ttg::meta::is_void_v<keyT>)
-          set_arg_local<i, keyT, Value>(key, std::forward<Value>(value));
+          set_arg_local_impl<i, keyT, Value>(key, std::forward<Value>(value), copy_in);
         else
-          set_arg_local<i, keyT, Value>(std::forward<Value>(value));
+          set_arg_local_impl<i, keyT, Value>(std::forward<Value>(value), copy_in);
 #if defined(PARSEC_PROF_TRACE) && defined(PARSEC_TTG_PROFILE_BACKEND)
           if(world.impl().profiling()) {
             parsec_profiling_ts_trace(world.impl().parsec_ttg_profile_backend_set_arg_end, 0, 0, NULL);
@@ -2178,11 +2195,13 @@ namespace ttg_parsec {
         if constexpr (!ttg::has_split_metadata<decvalueT>::value) {
           pos = pack(value, msg->bytes, pos);
         } else {
-          detail::ttg_data_copy_t *copy;
-          copy = detail::find_copy_in_task(detail::parsec_ttg_caller, &value);
+          detail::ttg_data_copy_t *copy = copy_in;
           if (nullptr == copy) {
-            // We need to create a copy for this data, as it does not exist yet.
-            copy = detail::create_new_datacopy(std::forward<Value>(value));
+            copy = detail::find_copy_in_task(detail::parsec_ttg_caller, &value);
+            if (nullptr == copy) {
+              // We need to create a copy for this data, as it does not exist yet.
+              copy = detail::create_new_datacopy(std::forward<Value>(value));
+            }
           }
           copy = detail::register_data_copy<decvalueT>(copy, nullptr, true);
 
@@ -3333,35 +3352,47 @@ namespace ttg_parsec {
         TTBase::invoke();
     }
 
-#if 0
+  private:
     template<typename Key, typename Arg, typename... Args, std::size_t I, std::size_t... Is>
-    void invoke_arglist(const Key& key, Arg&& arg, Args&&... args, std::index_sequence<I, Is...>) {
-      if constexpr (ttg::detail::view_trait<Arg>::is_view) {
-        /* need to unpack the view */
+    void invoke_arglist(std::index_sequence<I, Is...>, const Key& key, Arg&& arg, Args&&... args) {
+      using arg_type = std::decay_t<Arg>;
+      if constexpr (ttg::detail::is_ptr_v<arg_type>) {
+        /* add a reference to the object */
+        auto copy = ttg_parsec::detail::get_copy(arg);
+        copy->add_ref();
+        /* reset readers so that the value can flow without copying */
+        copy->reset_readers();
+        auto& val = *arg;
+        set_arg_impl<I>(key, val, copy);
         if constexpr (std::is_rvalue_reference_v<Arg>) {
-          ttg::detail::value_copy_handler
+          /* if the ptr was moved in we reset it */
+          arg.reset();
         }
-      } else {
-        ttg::detail::value_copy_handler<ttg::Runtime::PaRSEC> vch{arg};
+      } else if constexpr (!ttg::detail::is_ptr_v<arg_type>) {
+        set_arg<I>(key, std::forward<Arg>(arg));
       }
       if constexpr (sizeof...(Is) > 0) {
         /* recursive next argument */
-        invoke_arglist(key, std::forward<Args>(args)..., std::index_sequence<Is...>{});
+        invoke_arglist(std::index_sequence<Is...>{}, key, std::forward<Args>(args)...);
       }
     }
 
+  public:
     // Manual injection of a task with all input arguments specified as a tuple
     template <typename Key = keyT, typename Arg, typename... Args>
     std::enable_if_t<!ttg::meta::is_void_v<Key> && !ttg::meta::is_empty_tuple_v<input_values_tuple_type>, void> invoke(
         const Key &key, Arg&& arg, Args&&... args) {
+      static_assert(sizeof...(Args)+1 == std::tuple_size_v<actual_input_tuple_type>,
+                    "Number of arguments to invoke must match the number of task inputs.");
       TTG_OP_ASSERT_EXECUTABLE();
       /* trigger non-void inputs */
-      set_args(ttg::meta::nonvoid_index_seq<actual_input_tuple_type>{}, key, args);
+      invoke_arglist(ttg::meta::nonvoid_index_seq<actual_input_tuple_type>{}, key,
+                     std::forward<Arg>(arg), std::forward<Args>(args)...);
+      //set_args(ttg::meta::nonvoid_index_seq<actual_input_tuple_type>{}, key, args);
       /* trigger void inputs */
       using void_index_seq = ttg::meta::void_index_seq<actual_input_tuple_type>;
       set_args(void_index_seq{}, key, ttg::detail::make_void_tuple<void_index_seq::size()>());
     }
-#endif // 0
 
     void set_defer_writer(bool value) {
       m_defer_writer = value;
@@ -3442,106 +3473,6 @@ namespace ttg_parsec {
 
 #include "ttg/make_tt.h"
 
-
-  namespace detail {
-    template<typename... Views, std::size_t I, std::size_t... Is>
-    bool register_device_memory(std::tuple<Views&...> &views, std::index_sequence<I, Is...>) {
-      static_assert(I < MAX_PARAM_COUNT,
-                    "PaRSEC only supports MAX_PARAM_COUNT device input/outputs. "
-                    "Increase MAX_PARAM_COUNT and recompile PaRSEC/TTG.");
-      using view_type = std::remove_reference_t<std::tuple_element_t<I, std::tuple<Views&...>>>;
-      auto& view = std::get<I>(views);
-      bool is_current = false;
-      static_assert(ttg::is_buffer_v<view_type> || ttg_parsec::is_devicescratch_v<view_type>);
-      /* get_parsec_data is overloaded for buffer and devicescratch */
-      parsec_data_t* data = detail::get_parsec_data(view);
-      /* TODO: check whether the device is current */
-
-      auto flow_flags = PARSEC_FLOW_ACCESS_RW;
-      bool pushout = false;
-      if constexpr (std::is_const_v<view_type>) {
-        flow_flags = PARSEC_FLOW_ACCESS_READ;
-      } else if constexpr (ttg_parsec::is_devicescratch_v<view_type>) {
-        switch(view.scope()) {
-          case ttg::scope::Allocate:
-            flow_flags = PARSEC_FLOW_ACCESS_NONE;
-            break;
-          case ttg::scope::SyncIn:
-            flow_flags = PARSEC_FLOW_ACCESS_READ;
-            break;
-          case ttg::scope::SyncOut:
-            flow_flags = PARSEC_FLOW_ACCESS_WRITE;
-            pushout = true;
-            break;
-          case ttg::scope::SyncInOut:
-            flow_flags = PARSEC_FLOW_ACCESS_RW;
-            pushout = true;
-            break;
-        }
-      }
-      assert(nullptr != detail::parsec_ttg_caller->dev_ptr);
-      parsec_gpu_task_t *gpu_task = detail::parsec_ttg_caller->dev_ptr->gpu_task;
-      parsec_flow_t *flows = detail::parsec_ttg_caller->dev_ptr->flows;
-
-      std::cout << "register_device_memory task " << detail::parsec_ttg_caller << " data " << I << " "
-                << data << " size " << data->nb_elts << std::endl;
-
-      /* build the flow */
-      /* TODO: reuse the flows of the task class? How can we control the sync direction then? */
-      flows[I] = parsec_flow_t{.name = nullptr,
-                               .sym_type = PARSEC_SYM_INOUT,
-                               .flow_flags = static_cast<uint8_t>(flow_flags),
-                               .flow_index = I,
-                               .flow_datatype_mask = ~0 };
-
-      gpu_task->flow_nb_elts[I] = data->nb_elts; // size in bytes
-      gpu_task->flow[I] = &flows[I];
-
-      if (pushout) {
-        std::cout << "PUSHOUT " << I << std::endl;
-        gpu_task->pushout |= 1<<I;
-      }
-
-      /* set the input data copy, parsec will take care of the transfer
-       * and the buffer will look at the parsec_data_t for the current pointer */
-      //detail::parsec_ttg_caller->parsec_task.data[I].data_in = data->device_copies[data->owner_device];
-      detail::parsec_ttg_caller->parsec_task.data[I].data_in = data->device_copies[0];
-      detail::parsec_ttg_caller->parsec_task.data[I].source_repo_entry = NULL;
-
-      if constexpr (sizeof...(Is) > 0) {
-        is_current |= register_device_memory(views, std::index_sequence<Is...>{});
-      }
-      return is_current;
-    }
-  } // namespace detail
-
-  /* Takes a tuple of ttg::Views or ttg::buffers and register them
-   * with the currently executing task. Returns true if all memory
-   * is current on the target device, false if transfers are required. */
-  template<typename... Views>
-  bool register_device_memory(std::tuple<Views&...> &views) {
-    if (nullptr == detail::parsec_ttg_caller) {
-      throw std::runtime_error("register_device_memory may only be invoked from inside a task!");
-    }
-
-    if (nullptr == detail::parsec_ttg_caller->dev_ptr) {
-      throw std::runtime_error("register_device_memory called inside a non-gpu task!");
-    }
-
-    bool is_current = detail::register_device_memory(views, std::index_sequence_for<Views...>{});
-
-    /* reset all entries in the current task */
-    for (int i = sizeof...(Views); i < MAX_PARAM_COUNT; ++i) {
-      detail::parsec_ttg_caller->parsec_task.data[i].data_in = nullptr;
-      detail::parsec_ttg_caller->dev_ptr->flows[i].flow_flags = PARSEC_FLOW_ACCESS_NONE;
-      detail::parsec_ttg_caller->dev_ptr->flows[i].flow_index = i;
-      detail::parsec_ttg_caller->dev_ptr->gpu_task->flow[i] = &detail::parsec_ttg_caller->dev_ptr->flows[i];
-      detail::parsec_ttg_caller->dev_ptr->gpu_task->flow_nb_elts[i] = 0;
-    }
-
-    return is_current;
-  }
-
 }  // namespace ttg_parsec
 
 /**
diff --git a/ttg/ttg/ptr.h b/ttg/ttg/ptr.h
index f2265f65e..d0e27d654 100644
--- a/ttg/ttg/ptr.h
+++ b/ttg/ttg/ptr.h
@@ -6,18 +6,105 @@
 namespace ttg {
 
 template<typename T>
-using ptr = TTG_IMPL_NS::ptr<T>;
-
-template<typename T>
-ptr<T> get_ptr(const T& obj) {
-  return TTG_IMPL_NS::get_ptr(obj);
-}
+using Ptr = TTG_IMPL_NS::ptr<T>;
 
 template<typename T, typename... Args>
-ptr<T> make_ptr(Args&&... args) {
+Ptr<T> make_ptr(Args&&... args) {
   return TTG_IMPL_NS::make_ptr(std::forward<Args>(args)...);
 }
 
+namespace detail {
+    template<typename T>
+    struct is_ptr : std::false_type
+    { };
+
+    template<typename T>
+    struct is_ptr<ttg::Ptr<T>> : std::true_type
+    { };
+
+    template<typename T>
+    constexpr bool is_ptr_v = is_ptr<T>::value;
+
+
+    /* awaiter for ttg::get_ptr with multiple arguments
+     * operator co_wait will return the tuple of ttg::Ptr
+     */
+    template<typename... Ts>
+    struct get_ptr_tpl_t {
+    private:
+      std::tuple<ttg::Ptr<Ts>...> m_ptr_tuple;
+      bool m_is_ready = false;
+    public:
+      get_ptr_tpl_t(bool is_ready, std::tuple<ttg::ptr<Ts>...>&& ptrs)
+      : m_ptr_tuple(std::forward<std::tuple<ttg::Ptr<Ts>...>>(ptrs))
+      , m_is_ready(is_ready)
+      { }
+
+      bool await_ready() const noexcept {
+        return m_is_ready;
+      }
+
+      constexpr void await_suspend( std::coroutine_handle<> ) const noexcept {
+        /* TODO: anything to be done here? */
+      }
+
+      auto await_resume() const noexcept {
+        return std::move(m_ptr_tuple);
+      }
+    };
+
+
+    /* awaiter for ttg::get_ptr for a single argument */
+    template<typename T>
+    struct get_ptr_t {
+    private:
+      ttg::Ptr<T> m_ptr;
+      bool m_is_ready = false;
+    public:
+      get_ptr_t(bool is_ready, ttg::Ptr<T>&& ptr)
+      : m_ptr(std::forward<ttg::Ptr<T>>(ptr))
+      , m_is_ready(is_ready)
+      { }
+
+      bool await_ready() const noexcept {
+        return m_is_ready;
+      }
+
+      constexpr void await_suspend( std::coroutine_handle<> ) const noexcept {
+        /* TODO: anything to be done here? */
+      }
+
+      auto await_resume() const noexcept {
+        return std::move(m_ptr);
+      }
+    };
+  } // namespace detail
+
+
+  /**
+   * Get an awaiter that results in a ttg::Ptr to a task argument.
+   * Must only be called inside a task on a value that was passed
+   * to the task and has not yet been moved on.
+   * Should be used in conjunction with co_await, e.g.,
+   * ttg::Ptr<double> ptr = co_await ttg::get_ptr(val);
+   *
+   * Multiple value can be passed, which results in a tuple of ptr:
+   * ttg::Ptr<double> ptr1, ptr2;
+   * std::tie(ptr1, ptr2) = co_await ttg::get_ptr(val1, val2);
+   */
+  template<typename Arg, typename... Args>
+  auto get_ptr(Arg&& arg, Args&&... args) {
+    bool is_ready;
+    using tpl_type    = std::tuple<ttg::Ptr<std::decay_t<Arg>, std::decay<Args>...>>;
+    using result_type = std::pair<bool, tpl_type>;
+    result_type p = TTG_IMPL_NS::get_ptr(std::forward<Arg>(arg), std::forward<Args>(args)...);
+    if constexpr (sizeof...(Args) > 0) {
+      return detail::get_ptr_tpl_t<std::decay_t<Arg>, std::decay_t<Args>...>(p.first, std::move(p.second));
+    } else if constexpr (sizeof...(Args) == 0) {
+      return detail::get_ptr_t<std::decay_t<Arg>>(p.first, std::move(std::get<0>(p.second)));
+    }
+  }
+
 } // namespace ttg
 
 #endif // TTG_PTR_H
\ No newline at end of file
diff --git a/ttg/ttg/view.h b/ttg/ttg/view.h
index c8e48ae4c..f28e32ef0 100644
--- a/ttg/ttg/view.h
+++ b/ttg/ttg/view.h
@@ -5,6 +5,8 @@
 #include <type_traits>
 #include <span>
 
+#include "ttg/ptr.h"
+
 namespace ttg {
 
   enum class ViewScope {
@@ -389,7 +391,7 @@ namespace ttg {
   }
 
   template<typename... Args>
-  auto make_await(Args&&... args) {
+  inline auto make_await(Args&&... args) {
     return detail::await_t{std::tie(std::forward<Args>(args)...)};
   }
 
@@ -398,10 +400,10 @@ namespace ttg {
     struct to_device_t {
       std::tuple<Ts&...> ties;
     };
-  }
+  } // namespace detail
 
   template<typename... Args>
-  auto to_device(Args&&... args) {
+  inline auto to_device(Args&&... args) {
     return detail::to_device_t{std::tie(std::forward<Args>(args)...)};
   }
 
@@ -410,13 +412,39 @@ namespace ttg {
     struct to_host_t {
       std::tuple<Ts&...> ties;
     };
-  }
+  } // namespace detail
 
   template<typename... Args>
-  auto to_host(Args&&... args) {
+  inline auto to_host(Args&&... args) {
     return detail::to_host_t{std::tie(std::forward<Args>(args)...)};
   }
 
+  namespace detail {
+    template<typename... T>
+    struct wait_kernel_t;
+    template<>
+    struct wait_kernel_t<>
+    { };
+    template<typename T, typename... Ts>
+    struct wait_kernel_t<T, Ts...> {
+      std::tuple<T&, Ts&...> ties;
+    };
+  } // namespace detail
+
+  /* Wait for the kernel to complete */
+  inline auto wait_kernel() {
+    return detail::wait_kernel_t<>{};
+  }
+
+  /* Wait for kernel to complete and provided ttg::buffer
+   * to be transferred back to host */
+  template<typename... Buffers>
+  inline auto wait_kernel_out(Buffers&&... args) {
+    static_assert((ttg::detail::is_buffer_v<std::decay_t<Buffers>>&&...),
+                  "Only ttg::buffer can be explicitly waited on!");
+    return detail::wait_kernel_t<std::decay_t<Buffers>...>{std::tie(std::forward<Buffers>(args)...)};
+  }
+
   struct device_task_promise_type;
 
   using device_task_handle_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<device_task_promise_type>;
@@ -555,6 +583,26 @@ namespace ttg {
       return {};
     }
 
+    template<typename... Ts>
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(detail::wait_kernel_t<ttg::buffer<Ts>...>&& a) {
+      std::cout << "yield_value: wait_kernel_t" << std::endl;
+      if constexpr (sizeof...(Ts) > 0) {
+        TTG_IMPL_NS::mark_device_out(a.ties);
+      }
+      m_state = TTG_DEVICE_CORO_WAIT_KERNEL;
+      return {};
+    }
+
+    template<typename... Ts>
+    auto await_transform(ttg::detail::get_ptr_tpl_t<Ts...>&& a) {
+      return a;
+    }
+
+    template<typename T>
+    auto await_transform(ttg::detail::get_ptr_t<T>&& a) {
+      return a;
+    }
+
 
     void return_void() {
       m_state = TTG_DEVICE_CORO_COMPLETE;

From 542afc25a657bb0b3ac3ad0bdd9f95583d6ca007 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 3 Mar 2023 09:57:52 -0500
Subject: [PATCH 071/259] get_ptr is not an awaitable operation

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/fwd.h |  4 ++++
 ttg/ttg/parsec/ptr.h | 17 +++++++++--------
 ttg/ttg/ptr.h        | 12 ++++++++++--
 ttg/ttg/view.h       |  3 ++-
 4 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/ttg/ttg/parsec/fwd.h b/ttg/ttg/parsec/fwd.h
index f01439a1d..bc8bcfb72 100644
--- a/ttg/ttg/parsec/fwd.h
+++ b/ttg/ttg/parsec/fwd.h
@@ -68,8 +68,12 @@ namespace ttg_parsec {
   template <typename T>
   static void ttg_broadcast(ttg::World world, T &data, int source_rank);
 
+#if 0
   template<typename... Args>
   inline std::pair<bool, std::tuple<ptr<std::decay_t<Args>>...>> get_ptr(Args&&... args);
+#endif
+  template<typename T>
+  inline ptr<std::decay_t<T>> get_ptr(T&& obj);
 
   template<typename T, typename... Args>
   inline ptr<T> make_ptr(Args&&... args);
diff --git a/ttg/ttg/parsec/ptr.h b/ttg/ttg/parsec/ptr.h
index 1c43db418..6499e050b 100644
--- a/ttg/ttg/parsec/ptr.h
+++ b/ttg/ttg/parsec/ptr.h
@@ -127,10 +127,10 @@ namespace ttg_parsec {
   } // namespace detail
 
   template<typename T, typename... Args>
-  ptr<T> make_ptr(Args&&... args);
+  ptr<T> ttg_parsec::make_ptr(Args&&... args);
 
   template<typename T>
-  ptr<T> get_ptr(const T& obj);
+  ptr<std::decay_t<T>> ttg_parsec::get_ptr(T&& obj);
 
   template<typename T>
   struct ptr {
@@ -146,7 +146,7 @@ namespace ttg_parsec {
     template<typename... Args>
     friend ptr<T> make_ptr(Args&&... args);
     template<typename S>
-    friend ptr<S> get_ptr(const S& obj);
+    friend ptr<std::decay_t<S>> get_ptr(S&& obj);
     template<typename S>
     friend detail::ttg_data_copy_t* detail::get_copy(ptr<S>& p);
     friend ttg::detail::value_copy_handler<ttg::Runtime::PaRSEC>;
@@ -196,6 +196,7 @@ namespace ttg_parsec {
     }
   };
 
+#if 0
   namespace detail {
     template<typename Arg>
     inline auto get_ptr(Arg&& obj) {
@@ -243,25 +244,25 @@ namespace ttg_parsec {
     std::tuple<ptr<std::decay_t<Args>>...> tpl = {(fn(std::forward<Args>(args)))...};
     return {ready, std::move(tpl)};
   }
+#endif // 0
 
-#if 0
   template<typename T>
-  ptr<T> get_ptr(const T& obj) {
+  inline ptr<std::decay_t<T>> get_ptr(T&& obj) {
+    using ptr_type = ptr<std::decay_t<T>>;
     if (nullptr != detail::parsec_ttg_caller) {
       for (int i = 0; i < detail::parsec_ttg_caller->data_count; ++i) {
         detail::ttg_data_copy_t *copy = detail::parsec_ttg_caller->copies[i];
         if (nullptr != copy) {
           if (copy->get_ptr() == &obj) {
-            return ptr<T>(copy);
+            return ptr_type(copy);
           }
         }
       }
     }
     /* object not tracked, make a new ptr that is now tracked */
     detail::ttg_data_copy_t *copy = detail::create_new_datacopy(obj);
-    return ptr<T>(copy);
+    return ptr_type(copy);
   }
-#endif // 0
 
   template<typename T, typename... Args>
   inline ptr<T> make_ptr(Args&&... args) {
diff --git a/ttg/ttg/ptr.h b/ttg/ttg/ptr.h
index d0e27d654..3cc699f60 100644
--- a/ttg/ttg/ptr.h
+++ b/ttg/ttg/ptr.h
@@ -13,6 +13,11 @@ Ptr<T> make_ptr(Args&&... args) {
   return TTG_IMPL_NS::make_ptr(std::forward<Args>(args)...);
 }
 
+template<typename T>
+auto get_ptr(T&& obj) {
+  return TTG_IMPL_NS::get_ptr(std::forward<T>(obj));
+}
+
 namespace detail {
     template<typename T>
     struct is_ptr : std::false_type
@@ -25,6 +30,10 @@ namespace detail {
     template<typename T>
     constexpr bool is_ptr_v = is_ptr<T>::value;
 
+} // namespace detail
+
+#if 0
+namespace detail {
 
     /* awaiter for ttg::get_ptr with multiple arguments
      * operator co_wait will return the tuple of ttg::Ptr
@@ -80,7 +89,6 @@ namespace detail {
     };
   } // namespace detail
 
-
   /**
    * Get an awaiter that results in a ttg::Ptr to a task argument.
    * Must only be called inside a task on a value that was passed
@@ -104,7 +112,7 @@ namespace detail {
       return detail::get_ptr_t<std::decay_t<Arg>>(p.first, std::move(std::get<0>(p.second)));
     }
   }
-
+#endif // 0
 } // namespace ttg
 
 #endif // TTG_PTR_H
\ No newline at end of file
diff --git a/ttg/ttg/view.h b/ttg/ttg/view.h
index f28e32ef0..4275b3fa0 100644
--- a/ttg/ttg/view.h
+++ b/ttg/ttg/view.h
@@ -593,6 +593,7 @@ namespace ttg {
       return {};
     }
 
+#if 0
     template<typename... Ts>
     auto await_transform(ttg::detail::get_ptr_tpl_t<Ts...>&& a) {
       return a;
@@ -602,7 +603,7 @@ namespace ttg {
     auto await_transform(ttg::detail::get_ptr_t<T>&& a) {
       return a;
     }
-
+#endif // 0
 
     void return_void() {
       m_state = TTG_DEVICE_CORO_COMPLETE;

From 53fd5f7f251e627a7932503d1392b901123d9a28 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 8 Mar 2023 19:17:11 -0500
Subject: [PATCH 072/259] Extend CUDA test and add support for sending objects
 containing a buffer

---
 CMakeLists.txt                     |  24 +-
 cmake/modules/AddCUDAToolkit.cmake |   9 +
 tests/unit/CMakeLists.txt          |   4 +-
 tests/unit/cuda_kernel.cu          |  22 ++
 tests/unit/cuda_kernel.h           |   4 +
 tests/unit/device_coro.cc          |  24 +-
 ttg/ttg/config.in.h                |   2 +
 ttg/ttg/device/send.h              |  26 ++
 ttg/ttg/parsec/buffer.h            |  39 +-
 ttg/ttg/parsec/devicefunc.h        | 178 +++++++++
 ttg/ttg/parsec/fwd.h               |   3 +
 ttg/ttg/parsec/task.h              |   2 +-
 ttg/ttg/parsec/ttg.h               | 616 ++++++++++++-----------------
 ttg/ttg/parsec/ttg_data_copy.h     |  58 ++-
 ttg/ttg/view.h                     |  30 +-
 15 files changed, 640 insertions(+), 401 deletions(-)
 create mode 100644 cmake/modules/AddCUDAToolkit.cmake
 create mode 100644 tests/unit/cuda_kernel.cu
 create mode 100644 tests/unit/cuda_kernel.h
 create mode 100644 ttg/ttg/device/send.h
 create mode 100644 ttg/ttg/parsec/devicefunc.h

diff --git a/CMakeLists.txt b/CMakeLists.txt
index f16532188..c4c28b042 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -86,6 +86,20 @@ include(FindOrFetchBoost)
 # C++ coroutines
 find_package(CXXStdCoroutine MODULE REQUIRED COMPONENTS Final Experimental)
 
+
+##########################
+#### CUDA: must come before PaRSEC
+##########################
+include(CheckLanguage)
+check_language(CUDA)
+if(CMAKE_CUDA_COMPILER)
+    enable_language(CUDA)
+endif(CMAKE_CUDA_COMPILER)
+set(TTG_HAVE_CUDA ${CMAKE_CUDA_COMPILER} CACHE BOOL "True if TTG supports compiling .cu files")
+
+
+
+
 ##########################
 #### prerequisite runtimes
 ##########################
@@ -101,16 +115,6 @@ if (TARGET MADworld)
 endif(TARGET MADworld)
 
 
-##########################
-#### CUDA
-##########################
-check_language(CUDA)
-if(CMAKE_CUDA_COMPILER)
-    enable_language(CUDA)
-endif(CMAKE_CUDA_COMPILER)
-set(TTG_HAVE_CUDA ${CMAKE_CUDA_COMPILER} CACHE BOOL "True if TTG supports compiling .cu files")
-
-
 ##########################
 #### Examples
 ##########################
diff --git a/cmake/modules/AddCUDAToolkit.cmake b/cmake/modules/AddCUDAToolkit.cmake
new file mode 100644
index 000000000..5d682345c
--- /dev/null
+++ b/cmake/modules/AddCUDAToolkit.cmake
@@ -0,0 +1,9 @@
+find_package(CUDAToolkit)
+set(TTG_HAVE_CUDA ${CUDAToolkit_FOUND} CACHE BOOL "True if TTG supports CUDA")
+if (TTG_HAVE_CUDA)
+    check_language(CUDA)
+    if(CMAKE_CUDA_COMPILER)
+        enable_language(CUDA)
+    endif(CMAKE_CUDA_COMPILER)
+    set(TTG_HAVE_CU_COMPILER ${CMAKE_CUDA_COMPILER} CACHE BOOL "True if TTG supports compiling .cu files")
+endif(TTG_HAVE_CUDA)
\ No newline at end of file
diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt
index 73d8a152a..c2db7bd3f 100644
--- a/tests/unit/CMakeLists.txt
+++ b/tests/unit/CMakeLists.txt
@@ -11,7 +11,9 @@ set(ut_libs Catch2::Catch2)
 #if (TARGET std::coroutine)
     #list(APPEND ut_src fibonacci-coro.cc)
     list(APPEND ut_src device_coro.cc)
-    list(APPEND ut_src cuda_kernel.cu)
+    if (TTG_HAVE_CUDA)
+        list(APPEND ut_src cuda_kernel.cu)
+    endif(TTG_HAVE_CUDA)
     list(APPEND ut_libs std::coroutine)
 #endif()
 add_ttg_executable(core-unittests-ttg "${ut_src}" LINK_LIBRARIES "${ut_libs}")
diff --git a/tests/unit/cuda_kernel.cu b/tests/unit/cuda_kernel.cu
new file mode 100644
index 000000000..b97611fe5
--- /dev/null
+++ b/tests/unit/cuda_kernel.cu
@@ -0,0 +1,22 @@
+
+#include "cuda_kernel.h"
+
+#ifdef TTG_HAVE_CUDA
+
+static __global__ void cu_increment_buffer(double* buffer, double* scratch) {
+  // Thread index
+  int tx = threadIdx.x;
+
+  buffer[tx] += 1.0;
+  if (tx == 0 && scratch != nullptr) {
+    *scratch = buffer[tx];
+  }
+}
+
+void increment_buffer(double* buffer, std::size_t buffer_size, double* scratch, std::size_t scratch_size) {
+
+  cu_increment_buffer<<<1, buffer_size>>>(buffer, scratch);
+
+}
+
+#endif // TTG_HAVE_CUDA
\ No newline at end of file
diff --git a/tests/unit/cuda_kernel.h b/tests/unit/cuda_kernel.h
new file mode 100644
index 000000000..4fec87a99
--- /dev/null
+++ b/tests/unit/cuda_kernel.h
@@ -0,0 +1,4 @@
+#include "ttg/config.h"
+#include <cinttypes>
+
+void increment_buffer(double* buffer, std::size_t buffer_size, double* scratch, std::size_t scratch_size);
\ No newline at end of file
diff --git a/tests/unit/device_coro.cc b/tests/unit/device_coro.cc
index 11ed77d91..c4b8a19e6 100644
--- a/tests/unit/device_coro.cc
+++ b/tests/unit/device_coro.cc
@@ -26,9 +26,9 @@ namespace madness::archive {
 }  // namespace madness::archive
 
 
-
 TEST_CASE("Device", "coro") {
 
+#if 0
   SECTION("devicebuf") {
 
     ttg::Edge<int, value_t> edge;
@@ -73,13 +73,17 @@ TEST_CASE("Device", "coro") {
       CHECK(ds.device_ptr()  != nullptr);
 
       /* call a kernel */
+#ifdef TTG_HAVE_CUDA
       increment_buffer(val.db.current_device_ptr(), val.db.size(), ds.device_ptr(), ds.size());
+#endif // TTG_HAVE_CUDA
 
       /* here we suspend to wait for a kernel to complete */
       co_await ttg::wait_kernel();
 
+#ifdef TTG_HAVE_CUDA
       /* buffer is increment once per task, so it should be the same as key */
-      CHECK(static_cast<int>(scratch) == key);
+      CHECK((static_cast<int>(scratch)-1) == key);
+#endif // 0
 
       /* we're back, the kernel executed and we can send */
       if (key < 10) {
@@ -94,11 +98,13 @@ TEST_CASE("Device", "coro") {
     if (ttg::default_execution_context().rank() == 0) tt->invoke(0, value_t{});
     ttg::ttg_fence(ttg::default_execution_context());
   }
+#endif // 0
 
   SECTION("ptr") {
 
     ttg::Edge<int, value_t> edge;
     ttg::Ptr<value_t> ptr;
+    int last_key = 0;
     auto fn = [&](const int& key, value_t&& val) -> ttg::device_task {
       double scratch = 1.0;
       ttg::devicescratch<double> ds = ttg::make_scratch(&scratch, ttg::scope::SyncOut);
@@ -109,14 +115,18 @@ TEST_CASE("Device", "coro") {
       CHECK(ds.device_ptr()  != nullptr);
 
       /* KERNEL */
+#ifdef TTG_HAVE_CUDA
       increment_buffer(val.db.current_device_ptr(), val.db.size(), ds.device_ptr(), ds.size());
+#endif // TTG_HAVE_CUDA
 
       /* here we suspend to wait for a kernel and the out-transfer to complete */
       co_await ttg::wait_kernel_out(val.db);
 
+#ifdef TTG_HAVE_CUDA
       /* buffer is increment once per task, so it should be the same as key */
-      CHECK(static_cast<int>(scratch) == key);
-      CHECK(static_cast<int>(*val.db.host_ptr()) == key);
+      CHECK(static_cast<int>(scratch) == key+1);
+      CHECK(static_cast<int>(*val.db.host_ptr()) == key+1);
+#endif // TTG_HAVE_CUDA
 
       /* we're back, the kernel executed and we can send */
       if (key < 10 || scratch < 0.0) {
@@ -125,6 +135,7 @@ TEST_CASE("Device", "coro") {
         /* exfiltrate the value */
         /* TODO: what consistency do we expect from get_ptr? */
         ptr = ttg::get_ptr(val);
+        last_key = key;
       }
     };
 
@@ -138,13 +149,16 @@ TEST_CASE("Device", "coro") {
     CHECK(ptr.is_valid());
 
     /* feed the ptr back into a graph */
-    if (ttg::default_execution_context().rank() == 0) tt->invoke(11, ptr);
+    if (ttg::default_execution_context().rank() == 0) tt->invoke(last_key+1, ptr);
     ttg::ttg_fence(ttg::default_execution_context());
 
     ptr.reset();
   }
 
 
+
+
+
 #if 0
 
   SECTION("device_task") {
diff --git a/ttg/ttg/config.in.h b/ttg/ttg/config.in.h
index 87113b4e5..8f208c611 100644
--- a/ttg/ttg/config.in.h
+++ b/ttg/ttg/config.in.h
@@ -11,4 +11,6 @@
 /** the C++ namespace containing the coroutine API */
 #define TTG_CXX_COROUTINE_NAMESPACE @CXX_COROUTINE_NAMESPACE@
 
+#cmakedefine TTG_HAVE_CUDA
+
 #endif  // TTG_CONFIG_IN_H
diff --git a/ttg/ttg/device/send.h b/ttg/ttg/device/send.h
new file mode 100644
index 000000000..d125c87f2
--- /dev/null
+++ b/ttg/ttg/device/send.h
@@ -0,0 +1,26 @@
+#ifndef TTG_DEVICE_SEND_H
+#define TTG_DEVICE_SEND_H
+
+
+namespace ttg {
+
+  namespace detail {
+
+    /* structure holding references to the key and value, awaitable */
+    template<typename Key, typename Value>
+    struct await_send {
+        const Key& m_key;
+        Value& m_value;
+
+        await_send(const Key& key, Value& val)
+        : m_key(key)
+        , m_value(val)
+        { }
+    };
+
+  } // namespace ttg
+
+} // namespace ttg
+
+
+#endif // TTG_DEVICE_SEND_H
\ No newline at end of file
diff --git a/ttg/ttg/parsec/buffer.h b/ttg/ttg/parsec/buffer.h
index 6d1a0c16f..8a293738e 100644
--- a/ttg/ttg/parsec/buffer.h
+++ b/ttg/ttg/parsec/buffer.h
@@ -5,10 +5,12 @@
 #define TTG_PARSEC_MAX_NUM_DEVICES 4
 
 #include <array>
+#include <vector>
 #include <parsec.h>
 #include <parsec/data_internal.h>
 #include <parsec/mca/device/device.h>
 #include "ttg/parsec/ttg_data_copy.h"
+#include "ttg/util/iovec.h"
 
 namespace ttg_parsec {
 
@@ -136,11 +138,22 @@ struct buffer {
   : m_data(std::move(db.m_data))
   , m_host_data(std::move(db.m_host_data))
   , m_count(db.m_count)
+  , m_ttg_copy(db.m_ttg_copy)
   {
     db.m_count = 0;
-    /* don't update the ttg_copy, we keep the connection */
+
+    if (nullptr == m_ttg_copy && nullptr != detail::ttg_data_copy_container()) {
+      m_ttg_copy = detail::ttg_data_copy_container();
+      /* register with the new ttg_copy */
+      m_ttg_copy->add_device_data(m_data.get());
+    }
   }
 
+  /* explicitly disable copying of buffers
+   * TODO: should we allow this? What data to use?
+   */
+  buffer(const buffer& db) = delete;
+#if 0
   /* copy the host data but leave the devices untouched */
   buffer(const buffer& db)
   : m_data(db.m_count ? parsec_data_new() : nullptr,
@@ -157,6 +170,7 @@ struct buffer {
     /* create the host copy with the allocated memory */
     create_host_copy();
   }
+#endif // 0
 
   /* allow moving device buffers */
   buffer& operator=(buffer&& db) {
@@ -167,6 +181,12 @@ struct buffer {
     /* don't update the ttg_copy, we keep the connection */
   }
 
+  /* explicitly disable copying of buffers
+   * TODO: should we allow this? What data to use?
+   */
+  buffer& operator=(const buffer& db) = delete;
+
+#if 0
   /* copy the host buffer content but leave the devices untouched */
   buffer& operator=(const buffer& db) {
     if (db.m_count == 0) {
@@ -184,6 +204,7 @@ struct buffer {
     }
     m_count = db.m_count;
   }
+#endif // 0
 
   /* set the current device, useful when a device
    * buffer was modified outside of a TTG */
@@ -335,11 +356,15 @@ struct buffer {
   serialize(Archive& ar) {
     if constexpr (ttg::detail::is_output_archive_v<Archive>)
       std::size_t s = size();
+      assert(m_ttg_copy != nullptr); // only tracked objects allowed
+      m_ttg_copy->iovec_add(ttg::iovec{s*sizeof(T), current_device_ptr()});
       ar(s);
     else {
       std::size_t s;
       ar(s);
       reset(s);
+      assert(m_ttg_copy != nullptr); // only tracked objects allowed
+      m_ttg_copy->iovec_add(ttg::iovec{s*sizeof(T), current_device_ptr()});
     }
     ar(value);
   }
@@ -350,14 +375,18 @@ struct buffer {
   std::enable_if_t<std::is_base_of_v<madness::archive::BufferInputArchive, Archive> ||
                    std::is_base_of_v<madness::archive::BufferOutputArchive, Archive>>
   serialize(Archive& ar) {
-    if constexpr (ttg::detail::is_output_archive_v<Archive>)
-      ar& size();
-    else {
+    if constexpr (ttg::detail::is_output_archive_v<Archive>) {
+      std::size_t s = size();
+      ar& s;
+      assert(m_ttg_copy != nullptr); // only tracked objects allowed
+      m_ttg_copy->iovec_add(ttg::iovec{s*sizeof(T), current_device_ptr()});
+    } else {
       std::size_t s;
       ar & s;
       /* initialize internal pointers and then reset */
-
       reset(s);
+      assert(m_ttg_copy != nullptr); // only tracked objects allowed
+      m_ttg_copy->iovec_add(ttg::iovec{s*sizeof(T), current_device_ptr()});
     }
   }
 #endif // TTG_SERIALIZATION_SUPPORTS_MADNESS
diff --git a/ttg/ttg/parsec/devicefunc.h b/ttg/ttg/parsec/devicefunc.h
new file mode 100644
index 000000000..3273e8084
--- /dev/null
+++ b/ttg/ttg/parsec/devicefunc.h
@@ -0,0 +1,178 @@
+#ifndef TTG_PARSEC_DEVICEFUNC_H
+#define TTG_PARSEC_DEVICEFUNC_H
+
+#include "ttg/parsec/task.h"
+#include <parsec/mca/device/device_gpu.h>
+
+namespace ttg_parsec {
+  namespace detail {
+    template<typename... Views, std::size_t I, std::size_t... Is>
+    inline bool register_device_memory(std::tuple<Views&...> &views, std::index_sequence<I, Is...>) {
+      static_assert(I < MAX_PARAM_COUNT,
+                    "PaRSEC only supports MAX_PARAM_COUNT device input/outputs. "
+                    "Increase MAX_PARAM_COUNT and recompile PaRSEC/TTG.");
+      using view_type = std::remove_reference_t<std::tuple_element_t<I, std::tuple<Views&...>>>;
+      auto& view = std::get<I>(views);
+      bool is_current = false;
+      static_assert(ttg::is_buffer_v<view_type> || ttg_parsec::is_devicescratch_v<view_type>);
+      /* get_parsec_data is overloaded for buffer and devicescratch */
+      parsec_data_t* data = detail::get_parsec_data(view);
+      /* TODO: check whether the device is current */
+
+      auto flow_flags = PARSEC_FLOW_ACCESS_RW;
+      bool pushout = false;
+      if constexpr (std::is_const_v<view_type>) {
+        flow_flags = PARSEC_FLOW_ACCESS_READ;
+      } else if constexpr (ttg_parsec::is_devicescratch_v<view_type>) {
+        switch(view.scope()) {
+          case ttg::scope::Allocate:
+            flow_flags = PARSEC_FLOW_ACCESS_NONE;
+            break;
+          case ttg::scope::SyncIn:
+            flow_flags = PARSEC_FLOW_ACCESS_READ;
+            break;
+          case ttg::scope::SyncOut:
+            flow_flags = PARSEC_FLOW_ACCESS_WRITE;
+            pushout = true;
+            break;
+          case ttg::scope::SyncInOut:
+            flow_flags = PARSEC_FLOW_ACCESS_RW;
+            pushout = true;
+            break;
+        }
+      }
+      assert(nullptr != detail::parsec_ttg_caller->dev_ptr);
+      parsec_gpu_task_t *gpu_task = detail::parsec_ttg_caller->dev_ptr->gpu_task;
+      parsec_flow_t *flows = detail::parsec_ttg_caller->dev_ptr->flows;
+
+      std::cout << "register_device_memory task " << detail::parsec_ttg_caller << " data " << I << " "
+                << data << " size " << data->nb_elts << std::endl;
+
+      /* build the flow */
+      /* TODO: reuse the flows of the task class? How can we control the sync direction then? */
+      flows[I] = parsec_flow_t{.name = nullptr,
+                               .sym_type = PARSEC_SYM_INOUT,
+                               .flow_flags = static_cast<uint8_t>(flow_flags),
+                               .flow_index = I,
+                               .flow_datatype_mask = ~0 };
+
+      gpu_task->flow_nb_elts[I] = data->nb_elts; // size in bytes
+      gpu_task->flow[I] = &flows[I];
+
+      if (pushout) {
+        std::cout << "PUSHOUT " << I << std::endl;
+        gpu_task->pushout |= 1<<I;
+      }
+
+      /* set the input data copy, parsec will take care of the transfer
+       * and the buffer will look at the parsec_data_t for the current pointer */
+      //detail::parsec_ttg_caller->parsec_task.data[I].data_in = data->device_copies[data->owner_device];
+      detail::parsec_ttg_caller->parsec_task.data[I].data_in = data->device_copies[0];
+      detail::parsec_ttg_caller->parsec_task.data[I].source_repo_entry = NULL;
+
+      if constexpr (sizeof...(Is) > 0) {
+        is_current |= register_device_memory(views, std::index_sequence<Is...>{});
+      }
+      return is_current;
+    }
+  } // namespace detail
+
+  /* Takes a tuple of ttg::Views or ttg::buffers and register them
+   * with the currently executing task. Returns true if all memory
+   * is current on the target device, false if transfers are required. */
+  template<typename... Views>
+  inline bool register_device_memory(std::tuple<Views&...> &views) {
+    if (nullptr == detail::parsec_ttg_caller) {
+      throw std::runtime_error("register_device_memory may only be invoked from inside a task!");
+    }
+
+    if (nullptr == detail::parsec_ttg_caller->dev_ptr) {
+      throw std::runtime_error("register_device_memory called inside a non-gpu task!");
+    }
+
+    bool is_current = detail::register_device_memory(views, std::index_sequence_for<Views...>{});
+
+    /* reset all entries in the current task */
+    for (int i = sizeof...(Views); i < MAX_PARAM_COUNT; ++i) {
+      detail::parsec_ttg_caller->parsec_task.data[i].data_in = nullptr;
+      detail::parsec_ttg_caller->dev_ptr->flows[i].flow_flags = PARSEC_FLOW_ACCESS_NONE;
+      detail::parsec_ttg_caller->dev_ptr->flows[i].flow_index = i;
+      detail::parsec_ttg_caller->dev_ptr->gpu_task->flow[i] = &detail::parsec_ttg_caller->dev_ptr->flows[i];
+      detail::parsec_ttg_caller->dev_ptr->gpu_task->flow_nb_elts[i] = 0;
+    }
+
+    return is_current;
+  }
+
+  namespace detail {
+    template<typename... Views, std::size_t I, std::size_t... Is>
+    inline void mark_device_out(std::tuple<Views&...> &views, std::index_sequence<I, Is...>) {
+
+      using view_type = std::remove_reference_t<std::tuple_element_t<I, std::tuple<Views&...>>>;
+      auto& view = std::get<I>(views);
+
+      /* get_parsec_data is overloaded for buffer and devicescratch */
+      parsec_data_t* data = detail::get_parsec_data(view);
+      /* find the data copy and mark it as pushout */
+      int i = 0;
+      parsec_gpu_task_t *gpu_task = detail::parsec_ttg_caller->dev_ptr->gpu_task;
+      while (detail::parsec_ttg_caller->parsec_task.data[i].data_in != nullptr) {
+        if (detail::parsec_ttg_caller->parsec_task.data[i].data_in == data->device_copies[0]) {
+          gpu_task->pushout |= 1<<i;
+          break;
+        }
+        ++i;
+      }
+
+      if constexpr (sizeof...(Is) > 0) {
+        // recursion
+        return mark_device_out(views, std::index_sequence<Is...>{});
+      }
+    }
+  } // namespace detail
+
+  template<typename... Buffer>
+  inline void mark_device_out(std::tuple<Buffer&...> &b) {
+
+    if (nullptr == detail::parsec_ttg_caller) {
+      throw std::runtime_error("mark_device_out may only be invoked from inside a task!");
+    }
+
+    if (nullptr == detail::parsec_ttg_caller->dev_ptr) {
+      throw std::runtime_error("mark_device_out called inside a non-gpu task!");
+    }
+
+    detail::mark_device_out(b, std::index_sequence_for<Buffer...>{});
+  }
+
+  namespace detail {
+
+    template<typename... Views, std::size_t I, std::size_t... Is>
+    inline void post_device_out(std::tuple<Views&...> &views, std::index_sequence<I, Is...>) {
+
+      using view_type = std::remove_reference_t<std::tuple_element_t<I, std::tuple<Views&...>>>;
+
+      if constexpr (!std::is_const_v<view_type>) {
+        auto& view = std::get<I>(views);
+
+        /* get_parsec_data is overloaded for buffer and devicescratch */
+        parsec_data_t* data = detail::get_parsec_data(view);
+
+        data->device_copies[0]->version++;
+        data->owner_device = 0;
+      }
+
+      if constexpr (sizeof...(Is) > 0) {
+        // recursion
+        return mark_device_out(views, std::index_sequence<Is...>{});
+      }
+    }
+  } // namespace detail
+  template<typename... Buffer>
+  inline void post_device_out(std::tuple<Buffer&...> &b) {
+    detail::post_device_out(b, std::index_sequence_for<Buffer...>{});
+  }
+
+} // namespace ttg_parsec
+
+#endif // TTG_PARSEC_DEVICEFUNC_H
\ No newline at end of file
diff --git a/ttg/ttg/parsec/fwd.h b/ttg/ttg/parsec/fwd.h
index bc8bcfb72..efcfe795c 100644
--- a/ttg/ttg/parsec/fwd.h
+++ b/ttg/ttg/parsec/fwd.h
@@ -27,6 +27,9 @@ namespace ttg_parsec {
   template<typename... Buffer>
   inline void mark_device_out(std::tuple<Buffer&...> &b);
 
+  template<typename... Buffer>
+  inline void post_device_out(std::tuple<Buffer&...> &b);
+
   /// \internal the OG name
   template <typename keyT, typename output_terminalsT, typename derivedT, typename... input_valueTs>
   using Op [[deprecated("use TT instead")]] = TT<keyT, output_terminalsT, derivedT, ttg::typelist<input_valueTs...>>;
diff --git a/ttg/ttg/parsec/task.h b/ttg/ttg/parsec/task.h
index c8465b81c..7454c1a1e 100644
--- a/ttg/ttg/parsec/task.h
+++ b/ttg/ttg/parsec/task.h
@@ -20,7 +20,7 @@ namespace ttg_parsec {
     {
       static constexpr bool support_device = false;
       static constexpr size_t num_flows = 0;
-      device_state_t(parsec_task_t *parsec_task)
+      device_state_t()
       { }
       static constexpr device_ptr_t* dev_ptr() {
         return nullptr;
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 2a706ac77..aa8982855 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -122,16 +122,28 @@ namespace ttg_parsec {
   inline std::multimap<uint64_t, static_set_arg_fct_arg_t> delayed_unpack_actions;
 
   struct msg_header_t {
-    typedef enum {
+    typedef enum fn_id : std::int8_t {
       MSG_SET_ARG = 0,
       MSG_SET_ARGSTREAM_SIZE = 1,
       MSG_FINALIZE_ARGSTREAM_SIZE = 2,
-      MSG_GET_FROM_PULL =3 } fn_id_t;
+      MSG_GET_FROM_PULL = 3 } fn_id_t;
     uint32_t taskpool_id;
     uint64_t op_id;
+    std::size_t key_offset = 0;
     fn_id_t fn_id;
+    std::int8_t num_iovecs = 0;
     int32_t param_id;
-    int num_keys;
+    int num_keys = 0;
+    int sender;
+
+    msg_header_t(fn_id_t fid, uint32_t tid, uint64_t oid, int32_t pid, int nk, int sender)
+    : fn_id(fid)
+    , taskpool_id(tid)
+    , op_id(oid)
+    , param_id(pid)
+    , num_keys(num_keys)
+    , sender(sender)
+    { }
   };
 
   static void unregister_parsec_tags(void *_);
@@ -625,7 +637,13 @@ namespace ttg_parsec {
     template <typename Value>
     inline ttg_data_copy_t *create_new_datacopy(Value &&value) {
       using value_type = std::decay_t<Value>;
-      ttg_data_copy_t *copy = new ttg_data_value_copy_t<value_type>(std::forward<Value>(value));
+      ttg_data_copy_t *copy;
+      if constexpr (std::is_rvalue_reference_v<decltype(value)> ||
+                    std::is_copy_constructible_v<std::decay_t<Value>>) {
+        copy = new ttg_data_value_copy_t<value_type>(std::forward<Value>(value));
+      } else {
+        throw std::logic_error("Trying to copy-construct data that is not copy-constructible!");
+      }
 #if defined(PARSEC_PROF_TRACE) && defined(PARSEC_TTG_PROFILE_BACKEND)
       // Keep track of additional memory usage
       if(ttg::default_execution_context().impl().profiling()) {
@@ -818,7 +836,6 @@ namespace ttg_parsec {
            */
           copy_in->mark_mutable();
           assert(nullptr == copy_in->get_next_task());
-          assert(nullptr != task);
           copy_in->set_next_task(&task->parsec_task);
         } else {
           if (task->defer_writer && nullptr == copy_in->get_next_task()) {
@@ -964,8 +981,14 @@ namespace ttg_parsec {
       unsigned char bytes[WorldImpl::PARSEC_TTG_MAX_AM_SIZE - sizeof(msg_header_t)];
 
       msg_t() = default;
-      msg_t(uint64_t tt_id, uint32_t taskpool_id, msg_header_t::fn_id_t fn_id, int32_t param_id, int num_keys = 1)
-          : tt_id{taskpool_id, tt_id, fn_id, param_id, num_keys} {}
+      msg_t(uint64_t tt_id,
+            uint32_t taskpool_id,
+            msg_header_t::fn_id_t fn_id,
+            int32_t param_id,
+            int sender,
+            int num_keys = 1)
+      : tt_id(fn_id, taskpool_id, tt_id, param_id, sender, num_keys)
+      {}
     };
   }  // namespace detail
 
@@ -1158,7 +1181,8 @@ namespace ttg_parsec {
       auto &world_impl = world.impl();
       parsec_taskpool_t *tp = world_impl.taskpool();
       std::unique_ptr<msg_t> msg = std::make_unique<msg_t>(get_instance_id(), tp->taskpool_id,
-                                                            msg_header_t::MSG_GET_FROM_PULL, i, 1);
+                                                            msg_header_t::MSG_GET_FROM_PULL, i,
+                                                            world.rank(), 1);
       /* pack the key */
       size_t pos = 0;
       pos = pack(key, msg->bytes, pos);
@@ -1393,7 +1417,7 @@ namespace ttg_parsec {
 #else
         nullptr;
 #endif
-      std::cout << "static_op: suspended_task_address " << suspended_task_address << std::endl;
+      //std::cout << "static_op: suspended_task_address " << suspended_task_address << std::endl;
       if (suspended_task_address == nullptr) {  // task is a coroutine that has not started or an ordinary function
 
         ttT *baseobj = task->tt;
@@ -1680,9 +1704,12 @@ namespace ttg_parsec {
       msg_t *msg = static_cast<msg_t *>(data);
       if constexpr (!ttg::meta::is_void_v<keyT>) {
         /* unpack the keys */
-        uint64_t pos = 0;
+        /* TODO: can we avoid copying all the keys?! */
+        uint64_t pos = msg->tt_id.key_offset;
+        uint64_t key_end_pos;
         std::vector<keyT> keylist;
         int num_keys = msg->tt_id.num_keys;
+        std::cout << "set_arg_from_msg num_keys " << num_keys << std::endl;
         keylist.reserve(num_keys);
         auto rank = world.rank();
         for (int k = 0; k < num_keys; ++k) {
@@ -1691,91 +1718,101 @@ namespace ttg_parsec {
           assert(keymap(key) == rank);
           keylist.push_back(std::move(key));
         }
+        key_end_pos = pos;
+        /* jump back to the beginning of the message to get the value */
+        pos = 0;
         // case 1
         if constexpr (!ttg::meta::is_void_v<valueT>) {
           using decvalueT = std::decay_t<valueT>;
-          if constexpr (!ttg::has_split_metadata<decvalueT>::value) {
-            detail::ttg_data_copy_t *copy = detail::create_new_datacopy(decvalueT{});
-            unpack(*static_cast<decvalueT *>(copy->get_ptr()), msg->bytes, pos);
-
-            set_arg_from_msg_keylist<i, decvalueT>(ttg::span<keyT>(&keylist[0], num_keys), copy);
-          } else {
-            /* unpack the header and start the RMA transfers */
+          int32_t num_iovecs = msg->tt_id.num_iovecs;
+          std::span<parsec_ce_mem_reg_handle_t> tag_span;
+          detail::ttg_data_copy_t *copy;
+          if constexpr (ttg::has_split_metadata<decvalueT>::value) {
             ttg::SplitMetadataDescriptor<decvalueT> descr;
             using metadata_t = decltype(descr.get_metadata(std::declval<decvalueT>()));
-            size_t metadata_size = sizeof(metadata_t);
 
             /* unpack the metadata */
             metadata_t metadata;
-            std::memcpy(&metadata, msg->bytes + pos, metadata_size);
-            pos += metadata_size;
+            pos = unpack(metadata, msg->bytes, pos);
+
+            copy = detail::create_new_datacopy(descr.create_from_metadata(metadata));
+          } else if constexpr (!ttg::has_split_metadata<decvalueT>::value) {
+            copy = detail::create_new_datacopy(decvalueT{});
+            /* unpack the object, potentially discovering iovecs */
+            pos = unpack(*static_cast<decvalueT *>(copy->get_ptr()), msg->bytes, pos);
+            assert(std::distance(copy->iovec_begin(), copy->iovec_end()) == num_iovecs);
+          }
 
-            /* unpack the remote rank */
-            int remote;
-            std::memcpy(&remote, msg->bytes + pos, sizeof(remote));
-            pos += sizeof(remote);
+          if (num_iovecs == 0) {
+            set_arg_from_msg_keylist<i, decvalueT>(ttg::span<keyT>(&keylist[0], num_keys), copy);
+          } else {
+            /* unpack the header and start the RMA transfers */
 
+            /* get the remote rank */
+            int remote = msg->tt_id.sender;
             assert(remote < world.size());
 
-            /* extract the number of chunks */
-            int32_t num_iovecs;
-            std::memcpy(&num_iovecs, msg->bytes + pos, sizeof(num_iovecs));
-            pos += sizeof(num_iovecs);
-
-            detail::ttg_data_copy_t *copy = detail::create_new_datacopy(descr.create_from_metadata(metadata));
             /* nothing else to do if the object is empty */
-            if (0 == num_iovecs) {
-              set_arg_from_msg_keylist<i, decvalueT>(keylist, copy);
-            } else {
-              /* extract the callback tag */
-              parsec_ce_tag_t cbtag;
-              std::memcpy(&cbtag, msg->bytes + pos, sizeof(cbtag));
-              pos += sizeof(cbtag);
-
-              /* create the value from the metadata */
-              auto activation = new detail::rma_delayed_activate(
-                  std::move(keylist), copy, num_iovecs, [this](std::vector<keyT> &&keylist, detail::ttg_data_copy_t *copy) {
-                    set_arg_from_msg_keylist<i, decvalueT>(keylist, copy);
-                    this->world.impl().decrement_inflight_msg();
-                  });
-              auto &val = *static_cast<decvalueT *>(copy->get_ptr());
-
-              using ActivationT = std::decay_t<decltype(*activation)>;
-
-              int nv = 0;
-              /* process payload iovecs */
-              auto iovecs = descr.get_data(val);
-              /* start the RMA transfers */
-              for (auto &&iov : iovecs) {
-                ++nv;
-                parsec_ce_mem_reg_handle_t rreg;
-                int32_t rreg_size_i;
-                std::memcpy(&rreg_size_i, msg->bytes + pos, sizeof(rreg_size_i));
-                pos += sizeof(rreg_size_i);
-                rreg = static_cast<parsec_ce_mem_reg_handle_t>(msg->bytes + pos);
-                pos += rreg_size_i;
-                // std::intptr_t *fn_ptr = reinterpret_cast<std::intptr_t *>(msg->bytes + pos);
-                // pos += sizeof(*fn_ptr);
-                std::intptr_t fn_ptr;
-                std::memcpy(&fn_ptr, msg->bytes + pos, sizeof(fn_ptr));
-                pos += sizeof(fn_ptr);
-
-                /* register the local memory */
-                parsec_ce_mem_reg_handle_t lreg;
-                size_t lreg_size;
-                parsec_ce.mem_register(iov.data, PARSEC_MEM_TYPE_NONCONTIGUOUS, iov.num_bytes, parsec_datatype_int8_t,
-                                       iov.num_bytes, &lreg, &lreg_size);
-                world.impl().increment_inflight_msg();
-                /* TODO: PaRSEC should treat the remote callback as a tag, not a function pointer! */
-                parsec_ce.get(&parsec_ce, lreg, 0, rreg, 0, iov.num_bytes, remote,
-                              &detail::get_complete_cb<ActivationT>, activation,
-                              /*world.impl().parsec_ttg_rma_tag()*/
-                              cbtag, &fn_ptr, sizeof(std::intptr_t));
-              }
-
-              assert(num_iovecs == nv);
-              assert(size == (pos + sizeof(msg_header_t)));
+            /* extract the callback tag */
+            parsec_ce_tag_t cbtag;
+            std::memcpy(&cbtag, msg->bytes + pos, sizeof(cbtag));
+            pos += sizeof(cbtag);
+
+            std::cout << "set_arg_from_msg pos after cbtag " << pos << std::endl;
+
+            /* create the value from the metadata */
+            auto activation = new detail::rma_delayed_activate(
+                std::move(keylist), copy, num_iovecs, [this](std::vector<keyT> &&keylist, detail::ttg_data_copy_t *copy) {
+                  set_arg_from_msg_keylist<i, decvalueT>(keylist, copy);
+                  this->world.impl().decrement_inflight_msg();
+                });
+            auto &val = *static_cast<decvalueT *>(copy->get_ptr());
+
+            using ActivationT = std::decay_t<decltype(*activation)>;
+
+            int nv = 0;
+            /* start the RMA transfers */
+            auto handle_iovecs_fn =
+              [&](auto&& iovecs) {
+                  for (auto &&iov : iovecs) {
+                    ++nv;
+                    parsec_ce_mem_reg_handle_t rreg;
+                    int32_t rreg_size_i;
+                    std::memcpy(&rreg_size_i, msg->bytes + pos, sizeof(rreg_size_i));
+                    pos += sizeof(rreg_size_i);
+                    rreg = static_cast<parsec_ce_mem_reg_handle_t>(msg->bytes + pos);
+                    pos += rreg_size_i;
+                    // std::intptr_t *fn_ptr = reinterpret_cast<std::intptr_t *>(msg->bytes + pos);
+                    // pos += sizeof(*fn_ptr);
+                    std::intptr_t fn_ptr;
+                    std::memcpy(&fn_ptr, msg->bytes + pos, sizeof(fn_ptr));
+                    pos += sizeof(fn_ptr);
+
+                    /* register the local memory */
+                    parsec_ce_mem_reg_handle_t lreg;
+                    size_t lreg_size;
+                    parsec_ce.mem_register(iov.data, PARSEC_MEM_TYPE_NONCONTIGUOUS, iov.num_bytes, parsec_datatype_int8_t,
+                                            iov.num_bytes, &lreg, &lreg_size);
+                    world.impl().increment_inflight_msg();
+                    /* TODO: PaRSEC should treat the remote callback as a tag, not a function pointer! */
+                    parsec_ce.get(&parsec_ce, lreg, 0, rreg, 0, iov.num_bytes, remote,
+                                  &detail::get_complete_cb<ActivationT>, activation,
+                                  /*world.impl().parsec_ttg_rma_tag()*/
+                                  cbtag, &fn_ptr, sizeof(std::intptr_t));
+                  }
+            };
+            if constexpr (ttg::has_split_metadata<decvalueT>::value) {
+              ttg::SplitMetadataDescriptor<decvalueT> descr;
+              handle_iovecs_fn(descr.get_data(val));
+            } else if constexpr (!ttg::has_split_metadata<decvalueT>::value) {
+              handle_iovecs_fn(copy->iovec_span());
+              copy->iovec_reset();
             }
+            std::cout << "set_arg_from_msg pos after iovecs " << pos << " key_offset " << msg->tt_id.key_offset << std::endl;
+
+            assert(num_iovecs == nv);
+            if (size != (key_end_pos + sizeof(msg_header_t))) std::cout << "XXX pos + header + keys " << pos + sizeof(msg_header_t) + num_keys*sizeof(keyT) << " size " << size << std::endl;
+            assert(size == (key_end_pos + sizeof(msg_header_t)));
           }
           // case 2 and 3
         } else if constexpr (!ttg::meta::is_void_v<keyT> && std::is_void_v<valueT>) {
@@ -2048,8 +2085,7 @@ namespace ttg_parsec {
           /* if this is a host task make sure tracked buffers get copied to the host */
           if constexpr(!derived_has_cuda_op()) {
             int c = 0;
-            for (auto it : *copy) {
-              parsec_data_t *data = *it;
+            for (auto data : *copy) {
               if (data->owner_device != 0) {
                 task->parsec_task.data[c].data_in = data->device_copies[0];
                 task->parsec_task.data[c].source_repo_entry = NULL;
@@ -2165,9 +2201,9 @@ namespace ttg_parsec {
         owner = keymap();
       if (owner == world.rank()) {
         if constexpr (!ttg::meta::is_void_v<keyT>)
-          set_arg_local_impl<i, keyT, Value>(key, std::forward<Value>(value), copy_in);
+          set_arg_local_impl<i>(key, std::forward<Value>(value), copy_in);
         else
-          set_arg_local_impl<i, keyT, Value>(std::forward<Value>(value), copy_in);
+          set_arg_local_impl<i>(ttg::Void{}, std::forward<Value>(value), copy_in);
 #if defined(PARSEC_PROF_TRACE) && defined(PARSEC_TTG_PROFILE_BACKEND)
           if(world.impl().profiling()) {
             parsec_profiling_ts_trace(world.impl().parsec_ttg_profile_backend_set_arg_end, 0, 0, NULL);
@@ -2181,46 +2217,25 @@ namespace ttg_parsec {
       using msg_t = detail::msg_t;
       auto &world_impl = world.impl();
       uint64_t pos = 0;
+      int num_iovecs = 0;
       std::unique_ptr<msg_t> msg = std::make_unique<msg_t>(get_instance_id(), world_impl.taskpool()->taskpool_id,
-                                                           msg_header_t::MSG_SET_ARG, i, 1);
+                                                           msg_header_t::MSG_SET_ARG, i, world_impl.rank(), 1);
       using decvalueT = std::decay_t<Value>;
-      /* pack the key */
-      msg->tt_id.num_keys = 0;
-      if constexpr (!ttg::meta::is_void_v<Key>) {
-        pos = pack(key, msg->bytes, pos);
-        msg->tt_id.num_keys = 1;
-      }
+      msg->tt_id.sender = world_impl.rank();
 
       if constexpr (!ttg::meta::is_void_v<decvalueT>) {
-        if constexpr (!ttg::has_split_metadata<decvalueT>::value) {
-          pos = pack(value, msg->bytes, pos);
-        } else {
-          detail::ttg_data_copy_t *copy = copy_in;
+
+        detail::ttg_data_copy_t *copy = copy_in;
+        /* make sure we have a data copy to register with */
+        if (nullptr == copy) {
+          copy = detail::find_copy_in_task(detail::parsec_ttg_caller, &value);
           if (nullptr == copy) {
-            copy = detail::find_copy_in_task(detail::parsec_ttg_caller, &value);
-            if (nullptr == copy) {
-              // We need to create a copy for this data, as it does not exist yet.
-              copy = detail::create_new_datacopy(std::forward<Value>(value));
-            }
+            // We need to create a copy for this data, as it does not exist yet.
+            copy = detail::create_new_datacopy(std::forward<Value>(value));
           }
-          copy = detail::register_data_copy<decvalueT>(copy, nullptr, true);
-
-          ttg::SplitMetadataDescriptor<decvalueT> descr;
-          auto metadata = descr.get_metadata(value);
-          size_t metadata_size = sizeof(metadata);
-          /* pack the metadata */
-          std::memcpy(msg->bytes + pos, &metadata, metadata_size);
-          pos += metadata_size;
-          /* pack the local rank */
-          int rank = world.rank();
-          std::memcpy(msg->bytes + pos, &rank, sizeof(rank));
-          pos += sizeof(rank);
-
-          auto iovecs = descr.get_data(*static_cast<decvalueT *>(copy->get_ptr()));
+        }
 
-          int32_t num_iovs = std::distance(std::begin(iovecs), std::end(iovecs));
-          std::memcpy(msg->bytes + pos, &num_iovs, sizeof(num_iovs));
-          pos += sizeof(num_iovs);
+        auto handle_iovec_fn = [&](auto&& iovecs){
 
           /* TODO: at the moment, the tag argument to parsec_ce.get() is treated as a
            * raw function pointer instead of a preregistered AM tag, so play that game.
@@ -2234,6 +2249,7 @@ namespace ttg_parsec {
            * memory layout: [<lreg_size, lreg, release_cb_ptr>, ...]
            */
           for (auto &&iov : iovecs) {
+            copy = detail::register_data_copy<decvalueT>(copy, nullptr, true);
             parsec_ce_mem_reg_handle_t lreg;
             size_t lreg_size;
             /* TODO: only register once when we can broadcast the data! */
@@ -2259,12 +2275,47 @@ namespace ttg_parsec {
             std::memcpy(msg->bytes + pos, &fn_ptr, sizeof(fn_ptr));
             pos += sizeof(fn_ptr);
           }
+        };
+
+        if constexpr (ttg::has_split_metadata<std::decay_t<Value>>::value) {
+          ttg::SplitMetadataDescriptor<decvalueT> descr;
+          auto iovs = descr.get_data(*const_cast<decvalueT *>(&value));
+          num_iovecs = std::distance(std::begin(iovs), std::end(iovs));
+          /* pack the metadata */
+          auto metadata = descr.get_metadata(value);
+          size_t metadata_size = sizeof(metadata);
+          pos = pack(metadata, msg->bytes, pos);
+          handle_iovec_fn(iovs);
+        } else if constexpr (!ttg::has_split_metadata<std::decay_t<Value>>::value) {
+          /* serialize the object */
+          pos = pack(value, msg->bytes, pos);
+          num_iovecs = std::distance(copy->iovec_begin(), copy->iovec_end());
+          /* handle any iovecs contained in it */
+          handle_iovec_fn(copy->iovec_span());
+          copy->iovec_reset();
         }
+
+        msg->tt_id.num_iovecs = num_iovecs;
       }
+
+      /* pack the key */
+      msg->tt_id.num_keys = 0;
+      msg->tt_id.key_offset = pos;
+      std::cout << "set_arg_impl pos " << pos << " key_offset " << msg->tt_id.key_offset << " key size " << sizeof(key) << std::endl;
+      if constexpr (!ttg::meta::is_void_v<Key>) {
+        size_t tmppos = pack(key, msg->bytes, pos);
+        std::cout << "set_arg_impl pos before " << pos << " after pack " << tmppos << std::endl;
+        pos = tmppos;
+        msg->tt_id.num_keys = 1;
+      }
+
       parsec_taskpool_t *tp = world_impl.taskpool();
       tp->tdm.module->outgoing_message_start(tp, owner, NULL);
       tp->tdm.module->outgoing_message_pack(tp, owner, NULL, NULL, 0);
-      // std::cout << "Sending AM with " << msg->op_id.num_keys << " keys " << std::endl;
+      std::cout << "set_arg_impl num_keys " << msg->tt_id.num_keys
+                << ", total size " << sizeof(msg_header_t) + pos
+                << " pos " << pos << " key_offset " << msg->tt_id.key_offset
+                << " sizeof header" << sizeof(msg_header_t) << std::endl;
       parsec_ce.send_am(&parsec_ce, world_impl.parsec_ttg_tag(), owner, static_cast<void *>(msg.get()),
                         sizeof(msg_header_t) + pos);
 #if defined(PARSEC_PROF_TRACE) && defined(PARSEC_TTG_PROFILE_BACKEND)
@@ -2323,84 +2374,13 @@ namespace ttg_parsec {
     }
 
     template <std::size_t i, typename Key, typename Value>
-    std::enable_if_t<!ttg::meta::is_void_v<Key> && !std::is_void_v<std::decay_t<Value>> &&
-                         !ttg::has_split_metadata<std::decay_t<Value>>::value,
+    std::enable_if_t<!ttg::meta::is_void_v<Key> && !std::is_void_v<std::decay_t<Value>>,
                      void>
     broadcast_arg(const ttg::span<const Key> &keylist, const Value &value) {
-      auto world = ttg_default_execution_context();
-      int rank = world.rank();
-
-      bool have_remote = keylist.end() != std::find_if(keylist.begin(), keylist.end(),
-                                                       [&](const Key &key) { return keymap(key) != rank; });
-
-      if (have_remote) {
-        std::vector<Key> keylist_sorted(keylist.begin(), keylist.end());
-
-        /* Assuming there are no local keys, will be updated while processing remote keys */
-        auto local_begin = keylist_sorted.end();
-        auto local_end = keylist_sorted.end();
-
-        /* sort the input key list by owner and check whether there are remote keys */
-        std::sort(keylist_sorted.begin(), keylist_sorted.end(), [&](const Key &a, const Key &b) mutable {
-          int rank_a = keymap(a);
-          int rank_b = keymap(b);
-          return rank_a < rank_b;
-        });
-
-        using msg_t = detail::msg_t;
-        local_begin = keylist_sorted.end();
-        auto &world_impl = world.impl();
-        std::unique_ptr<msg_t> msg = std::make_unique<msg_t>(get_instance_id(), world_impl.taskpool()->taskpool_id,
-                                                             msg_header_t::MSG_SET_ARG, i);
-
-        parsec_taskpool_t *tp = world_impl.taskpool();
-
-        for (auto it = keylist_sorted.begin(); it < keylist_sorted.end(); /* increment inline */) {
-          auto owner = keymap(*it);
-          if (owner == rank) {
-            /* make sure we don't lose local keys */
-            local_begin = it;
-            local_end =
-                std::find_if_not(++it, keylist_sorted.end(), [&](const Key &key) { return keymap(key) == rank; });
-            it = local_end;
-            continue;
-          }
-
-          /* pack all keys for this owner */
-          int num_keys = 0;
-          uint64_t pos = 0;
-          do {
-            ++num_keys;
-            pos = pack(*it, msg->bytes, pos);
-            ++it;
-          } while (it < keylist_sorted.end() && keymap(*it) == owner);
-          msg->tt_id.num_keys = num_keys;
-
-          /* TODO: use RMA to transfer the value */
-          pos = pack(value, msg->bytes, pos);
-
-          /* Send the message */
-          tp->tdm.module->outgoing_message_start(tp, owner, NULL);
-          tp->tdm.module->outgoing_message_pack(tp, owner, NULL, NULL, 0);
-          parsec_ce.send_am(&parsec_ce, world_impl.parsec_ttg_tag(), owner, static_cast<void *>(msg.get()),
-                            sizeof(msg_header_t) + pos);
-        }
-        /* handle local keys */
-        broadcast_arg_local<i>(local_begin, local_end, value);
-      } else {
-        /* only local keys */
-        broadcast_arg_local<i>(keylist.begin(), keylist.end(), value);
-      }
-    }
-
-    template <std::size_t i, typename Key, typename Value>
-    std::enable_if_t<!ttg::meta::is_void_v<Key> && !std::is_void_v<std::decay_t<Value>> &&
-                         ttg::has_split_metadata<std::decay_t<Value>>::value,
-                     void>
-    splitmd_broadcast_arg(const ttg::span<const Key> &keylist, const Value &value) {
       using valueT = std::tuple_element_t<i, input_values_full_tuple_type>;
       auto world = ttg_default_execution_context();
       int rank = world.rank();
+      uint64_t pos = 0;
       bool have_remote = keylist.end() != std::find_if(keylist.begin(), keylist.end(),
                                                        [&](const Key &key) { return keymap(key) != rank; });
 
@@ -2419,41 +2399,68 @@ namespace ttg_parsec {
         auto local_begin = keylist_sorted.end();
         auto local_end = keylist_sorted.end();
 
-        ttg::SplitMetadataDescriptor<decvalueT> descr;
-        auto iovs = descr.get_data(*const_cast<decvalueT *>(&value));
-        int32_t num_iovs = std::distance(std::begin(iovs), std::end(iovs));
+        int32_t num_iovs = 0;
+
+        detail::ttg_data_copy_t *copy;
+        copy = detail::find_copy_in_task(detail::parsec_ttg_caller, &value);
+        assert(nullptr != copy);
+
         std::vector<std::pair<int32_t, std::shared_ptr<void>>> memregs;
-        memregs.reserve(num_iovs);
-
-        /* register all iovs so the registration can be reused */
-        for (auto &&iov : iovs) {
-          parsec_ce_mem_reg_handle_t lreg;
-          size_t lreg_size;
-          parsec_ce.mem_register(iov.data, PARSEC_MEM_TYPE_NONCONTIGUOUS, iov.num_bytes, parsec_datatype_int8_t,
-                                 iov.num_bytes, &lreg, &lreg_size);
-          /* TODO: use a static function for deregistration here? */
-          memregs.push_back(std::make_pair(static_cast<int32_t>(lreg_size),
-                                           /* TODO: this assumes that parsec_ce_mem_reg_handle_t is void* */
-                                           std::shared_ptr<void>{lreg, [](void *ptr) {
-                                                                   parsec_ce_mem_reg_handle_t memreg =
-                                                                       (parsec_ce_mem_reg_handle_t)ptr;
-                                                                   parsec_ce.mem_unregister(&memreg);
-                                                                 }}));
-        }
+        auto register_iovs_fn = [&memregs](auto&& iovs){
+          for (auto &&iov : iovs) {
+            parsec_ce_mem_reg_handle_t lreg;
+            size_t lreg_size;
+            parsec_ce.mem_register(iov.data, PARSEC_MEM_TYPE_NONCONTIGUOUS, iov.num_bytes, parsec_datatype_int8_t,
+                                  iov.num_bytes, &lreg, &lreg_size);
+            /* TODO: use a static function for deregistration here? */
+            memregs.push_back(std::make_pair(static_cast<int32_t>(lreg_size),
+                                            /* TODO: this assumes that parsec_ce_mem_reg_handle_t is void* */
+                                            std::shared_ptr<void>{lreg, [](void *ptr) {
+                                                                    parsec_ce_mem_reg_handle_t memreg =
+                                                                        (parsec_ce_mem_reg_handle_t)ptr;
+                                                                    parsec_ce.mem_unregister(&memreg);
+                                                                  }}));
+          }
+        };
 
         using msg_t = detail::msg_t;
         auto &world_impl = world.impl();
         std::unique_ptr<msg_t> msg = std::make_unique<msg_t>(get_instance_id(), world_impl.taskpool()->taskpool_id,
-                                                             msg_header_t::MSG_SET_ARG, i);
-        auto metadata = descr.get_metadata(value);
-        size_t metadata_size = sizeof(metadata);
+                                                             msg_header_t::MSG_SET_ARG, i, world_impl.rank());
 
-        detail::ttg_data_copy_t *copy;
-        copy = detail::find_copy_in_task(detail::parsec_ttg_caller, &value);
-        assert(nullptr != copy);
+        if constexpr (ttg::has_split_metadata<std::decay_t<Value>>::value) {
+          ttg::SplitMetadataDescriptor<decvalueT> descr;
+          auto iovs = descr.get_data(*const_cast<decvalueT *>(&value));
+          num_iovs = std::distance(std::begin(iovs), std::end(iovs));
+          memregs.reserve(num_iovs);
+          register_iovs_fn(iovs);
+          /* pack the metadata */
+          auto metadata = descr.get_metadata(value);
+          size_t metadata_size = sizeof(metadata);
+          pos = pack(metadata, msg->bytes, pos);
+        } else if constexpr (!ttg::has_split_metadata<std::decay_t<Value>>::value) {
+          /* serialize the object once */
+          pos = pack(value, msg->bytes, pos);
+          num_iovs = std::distance(copy->iovec_begin(), copy->iovec_end());
+          register_iovs_fn(copy->iovec_span());
+          copy->iovec_reset();
+        }
+
+        /* TODO: at the moment, the tag argument to parsec_ce.get() is treated as a
+          * raw function pointer instead of a preregistered AM tag, so play that game.
+          * Once this is fixed in PaRSEC we need to use parsec_ttg_rma_tag instead! */
+        parsec_ce_tag_t cbtag = reinterpret_cast<parsec_ce_tag_t>(&detail::get_remote_complete_cb);
+        std::memcpy(msg->bytes + pos, &cbtag, sizeof(cbtag));
+        pos += sizeof(cbtag);
+
+        msg->tt_id.num_iovecs = num_iovs;
+
+        std::size_t save_pos = pos;
+        std::cout << "broadcast_arg save_pos " << save_pos <<std::endl;
 
         parsec_taskpool_t *tp = world_impl.taskpool();
         for (auto it = keylist_sorted.begin(); it < keylist_sorted.end(); /* increment done inline */) {
+
           auto owner = keymap(*it);
           if (owner == rank) {
             local_begin = it;
@@ -2464,41 +2471,14 @@ namespace ttg_parsec {
             continue;
           }
 
-          /* count keys and set it afterwards */
-          uint64_t pos = 0;
-          /* pack all keys for this owner */
-          int num_keys = 0;
-          do {
-            ++num_keys;
-            pos = pack(*it, msg->bytes, pos);
-            ++it;
-          } while (it < keylist_sorted.end() && keymap(*it) == owner);
-          msg->tt_id.num_keys = num_keys;
-
-          /* pack the metadata */
-          std::memcpy(msg->bytes + pos, &metadata, metadata_size);
-          pos += metadata_size;
-          /* pack the local rank */
-          int rank = world.rank();
-          std::memcpy(msg->bytes + pos, &rank, sizeof(rank));
-          pos += sizeof(rank);
-          /* pack the number of iovecs */
-          std::memcpy(msg->bytes + pos, &num_iovs, sizeof(num_iovs));
-          pos += sizeof(num_iovs);
-
-          /* TODO: at the moment, the tag argument to parsec_ce.get() is treated as a
-           * raw function pointer instead of a preregistered AM tag, so play that game.
-           * Once this is fixed in PaRSEC we need to use parsec_ttg_rma_tag instead! */
-          parsec_ce_tag_t cbtag = reinterpret_cast<parsec_ce_tag_t>(&detail::get_remote_complete_cb);
-          std::memcpy(msg->bytes + pos, &cbtag, sizeof(cbtag));
-          pos += sizeof(cbtag);
-
+          /* rewind the buffer and start packing a new set of memregs and keys */
+          pos = save_pos;
           /**
            * pack the registration handles
            * memory layout: [<lreg_size, lreg, lreg_fn>, ...]
+           * NOTE: we need to pack these for every receiver to ensure correct ref-counting of the registration
            */
-          int idx = 0;
-          for (auto &&iov : iovs) {
+          for (int idx = 0; idx < num_iovs; ++idx) {
             // auto [lreg_size, lreg_ptr] = memregs[idx];
             int32_t lreg_size;
             std::shared_ptr<void> lreg_ptr;
@@ -2513,17 +2493,31 @@ namespace ttg_parsec {
             copy = detail::register_data_copy<valueT>(copy, nullptr, true);
             std::function<void(void)> *fn = new std::function<void(void)>([=]() mutable {
               /* shared_ptr of value and registration captured by value so resetting
-               * them here will eventually release the memory/registration */
+                * them here will eventually release the memory/registration */
               detail::release_data_copy(copy);
               lreg_ptr_v.reset();
             });
             std::intptr_t fn_ptr{reinterpret_cast<std::intptr_t>(fn)};
             std::memcpy(msg->bytes + pos, &fn_ptr, sizeof(fn_ptr));
             pos += sizeof(fn_ptr);
-            ++idx;
           }
+
+          /* mark the beginning of the keys */
+          msg->tt_id.key_offset = pos;
+          std::cout << "broadcast_arg key_offset " << msg->tt_id.key_offset <<std::endl;
+
+          /* pack all keys for this owner */
+          int num_keys = 0;
+          do {
+            ++num_keys;
+            pos = pack(*it, msg->bytes, pos);
+            ++it;
+          } while (it < keylist_sorted.end() && keymap(*it) == owner);
+          msg->tt_id.num_keys = num_keys;
+
           tp->tdm.module->outgoing_message_start(tp, owner, NULL);
           tp->tdm.module->outgoing_message_pack(tp, owner, NULL, NULL, 0);
+          std::cout << "broadcast_arg total size " << sizeof(msg_header_t) + pos << " pos " << pos <<std::endl;
           parsec_ce.send_am(&parsec_ce, world_impl.parsec_ttg_tag(), owner, static_cast<void *>(msg.get()),
                             sizeof(msg_header_t) + pos);
         }
@@ -2612,10 +2606,10 @@ namespace ttg_parsec {
         auto &world_impl = world.impl();
         uint64_t pos = 0;
         std::unique_ptr<msg_t> msg = std::make_unique<msg_t>(get_instance_id(), world_impl.taskpool()->taskpool_id,
-                                                             msg_header_t::MSG_SET_ARGSTREAM_SIZE, i, 1);
+                                                             msg_header_t::MSG_SET_ARGSTREAM_SIZE, i,
+                                                             world_impl.rank(), 1);
         /* pack the key */
         pos = pack(key, msg->bytes, pos);
-        msg->tt_id.num_keys = 1;
         pos = pack(size, msg->bytes, pos);
         parsec_taskpool_t *tp = world_impl.taskpool();
         tp->tdm.module->outgoing_message_start(tp, owner, NULL);
@@ -2667,9 +2661,8 @@ namespace ttg_parsec {
         auto &world_impl = world.impl();
         uint64_t pos = 0;
         std::unique_ptr<msg_t> msg = std::make_unique<msg_t>(get_instance_id(), world_impl.taskpool()->taskpool_id,
-                                                             msg_header_t::MSG_SET_ARGSTREAM_SIZE, i, 1);
-        /* pack the key */
-        msg->tt_id.num_keys = 0;
+                                                             msg_header_t::MSG_SET_ARGSTREAM_SIZE, i,
+                                                             world_impl.rank(), 0);
         pos = pack(size, msg->bytes, pos);
         parsec_taskpool_t *tp = world_impl.taskpool();
         tp->tdm.module->outgoing_message_start(tp, owner, NULL);
@@ -2719,10 +2712,10 @@ namespace ttg_parsec {
         auto &world_impl = world.impl();
         uint64_t pos = 0;
         std::unique_ptr<msg_t> msg = std::make_unique<msg_t>(get_instance_id(), world_impl.taskpool()->taskpool_id,
-                                                             msg_header_t::MSG_FINALIZE_ARGSTREAM_SIZE, i, 1);
+                                                             msg_header_t::MSG_FINALIZE_ARGSTREAM_SIZE, i,
+                                                             world_impl.rank(), 1);
         /* pack the key */
         pos = pack(key, msg->bytes, pos);
-        msg->tt_id.num_keys = 1;
         parsec_taskpool_t *tp = world_impl.taskpool();
         tp->tdm.module->outgoing_message_start(tp, owner, NULL);
         tp->tdm.module->outgoing_message_pack(tp, owner, NULL, NULL, 0);
@@ -2765,8 +2758,8 @@ namespace ttg_parsec {
         auto &world_impl = world.impl();
         uint64_t pos = 0;
         std::unique_ptr<msg_t> msg = std::make_unique<msg_t>(get_instance_id(), world_impl.taskpool()->taskpool_id,
-                                                             msg_header_t::MSG_FINALIZE_ARGSTREAM_SIZE, i, 1);
-        msg->tt_id.num_keys = 0;
+                                                             msg_header_t::MSG_FINALIZE_ARGSTREAM_SIZE, i,
+                                                             world_impl.rank(), 0);
         parsec_taskpool_t *tp = world_impl.taskpool();
         tp->tdm.module->outgoing_message_start(tp, owner, NULL);
         tp->tdm.module->outgoing_message_pack(tp, owner, NULL, NULL, 0);
@@ -2828,11 +2821,7 @@ namespace ttg_parsec {
           set_arg<i, keyT, const valueT &>(key, value);
         };
         auto broadcast_callback = [this](const ttg::span<const keyT> &keylist, const valueT &value) {
-          if constexpr (ttg::has_split_metadata<std::decay_t<valueT>>::value) {
-            splitmd_broadcast_arg<i, keyT, valueT>(keylist, value);
-          } else {
             broadcast_arg<i, keyT, valueT>(keylist, value);
-          }
         };
         auto setsize_callback = [this](const keyT &key, std::size_t size) { set_argstream_size<i>(key, size); };
         auto finalize_callback = [this](const keyT &key) { finalize_argstream<i>(key); };
@@ -3006,7 +2995,7 @@ namespace ttg_parsec {
 
     static parsec_hook_return_t complete_task_and_release(parsec_execution_stream_t *es, parsec_task_t *parsec_task) {
 
-      std::cout << "complete_task_and_release: task " << parsec_task << std::endl;
+      //std::cout << "complete_task_and_release: task " << parsec_task << std::endl;
 
       task_t *task = (task_t*)parsec_task;
 
@@ -3364,6 +3353,7 @@ namespace ttg_parsec {
         copy->reset_readers();
         auto& val = *arg;
         set_arg_impl<I>(key, val, copy);
+        ttg_parsec::detail::release_data_copy(copy);
         if constexpr (std::is_rvalue_reference_v<Arg>) {
           /* if the ptr was moved in we reset it */
           arg.reset();
@@ -3378,7 +3368,7 @@ namespace ttg_parsec {
     }
 
   public:
-    // Manual injection of a task with all input arguments specified as a tuple
+    // Manual injection of a task with all input arguments specified as variadic arguments
     template <typename Key = keyT, typename Arg, typename... Args>
     std::enable_if_t<!ttg::meta::is_void_v<Key> && !ttg::meta::is_empty_tuple_v<input_values_tuple_type>, void> invoke(
         const Key &key, Arg&& arg, Args&&... args) {
@@ -3497,6 +3487,9 @@ struct ttg::detail::value_copy_handler<ttg::Runtime::PaRSEC> {
 
   template <typename Value>
   inline Value &&operator()(Value &&value) {
+    static_assert(std::is_rvalue_reference_v<decltype(value)> ||
+                  std::is_copy_constructible_v<std::decay_t<Value>>,
+                  "Data sent without being moved must be copy-constructible!");
     if (nullptr == ttg_parsec::detail::parsec_ttg_caller) {
       ttg::print("ERROR: ttg_send or ttg_broadcast called outside of a task!\n");
     }
@@ -3523,6 +3516,8 @@ struct ttg::detail::value_copy_handler<ttg::Runtime::PaRSEC> {
 
   template <typename Value>
   inline const Value &operator()(const Value &value) {
+    static_assert(std::is_copy_constructible_v<std::decay_t<Value>>,
+                  "Data sent without being moved must be copy-constructible!");
     if (nullptr == ttg_parsec::detail::parsec_ttg_caller) {
       ttg::print("ERROR: ttg_send or ttg_broadcast called outside of a task!\n");
     }
@@ -3543,95 +3538,6 @@ struct ttg::detail::value_copy_handler<ttg::Runtime::PaRSEC> {
     return *value_ptr;
   }
 
-#if 0
-  /* we have to make a copy of non-const data as the user may modify it after
-   * send/broadcast */
-  template <typename Value, typename Enabler = std::enable_if_t<!std::is_const_v<Value>>>
-  inline Value &operator()(Value &value) {
-    if (nullptr == ttg_parsec::detail::parsec_ttg_caller) {
-      ttg::print("ERROR: ttg_send or ttg_broadcast called outside of a task!\n");
-    }
-    /* the value is not known, create a copy that we can track */
-    ttg_parsec::detail::ttg_data_copy_t *copy;
-    copy = ttg_parsec::detail::create_new_datacopy(value);
-    bool inserted = ttg_parsec::detail::add_copy_to_task(copy, ttg_parsec::detail::parsec_ttg_caller);
-    assert(inserted);
-    Value *value_ptr = reinterpret_cast<Value *>(copy->get_ptr());
-    copy_to_remove = copy;
-    return *value_ptr;
-  }
-
-  /**
-   * Overload for PersistentView objects
-   *
-   * TODO: make sure the device copy is current?!
-   */
-
-  template <typename Value>
-  inline Value &operator()(ttg::PersistentView<Value> &&ptr) {
-    Value& value_ref = *ptr;
-
-    /* register the copy with the task so that the TT can find it */
-    ttg_parsec::detail::ttg_data_copy_t *copy;
-    copy = ptr.get_copy();
-    bool inserted = ttg_parsec::detail::add_copy_to_task(copy, ttg_parsec::detail::parsec_ttg_caller);
-    if (inserted) {
-      copy_to_remove = copy;
-      /* if this is the first time we see this copy again we add the TTG reference */
-      if (0 == copy->num_readers()) {
-        copy->add_ref();
-      }
-      /* steal the copy from the ptr */
-      ptr.reset();
-    }
-
-    return value_ref;
-  }
-
-  template <typename Value>
-  inline Value &operator()(ttg::PersistentView<Value> &ptr) {
-    Value& value_ref = *ptr;
-
-    /* register the copy with the task so that the TT can find it */
-    ttg_parsec::detail::ttg_data_copy_t *copy;
-    copy = ptr.get_copy();
-    bool inserted = ttg_parsec::detail::add_copy_to_task(copy, ttg_parsec::detail::parsec_ttg_caller);
-    if (inserted) {
-      copy_to_remove = copy;
-      /* if this is the first time we see this copy again we add the TTG reference */
-      if (0 == copy->readers) {
-        copy->add_ref();
-      }
-    }
-    return value_ref;
-  }
-
-  template <typename Value>
-  inline const Value &operator()(const ttg::PersistentView<Value> &ptr) {
-    const Value& value_ref = *ptr;
-    /* register the copy with the task so that the TT can find it */
-    ttg_parsec::detail::ttg_data_copy_t *copy;
-    copy = ptr.get_copy();
-    bool inserted = ttg_parsec::detail::add_copy_to_task(copy, ttg_parsec::detail::parsec_ttg_caller);
-    if (inserted) {
-      copy_to_remove = copy;
-      /* if this is the first time we see this copy again we add the TTG reference */
-      if (0 == copy->num_readers()) {
-        copy->add_ref();
-      }
-    }
-    return value_ref;
-  }
-
-  /**
-   * Overload for ttg::ptr objects
-   *
-   * TODO: implement :)
-   *
-   * TODO: make sure the host copies are current?!
-   */
-#endif // 0
-
 };
 
 #endif  // PARSEC_TTG_H_INCLUDED
diff --git a/ttg/ttg/parsec/ttg_data_copy.h b/ttg/ttg/parsec/ttg_data_copy.h
index 66c907356..e97184b28 100644
--- a/ttg/ttg/parsec/ttg_data_copy.h
+++ b/ttg/ttg/parsec/ttg_data_copy.h
@@ -11,40 +11,47 @@
 #include <parsec.h>
 
 #include "ttg/parsec/thread_local.h"
+#include "ttg/util/span.h"
 
 
 namespace ttg_parsec {
 
   namespace detail {
 
+    template<typename T>
+    struct ttg_data_copy_container_setter {
+      ttg_data_copy_container_setter(T* ptr) {
+        /* set the container ptr here, will be reset in the the ttg_data_value_copy_t ctor */
+        ttg_data_copy_container() = ptr;
+      }
+    };
+
     /* Non-owning copy-tracking wrapper, accounting for N readers or 1 writer.
      * Also counts external references, which are not treated as
      * readers or writers but merely prevent the object from being
      * destroyed once no readers/writers exist.
      */
-    struct ttg_data_copy_t {
+    struct ttg_data_copy_t : private ttg_data_copy_container_setter<ttg_data_copy_t> {
 
       /* special value assigned to parsec_data_copy_t::readers to mark the copy as
       * mutable, i.e., a task will modify it */
       static constexpr int mutable_tag = std::numeric_limits<int>::min();
 
       ttg_data_copy_t()
-      {
-        /* set the container ptr here, will be reset in the the ttg_data_value_copy_t ctor */
-        ttg_data_copy_container() = this;
-      }
+      : ttg_data_copy_container_setter(this)
+      { }
 
-      ttg_data_copy_t(const ttg_data_copy_t& c) {
+      ttg_data_copy_t(const ttg_data_copy_t& c)
+      : ttg_data_copy_container_setter(this)
+      {
         /* we allow copying but do not copy any data over from the original
          * device copies will have to be allocated again
          * and it's a new object to reference */
-
-        /* set the container ptr here, will be reset in the the ttg_data_value_copy_t ctor */
-        ttg_data_copy_container() = this;
       }
 
       ttg_data_copy_t(ttg_data_copy_t&& c)
-      : m_ptr(c.m_ptr)
+      : ttg_data_copy_container_setter(this)
+      , m_ptr(c.m_ptr)
       , m_next_task(c.m_next_task)
       , m_readers(c.m_readers)
       , m_refs(c.m_refs.load(std::memory_order_relaxed))
@@ -55,9 +62,6 @@ namespace ttg_parsec {
         c.m_num_dev_data = 0;
         c.m_readers = 0;
         c.m_single_dev_data = nullptr;
-
-        /* set the container ptr here, will be reset in the the ttg_data_value_copy_t ctor */
-        ttg_data_copy_container() = this;
       }
 
       ttg_data_copy_t& operator=(ttg_data_copy_t&& c)
@@ -232,6 +236,32 @@ namespace ttg_parsec {
         }
       }
 
+      using iovec_iterator = typename std::vector<ttg::iovec>::iterator;
+
+      iovec_iterator iovec_begin() {
+        return m_iovecs.begin();
+      }
+
+      iovec_iterator iovec_end() {
+        return m_iovecs.end();
+      }
+
+      void iovec_reset() {
+        m_iovecs.clear();
+      }
+
+      void iovec_add(const ttg::iovec& iov) {
+        m_iovecs.push_back(iov);
+      }
+
+      ttg::span<ttg::iovec> iovec_span() {
+        return ttg::span<ttg::iovec>(m_iovecs.data(), m_iovecs.size());
+      }
+
+      std::size_t iovec_count() const {
+        return m_iovecs.size();
+      }
+
 #if defined(PARSEC_PROF_TRACE) && defined(PARSEC_TTG_PROFILE_BACKEND)
       int64_t size;
       int64_t uid;
@@ -242,6 +272,8 @@ namespace ttg_parsec {
       int32_t        m_readers  = 1;
       std::atomic<int32_t>  m_refs = 1; // number of entities referencing this copy (TTGs, external)
 
+      std::vector<ttg::iovec> m_iovecs;
+
       std::vector<parsec_data_t*> m_dev_data;   //< used if there are multiple device copies
                                                   //  that belong to this object
       parsec_data_t *m_single_dev_data;           //< used if there is a single device copy
diff --git a/ttg/ttg/view.h b/ttg/ttg/view.h
index 4275b3fa0..98c012ce1 100644
--- a/ttg/ttg/view.h
+++ b/ttg/ttg/view.h
@@ -420,14 +420,22 @@ namespace ttg {
   }
 
   namespace detail {
-    template<typename... T>
-    struct wait_kernel_t;
-    template<>
-    struct wait_kernel_t<>
-    { };
-    template<typename T, typename... Ts>
-    struct wait_kernel_t<T, Ts...> {
-      std::tuple<T&, Ts&...> ties;
+    template<typename... Ts>
+    struct wait_kernel_t {
+      std::tuple<Ts&...> ties;
+
+      /* always suspend */
+      constexpr bool await_ready() const noexcept { return false; }
+
+      /* always suspend */
+      constexpr void await_suspend( std::coroutine_handle<> ) const noexcept {}
+
+      void await_resume() noexcept {
+        if constexpr (sizeof...(Ts) > 0) {
+          /* hook to allow the backend to handle the data after pushout */
+          TTG_IMPL_NS::post_device_out(ties);
+        }
+      }
     };
   } // namespace detail
 
@@ -442,7 +450,7 @@ namespace ttg {
   inline auto wait_kernel_out(Buffers&&... args) {
     static_assert((ttg::detail::is_buffer_v<std::decay_t<Buffers>>&&...),
                   "Only ttg::buffer can be explicitly waited on!");
-    return detail::wait_kernel_t<std::decay_t<Buffers>...>{std::tie(std::forward<Buffers>(args)...)};
+    return detail::wait_kernel_t<std::remove_reference_t<Buffers>...>{std::tie(std::forward<Buffers>(args)...)};
   }
 
   struct device_task_promise_type;
@@ -584,13 +592,13 @@ namespace ttg {
     }
 
     template<typename... Ts>
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(detail::wait_kernel_t<ttg::buffer<Ts>...>&& a) {
+    auto await_transform(detail::wait_kernel_t<ttg::buffer<Ts>...>&& a) {
       std::cout << "yield_value: wait_kernel_t" << std::endl;
       if constexpr (sizeof...(Ts) > 0) {
         TTG_IMPL_NS::mark_device_out(a.ties);
       }
       m_state = TTG_DEVICE_CORO_WAIT_KERNEL;
-      return {};
+      return a;
     }
 
 #if 0

From c6b041654dd97162bf0720080ae443fb30871f2e Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 9 Mar 2023 09:54:42 -0500
Subject: [PATCH 073/259] Fix msg header ctor and non-device tasks

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/ttg.h | 38 ++++++++++++++------------------------
 1 file changed, 14 insertions(+), 24 deletions(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index aa8982855..a45cec794 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -136,12 +136,12 @@ namespace ttg_parsec {
     int num_keys = 0;
     int sender;
 
-    msg_header_t(fn_id_t fid, uint32_t tid, uint64_t oid, int32_t pid, int nk, int sender)
+    msg_header_t(fn_id_t fid, uint32_t tid, uint64_t oid, int32_t pid, int sender, int nk)
     : fn_id(fid)
     , taskpool_id(tid)
     , op_id(oid)
     , param_id(pid)
-    , num_keys(num_keys)
+    , num_keys(nk)
     , sender(sender)
     { }
   };
@@ -225,7 +225,7 @@ namespace ttg_parsec {
     int parsec_ttg_profile_backend_allocate_datacopy, parsec_ttg_profile_backend_free_datacopy;
 #endif
 
-    static constexpr const int PARSEC_TTG_MAX_AM_SIZE = 1024 * 1024;
+    static constexpr const int PARSEC_TTG_MAX_AM_SIZE = 4 * 1024;
     WorldImpl(int *argc, char **argv[], int ncores, parsec_context_t *c = nullptr)
         : WorldImplBase(query_comm_size(), query_comm_rank())
         , ctx(c)
@@ -1709,7 +1709,6 @@ namespace ttg_parsec {
         uint64_t key_end_pos;
         std::vector<keyT> keylist;
         int num_keys = msg->tt_id.num_keys;
-        std::cout << "set_arg_from_msg num_keys " << num_keys << std::endl;
         keylist.reserve(num_keys);
         auto rank = world.rank();
         for (int k = 0; k < num_keys; ++k) {
@@ -1725,7 +1724,6 @@ namespace ttg_parsec {
         if constexpr (!ttg::meta::is_void_v<valueT>) {
           using decvalueT = std::decay_t<valueT>;
           int32_t num_iovecs = msg->tt_id.num_iovecs;
-          std::span<parsec_ce_mem_reg_handle_t> tag_span;
           detail::ttg_data_copy_t *copy;
           if constexpr (ttg::has_split_metadata<decvalueT>::value) {
             ttg::SplitMetadataDescriptor<decvalueT> descr;
@@ -1758,8 +1756,6 @@ namespace ttg_parsec {
             std::memcpy(&cbtag, msg->bytes + pos, sizeof(cbtag));
             pos += sizeof(cbtag);
 
-            std::cout << "set_arg_from_msg pos after cbtag " << pos << std::endl;
-
             /* create the value from the metadata */
             auto activation = new detail::rma_delayed_activate(
                 std::move(keylist), copy, num_iovecs, [this](std::vector<keyT> &&keylist, detail::ttg_data_copy_t *copy) {
@@ -1795,6 +1791,7 @@ namespace ttg_parsec {
                                             iov.num_bytes, &lreg, &lreg_size);
                     world.impl().increment_inflight_msg();
                     /* TODO: PaRSEC should treat the remote callback as a tag, not a function pointer! */
+                    std::cout << "set_arg_from_msg: get rreg " << rreg << " remote " << remote << std::endl;
                     parsec_ce.get(&parsec_ce, lreg, 0, rreg, 0, iov.num_bytes, remote,
                                   &detail::get_complete_cb<ActivationT>, activation,
                                   /*world.impl().parsec_ttg_rma_tag()*/
@@ -1808,10 +1805,8 @@ namespace ttg_parsec {
               handle_iovecs_fn(copy->iovec_span());
               copy->iovec_reset();
             }
-            std::cout << "set_arg_from_msg pos after iovecs " << pos << " key_offset " << msg->tt_id.key_offset << std::endl;
 
             assert(num_iovecs == nv);
-            if (size != (key_end_pos + sizeof(msg_header_t))) std::cout << "XXX pos + header + keys " << pos + sizeof(msg_header_t) + num_keys*sizeof(keyT) << " size " << size << std::endl;
             assert(size == (key_end_pos + sizeof(msg_header_t)));
           }
           // case 2 and 3
@@ -2221,7 +2216,6 @@ namespace ttg_parsec {
       std::unique_ptr<msg_t> msg = std::make_unique<msg_t>(get_instance_id(), world_impl.taskpool()->taskpool_id,
                                                            msg_header_t::MSG_SET_ARG, i, world_impl.rank(), 1);
       using decvalueT = std::decay_t<Value>;
-      msg->tt_id.sender = world_impl.rank();
 
       if constexpr (!ttg::meta::is_void_v<decvalueT>) {
 
@@ -2262,8 +2256,9 @@ namespace ttg_parsec {
             int32_t lreg_size_i = lreg_size;
             std::memcpy(msg->bytes + pos, &lreg_size_i, sizeof(lreg_size_i));
             pos += sizeof(lreg_size_i);
-            std::memcpy(msg->bytes + pos, lreg, lreg_size_i);
-            pos += lreg_size_i;
+            std::memcpy(msg->bytes + pos, lreg, lreg_size);
+            pos += lreg_size;
+            std::cout << "set_arg_impl lreg " << lreg << std::endl;
             /* TODO: can we avoid the extra indirection of going through std::function? */
             std::function<void(void)> *fn = new std::function<void(void)>([=]() mutable {
               /* shared_ptr of value and registration captured by value so resetting
@@ -2301,10 +2296,8 @@ namespace ttg_parsec {
       /* pack the key */
       msg->tt_id.num_keys = 0;
       msg->tt_id.key_offset = pos;
-      std::cout << "set_arg_impl pos " << pos << " key_offset " << msg->tt_id.key_offset << " key size " << sizeof(key) << std::endl;
       if constexpr (!ttg::meta::is_void_v<Key>) {
         size_t tmppos = pack(key, msg->bytes, pos);
-        std::cout << "set_arg_impl pos before " << pos << " after pack " << tmppos << std::endl;
         pos = tmppos;
         msg->tt_id.num_keys = 1;
       }
@@ -2312,10 +2305,7 @@ namespace ttg_parsec {
       parsec_taskpool_t *tp = world_impl.taskpool();
       tp->tdm.module->outgoing_message_start(tp, owner, NULL);
       tp->tdm.module->outgoing_message_pack(tp, owner, NULL, NULL, 0);
-      std::cout << "set_arg_impl num_keys " << msg->tt_id.num_keys
-                << ", total size " << sizeof(msg_header_t) + pos
-                << " pos " << pos << " key_offset " << msg->tt_id.key_offset
-                << " sizeof header" << sizeof(msg_header_t) << std::endl;
+      std::cout << "set_arg_impl send_am owner " << owner << " sender " << msg->tt_id.sender << std::endl;
       parsec_ce.send_am(&parsec_ce, world_impl.parsec_ttg_tag(), owner, static_cast<void *>(msg.get()),
                         sizeof(msg_header_t) + pos);
 #if defined(PARSEC_PROF_TRACE) && defined(PARSEC_TTG_PROFILE_BACKEND)
@@ -2418,8 +2408,10 @@ namespace ttg_parsec {
                                             std::shared_ptr<void>{lreg, [](void *ptr) {
                                                                     parsec_ce_mem_reg_handle_t memreg =
                                                                         (parsec_ce_mem_reg_handle_t)ptr;
+                                                                    std::cout << "broadcast_arg memunreg lreg " << memreg << std::endl;
                                                                     parsec_ce.mem_unregister(&memreg);
                                                                   }}));
+            std::cout << "broadcast_arg memreg lreg " << lreg << std::endl;
           }
         };
 
@@ -2456,7 +2448,6 @@ namespace ttg_parsec {
         msg->tt_id.num_iovecs = num_iovs;
 
         std::size_t save_pos = pos;
-        std::cout << "broadcast_arg save_pos " << save_pos <<std::endl;
 
         parsec_taskpool_t *tp = world_impl.taskpool();
         for (auto it = keylist_sorted.begin(); it < keylist_sorted.end(); /* increment done inline */) {
@@ -2487,15 +2478,15 @@ namespace ttg_parsec {
             pos += sizeof(lreg_size);
             std::memcpy(msg->bytes + pos, lreg_ptr.get(), lreg_size);
             pos += lreg_size;
-            /* create a function that will be invoked upon RMA completion at the target */
-            std::shared_ptr<void> lreg_ptr_v = lreg_ptr;
+            std::cout << "broadcast_arg lreg_ptr " << lreg_ptr.get() << std::endl;
             /* mark another reader on the copy */
             copy = detail::register_data_copy<valueT>(copy, nullptr, true);
+            /* create a function that will be invoked upon RMA completion at the target */
             std::function<void(void)> *fn = new std::function<void(void)>([=]() mutable {
               /* shared_ptr of value and registration captured by value so resetting
                 * them here will eventually release the memory/registration */
               detail::release_data_copy(copy);
-              lreg_ptr_v.reset();
+              lreg_ptr.reset();
             });
             std::intptr_t fn_ptr{reinterpret_cast<std::intptr_t>(fn)};
             std::memcpy(msg->bytes + pos, &fn_ptr, sizeof(fn_ptr));
@@ -2504,7 +2495,6 @@ namespace ttg_parsec {
 
           /* mark the beginning of the keys */
           msg->tt_id.key_offset = pos;
-          std::cout << "broadcast_arg key_offset " << msg->tt_id.key_offset <<std::endl;
 
           /* pack all keys for this owner */
           int num_keys = 0;
@@ -2517,7 +2507,7 @@ namespace ttg_parsec {
 
           tp->tdm.module->outgoing_message_start(tp, owner, NULL);
           tp->tdm.module->outgoing_message_pack(tp, owner, NULL, NULL, 0);
-          std::cout << "broadcast_arg total size " << sizeof(msg_header_t) + pos << " pos " << pos <<std::endl;
+          std::cout << "broadcast_arg send_am owner " << owner << std::endl;
           parsec_ce.send_am(&parsec_ce, world_impl.parsec_ttg_tag(), owner, static_cast<void *>(msg.get()),
                             sizeof(msg_header_t) + pos);
         }

From b9b7bdb940e78b9547f693027a04fcd24f07f63e Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 15 Mar 2023 14:41:08 -0400
Subject: [PATCH 074/259] Draft for sending out of a device task

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 CMakeLists.txt              |   5 +-
 tests/unit/device_coro.cc   | 299 ++++++--------------------------
 ttg/CMakeLists.txt          |   4 +
 ttg/ttg/devicescope.h       |   6 +-
 ttg/ttg/devicescratch.h     |  15 ++
 ttg/ttg/parsec/devicefunc.h |  38 ++--
 ttg/ttg/parsec/fwd.h        |   3 +
 ttg/ttg/parsec/task.h       |   9 +-
 ttg/ttg/parsec/ttg.h        | 163 ++++++++++++++----
 ttg/ttg/terminal.h          |  65 ++++++-
 ttg/ttg/util/meta.h         |  27 +++
 ttg/ttg/view.h              | 333 +++++++++++++++++++++++++++++++++---
 12 files changed, 634 insertions(+), 333 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c4c28b042..bc106c6ee 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -97,7 +97,10 @@ if(CMAKE_CUDA_COMPILER)
 endif(CMAKE_CUDA_COMPILER)
 set(TTG_HAVE_CUDA ${CMAKE_CUDA_COMPILER} CACHE BOOL "True if TTG supports compiling .cu files")
 
-
+find_package(CUDAToolkit)
+if (TARGET CUDA::cudart)
+  set(TTG_HAVE_CUDART True CACHE BOOL "True support for CUDART was found.")
+endif(TARGET CUDA::cudart)
 
 
 ##########################
diff --git a/tests/unit/device_coro.cc b/tests/unit/device_coro.cc
index c4b8a19e6..5b51b0724 100644
--- a/tests/unit/device_coro.cc
+++ b/tests/unit/device_coro.cc
@@ -28,7 +28,6 @@ namespace madness::archive {
 
 TEST_CASE("Device", "coro") {
 
-#if 0
   SECTION("devicebuf") {
 
     ttg::Edge<int, value_t> edge;
@@ -45,9 +44,10 @@ TEST_CASE("Device", "coro") {
       co_await ttg::wait_kernel();
 
       /* we're back, the kernel executed and we can send */
-      if (key < 1) {
+      if (key < 10) {
         /* TODO: should we move the view in here if we want to get the device side data */
-        ttg::send<0>(key+1, std::move(val));
+        //ttg::send<0>(key+1, std::move(val));
+        co_await ttg::device::forward(ttg::device::send<0>(key+1, std::move(val)));
       }
     };
 
@@ -65,7 +65,7 @@ TEST_CASE("Device", "coro") {
     ttg::Edge<int, value_t> edge;
     auto fn = [&](const int& key, value_t&& val) -> ttg::device_task {
       double scratch = 0.0;
-      ttg::devicescratch<double> ds = ttg::make_scratch(&scratch, ttg::scope::SyncOut);
+      ttg::devicescratch<double> ds = ttg::make_scratch(&scratch, ttg::scope::Allocate);
 
       /* wait for the view to be available on the device */
       co_await ttg::to_device(ds, val.db);
@@ -78,7 +78,7 @@ TEST_CASE("Device", "coro") {
 #endif // TTG_HAVE_CUDA
 
       /* here we suspend to wait for a kernel to complete */
-      co_await ttg::wait_kernel();
+      co_await ttg::wait_kernel(ds);
 
 #ifdef TTG_HAVE_CUDA
       /* buffer is increment once per task, so it should be the same as key */
@@ -88,7 +88,10 @@ TEST_CASE("Device", "coro") {
       /* we're back, the kernel executed and we can send */
       if (key < 10) {
         /* TODO: should we move the view in here if we want to get the device side data */
-        ttg::send<0>(key+1, std::move(val));
+        //ttg::send<0>(key+1, std::move(val));
+        /* NOTE: we use co_await here instead of co_return because co_return destroys all local variables first;
+         *       we will not return from this co_await!*/
+        co_await ttg::device::forward(ttg::device::send<0>(key+1, std::move(val)));
       }
     };
 
@@ -98,7 +101,6 @@ TEST_CASE("Device", "coro") {
     if (ttg::default_execution_context().rank() == 0) tt->invoke(0, value_t{});
     ttg::ttg_fence(ttg::default_execution_context());
   }
-#endif // 0
 
   SECTION("ptr") {
 
@@ -107,7 +109,7 @@ TEST_CASE("Device", "coro") {
     int last_key = 0;
     auto fn = [&](const int& key, value_t&& val) -> ttg::device_task {
       double scratch = 1.0;
-      ttg::devicescratch<double> ds = ttg::make_scratch(&scratch, ttg::scope::SyncOut);
+      ttg::devicescratch<double> ds = ttg::make_scratch(&scratch, ttg::scope::SyncIn);
 
       /* wait for the view to be available on the device */
       co_await ttg::to_device(ds, val.db);
@@ -120,7 +122,7 @@ TEST_CASE("Device", "coro") {
 #endif // TTG_HAVE_CUDA
 
       /* here we suspend to wait for a kernel and the out-transfer to complete */
-      co_await ttg::wait_kernel_out(val.db);
+      co_await ttg::wait_kernel(val.db, ds);
 
 #ifdef TTG_HAVE_CUDA
       /* buffer is increment once per task, so it should be the same as key */
@@ -130,7 +132,8 @@ TEST_CASE("Device", "coro") {
 
       /* we're back, the kernel executed and we can send */
       if (key < 10 || scratch < 0.0) {
-        ttg::send<0>(key+1, std::move(val));
+        //ttg::send<0>(key+1, std::move(val));
+        co_await ttg::device::forward(ttg::device::send<0>(key+1, std::move(val)));
       } else {
         /* exfiltrate the value */
         /* TODO: what consistency do we expect from get_ptr? */
@@ -155,269 +158,63 @@ TEST_CASE("Device", "coro") {
     ptr.reset();
   }
 
-
-
-
-
 #if 0
+  /* TODO: enabel this test once we control the PaRSEC state machine! */
+  SECTION("device_host_tasks") {
 
-  SECTION("device_task") {
-    ttg::Edge<int, double> edge;
-    auto fn = [&](const int& key, double&& val) -> ttg::device_task {
-      ttg::View<double> view = ttg::make_view(val, ttg::ViewScope::SyncInOut);
-      ttg::print("device_task key ", key, ", value ", val);
-      /* wait for the view to be available on the device */
-      co_yield view;
-      // co_yield std::tie(view1, view2);
-      // TTG_WAIT_VIEW(view);
-      /* once we're back here the data has been transferred */
-      CHECK(view.get_device_ptr<0>()  != nullptr);
-      CHECK(view.get_device_size<0>() == sizeof(val));
-      CHECK(&view.get_host_object() == &val);
+    ttg::Edge<int, value_t> h2d, d2h;
 
-      ttg::print("device_task key ", key, ", device pointer ", view.get_device_ptr<0>());
+    auto host_fn = [&](const int& key, value_t&& val) {
+      /* check that the data has been synced back */
+      CHECK(static_cast<int>(*val.db.host_ptr()) == key);
 
-      /* here we suspend to wait for a kernel to complete */
-      co_yield ttg::device_op_wait_kernel{};
-      // TTG_WAIT_KERNEL();
+      /* modify the data */
+      *val.db.host_ptr() += 1.0;
+      CHECK(static_cast<int>(*val.db.host_ptr()) == key+1);
 
-      /* we're back, the kernel executed and we can send */
-      if (key < 10) {
-        ttg::send<0>(key+1, std::move(val));
-      }
+      /* send back to the device */
+      ttg::send<0>(key+1, std::move(val));
     };
-    auto tt = ttg::make_tt<ttg::ExecutionSpace::CUDA>(fn, ttg::edges(edge), ttg::edges(edge),
-                                                      "device_task", {"edge_in"}, {"edge_out"});
-    make_graph_executable(tt);
-    if (ttg::default_execution_context().rank() == 0) tt->invoke(0, 0.0);
-    ttg::ttg_fence(ttg::default_execution_context());
-  }
+    auto htt = ttg::make_tt(host_fn, ttg::edges(d2h), ttg::edges(h2d),
+                            "host_task", {"d2h"}, {"h2d"});
+
+    auto device_fn = [&](const int& key, value_t&& val) -> ttg::device_task {
+      double scratch = 0.0;
+      ttg::devicescratch<double> ds = ttg::make_scratch(&scratch, ttg::scope::SyncOut);
 
-  SECTION("get_ptr") {
-    ttg::Edge<int, double> edge;
-    ttg::ptr<double> ptr;
-    auto fn = [&](const int& key, double&& val) -> ttg::device_task {
-      ttg::View<double> view = ttg::make_view(val, ttg::ViewScope::SyncInOut);
-      ttg::print("device_task key ", key, ", value ", val);
       /* wait for the view to be available on the device */
-      co_yield view;
-      // co_yield std::tie(view1, view2);
-      // TTG_WAIT_VIEW(view);
+      co_await ttg::to_device(ds, val.db);
       /* once we're back here the data has been transferred */
-      //CHECK(view.get_rw_device_ptr<0>()  != nullptr);
-      CHECK(view.get_device_ptr<0>()  != nullptr);
-      CHECK(view.get_device_size<0>() == sizeof(val));
-      CHECK(&view.get_host_object() == &val);
+      CHECK(ds.device_ptr()  != nullptr);
 
-      ttg::print("device_task key ", key, ", device pointer ", view.get_device_ptr<0>());
+      /* call a kernel */
+#ifdef TTG_HAVE_CUDA
+      increment_buffer(val.db.current_device_ptr(), val.db.size(), ds.device_ptr(), ds.size());
+#endif // TTG_HAVE_CUDA
 
       /* here we suspend to wait for a kernel to complete */
-      co_yield ttg::device_op_wait_kernel{};
-      // TTG_WAIT_KERNEL();
-
-      /* we're back, the kernel executed and we can send */
-      if (key < 10) {
-        /* TODO: should we move the view in here if we want to get the device side data */
-        ttg::send<0>(key+1, std::move(val));
-      } else {
-        /* exfiltrate the value */
-        ptr = ttg::get_ptr(val);
-      }
-    };
-
-    //ptr.get_view<ttg::ExecutionSpace::CUDA>(device_id);
-
-    auto tt = ttg::make_tt<ttg::ExecutionSpace::CUDA>(fn, ttg::edges(edge), ttg::edges(edge),
-                                                      "device_task", {"edge_in"}, {"edge_out"});
-    make_graph_executable(tt);
-    if (ttg::default_execution_context().rank() == 0) tt->invoke(0, 0.0);
-    ttg::ttg_fence(ttg::default_execution_context());
-
-#if 0
-    /* feed the host-side value back into the graph */
-    if (ttg::default_execution_context().rank() == 0) tt->invoke(0, pview.get_ptr());
-    ttg::ttg_fence(ttg::default_execution_context());
+      co_await ttg::wait_kernel();
 
-    /* feed the device-side value back into the graph */
-    if (ttg::default_execution_context().rank() == 0) tt->invoke(0, pview);
-    ttg::ttg_fence(ttg::default_execution_context());
+#ifdef TTG_HAVE_CUDA
+      /* buffer is increment once per task, so it should be the same as key */
+      CHECK((static_cast<int>(scratch)-1) == key);
 #endif // 0
-  }
-
-
-
-
-  SECTION("device_task") {
-    ttg::Edge<int, double> edge;
-    auto fn = [&](const int& key, double&& val) -> ttg::device_task {
-      // will schedule the view for transfer to and from the device
-      ttg::View<double> view = ttg::make_view(val, ttg::ViewScope::SyncInOut);
-      ttg::print("device_task key ", key, ", value ", val);
-      /* wait for the view to be available on the device */
-      co_await ttg::device_task_wait_views{};
-
-      /* once we're back here the data has been transferred */
-      CHECK(view.get_device_ptr<0>()  != nullptr);
-      CHECK(view.get_device_size<0>() == sizeof(val));
-      CHECK(&view.get_host_object() == &val);
-
-      ttg::print("device_task key ", key, ", device pointer ", view.get_device_ptr<0>());
-
-      while (val < 10.0) {
-
-        view.set_scope(ttg::ViewScope::SyncOut);
-
-        /* <submit kernel here> */
-
-        /* here we suspend to wait for a kernel to complete */
-        co_await ttg::device_task_wait_kernel{};
-
-        // TTG_WAIT_KERNEL();
-      }
 
       /* we're back, the kernel executed and we can send */
       if (key < 10) {
-        ttg::send<0>(key+1, val);
-      }
-    };
-    auto tt = ttg::make_tt<ttg::ExecutionSpace::CUDA>(fn, ttg::edges(edge), ttg::edges(edge),
-                                                      "device_task", {"edge_in"}, {"edge_out"});
-    make_graph_executable(tt);
-    if (ttg::default_execution_context().rank() == 0) tt->invoke(0, 0.0);
-    ttg::ttg_fence(ttg::default_execution_context());
-  }
-
-  struct A {
-    double norm;
-    std::vector<double> d;
-  };
-
-  SECTION("device_task") {
-    ttg::Edge<int, double> edge;
-    auto fn = [&](const int& key, ttg::ptr<A>&& a) -> ttg::device_task {
-      // will schedule the view for transfer to and from the device
-      View<double> norm_view = a.to_host(&A::norm);
-
-      View<A::norm> norm_view{a};
-      co_await ttg::device::wait_transfer{};
-
-
-
-
-      if (val)
-      val += 1.0;
-      ptr.sync_to_device();
-      /* wait for the view to be available on the device */
-      co_await ttg::device_task_wait_views{};
-
-      /* once we're back here the data has been transferred */
-      CHECK(view.get_device_ptr<0>()  != nullptr);
-      CHECK(view.get_device_size<0>() == sizeof(val));
-      CHECK(&view.get_host_object() == &val);
-
-      ttg::print("device_task key ", key, ", device pointer ", view.get_device_ptr<0>());
-
-      while (val < 10.0) {
-
-        view.set_scope(ttg::ViewScope::SyncOut);
-
-        /* <submit kernel here> */
-
-        /* here we suspend to wait for a kernel to complete */
-        co_await ttg::device_task_wait_kernel{};
-
-        // TTG_WAIT_KERNEL();
-      }
-
-      /* we're back, the kernel executed and we can send */
-      if (key < 10) {
-        ttg::send<0>(key+1, val);
+        /* TODO: should we move the view in here if we want to get the device side data */
+        ttg::send<0>(key+1, std::move(val));
       }
     };
-    auto tt = ttg::make_tt<ttg::ExecutionSpace::CUDA>(fn, ttg::edges(edge), ttg::edges(edge),
-                                                      "device_task", {"edge_in"}, {"edge_out"});
-    make_graph_executable(tt);
-    if (ttg::default_execution_context().rank() == 0) tt->invoke(0, 0.0);
-    ttg::ttg_fence(ttg::default_execution_context());
-  }
-
-  struct A {
-    int a[10];
-    double b[10];
-  };
-
-
-  SECTION("device_task_struct") {
-    ttg::Edge<int, double> edge;
-    auto fn = [](const int& key, A&& val) -> ttg::device_task {
-      auto view = ttg::make_view(val, {val.a, 10, ttg::ViewScope::SyncIn},
-                                      {val.b, 10, ttg::ViewScope::SyncIn});
-      /* wait for the view to be available on the device */
-      co_yield view;
-      // co_yield std::tie(view1, view2);
-      // TTG_WAIT_VIEW(view);
-      /* once we're back here the data has been transferred */
-      CHECK(view.get_device_ptr<0>() != nullptr);
-      CHECK(view.size<0>() == 10*sizeof(int));
-      CHECK(&view.get_host_object() == &val);
-
-      // <submit kernel here>
-
-      /* here we suspend to wait for a kernel to complete */
-      co_yield ttg::device_op_wait_kernel{};
-      // TTG_WAIT_KERNEL();
 
-      /* we're back, the kernel executed and we can send */
-      if (key < 10) {
-        ttg::send<0>(key+1, val);
-      }
-    };
-    auto tt = ttg::make_tt(fn, ttg::edges(edge), ttg::edges(edge),
-                           "device_task", {"edge_in"}, {"edge_out"});
-    make_graph_executable(tt);
-    if (ttg::default_execution_context().rank() == 0) tt->invoke(0, 0.0);
+    auto dtt = ttg::make_tt<ttg::ExecutionSpace::CUDA>(device_fn, ttg::edges(h2d), ttg::edges(d2h),
+                                                      "device_task", {"h2d"}, {"d2h"});
+    make_graph_executable(dtt);
+    if (ttg::default_execution_context().rank() == 0) htt->invoke(0, value_t{});
     ttg::ttg_fence(ttg::default_execution_context());
   }
 
 
-
-  SECTION("device_task_struct_multi") {
-    ttg::Edge<int, double> edge;
-    auto fn = [](const int& key, A&& val) -> ttg::device_task {
-      auto view = ttg::make_view(val, {val.a, 10},
-                                      {val.b, 10, ttg::ViewScope::SyncOut});
-      /* wait for the view to be available on the device */
-      co_yield view;
-      // co_yield std::tie(view1, view2);
-      // TTG_WAIT_VIEW(view);
-      /* once we're back here the data has been transferred */
-      CHECK(view.get_device_ptr<0>() != nullptr);
-      CHECK(view.size<0>() == 10*sizeof(int));
-      CHECK(&view.get_host_object() == &val);
-
-      // <submit kernel here>
-
-      /* here we suspend to wait for a kernel to complete */
-      co_yield ttg::device_op_wait_kernel{};
-      // TTG_WAIT_KERNEL();
-
-      if (val.b[0] < 100.0) {
-        // <submit another kernel here>
-        view.sync_out<0>();
-        // anything submitted before this point must finish before the data is returned
-        co_yield view;
-      }
-
-      /* we're back, the kernel executed and we can send */
-      if (key < 10) {
-        ttg::send<0>(key+1, val);
-      }
-    };
-    auto tt = ttg::make_tt(fn, ttg::edges(edge), ttg::edges(edge),
-                           "device_task", {"edge_in"}, {"edge_out"});
-    make_graph_executable(tt);
-    if (ttg::default_execution_context().rank() == 0) tt->invoke(0, 0.0);
-    ttg::ttg_fence(ttg::default_execution_context());
-  }
 #endif // 0
-}
+
+}
\ No newline at end of file
diff --git a/ttg/CMakeLists.txt b/ttg/CMakeLists.txt
index ef868f556..4f94ede42 100644
--- a/ttg/CMakeLists.txt
+++ b/ttg/CMakeLists.txt
@@ -109,6 +109,10 @@ if (TARGET std::coroutine)
           ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/coroutine.h
           )
 endif ()
+if (TARGET CUDA::cudart)
+  list(APPEND ttg-deps CUDA::cudart)
+  list(APPEND ttg-defs "TTG_HAVE_CUDART=1")
+endif (TARGET CUDA::cudart)
 
 set(ttg-public-headers ${ttg-headers};${ttg-impl-headers};${ttg-base-headers};${ttg-util-headers})
 if (NOT TTG_IGNORE_BUNDLED_EXTERNALS)
diff --git a/ttg/ttg/devicescope.h b/ttg/ttg/devicescope.h
index a8427fb56..594e6db0b 100644
--- a/ttg/ttg/devicescope.h
+++ b/ttg/ttg/devicescope.h
@@ -4,11 +4,7 @@
 namespace ttg {
   enum class scope {
     Allocate     = 0x0,  //< memory allocated as scratch, but not moved in or out
-    SyncIn       = 0x2,  //< data will be allocated on and transferred to device
-                         //< if latest version resides on the device (no previous sync-out) the data will
-                         //< not be transferred again
-    SyncOut      = 0x4,  //< value will be transferred from device to host after kernel completes
-    SyncInOut    = 0x8,  //< data will be moved in and synchronized back out after the kernel completes
+    SyncIn       = 0x2,  //< memory allocated as scratch and data transferred to device
   };
 } // namespace ttg
 
diff --git a/ttg/ttg/devicescratch.h b/ttg/ttg/devicescratch.h
index 1478d82cf..37510436f 100644
--- a/ttg/ttg/devicescratch.h
+++ b/ttg/ttg/devicescratch.h
@@ -14,6 +14,21 @@ auto make_scratch(T* val, ttg::scope scope, std::size_t count = 1) {
   return devicescratch<T>(val, scope, 1);
 }
 
+namespace detail {
+
+  template<typename T>
+  struct is_devicescratch : std::false_type
+  { };
+
+  template<typename T>
+  struct is_devicescratch<ttg::devicescratch<T>> : std::true_type
+  { };
+
+  template<typename T>
+  constexpr bool is_devicescratch_v = is_devicescratch<T>::value;
+
+} // namespace detail
+
 } // namespace ttg
 
 #endif // TTG_DEVICESCRATCH_H
\ No newline at end of file
diff --git a/ttg/ttg/parsec/devicefunc.h b/ttg/ttg/parsec/devicefunc.h
index 3273e8084..f509138f9 100644
--- a/ttg/ttg/parsec/devicefunc.h
+++ b/ttg/ttg/parsec/devicefunc.h
@@ -1,9 +1,17 @@
 #ifndef TTG_PARSEC_DEVICEFUNC_H
 #define TTG_PARSEC_DEVICEFUNC_H
 
+#if defined(TTG_HAVE_CUDART)
+#include <cuda.h>
+#endif
+
 #include "ttg/parsec/task.h"
 #include <parsec/mca/device/device_gpu.h>
 
+#if defined(PARSEC_HAVE_CUDA)
+#include <parsec/mca/device/cuda/device_cuda.h>
+#endif // PARSEC_HAVE_CUDA
+
 namespace ttg_parsec {
   namespace detail {
     template<typename... Views, std::size_t I, std::size_t... Is>
@@ -31,14 +39,6 @@ namespace ttg_parsec {
           case ttg::scope::SyncIn:
             flow_flags = PARSEC_FLOW_ACCESS_READ;
             break;
-          case ttg::scope::SyncOut:
-            flow_flags = PARSEC_FLOW_ACCESS_WRITE;
-            pushout = true;
-            break;
-          case ttg::scope::SyncInOut:
-            flow_flags = PARSEC_FLOW_ACCESS_RW;
-            pushout = true;
-            break;
         }
       }
       assert(nullptr != detail::parsec_ttg_caller->dev_ptr);
@@ -105,7 +105,7 @@ namespace ttg_parsec {
   }
 
   namespace detail {
-    template<typename... Views, std::size_t I, std::size_t... Is>
+    template<typename... Views, std::size_t I, std::size_t... Is, bool DeviceAvail = false>
     inline void mark_device_out(std::tuple<Views&...> &views, std::index_sequence<I, Is...>) {
 
       using view_type = std::remove_reference_t<std::tuple_element_t<I, std::tuple<Views&...>>>;
@@ -116,6 +116,19 @@ namespace ttg_parsec {
       /* find the data copy and mark it as pushout */
       int i = 0;
       parsec_gpu_task_t *gpu_task = detail::parsec_ttg_caller->dev_ptr->gpu_task;
+      parsec_gpu_exec_stream_t *stream = detail::parsec_ttg_caller->dev_ptr->stream;
+      /* enqueue the transfer into the compute stream to come back once the compute and transfer are complete */
+
+#if defined(TTG_HAVE_CUDART) && defined(PARSEC_HAVE_CUDA)
+      parsec_cuda_exec_stream_t *cuda_stream = (parsec_cuda_exec_stream_t *)stream;
+      cudaMemcpyAsync(data->device_copies[0]->device_private,
+                      data->device_copies[data->owner_device]->device_private,
+                      data->nb_elts, cudaMemcpyDeviceToHost, cuda_stream->cuda_stream);
+#else
+      static_assert(DeviceAvail, "No device implementation detected!");
+#endif // defined(PARSEC_HAVE_CUDA)
+
+#if 0
       while (detail::parsec_ttg_caller->parsec_task.data[i].data_in != nullptr) {
         if (detail::parsec_ttg_caller->parsec_task.data[i].data_in == data->device_copies[0]) {
           gpu_task->pushout |= 1<<i;
@@ -123,10 +136,10 @@ namespace ttg_parsec {
         }
         ++i;
       }
-
+#endif // 0
       if constexpr (sizeof...(Is) > 0) {
         // recursion
-        return mark_device_out(views, std::index_sequence<Is...>{});
+        mark_device_out(views, std::index_sequence<Is...>{});
       }
     }
   } // namespace detail
@@ -164,7 +177,7 @@ namespace ttg_parsec {
 
       if constexpr (sizeof...(Is) > 0) {
         // recursion
-        return mark_device_out(views, std::index_sequence<Is...>{});
+        post_device_out(views, std::index_sequence<Is...>{});
       }
     }
   } // namespace detail
@@ -173,6 +186,7 @@ namespace ttg_parsec {
     detail::post_device_out(b, std::index_sequence_for<Buffer...>{});
   }
 
+
 } // namespace ttg_parsec
 
 #endif // TTG_PARSEC_DEVICEFUNC_H
\ No newline at end of file
diff --git a/ttg/ttg/parsec/fwd.h b/ttg/ttg/parsec/fwd.h
index efcfe795c..f184548d4 100644
--- a/ttg/ttg/parsec/fwd.h
+++ b/ttg/ttg/parsec/fwd.h
@@ -30,6 +30,9 @@ namespace ttg_parsec {
   template<typename... Buffer>
   inline void post_device_out(std::tuple<Buffer&...> &b);
 
+  /* the query of the parsec backend only returns whether the data should be marked for pushout */
+  using query_result_type = bool;
+
   /// \internal the OG name
   template <typename keyT, typename output_terminalsT, typename derivedT, typename... input_valueTs>
   using Op [[deprecated("use TT instead")]] = TT<keyT, output_terminalsT, derivedT, ttg::typelist<input_valueTs...>>;
diff --git a/ttg/ttg/parsec/task.h b/ttg/ttg/parsec/task.h
index 7454c1a1e..3636e903a 100644
--- a/ttg/ttg/parsec/task.h
+++ b/ttg/ttg/parsec/task.h
@@ -13,6 +13,7 @@ namespace ttg_parsec {
     struct device_ptr_t {
       parsec_gpu_task_t* gpu_task = nullptr;
       parsec_flow_t* flows = nullptr;
+      parsec_gpu_exec_stream_t* stream = nullptr;
     };
 
     template<bool SupportDevice>
@@ -32,7 +33,7 @@ namespace ttg_parsec {
       static constexpr bool support_device = false;
       static constexpr size_t num_flows = MAX_PARAM_COUNT;
       parsec_flow_t m_flows[num_flows];
-      device_ptr_t m_dev_ptr = {nullptr, &m_flows[0]}; // gpu_task will be allocated in each task
+      device_ptr_t m_dev_ptr = {nullptr, &m_flows[0], nullptr}; // gpu_task will be allocated in each task
       device_ptr_t* dev_ptr() {
         return &m_dev_ptr;
       }
@@ -121,14 +122,16 @@ namespace ttg_parsec {
     struct parsec_ttg_task_t : public parsec_ttg_task_base_t {
       using key_type = typename TT::key_type;
       static constexpr size_t num_streams = TT::numins;
+      /* device tasks may have to store more copies than it's inputs as their sends are aggregated */
+      static constexpr size_t num_copies  = TT::derived_has_cuda_op() ? static_cast<size_t>(MAX_PARAM_COUNT)
+                                                                      : (num_streams+1);
       TT* tt;
       key_type key;
       size_goal_t stream[num_streams] = {};
 #ifdef TTG_HAS_COROUTINE
       void* suspended_task_address = nullptr;  // if not null the function is suspended
 #endif
-      ttg_data_copy_t *copies[num_streams+1] = { nullptr };  // the data copies tracked by this task
-                                                             // +1 for the copy needed during send/bcast
+      ttg_data_copy_t *copies[num_copies] = { nullptr };  // the data copies tracked by this task
       device_state_t<TT::derived_has_cuda_op()> dev_state;
 
       parsec_ttg_task_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class)
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index a45cec794..c6e01c2f2 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -62,6 +62,11 @@
 #include <tuple>
 #include <vector>
 
+// needed for MPIX_CUDA_AWARE_SUPPORT
+#include <mpi.h>
+#include <mpi-ext.h>
+
+
 /* TODO: remove once we use PaRSEC master */
 #ifndef PARSEC_HAVE_CUDA
 #define PARSEC_HAVE_CUDA 1
@@ -1219,6 +1224,8 @@ namespace ttg_parsec {
       // get the device task from the coroutine handle
       ttg::device_task dev_task = ttg::device_task_handle_type::from_address(task->suspended_task_address);
 
+      task->dev_ptr->stream = gpu_stream;
+
       std::cout << "device_static_submit task " << task << std::endl;
 
       // get the promise which contains the views
@@ -1227,28 +1234,23 @@ namespace ttg_parsec {
       /* we should still be waiting for the transfer to complete */
       assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_TRANSFER);
 
-#if 0
-      /* update the device pointers in the device views */
-      int i = 0;
-      for (auto& view : dev_data) {
-        /* iterate over all viewspans of this view (i.e., all memory ranges in this view) */
-        for (auto& view_span : view) {
-          view_span.set_data(task->parsec_task.data[i].data_out);
-          ++i;
-        }
-      }
-#endif // 0
-
       /* Here we call back into the coroutine again after the transfers have completed */
       static_op<Space>(&task->parsec_task);
 
+      /* Get a new handle for the promise*/
       dev_task = ttg::device_task_handle_type::from_address(task->suspended_task_address);
       dev_data = dev_task.promise();
-      /* for now make sure we're waiting for the kernel to complete and the coro hasn't skipped this step */
-      assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_KERNEL);
 
-      /* next time we will come back into complete_task_and_release */
-      return PARSEC_HOOK_RETURN_DONE;
+      assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_KERNEL ||
+             dev_data.state() == ttg::TTG_DEVICE_CORO_COMPLETE);
+
+      /* we will come back into this function once the kernel and transfers are */
+      int rc = PARSEC_HOOK_RETURN_AGAIN;
+      if (ttg::TTG_DEVICE_CORO_COMPLETE == dev_data.state()) {
+        /* the task started sending so we won't come back here */
+        rc = PARSEC_HOOK_RETURN_DONE;
+      }
+      return rc;
     }
 
     template <ttg::ExecutionSpace Space>
@@ -1584,9 +1586,14 @@ namespace ttg_parsec {
     }
 
     template <typename T>
-    uint64_t pack(T &obj, void *bytes, uint64_t pos) {
+    uint64_t pack(T &obj, void *bytes, uint64_t pos, detail::ttg_data_copy_t *copy = nullptr) {
       const ttg_data_descriptor *dObj = ttg::get_data_descriptor<ttg::meta::remove_cvr_t<T>>();
       uint64_t payload_size = dObj->payload_size(&obj);
+      if (copy) {
+        /* reset any tracked data, we don't care about the packing from the payload size */
+        copy->iovec_reset();
+      }
+
       if constexpr (!ttg::default_data_descriptor<ttg::meta::remove_cvr_t<T>>::serialize_size_is_const) {
         const ttg_data_descriptor *dSiz = ttg::get_data_descriptor<uint64_t>();
         dSiz->pack_payload(&payload_size, sizeof(uint64_t), pos, bytes);
@@ -1738,6 +1745,7 @@ namespace ttg_parsec {
             copy = detail::create_new_datacopy(decvalueT{});
             /* unpack the object, potentially discovering iovecs */
             pos = unpack(*static_cast<decvalueT *>(copy->get_ptr()), msg->bytes, pos);
+            std::cout << "num_iovecs " << num_iovecs << " distance " << std::distance(copy->iovec_begin(), copy->iovec_end()) << std::endl;
             assert(std::distance(copy->iovec_begin(), copy->iovec_end()) == num_iovecs);
           }
 
@@ -2077,17 +2085,6 @@ namespace ttg_parsec {
             copy = detail::create_new_datacopy(std::forward<Value>(value));
           }
 
-          /* if this is a host task make sure tracked buffers get copied to the host */
-          if constexpr(!derived_has_cuda_op()) {
-            int c = 0;
-            for (auto data : *copy) {
-              if (data->owner_device != 0) {
-                task->parsec_task.data[c].data_in = data->device_copies[0];
-                task->parsec_task.data[c].source_repo_entry = NULL;
-                ++c;
-              }
-            }
-          }
           /* if we registered as a writer and were the first to register with this copy
            * we need to defer the release of this task to give other tasks a chance to
            * make a copy of the original data */
@@ -2283,8 +2280,10 @@ namespace ttg_parsec {
           handle_iovec_fn(iovs);
         } else if constexpr (!ttg::has_split_metadata<std::decay_t<Value>>::value) {
           /* serialize the object */
-          pos = pack(value, msg->bytes, pos);
+          std::cout << "PRE pack num_iovecs " << std::distance(copy->iovec_begin(), copy->iovec_end()) << std::endl;
+          pos = pack(value, msg->bytes, pos, copy);
           num_iovecs = std::distance(copy->iovec_begin(), copy->iovec_end());
+          std::cout << "POST pack num_iovecs " << num_iovecs << std::endl;
           /* handle any iovecs contained in it */
           handle_iovec_fn(copy->iovec_span());
           copy->iovec_reset();
@@ -2432,7 +2431,7 @@ namespace ttg_parsec {
           pos = pack(metadata, msg->bytes, pos);
         } else if constexpr (!ttg::has_split_metadata<std::decay_t<Value>>::value) {
           /* serialize the object once */
-          pos = pack(value, msg->bytes, pos);
+          pos = pack(value, msg->bytes, pos, copy);
           num_iovs = std::distance(copy->iovec_begin(), copy->iovec_end());
           register_iovs_fn(copy->iovec_span());
           copy->iovec_reset();
@@ -2778,6 +2777,84 @@ namespace ttg_parsec {
       }
     }
 
+    void copy_mark_pushout(detail::ttg_data_copy_t *copy) {
+
+      assert(detail::parsec_ttg_caller->dev_ptr && detail::parsec_ttg_caller->dev_ptr->gpu_task);
+      parsec_gpu_task_t *gpu_task = detail::parsec_ttg_caller->dev_ptr->gpu_task;
+      for (auto data : *copy) {
+        if (data->owner_device != 0 &&
+            data->device_copies[0]->version < data->device_copies[data->owner_device]->version) {
+          /* find the flow */
+          int flowidx = 0;
+          while (flowidx < MAX_PARAM_COUNT &&
+                gpu_task->flow[flowidx]->flow_flags != PARSEC_FLOW_ACCESS_NONE) {
+            if (detail::parsec_ttg_caller->parsec_task.data[flowidx].data_in->original == data) {
+              /* found the right data, set the corresponding flow as pushout */
+              break;
+            }
+            ++flowidx;
+          }
+          if (flowidx == MAX_PARAM_COUNT) {
+            throw std::runtime_error("Cannot add more than MAX_PARAM_COUNT flows to a task!");
+          }
+          if (gpu_task->flow[flowidx]->flow_flags == PARSEC_FLOW_ACCESS_NONE) {
+            /* no flow found, add one and mark it pushout */
+            detail::parsec_ttg_caller->parsec_task.data[flowidx].data_in = data->device_copies[0];
+            ((parsec_flow_t *)gpu_task->flow[flowidx])->flow_flags |= PARSEC_FLOW_ACCESS_RW;
+            gpu_task->flow_nb_elts[flowidx] = data->nb_elts;
+          }
+          gpu_task->pushout |= 1<<flowidx;
+        }
+      }
+    }
+
+    template <std::size_t i, typename Key, typename Value>
+    std::enable_if_t<!ttg::meta::is_void_v<Key> && !std::is_void_v<std::decay_t<Value>>,
+                     void>
+    prepare_send(const ttg::span<const Key> &keylist, const Value &value) {
+      using valueT = std::tuple_element_t<i, input_values_full_tuple_type>;
+
+      /* get the copy */
+      detail::ttg_data_copy_t *copy;
+      copy = detail::find_copy_in_task(detail::parsec_ttg_caller, &value);
+      assert(nullptr != copy);
+
+      /* check if there are non-local successors if it's a cuda task */
+      bool have_remote = false;
+      if constexpr (derived_has_cuda_op()) {
+        auto world = ttg_default_execution_context();
+        int rank = world.rank();
+        uint64_t pos = 0;
+        bool have_remote = keylist.end() != std::find_if(keylist.begin(), keylist.end(),
+                                                        [&](const Key &key) { return keymap(key) != rank; });
+      }
+      if (!derived_has_cuda_op() || have_remote) {
+        /* non-gpu task, check if we need to push back to the host */
+        copy_mark_pushout(copy);
+      }
+    }
+
+    template <std::size_t i, typename Key, typename Value>
+    std::enable_if_t<ttg::meta::is_void_v<Key> && !std::is_void_v<std::decay_t<Value>>,
+                     void>
+    prepare_send(const Value &value) {
+      using valueT = std::tuple_element_t<i, input_values_full_tuple_type>;
+
+      /* get the copy */
+      detail::ttg_data_copy_t *copy;
+      copy = detail::find_copy_in_task(detail::parsec_ttg_caller, &value);
+      assert(nullptr != copy);
+
+      /* check if there are non-local successors if it's a cuda task */
+      auto world = ttg_default_execution_context();
+      int rank = world.rank();
+      bool have_remote = (keymap() != rank);
+      if (!derived_has_cuda_op() || have_remote) {
+        /* non-gpu task, check if we need to push back to the host */
+        copy_mark_pushout(copy);
+      }
+    }
+
    private:
     // Copy/assign/move forbidden ... we could make it work using
     // PIMPL for this base class.  However, this instance of the base
@@ -2813,9 +2890,13 @@ namespace ttg_parsec {
         auto broadcast_callback = [this](const ttg::span<const keyT> &keylist, const valueT &value) {
             broadcast_arg<i, keyT, valueT>(keylist, value);
         };
+        auto prepare_send_callback = [this](const ttg::span<const keyT> &keylist, const valueT &value) {
+            prepare_send<i, keyT, valueT>(keylist, value);
+        };
         auto setsize_callback = [this](const keyT &key, std::size_t size) { set_argstream_size<i>(key, size); };
         auto finalize_callback = [this](const keyT &key) { finalize_argstream<i>(key); };
-        input.set_callback(send_callback, move_callback, broadcast_callback, setsize_callback, finalize_callback);
+        input.set_callback(send_callback, move_callback, broadcast_callback,
+                           setsize_callback, finalize_callback, prepare_send_callback);
       }
       //////////////////////////////////////////////////////////////////
       // case 2: nonvoid key, void value, mixed inputs
@@ -2838,7 +2919,10 @@ namespace ttg_parsec {
         auto send_callback = [this](const valueT &value) { set_arg<i, keyT, const valueT &>(value); };
         auto setsize_callback = [this](std::size_t size) { set_argstream_size<i>(size); };
         auto finalize_callback = [this]() { finalize_argstream<i>(); };
-        input.set_callback(send_callback, move_callback, {}, setsize_callback, finalize_callback);
+        auto prepare_send_callback = [this](const valueT &value) {
+            prepare_send<i, void>(value);
+        };
+        input.set_callback(send_callback, move_callback, {}, setsize_callback, finalize_callback, prepare_send_callback);
       }
       //////////////////////////////////////////////////////////////////
       // case 5: void key, void value, mixed inputs
@@ -3465,9 +3549,22 @@ struct ttg::detail::value_copy_handler<ttg::Runtime::PaRSEC> {
  private:
   ttg_parsec::detail::ttg_data_copy_t *copy_to_remove = nullptr;
 
+ public:
+  value_copy_handler() = default;
+  value_copy_handler(const value_copy_handler& h) = delete;
+  value_copy_handler(value_copy_handler&& h)
+  : copy_to_remove(h.copy_to_remove)
+  {
+    h.copy_to_remove = nullptr;
+  }
 
+  value_copy_handler& operator=(const value_copy_handler& h) = delete;
+  value_copy_handler& operator=(value_copy_handler&& h)
+  {
+    std::swap(copy_to_remove, h.copy_to_remove);
+    return *this;
+  }
 
- public:
   ~value_copy_handler() {
     if (nullptr != copy_to_remove) {
       ttg_parsec::detail::remove_data_copy(copy_to_remove, ttg_parsec::detail::parsec_ttg_caller);
diff --git a/ttg/ttg/terminal.h b/ttg/ttg/terminal.h
index 09ae64cad..9a86bf7a3 100644
--- a/ttg/ttg/terminal.h
+++ b/ttg/ttg/terminal.h
@@ -155,6 +155,7 @@ namespace ttg {
     using broadcast_callback_type = meta::detail::broadcast_callback_t<keyT, std::decay_t<valueT>>;
     using setsize_callback_type = typename base_type::setsize_callback_type;
     using finalize_callback_type = typename base_type::finalize_callback_type;
+    using prepare_send_callback_type = meta::detail::prepare_send_callback_t<keyT, std::decay_t<valueT>>;
     static constexpr bool is_an_input_terminal = true;
     ttg::detail::ContainerWrapper<keyT, valueT> container;
 
@@ -162,6 +163,7 @@ namespace ttg {
     send_callback_type send_callback;
     move_callback_type move_callback;
     broadcast_callback_type broadcast_callback;
+    prepare_send_callback_type prepare_send_callback;
 
     // No moving, copying, assigning permitted
     In(In &&other) = delete;
@@ -189,10 +191,12 @@ namespace ttg {
     void set_callback(const send_callback_type &send_callback, const move_callback_type &move_callback,
                       const broadcast_callback_type &bcast_callback = broadcast_callback_type{},
                       const setsize_callback_type &setsize_callback = setsize_callback_type{},
-                      const finalize_callback_type &finalize_callback = finalize_callback_type{}) {
+                      const finalize_callback_type &finalize_callback = finalize_callback_type{},
+                      const prepare_send_callback_type &prepare_send_callback = prepare_send_callback_type{}) {
       this->send_callback = send_callback;
       this->move_callback = move_callback;
       this->broadcast_callback = bcast_callback;
+      this->prepare_send_callback = prepare_send_callback;
       base_type::set_callback(setsize_callback, finalize_callback);
     }
 
@@ -271,7 +275,7 @@ namespace ttg {
           for (auto &&key : keylist) send(key, v);
         } else {
           /* got something we cannot iterate over (single element?) so put one element in the span */
-          broadcast_callback(ttg::span<const keyT>(&keylist, 1), v);
+          send(ttg::span<const keyT>(&keylist, 1), v);
         }
       }
     }
@@ -291,7 +295,36 @@ namespace ttg {
           for (auto &&key : keylist) sendk(key);
         } else {
           /* got something we cannot iterate over (single element?) so put one element in the span */
-          broadcast_callback(ttg::span<const keyT>(&keylist, 1));
+          sendk(ttg::span<const keyT>(&keylist, 1));
+        }
+      }
+    }
+
+
+    template <typename rangeT, typename Value>
+    void prepare_send(const rangeT &keylist, Value &&value) {
+      const Value &v = value;
+      if (prepare_send_callback) {
+        if constexpr (ttg::meta::is_iterable_v<rangeT>) {
+          prepare_send_callback(ttg::span<const keyT>(&(*std::begin(keylist)),
+                                                      std::distance(std::begin(keylist), std::end(keylist))),
+                                v);
+        } else {
+          /* got something we cannot iterate over (single element?) so put one element in the span */
+          prepare_send_callback(ttg::span<const keyT>(&keylist, 1), v);
+        }
+      }
+    }
+
+    template <typename rangeT, typename Value>
+    void prepare_send(Value &&value) {
+      const Value &v = value;
+      if (prepare_send_callback) {
+        if constexpr (ttg::meta::is_iterable_v<rangeT>) {
+          prepare_send_callback(v);
+        } else {
+          /* got something we cannot iterate over (single element?) so put one element in the span */
+          prepare_send_callback(v);
         }
       }
     }
@@ -531,6 +564,32 @@ namespace ttg {
         }
       }
     }
+
+    template <typename rangeT, typename Key = keyT, typename Value = valueT>
+    std::enable_if_t<meta::is_none_void_v<Key> && !meta::is_void_v<valueT>, void>
+    prepare_send(const rangeT &keylist, const Value &value) {
+      for (auto &&successor : this->successors()) {
+        assert(successor->get_type() != TerminalBase::Type::Write);
+        if (successor->get_type() == TerminalBase::Type::Read) {
+          return static_cast<In<keyT, std::add_const_t<valueT>> *>(successor)->prepare_send(keylist, value);
+        } else if (successor->get_type() == TerminalBase::Type::Consume) {
+          return static_cast<In<keyT, valueT> *>(successor)->prepare_send(keylist, value);
+        }
+      }
+    }
+
+    template <typename rangeT, typename Key = keyT, typename Value = valueT>
+    std::enable_if_t<meta::is_none_void_v<Key> && !meta::is_void_v<valueT>, void>
+    prepare_send(const Value &value) {
+      for (auto &&successor : this->successors()) {
+        assert(successor->get_type() != TerminalBase::Type::Write);
+        if (successor->get_type() == TerminalBase::Type::Read) {
+          return static_cast<In<keyT, std::add_const_t<valueT>> *>(successor)->prepare_send(value);
+        } else if (successor->get_type() == TerminalBase::Type::Consume) {
+          return static_cast<In<keyT, valueT> *>(successor)->prepare_send(value);
+        }
+      }
+    }
   };
 
   namespace meta {
diff --git a/ttg/ttg/util/meta.h b/ttg/ttg/util/meta.h
index 3637f66ae..389a9c8d5 100644
--- a/ttg/ttg/util/meta.h
+++ b/ttg/ttg/util/meta.h
@@ -761,6 +761,8 @@ namespace ttg {
       template <typename Key, typename Value>
       using broadcast_callback_t = typename broadcast_callback<Key, Value>::type;
 
+
+
       ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       // setsize_callback_t<key> = std::function<void(const keyT &, std::size_t)> protected against void key
       ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -836,6 +838,31 @@ namespace ttg {
       template <typename... valueTs>
       using input_reducers_t = typename input_reducers<valueTs...>::type;
 
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // prepare_send_callback_t<Key, Value> = std::function<int(const ttg::span<Key> &, const Value &)> protected against void key
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      template <typename Key, typename Value, typename Enabler = void>
+      struct prepare_send_callback;
+      template <typename Key, typename Value>
+      struct prepare_send_callback<Key, Value, std::enable_if_t<!is_void_v<Key> && !is_void_v<Value>>> {
+        using type = std::function<void(const ttg::span<const Key> &, const Value &)>;
+      };
+      template <typename Key, typename Value>
+      struct prepare_send_callback<Key, Value, std::enable_if_t<!is_void_v<Key> && is_void_v<Value>>> {
+        using type = std::function<void(const ttg::span<const Key> &)>;
+      };
+      template <typename Key, typename Value>
+      struct prepare_send_callback<Key, Value, std::enable_if_t<is_void_v<Key> && !is_void_v<Value>>> {
+        using type = std::function<void(const Value &)>;
+      };
+      template <typename Key, typename Value>
+      struct prepare_send_callback<Key, Value, std::enable_if_t<is_void_v<Key> && is_void_v<Value>>> {
+        using type = std::function<void()>;
+      };
+      template <typename Key, typename Value>
+      using prepare_send_callback_t = typename prepare_send_callback<Key, Value>::type;
+
+
     }  // namespace detail
 
     ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
diff --git a/ttg/ttg/view.h b/ttg/ttg/view.h
index 98c012ce1..b92111932 100644
--- a/ttg/ttg/view.h
+++ b/ttg/ttg/view.h
@@ -5,6 +5,8 @@
 #include <type_traits>
 #include <span>
 
+#include "ttg/fwd.h"
+#include "ttg/impl_selector.h"
 #include "ttg/ptr.h"
 
 namespace ttg {
@@ -439,20 +441,299 @@ namespace ttg {
     };
   } // namespace detail
 
-  /* Wait for the kernel to complete */
-  inline auto wait_kernel() {
-    return detail::wait_kernel_t<>{};
-  }
-
-  /* Wait for kernel to complete and provided ttg::buffer
+  /* Wait for kernel to complete and provided ttg::buffer and ttg::devicescratch
    * to be transferred back to host */
   template<typename... Buffers>
-  inline auto wait_kernel_out(Buffers&&... args) {
-    static_assert((ttg::detail::is_buffer_v<std::decay_t<Buffers>>&&...),
-                  "Only ttg::buffer can be explicitly waited on!");
+  inline auto wait_kernel(Buffers&&... args) {
+    static_assert(((ttg::detail::is_buffer_v<std::decay_t<Buffers>>
+                    ||ttg::detail::is_devicescratch_v<std::decay_t<Buffers>>)&&...),
+                  "Only ttg::buffer and ttg::devicescratch can be waited on!");
     return detail::wait_kernel_t<std::remove_reference_t<Buffers>...>{std::tie(std::forward<Buffers>(args)...)};
   }
 
+
+  /* TODO: move all device code into ttg::device */
+  namespace device {
+
+    /******************************
+     * Send/Broadcast handling
+     * We pass the value returned by the backend's copy handler into a coroutine
+     * and execute the first part (prepare), before suspending it.
+     * The second part (send/broadcast) is executed after the task completed.
+     ******************************/
+
+    namespace detail {
+      struct send_coro_promise_type;
+
+      using send_coro_handle_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<send_coro_promise_type>;
+
+      /// task that can be resumed after some events occur
+      struct send_coro_state : public send_coro_handle_type {
+        using base_type = send_coro_handle_type;
+
+        /// these are members mandated by the promise_type concept
+        ///@{
+
+        using promise_type = send_coro_promise_type;
+
+        ///@}
+
+        send_coro_state(base_type base) : base_type(std::move(base)) {}
+
+        base_type& handle() { return *this; }
+
+        /// @return true if ready to resume
+        inline bool ready() {
+          return true;
+        }
+
+        /// @return true if task completed and can be destroyed
+        inline bool completed();
+      };
+
+      struct send_coro_promise_type {
+
+        /* do not suspend the coroutine on first invocation, we want to run
+        * the coroutine immediately and suspend only once.
+        */
+        TTG_CXX_COROUTINE_NAMESPACE::suspend_never initial_suspend() {
+          return {};
+        }
+
+        /* we don't suspend the coroutine at the end.
+        * it can be destroyed once the send/broadcast is done
+        */
+        TTG_CXX_COROUTINE_NAMESPACE::suspend_never final_suspend() noexcept {
+          return {};
+        }
+
+        send_coro_state get_return_object() { return send_coro_state{send_coro_handle_type::from_promise(*this)}; }
+
+        /* the send coros only have an empty co_await */
+        TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(ttg::Void) {
+          return {};
+        }
+
+        void unhandled_exception() {
+
+        }
+
+      };
+
+      template<typename Key, typename Value, ttg::Runtime Runtime = ttg::ttg_runtime>
+      inline send_coro_state send_coro (const Key& key, Value&& value, ttg::Out<Key, Value> &t,
+                                        ttg::detail::value_copy_handler<Runtime>& ch) {
+        ttg::detail::value_copy_handler<Runtime> copy_handler = std::move(ch); // destroyed at the end of the coro
+        Key k = key;
+        t.prepare_send(k, std::forward<Value>(value));
+        co_await ttg::Void{}; // we'll come back once the task is done
+        t.send(k, std::forward<Value>(value));
+      };
+
+      template<typename Value, ttg::Runtime Runtime = ttg::ttg_runtime>
+      inline send_coro_state sendv_coro (Value&& value, ttg::Out<void, Value> &t,
+                                         ttg::detail::value_copy_handler<Runtime>& ch) {
+        ttg::detail::value_copy_handler<Runtime> copy_handler = std::move(ch); // destroyed at the end of the coro
+        t.prepare_send(std::forward<Value>(value));
+        co_await ttg::Void{}; // we'll come back once the task is done
+        t.sendv(std::forward<Value>(value));
+      };
+    } // namespace detail
+
+
+    /* functionality to deal with sending/broadcasting */
+    namespace detail {
+      struct send_t {
+        send_coro_state coro;
+      };
+    } // namespace detail
+
+    template <size_t i, typename keyT, typename valueT, typename... out_keysT, typename... out_valuesT,
+              ttg::Runtime Runtime = ttg::ttg_runtime>
+    inline detail::send_t send(const keyT &key, valueT &&value,
+                                        std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
+      ttg::detail::value_copy_handler<Runtime> copy_handler;
+      return detail::send_t{detail::send_coro(key, copy_handler(std::forward<valueT>(value)), std::get<i>(t), copy_handler)};
+    }
+
+    template <size_t i, typename valueT, typename... out_keysT, typename... out_valuesT,
+              ttg::Runtime Runtime = ttg::ttg_runtime>
+    inline detail::send_t sendv(
+        valueT &&value, std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
+      ttg::detail::value_copy_handler<Runtime> copy_handler;
+      return detail::send_t{detail::sendv_coro(copy_handler(std::forward<valueT>(value)), std::get<i>(t), copy_handler)};
+    }
+
+
+    // clang-format off
+    /// \brief Sends a task id and a value to the template tasks attached to the output terminal of this template task
+    /// \param[in] i Identifies which output terminal of this template task to select for sending
+    /// \param[in] key: the id of the task(s) receiving the value
+    /// \param[in] value: the value to send to the receiving task(s)
+    // clang-format on
+    template <typename keyT, typename valueT, ttg::Runtime Runtime = ttg::ttg_runtime>
+    inline detail::send_t send(size_t i, const keyT &key, valueT &&value) {
+      ttg::detail::value_copy_handler<Runtime> copy_handler;
+      auto *terminal_ptr = ttg::detail::get_out_terminal<keyT, valueT>(i, "ttg::device::send(i, key, value)");
+      return detail::send_t{detail::send_coro(key, copy_handler(std::forward<valueT>(value)), *terminal_ptr, copy_handler)};
+    }
+
+    // clang-format off
+    /// \brief Sends a task id and a value to the template tasks attached to the output terminal of this template task
+    /// \note this is provided to support `send<i>` with and without explicitly-passed terminal tuple
+    /// \tparam <i> Identifies which output terminal of this template task to select for sending
+    /// \param[in] key: the id of the task(s) receiving the value
+    /// \param[in] value: the value to send to the receiving task(s)
+    // clang-format on
+    template <size_t i, typename keyT, typename valueT>
+    inline auto send(const keyT &key, valueT &&value) {
+      return send(i, key, std::forward<valueT>(value));
+    }
+
+
+
+    namespace detail {
+
+      template<typename T, typename Enabler = void>
+      struct broadcast_keylist_trait {
+        using type = T;
+      };
+
+      /* overload for iterable types that extracts the type of the first element */
+      template<typename T>
+      struct broadcast_keylist_trait<T, std::enable_if_t<ttg::meta::is_iterable_v<T>>> {
+        using key_type = decltype(*std::begin(std::get<0>(std::declval<T>())));
+      };
+
+      template <size_t KeyId, size_t I, size_t... Is, typename... RangesT, typename valueT,
+                typename... out_keysT, typename... out_valuesT>
+      inline void prepare_broadcast(const std::tuple<RangesT...> &keylists, valueT &&value,
+                                    std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
+        std::get<I>(t)->prepare_send(std::get<KeyId>(keylists), std::forward<valueT>(value));
+        if constexpr (sizeof...(Is) > 0) {
+          detail::prepare_broadcast<KeyId+1, Is...>(keylists, std::forward<valueT>(value), t);
+        }
+      }
+
+      template <size_t KeyId, size_t I, size_t... Is, typename... RangesT, typename valueT,
+                typename... out_keysT, typename... out_valuesT>
+      inline void prepare_broadcast(const std::tuple<RangesT...> &keylists, valueT &&value) {
+        using key_t = broadcast_keylist_trait<
+                        std::tuple_element_t<KeyId, std::tuple<std::remove_reference_t<RangesT>...>>
+                      >::key_type;
+        auto *terminal_ptr = ttg::detail::get_out_terminal<key_t, valueT>(I, "ttg::device::broadcast(keylists, value)");
+        terminal_ptr->prepare_send(std::get<KeyId>(keylists), value);
+        if constexpr (sizeof...(Is) > 0) {
+          detail::prepare_broadcast<KeyId+1, Is...>(keylists, std::forward<valueT>(value));
+        }
+      }
+
+      template <size_t KeyId, size_t I, size_t... Is, typename... RangesT, typename valueT,
+                typename... out_keysT, typename... out_valuesT>
+      inline void broadcast(const std::tuple<RangesT...> &keylists, valueT &&value,
+                                    std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
+        std::get<I>(t)->broadcast(std::get<KeyId>(keylists), std::forward<valueT>(value));
+        if constexpr (sizeof...(Is) > 0) {
+          detail::broadcast<KeyId+1, Is...>(keylists, std::forward<valueT>(value), t);
+        }
+      }
+
+      template <size_t KeyId, size_t I, size_t... Is, typename... RangesT, typename valueT,
+                typename... out_keysT, typename... out_valuesT>
+      inline void broadcast(const std::tuple<RangesT...> &keylists, valueT &&value) {
+        using key_t = broadcast_keylist_trait<
+                        std::tuple_element_t<KeyId, std::tuple<std::remove_reference_t<RangesT>...>>
+                      >::key_type;
+        auto *terminal_ptr = ttg::detail::get_out_terminal<key_t, valueT>(I, "ttg::device::broadcast(keylists, value)");
+        terminal_ptr->broadcast(std::get<KeyId>(keylists), value);
+        if constexpr (sizeof...(Is) > 0) {
+          detail::broadcast<KeyId+1, Is...>(keylists, std::forward<valueT>(value));
+        }
+      }
+
+      /* overload with explicit terminals */
+      template <size_t I, size_t... Is, typename RangesT, typename valueT,
+                typename... out_keysT, typename... out_valuesT,
+                ttg::Runtime Runtime = ttg::ttg_runtime>
+      inline send_coro_state
+      broadcast_coro(RangesT &&keylists, valueT &&value,
+                     std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t,
+                     ttg::detail::value_copy_handler<Runtime>&& ch) {
+        ttg::detail::value_copy_handler<Runtime> copy_handler = std::move(ch); // destroyed at the end of the coro
+        RangesT kl = std::forward<RangesT>(keylists); // capture the keylist(s)
+        if constexpr (ttg::meta::is_tuple_v<RangesT>) {
+          // treat as tuple
+          prepare_broadcast<0, I, Is...>(kl, std::forward<std::decay_t<decltype(value)>>(value), t);
+          co_await ttg::Void{}; // we'll come back once the task is done
+          broadcast<0, I, Is...>(kl, std::forward<std::decay_t<decltype(value)>>(value), t);
+        } else if constexpr (!ttg::meta::is_tuple_v<RangesT>) {
+          // create a tie to the captured keylist
+          prepare_broadcast<0, I, Is...>(std::tie(kl), std::forward<std::decay_t<decltype(value)>>(value), t);
+          co_await ttg::Void{}; // we'll come back once the task is done
+          broadcast<0, I, Is...>(std::tie(kl), std::forward<std::decay_t<decltype(value)>>(value), t);
+        }
+      }
+
+      /* overload with implicit terminals */
+      template <size_t I, size_t... Is, typename RangesT, typename valueT,
+                ttg::Runtime Runtime = ttg::ttg_runtime>
+      inline send_coro_state
+      broadcast_coro(RangesT &&keylists, valueT &&value,
+                     ttg::detail::value_copy_handler<Runtime>&& ch) {
+        ttg::detail::value_copy_handler<Runtime> copy_handler = std::move(ch); // destroyed at the end of the coro
+        RangesT kl = std::forward<RangesT>(keylists); // capture the keylist(s)
+        if constexpr (ttg::meta::is_tuple_v<RangesT>) {
+          // treat as tuple
+          static_assert(sizeof...(Is)+1 == std::tuple_size_v<RangesT>,
+                        "Size of keylist tuple must match the number of output terminals");
+          prepare_broadcast<0, I, Is...>(kl, std::forward<std::decay_t<decltype(value)>>(value));
+          co_await ttg::Void{}; // we'll come back once the task is done
+          broadcast<0, I, Is...>(kl, std::forward<std::decay_t<decltype(value)>>(value));
+        } else if constexpr (!ttg::meta::is_tuple_v<RangesT>) {
+          // create a tie to the captured keylist
+          prepare_broadcast<0, I, Is...>(std::tie(kl), std::forward<std::decay_t<decltype(value)>>(value));
+          co_await ttg::Void{}; // we'll come back once the task is done
+          broadcast<0, I, Is...>(std::tie(kl), std::forward<std::decay_t<decltype(value)>>(value));
+        }
+      }
+    }  // namespace detail
+
+    /* overload with explicit terminals and keylist passed by const reference */
+    template <size_t I, size_t... Is, typename rangeT, typename valueT, typename... out_keysT, typename... out_valuesT,
+              ttg::Runtime Runtime = ttg::ttg_runtime>
+    inline detail::send_t broadcast(rangeT &&keylist,
+                                    valueT &&value,
+                                    const std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
+      ttg::detail::value_copy_handler<Runtime> copy_handler;
+      return detail::send_t{
+              broadcast_coro<0, I, Is...>(std::forward<rangeT>(keylist),
+                                          copy_handler(std::forward<valueT>(value)),
+                                          t, std::move(copy_handler))};
+
+    }
+
+    /* overload with implicit terminals and keylist passed by const reference */
+    template <size_t i, typename rangeT, typename valueT,
+              ttg::Runtime Runtime = ttg::ttg_runtime>
+    inline detail::send_t broadcast(rangeT &&keylist, valueT &&value) {
+      ttg::detail::value_copy_handler<Runtime> copy_handler;
+      return detail::send_t{broadcast_coro<0, i>(std::tie(keylist), copy_handler(std::forward<valueT>(value)),
+                                                 std::move(copy_handler))};
+    }
+
+    template<typename... Args, ttg::Runtime Runtime = ttg::ttg_runtime>
+    std::vector<device::detail::send_t> forward(Args&&... args) {
+      // TODO: check the cost of this!
+      return std::vector{std::forward<Args>(args)...};
+    }
+
+  } // namespace device
+
+  /*******************************************
+   * Device task promise and coroutine handle
+   *******************************************/
+
+
   struct device_task_promise_type;
 
   using device_task_handle_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<device_task_promise_type>;
@@ -504,6 +785,7 @@ namespace ttg {
       return {};
     }
 
+#if 0
     /* waiting for transfers to complete should always suspend
      * TODO: as an optimization, we could check here if all data
      *       is already available and avoid suspending...
@@ -526,6 +808,7 @@ namespace ttg {
       m_state = TTG_DEVICE_CORO_WAIT_TRANSFER;
       return {};
     }
+#endif // 0
 
     /* convenience-function to yield a single view */
     template<typename HostT, typename... DeviceViewTs>
@@ -592,7 +875,7 @@ namespace ttg {
     }
 
     template<typename... Ts>
-    auto await_transform(detail::wait_kernel_t<ttg::buffer<Ts>...>&& a) {
+    auto await_transform(detail::wait_kernel_t<Ts...>&& a) {
       std::cout << "yield_value: wait_kernel_t" << std::endl;
       if constexpr (sizeof...(Ts) > 0) {
         TTG_IMPL_NS::mark_device_out(a.ties);
@@ -601,15 +884,16 @@ namespace ttg {
       return a;
     }
 
-#if 0
-    template<typename... Ts>
-    auto await_transform(ttg::detail::get_ptr_tpl_t<Ts...>&& a) {
-      return a;
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(std::vector<device::detail::send_t>&& v) {
+      m_sends = std::forward<std::vector<device::detail::send_t>>(v);
+      m_state = TTG_DEVICE_CORO_COMPLETE;
+      return {};
     }
 
-    template<typename T>
-    auto await_transform(ttg::detail::get_ptr_t<T>&& a) {
-      return a;
+#if 0
+    void return_value(std::vector<device::detail::send_t>&& sends) {
+      m_sends = std::forward<std::vector<device::detail::send_t>>(sends);
+      m_state = TTG_DEVICE_CORO_COMPLETE;
     }
 #endif // 0
 
@@ -629,12 +913,12 @@ namespace ttg {
 
     using iterator = std::vector<device_obj_view>::iterator;
 
-    auto begin() {
-      return m_spans.begin();
-    }
-
-    auto end() {
-      return m_spans.end();
+    /* execute all pending send and broadcast operations */
+    void do_sends() {
+      for (auto& send : m_sends) {
+        send.coro();
+      }
+      m_sends.clear();
     }
 
     auto state() {
@@ -642,8 +926,7 @@ namespace ttg {
     }
 
   private:
-
-    std::vector<device_obj_view> m_spans;
+    std::vector<device::detail::send_t> m_sends;
     ttg_device_coro_state m_state = TTG_DEVICE_CORO_STATE_NONE;
 
   };

From 514acf8d3c8aef45cb5a1ab33e55ccc0d480a122 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Thu, 20 Apr 2023 11:53:53 -0400
Subject: [PATCH 075/259] compile fixups for clang (N.B. clang is still
 unusable due to missing std::atomic_ref in libc++)

---
 ttg/ttg/run.h  |  1 +
 ttg/ttg/view.h | 14 +++++++-------
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/ttg/ttg/run.h b/ttg/ttg/run.h
index b4dc593de..06111e05e 100644
--- a/ttg/ttg/run.h
+++ b/ttg/ttg/run.h
@@ -9,6 +9,7 @@
 
 #include "ttg/util/bug.h"
 #include "ttg/util/env.h"
+#include "ttg/world.h"
 
 namespace ttg {
 
diff --git a/ttg/ttg/view.h b/ttg/ttg/view.h
index b92111932..1ea95fe7e 100644
--- a/ttg/ttg/view.h
+++ b/ttg/ttg/view.h
@@ -156,12 +156,12 @@ namespace ttg {
 
     template<std::size_t i>
     auto get_device_ptr() {
-      return static_cast<std::tuple_element_t<i, span_tuple_type>::value_type*>(std::get<i>(m_spans).data());
+      return static_cast<typename std::tuple_element_t<i, span_tuple_type>::value_type*>(std::get<i>(m_spans).data());
     }
 
     template<std::size_t i>
     const auto get_device_ptr() const {
-      return static_cast<std::tuple_element_t<i, span_tuple_type>::value_type>(std::get<i>(m_spans).data());
+      return static_cast<typename std::tuple_element_t<i, span_tuple_type>::value_type>(std::get<i>(m_spans).data());
     }
 
     template<std::size_t i>
@@ -311,12 +311,12 @@ namespace ttg {
 
     template<std::size_t i>
     auto get_device_ptr() {
-      return static_cast<std::tuple_element_t<i, span_tuple_type>::value_type*>(std::get<i>(m_spans).data());
+      return static_cast<typename std::tuple_element_t<i, span_tuple_type>::value_type*>(std::get<i>(m_spans).data());
     }
 
     template<std::size_t i>
     const auto get_device_ptr() const {
-      return static_cast<std::tuple_element_t<i, span_tuple_type>::value_type>(std::get<i>(m_spans).data());
+      return static_cast<typename std::tuple_element_t<i, span_tuple_type>::value_type>(std::get<i>(m_spans).data());
     }
 
     template<std::size_t i>
@@ -430,7 +430,7 @@ namespace ttg {
       constexpr bool await_ready() const noexcept { return false; }
 
       /* always suspend */
-      constexpr void await_suspend( std::coroutine_handle<> ) const noexcept {}
+      constexpr void await_suspend( TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<> ) const noexcept {}
 
       void await_resume() noexcept {
         if constexpr (sizeof...(Ts) > 0) {
@@ -618,7 +618,7 @@ namespace ttg {
       template <size_t KeyId, size_t I, size_t... Is, typename... RangesT, typename valueT,
                 typename... out_keysT, typename... out_valuesT>
       inline void prepare_broadcast(const std::tuple<RangesT...> &keylists, valueT &&value) {
-        using key_t = broadcast_keylist_trait<
+        using key_t = typename broadcast_keylist_trait<
                         std::tuple_element_t<KeyId, std::tuple<std::remove_reference_t<RangesT>...>>
                       >::key_type;
         auto *terminal_ptr = ttg::detail::get_out_terminal<key_t, valueT>(I, "ttg::device::broadcast(keylists, value)");
@@ -641,7 +641,7 @@ namespace ttg {
       template <size_t KeyId, size_t I, size_t... Is, typename... RangesT, typename valueT,
                 typename... out_keysT, typename... out_valuesT>
       inline void broadcast(const std::tuple<RangesT...> &keylists, valueT &&value) {
-        using key_t = broadcast_keylist_trait<
+        using key_t = typename broadcast_keylist_trait<
                         std::tuple_element_t<KeyId, std::tuple<std::remove_reference_t<RangesT>...>>
                       >::key_type;
         auto *terminal_ptr = ttg::detail::get_out_terminal<key_t, valueT>(I, "ttg::device::broadcast(keylists, value)");

From 0c889ba3dd806e29171a9ea8b03aa4c6d171a8bc Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Thu, 20 Apr 2023 14:53:42 -0400
Subject: [PATCH 076/259] Wrap various C/C++-only options using
 language-specific generator expressions to avoid pissing CUDA off

---
 CMakeLists.txt                          | 4 ++--
 cmake/modules/FindCXXStdCoroutine.cmake | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index bc106c6ee..7245600de 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -59,9 +59,9 @@ option(TTG_ENABLE_TRACE "Whether to enable ttg::trace() output" OFF)
 option (FORCE_COLORED_OUTPUT "Always produce ANSI-colored output (GNU/Clang only)." TRUE)
 if (FORCE_COLORED_OUTPUT)
     if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
-       add_compile_options (-fdiagnostics-color=always)
+       add_compile_options ($<$<COMPILE_LANGUAGE:CXX,C>:-fdiagnostics-color=always>)
     elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
-       add_compile_options (-fcolor-diagnostics)
+       add_compile_options ($<$<COMPILE_LANGUAGE:CXX,C>:-fcolor-diagnostics>)
     endif ()
 endif (FORCE_COLORED_OUTPUT)
 
diff --git a/cmake/modules/FindCXXStdCoroutine.cmake b/cmake/modules/FindCXXStdCoroutine.cmake
index 3a86d7a4d..dbe0b394f 100644
--- a/cmake/modules/FindCXXStdCoroutine.cmake
+++ b/cmake/modules/FindCXXStdCoroutine.cmake
@@ -164,7 +164,7 @@ foreach(component IN LISTS CXXStdCoroutines_want_components_ordered)
         add_library(std::coroutine INTERFACE IMPORTED GLOBAL)
         target_compile_features(std::coroutine INTERFACE cxx_std_20)
         if (option)
-          target_compile_options(std::coroutine INTERFACE "${option}")
+          target_compile_options(std::coroutine INTERFACE "$<$<COMPILE_LANGUAGE:CXX>:${option}>")
         endif()
         set(CXX_COROUTINE_COMPONENT "${component}" CACHE STRING "The component of CXXStdCoroutine package found")
         # break out of this loop

From 809485449315eba47566a84809b83b1a9f26595d Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Thu, 20 Apr 2023 16:06:52 -0400
Subject: [PATCH 077/259] make ttg::buffer and device_coro's value_t
 serializable by boost

---
 tests/unit/device_coro.cc |  7 +++++--
 ttg/ttg/parsec/buffer.h   | 19 +++++++++++++++++++
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/tests/unit/device_coro.cc b/tests/unit/device_coro.cc
index 5b51b0724..c85b2287f 100644
--- a/tests/unit/device_coro.cc
+++ b/tests/unit/device_coro.cc
@@ -3,6 +3,8 @@
 #include "ttg.h"
 #include "ttg/view.h"
 
+#include "ttg/serialization.h"
+
 #include "cuda_kernel.h"
 
 struct value_t {
@@ -10,12 +12,13 @@ struct value_t {
   int quark;
 
   template<typename Archive>
-  void ttg_serialize(Archive& ar) {
+  void serialize(Archive& ar, const unsigned int version) {
     ar& quark;
     ar& db; // input:
   }
 };
 
+#ifdef TTG_SERIALIZATION_SUPPORTS_MADNESS
 /* devicebuf is non-POD so provide serialization
  * information for members not a devicebuf */
 namespace madness::archive {
@@ -24,7 +27,7 @@ namespace madness::archive {
     static inline void serialize(const Archive& ar, value_t& obj) { ar& obj.quark & obj.db; };
   };
 }  // namespace madness::archive
-
+#endif  // TTG_SERIALIZATION_SUPPORTS_MADNESS
 
 TEST_CASE("Device", "coro") {
 
diff --git a/ttg/ttg/parsec/buffer.h b/ttg/ttg/parsec/buffer.h
index 8a293738e..4a156e4fe 100644
--- a/ttg/ttg/parsec/buffer.h
+++ b/ttg/ttg/parsec/buffer.h
@@ -349,6 +349,25 @@ struct buffer {
 
   /* serialization support */
 
+#ifdef TTG_SERIALIZATION_SUPPORTS_BOOST
+  template <typename Archive>
+  void serialize(Archive& ar, const unsigned int version) {
+    if constexpr (ttg::detail::is_output_archive_v<Archive>) {
+      std::size_t s = size();
+      ar& s;
+      assert(m_ttg_copy != nullptr); // only tracked objects allowed
+      m_ttg_copy->iovec_add(ttg::iovec{s*sizeof(T), current_device_ptr()});
+    } else {
+      std::size_t s;
+      ar & s;
+      /* initialize internal pointers and then reset */
+      reset(s);
+      assert(m_ttg_copy != nullptr); // only tracked objects allowed
+      m_ttg_copy->iovec_add(ttg::iovec{s*sizeof(T), current_device_ptr()});
+    }
+  }
+#endif // TTG_SERIALIZATION_SUPPORTS_BOOST
+
 #ifdef TTG_SERIALIZATION_SUPPORTS_CEREAL
   template <class Archive>
   std::enable_if_t<std::is_base_of_v<cereal::detail::InputArchiveBase, Archive> ||

From 96e0d67aff78bf651da3eb88b7ca77249d6fca33 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Tue, 20 Jun 2023 22:35:10 +0000
Subject: [PATCH 078/259] CUDA tests: increment scratch buffer if provided

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 tests/unit/cuda_kernel.cu | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/unit/cuda_kernel.cu b/tests/unit/cuda_kernel.cu
index b97611fe5..f6f00d172 100644
--- a/tests/unit/cuda_kernel.cu
+++ b/tests/unit/cuda_kernel.cu
@@ -9,7 +9,7 @@ static __global__ void cu_increment_buffer(double* buffer, double* scratch) {
 
   buffer[tx] += 1.0;
   if (tx == 0 && scratch != nullptr) {
-    *scratch = buffer[tx];
+    *scratch += 1.0;
   }
 }
 

From 71f123c0faa3242d81ae0777b0934c10c151138b Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Tue, 20 Jun 2023 22:36:01 +0000
Subject: [PATCH 079/259] Disable catching POSIX signals in Catch2 tests

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 tests/unit/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt
index c2db7bd3f..67c6e6143 100644
--- a/tests/unit/CMakeLists.txt
+++ b/tests/unit/CMakeLists.txt
@@ -16,7 +16,7 @@ set(ut_libs Catch2::Catch2)
     endif(TTG_HAVE_CUDA)
     list(APPEND ut_libs std::coroutine)
 #endif()
-add_ttg_executable(core-unittests-ttg "${ut_src}" LINK_LIBRARIES "${ut_libs}")
+add_ttg_executable(core-unittests-ttg "${ut_src}" LINK_LIBRARIES "${ut_libs}" COMPILE_DEFINITIONS "-DCATCH_CONFIG_NO_POSIX_SIGNALS=1" )
 
 
 # serialization test: probes serialization via all supported serialization methods (MADNESS, Boost::serialization, cereal) that are available

From 4b6970ae42b37ac46396a3d9637564e45982a8a2 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Tue, 20 Jun 2023 22:37:20 +0000
Subject: [PATCH 080/259] Make CUDA tasks work in distributed, incl tests

---
 tests/unit/device_coro.cc      | 254 ++++++++++++++++++++++++++++++---
 ttg/ttg/parsec/buffer.h        | 127 +++++++----------
 ttg/ttg/parsec/devicefunc.h    |  59 +++++---
 ttg/ttg/parsec/devicescratch.h |  58 +++++---
 ttg/ttg/parsec/ttg.h           | 229 ++++++++++++-----------------
 ttg/ttg/parsec/ttg_data_copy.h |  37 ++++-
 ttg/ttg/view.h                 |  43 +-----
 7 files changed, 492 insertions(+), 315 deletions(-)

diff --git a/tests/unit/device_coro.cc b/tests/unit/device_coro.cc
index c85b2287f..ae2fb96bb 100644
--- a/tests/unit/device_coro.cc
+++ b/tests/unit/device_coro.cc
@@ -35,7 +35,8 @@ TEST_CASE("Device", "coro") {
 
     ttg::Edge<int, value_t> edge;
     auto fn = [&](const int& key, value_t&& val) -> ttg::device_task {
-      ttg::print("device_task key ", key);
+      //ttg::print("device_task key ", key);
+
       /* wait for the view to be available on the device */
       co_await ttg::to_device(val.db);
       /* once we're back here the data has been transferred */
@@ -60,6 +61,55 @@ TEST_CASE("Device", "coro") {
                                                       "device_task", {"edge_in"}, {"edge_out"});
     make_graph_executable(tt);
     if (ttg::default_execution_context().rank() == 0) tt->invoke(0, value_t{});
+    std::cout << "Entering fence" << std::endl;
+    ttg::ttg_fence(ttg::default_execution_context());
+  }
+
+  SECTION("devicebuf-inc") {
+
+    ttg::Edge<int, value_t> edge;
+    auto fn = [&](const int& key, value_t&& val) -> ttg::device_task {
+      //ttg::print("device_task key ", key);
+
+      /* wait for the view to be available on the device */
+      co_await ttg::to_device(val.db);
+      /* once we're back here the data has been transferred */
+      CHECK(val.db.current_device_ptr() != nullptr);
+
+      std::cout << "KEY " << key << " VAL IN DEV " << *val.db.current_device_ptr() << " VAL IN HOST " << *val.db.host_ptr() << std::endl;
+
+      /* call a kernel */
+#ifdef TTG_HAVE_CUDA
+      increment_buffer(val.db.current_device_ptr(), val.db.size(), nullptr, 0);
+#endif // TTG_HAVE_CUDA
+
+      /* here we suspend to wait for a kernel to complete */
+      co_await ttg::wait_kernel(val.db);
+
+      std::cout << "KEY " << key << " VAL OUT DEV " << *val.db.current_device_ptr() << " VAL OUT HOST " << *val.db.host_ptr() << std::endl;
+
+#ifdef TTG_HAVE_CUDA
+      /* buffer is increment once per task, so it should be the same as key */
+      CHECK(static_cast<int>(*val.db.host_ptr()) == key+1);
+#endif // TTG_HAVE_CUDA
+
+      /* we're back, the kernel executed and we can send */
+      if (key < 10) {
+        /* TODO: should we move the view in here if we want to get the device side data */
+        //ttg::send<0>(key+1, std::move(val));
+        co_await ttg::device::forward(ttg::device::send<0>(key+1, std::move(val)));
+      }
+    };
+
+    //ptr.get_view<ttg::ExecutionSpace::CUDA>(device_id);
+
+    auto tt = ttg::make_tt<ttg::ExecutionSpace::CUDA>(fn, ttg::edges(edge), ttg::edges(edge),
+                                                      "device_task", {"edge_in"}, {"edge_out"});
+    make_graph_executable(tt);
+    value_t v;
+    *v.db.host_ptr() = 2.0; // start from non-zero value
+    if (ttg::default_execution_context().rank() == 0) tt->invoke(2, std::move(v));
+    std::cout << "Entering fence" << std::endl;
     ttg::ttg_fence(ttg::default_execution_context());
   }
 
@@ -84,8 +134,92 @@ TEST_CASE("Device", "coro") {
       co_await ttg::wait_kernel(ds);
 
 #ifdef TTG_HAVE_CUDA
-      /* buffer is increment once per task, so it should be the same as key */
-      CHECK((static_cast<int>(scratch)-1) == key);
+      /* the scratch is allocated but no data is transferred in; it's incremented once */
+      CHECK((static_cast<int>(scratch)-1) == 0);
+#endif // 0
+
+      /* we're back, the kernel executed and we can send */
+      if (key < 10) {
+        /* TODO: should we move the view in here if we want to get the device side data */
+        //ttg::send<0>(key+1, std::move(val));
+        /* NOTE: we use co_await here instead of co_return because co_return destroys all local variables first;
+         *       we will not return from this co_await!*/
+        co_await ttg::device::forward(ttg::device::send<0>(key+1, std::move(val)));
+      }
+    };
+
+    auto tt = ttg::make_tt<ttg::ExecutionSpace::CUDA>(fn, ttg::edges(edge), ttg::edges(edge),
+                                                      "device_task", {"edge_in"}, {"edge_out"});
+    make_graph_executable(tt);
+    if (ttg::default_execution_context().rank() == 0) tt->invoke(0, value_t{});
+    ttg::ttg_fence(ttg::default_execution_context());
+  }
+
+  SECTION("scratch-syncin") {
+
+    ttg::Edge<int, value_t> edge;
+    auto fn = [&](const int& key, value_t&& val) -> ttg::device_task {
+      double scratch = key;
+      ttg::devicescratch<double> ds = ttg::make_scratch(&scratch, ttg::scope::SyncIn);
+
+      /* wait for the view to be available on the device */
+      co_await ttg::to_device(ds, val.db);
+      /* once we're back here the data has been transferred */
+      CHECK(ds.device_ptr()  != nullptr);
+
+      /* call a kernel */
+#ifdef TTG_HAVE_CUDA
+      increment_buffer(val.db.current_device_ptr(), val.db.size(), ds.device_ptr(), ds.size());
+#endif // TTG_HAVE_CUDA
+
+      /* here we suspend to wait for a kernel to complete */
+      co_await ttg::wait_kernel(ds);
+
+#ifdef TTG_HAVE_CUDA
+      /* scratch is increment once per task, so it should be the same as key */
+      CHECK((static_cast<int>(scratch))-1 == key);
+#endif // 0
+
+      /* we're back, the kernel executed and we can send */
+      if (key < 10) {
+        /* TODO: should we move the view in here if we want to get the device side data */
+        //ttg::send<0>(key+1, std::move(val));
+        /* NOTE: we use co_await here instead of co_return because co_return destroys all local variables first;
+         *       we will not return from this co_await!*/
+        co_await ttg::device::forward(ttg::device::send<0>(key+1, std::move(val)));
+      }
+    };
+
+    auto tt = ttg::make_tt<ttg::ExecutionSpace::CUDA>(fn, ttg::edges(edge), ttg::edges(edge),
+                                                      "device_task", {"edge_in"}, {"edge_out"});
+    make_graph_executable(tt);
+    if (ttg::default_execution_context().rank() == 0) tt->invoke(0, value_t{});
+    ttg::ttg_fence(ttg::default_execution_context());
+  }
+
+  SECTION("scratch-value-out") {
+
+    ttg::Edge<int, value_t> edge;
+    auto fn = [&](const int& key, value_t&& val) -> ttg::device_task {
+      double scratch = 0.0;
+      ttg::devicescratch<double> ds = ttg::make_scratch(&scratch, ttg::scope::Allocate);
+
+      /* wait for the view to be available on the device */
+      co_await ttg::to_device(ds, val.db);
+      /* once we're back here the data has been transferred */
+      CHECK(ds.device_ptr()  != nullptr);
+
+      /* call a kernel */
+#ifdef TTG_HAVE_CUDA
+      increment_buffer(val.db.current_device_ptr(), val.db.size(), ds.device_ptr(), ds.size());
+#endif // TTG_HAVE_CUDA
+
+      /* here we suspend to wait for a kernel to complete */
+      co_await ttg::wait_kernel(ds, val.db);
+
+#ifdef TTG_HAVE_CUDA
+      /* buffer is increment once per task, so it should be 1 */
+      CHECK((static_cast<int>(scratch)-1) == 0);
 #endif // 0
 
       /* we're back, the kernel executed and we can send */
@@ -110,8 +244,9 @@ TEST_CASE("Device", "coro") {
     ttg::Edge<int, value_t> edge;
     ttg::Ptr<value_t> ptr;
     int last_key = 0;
+    constexpr const int num_iter = 10;
     auto fn = [&](const int& key, value_t&& val) -> ttg::device_task {
-      double scratch = 1.0;
+      double scratch = key;
       ttg::devicescratch<double> ds = ttg::make_scratch(&scratch, ttg::scope::SyncIn);
 
       /* wait for the view to be available on the device */
@@ -134,7 +269,7 @@ TEST_CASE("Device", "coro") {
 #endif // TTG_HAVE_CUDA
 
       /* we're back, the kernel executed and we can send */
-      if (key < 10 || scratch < 0.0) {
+      if (key < num_iter) {
         //ttg::send<0>(key+1, std::move(val));
         co_await ttg::device::forward(ttg::device::send<0>(key+1, std::move(val)));
       } else {
@@ -152,7 +287,10 @@ TEST_CASE("Device", "coro") {
     make_graph_executable(tt);
     if (ttg::default_execution_context().rank() == 0) tt->invoke(0, value_t{});
     ttg::ttg_fence(ttg::default_execution_context());
-    CHECK(ptr.is_valid());
+    if (num_iter == last_key) {
+      CHECK(ptr.is_valid());
+      assert(ptr.is_valid());
+    }
 
     /* feed the ptr back into a graph */
     if (ttg::default_execution_context().rank() == 0) tt->invoke(last_key+1, ptr);
@@ -161,19 +299,22 @@ TEST_CASE("Device", "coro") {
     ptr.reset();
   }
 
-#if 0
   /* TODO: enabel this test once we control the PaRSEC state machine! */
-  SECTION("device_host_tasks") {
+  SECTION("device-host-tasks") {
 
     ttg::Edge<int, value_t> h2d, d2h;
 
     auto host_fn = [&](const int& key, value_t&& val) {
       /* check that the data has been synced back */
+#ifdef TTG_HAVE_CUDA
       CHECK(static_cast<int>(*val.db.host_ptr()) == key);
+#endif // TTG_HAVE_CUDA
 
       /* modify the data */
       *val.db.host_ptr() += 1.0;
+#ifdef TTG_HAVE_CUDA
       CHECK(static_cast<int>(*val.db.host_ptr()) == key+1);
+#endif // TTG_HAVE_CUDA
 
       /* send back to the device */
       ttg::send<0>(key+1, std::move(val));
@@ -182,31 +323,24 @@ TEST_CASE("Device", "coro") {
                             "host_task", {"d2h"}, {"h2d"});
 
     auto device_fn = [&](const int& key, value_t&& val) -> ttg::device_task {
-      double scratch = 0.0;
-      ttg::devicescratch<double> ds = ttg::make_scratch(&scratch, ttg::scope::SyncOut);
 
       /* wait for the view to be available on the device */
-      co_await ttg::to_device(ds, val.db);
-      /* once we're back here the data has been transferred */
-      CHECK(ds.device_ptr()  != nullptr);
+      co_await ttg::to_device(val.db);
 
       /* call a kernel */
 #ifdef TTG_HAVE_CUDA
-      increment_buffer(val.db.current_device_ptr(), val.db.size(), ds.device_ptr(), ds.size());
+      increment_buffer(val.db.current_device_ptr(), val.db.size(), nullptr, 0);
 #endif // TTG_HAVE_CUDA
 
       /* here we suspend to wait for a kernel to complete */
+      //co_await ttg::wait_kernel(val.db);
       co_await ttg::wait_kernel();
 
-#ifdef TTG_HAVE_CUDA
-      /* buffer is increment once per task, so it should be the same as key */
-      CHECK((static_cast<int>(scratch)-1) == key);
-#endif // 0
-
       /* we're back, the kernel executed and we can send */
       if (key < 10) {
         /* TODO: should we move the view in here if we want to get the device side data */
-        ttg::send<0>(key+1, std::move(val));
+        std::cout << "Sending to host key " << key+1 <<std::endl;
+        co_await ttg::device::forward(ttg::device::send<0>(key+1, std::move(val)));
       }
     };
 
@@ -217,7 +351,85 @@ TEST_CASE("Device", "coro") {
     ttg::ttg_fence(ttg::default_execution_context());
   }
 
+  SECTION("loop") {
 
-#endif // 0
+    ttg::Edge<int, value_t> edge;
+    auto fn = [&](int key, value_t&& val) -> ttg::device_task {
+      double scratch = 1.0;
+      ttg::devicescratch<double> ds = ttg::make_scratch(&scratch, ttg::scope::Allocate);
+
+      /* wait for the view to be available on the device */
+      co_await ttg::to_device(ds, val.db);
+      /* once we're back here the data has been transferred */
+      CHECK(ds.device_ptr()  != nullptr);
+
+      for (int i = 0; i < 10; ++i) {
+
+        CHECK(ds.device_ptr() != nullptr);
+        CHECK(val.db.current_device_ptr() != nullptr);
+
+        /* KERNEL */
+#ifdef TTG_HAVE_CUDA
+        increment_buffer(val.db.current_device_ptr(), val.db.size(), ds.device_ptr(), ds.size());
+        //increment_buffer(val.db.current_device_ptr(), val.db.size(), 0, 0);
+#endif // TTG_HAVE_CUDA
+
+        /* here we suspend to wait for a kernel and the out-transfer to complete */
+        co_await ttg::wait_kernel(val.db);
+
+#ifdef TTG_HAVE_CUDA
+        /* buffer is increment once per task, so it should be the same as key */
+        //CHECK(static_cast<int>(scratch) == i);
+        CHECK(static_cast<int>(*val.db.host_ptr()) == i+1);
+#endif // TTG_HAVE_CUDA
+      }
+    };
+
+    auto tt = ttg::make_tt<ttg::ExecutionSpace::CUDA>(fn, ttg::edges(edge), ttg::edges(edge),
+                                                      "device_task", {"edge_in"}, {"edge_out"});
+    make_graph_executable(tt);
+    if (ttg::default_execution_context().rank() == 0) tt->invoke(0, value_t{});
+    ttg::ttg_fence(ttg::default_execution_context());
+  }
+
+  SECTION("loop-scratchout") {
+
+    ttg::Edge<int, value_t> edge;
+    auto fn = [&](int key, value_t&& val) -> ttg::device_task {
+      double scratch = -10.0;
+      ttg::devicescratch<double> ds = ttg::make_scratch(&scratch, ttg::scope::SyncIn);
+
+      /* wait for the view to be available on the device */
+      co_await ttg::to_device(ds, val.db);
+      /* once we're back here the data has been transferred */
+      CHECK(ds.device_ptr()  != nullptr);
+
+      for (int i = 0; i < 10; ++i) {
+
+        CHECK(ds.device_ptr() != nullptr);
+        CHECK(val.db.current_device_ptr() != nullptr);
+
+        /* KERNEL */
+#ifdef TTG_HAVE_CUDA
+        increment_buffer(val.db.current_device_ptr(), val.db.size(), ds.device_ptr(), ds.size());
+        //increment_buffer(val.db.current_device_ptr(), val.db.size(), 0, 0);
+#endif // TTG_HAVE_CUDA
+
+        /* here we suspend to wait for a kernel and the out-transfer to complete */
+        co_await ttg::wait_kernel(val.db, ds);
 
+#ifdef TTG_HAVE_CUDA
+        /* buffer is increment once per task, so it should be the same as key */
+        CHECK(static_cast<int>(scratch) == (-10+i+1));
+        CHECK(static_cast<int>(*val.db.host_ptr()) == i+1);
+#endif // TTG_HAVE_CUDA
+      }
+    };
+
+    auto tt = ttg::make_tt<ttg::ExecutionSpace::CUDA>(fn, ttg::edges(edge), ttg::edges(edge),
+                                                      "device_task", {"edge_in"}, {"edge_out"});
+    make_graph_executable(tt);
+    if (ttg::default_execution_context().rank() == 0) tt->invoke(0, value_t{});
+    ttg::ttg_fence(ttg::default_execution_context());
+  }
 }
\ No newline at end of file
diff --git a/ttg/ttg/parsec/buffer.h b/ttg/ttg/parsec/buffer.h
index 4a156e4fe..401b1bc70 100644
--- a/ttg/ttg/parsec/buffer.h
+++ b/ttg/ttg/parsec/buffer.h
@@ -45,10 +45,10 @@ struct buffer {
 private:
   using delete_fn_t = std::add_pointer_t<void(element_type*)>;
 
-  using parsec_data_ptr = std::unique_ptr<parsec_data_t, decltype(&parsec_data_destroy)>;
   using host_data_ptr   = std::unique_ptr<element_type[], delete_fn_t>;
-  parsec_data_ptr m_data;
+  using parsec_data_ptr = std::unique_ptr<parsec_data_t, decltype(&parsec_data_destroy)>;
   host_data_ptr m_host_data;
+  parsec_data_ptr m_data;
   std::size_t m_count = 0;
   detail::ttg_data_copy_t *m_ttg_copy = nullptr;
 
@@ -61,7 +61,8 @@ struct buffer {
   }
 
   static void delete_parsec_data(parsec_data_t *data) {
-    std::cout << "delete parsec_data " << data << std::endl;
+    //std::cout << "delete parsec_data " << data << std::endl;
+    //assert(0);
     parsec_data_destroy(data);
   }
 
@@ -69,19 +70,15 @@ struct buffer {
     // nothing to be done, only used for nullptr
   }
 
-  void create_host_copy() {
-    /* create a new copy for the host object */
-    parsec_data_copy_t* copy;
-    copy = parsec_data_copy_new(m_data.get(), 0, parsec_datatype_int8_t, PARSEC_DATA_FLAG_PARSEC_MANAGED);
-    copy->device_private = m_host_data.get();
-    copy->coherency_state = PARSEC_DATA_COHERENCY_SHARED;
-    copy->version = 1; // this version is valid
-    m_data->nb_elts = sizeof(element_type)*m_count;
-    m_data->owner_device = 0;
-    /* register the new data with the host copy */
-    if (nullptr != m_ttg_copy) {
-      m_ttg_copy->add_device_data(m_data.get());
-    }
+  static parsec_data_t* create_parsec_data(void *ptr, size_t count) {
+    assert(ptr != nullptr);
+    parsec_data_t *data = parsec_data_create_with_type(nullptr, 0, ptr,
+                                                       sizeof(element_type)*count,
+                                                       parsec_datatype_int8_t);
+    data->device_copies[0]->flags |= PARSEC_DATA_FLAG_PARSEC_MANAGED;
+    data->device_copies[0]->coherency_state = PARSEC_DATA_COHERENCY_SHARED;
+    data->device_copies[0]->version = 1;
+    return data;
   }
 
   void reset() {
@@ -105,25 +102,21 @@ struct buffer {
   { }
 
   buffer(std::size_t count)
-  : m_data(parsec_data_new(), &delete_parsec_data)
-  , m_host_data(new element_type[count](), &delete_owned)
+  : m_host_data(new element_type[count](), &delete_owned)
+  , m_data(create_parsec_data(m_host_data.get(), count), &delete_parsec_data)
   , m_count(count)
   , m_ttg_copy(detail::ttg_data_copy_container())
-  {
-    create_host_copy();
-  }
+  { }
 
   /* Constructing a buffer using application-managed memory.
    * The memory pointed to by ptr must be accessible during
    * the life-time of the buffer. */
   buffer(element_type* ptr, std::size_t count = 1)
-  : m_data(parsec_data_new(), &parsec_data_destroy)
-  , m_host_data(ptr, &delete_non_owned)
+  : m_host_data(ptr, &delete_non_owned)
+  , m_data(create_parsec_data(ptr, count), &delete_parsec_data)
   , m_count(count)
   , m_ttg_copy(detail::ttg_data_copy_container())
-  {
-    create_host_copy();
-  }
+  { }
 
   ~buffer() {
     unpin(); // make sure the copies are not pinned
@@ -135,15 +128,25 @@ struct buffer {
 
   /* allow moving device buffers */
   buffer(buffer&& db)
-  : m_data(std::move(db.m_data))
-  , m_host_data(std::move(db.m_host_data))
+  : m_host_data(std::move(db.m_host_data))
+  , m_data(std::move(db.m_data))
   , m_count(db.m_count)
-  , m_ttg_copy(db.m_ttg_copy)
+  , m_ttg_copy(detail::ttg_data_copy_container())
   {
     db.m_count = 0;
 
-    if (nullptr == m_ttg_copy && nullptr != detail::ttg_data_copy_container()) {
-      m_ttg_copy = detail::ttg_data_copy_container();
+    //std::cout << "buffer::move-ctor from " << &db << " ttg-copy " << db.m_ttg_copy
+    //          << " to " << this << " ttg-copy " << m_ttg_copy
+    //          << " parsec-data " << m_data.get()
+    //          << std::endl;
+
+    if (nullptr != db.m_ttg_copy) {
+      /* remove from old ttg copy */
+      db.m_ttg_copy->remove_device_data(m_data.get());
+      db.m_ttg_copy = nullptr;
+    }
+
+    if (nullptr != m_ttg_copy) {
       /* register with the new ttg_copy */
       m_ttg_copy->add_device_data(m_data.get());
     }
@@ -153,24 +156,6 @@ struct buffer {
    * TODO: should we allow this? What data to use?
    */
   buffer(const buffer& db) = delete;
-#if 0
-  /* copy the host data but leave the devices untouched */
-  buffer(const buffer& db)
-  : m_data(db.m_count ? parsec_data_new() : nullptr,
-           db.m_count ? &parsec_data_destroy : &delete_null_parsec_data)
-  , m_host_data(db.m_count ? new element_type[db.m_count] : nullptr,
-                db.m_count ? &delete_owned : delete_non_owned)
-  , m_count(db.m_count)
-  , m_ttg_copy(detail::ttg_data_copy_container())
-  {
-    /* copy host data */
-    std::copy(db.m_host_data.get(),
-              db.m_host_data.get() + m_count,
-              m_host_data.get());
-    /* create the host copy with the allocated memory */
-    create_host_copy();
-  }
-#endif // 0
 
   /* allow moving device buffers */
   buffer& operator=(buffer&& db) {
@@ -178,6 +163,10 @@ struct buffer {
     m_host_data = std::move(db.m_host_data);
     m_count = db.m_count;
     db.m_count = 0;
+    //std::cout << "buffer::move-assign from " << &db << " ttg-copy " << db.m_ttg_copy
+    //          << " to " << this << " ttg-copy " << m_ttg_copy
+    //          << " parsec-data " << m_data.get()
+    //          << std::endl;
     /* don't update the ttg_copy, we keep the connection */
   }
 
@@ -186,26 +175,6 @@ struct buffer {
    */
   buffer& operator=(const buffer& db) = delete;
 
-#if 0
-  /* copy the host buffer content but leave the devices untouched */
-  buffer& operator=(const buffer& db) {
-    if (db.m_count == 0) {
-      m_data = parsec_data_ptr(nullptr, &delete_null_parsec_data);
-      m_host_data = host_data_ptr(nullptr, &delete_non_owned);
-    } else {
-      m_data = parsec_data_ptr(parsec_data_new(), &parsec_data_destroy);
-      m_host_data = host_data_ptr(new element_type[db.m_count], &delete_owned);
-      /* copy host data */
-      std::copy(db.m_host_data.get(),
-                db.m_host_data.get() + db.m_count,
-                m_host_data.get());
-      /* create the host copy with the allocated memory */
-      create_host_copy();
-    }
-    m_count = db.m_count;
-  }
-#endif // 0
-
   /* set the current device, useful when a device
    * buffer was modified outside of a TTG */
   void set_current_device(int device_id) {
@@ -319,10 +288,11 @@ struct buffer {
       m_data = parsec_data_ptr(nullptr, &delete_null_parsec_data);
       m_host_data = host_data_ptr(nullptr, &delete_non_owned);
     } else {
-      m_data = parsec_data_ptr(parsec_data_new(), &parsec_data_destroy);
       m_host_data = host_data_ptr(new element_type[count], &delete_owned);
-      /* create the host copy with the allocated memory */
-      create_host_copy();
+      m_data = parsec_data_ptr(create_parsec_data(m_host_data.get(), count), &delete_parsec_data);
+      if (nullptr != m_ttg_copy) {
+        m_ttg_copy->add_device_data(m_data.get());
+      }
     }
     m_count = count;
     /* don't touch the ttg_copy, we still belong to the same container */
@@ -331,17 +301,21 @@ struct buffer {
   /* Reset the buffer to use the ptr to count elements */
   void reset(T* ptr, std::size_t count = 1) {
     /* TODO: can we resize if count is smaller than m_count? */
+    if (count == m_count) {
+      return;
+    }
     /* drop the current data and reallocate */
     reset();
     if (nullptr == ptr) {
-      m_data = parsec_data_ptr(nullptr, &delete_null_parsec_data);
       m_host_data = host_data_ptr(nullptr, &delete_non_owned);
+      m_data = parsec_data_ptr(nullptr, &delete_null_parsec_data);
       m_count = 0;
     } else {
-      m_data = parsec_data_ptr(parsec_data_new(), &parsec_data_destroy);
       m_host_data = host_data_ptr(ptr, &delete_non_owned);
-      /* create the host copy with the allocated memory */
-      create_host_copy();
+      m_data = parsec_data_ptr(create_parsec_data(m_host_data.get()), &delete_parsec_data);
+      if (nullptr != m_ttg_copy) {
+        m_ttg_copy->add_device_data(m_data.get());
+      }
       m_count = count;
     }
     /* don't touch the ttg_copy, we still belong to the same container */
@@ -402,6 +376,7 @@ struct buffer {
     } else {
       std::size_t s;
       ar & s;
+      //std::cout << "serialize(IN) buffer " << this << " size " << s << std::endl;
       /* initialize internal pointers and then reset */
       reset(s);
       assert(m_ttg_copy != nullptr); // only tracked objects allowed
diff --git a/ttg/ttg/parsec/devicefunc.h b/ttg/ttg/parsec/devicefunc.h
index f509138f9..9509f5dcf 100644
--- a/ttg/ttg/parsec/devicefunc.h
+++ b/ttg/ttg/parsec/devicefunc.h
@@ -6,11 +6,12 @@
 #endif
 
 #include "ttg/parsec/task.h"
+#include <parsec.h>
 #include <parsec/mca/device/device_gpu.h>
 
-#if defined(PARSEC_HAVE_CUDA)
+#if defined(PARSEC_HAVE_DEV_CUDA_SUPPORT)
 #include <parsec/mca/device/cuda/device_cuda.h>
-#endif // PARSEC_HAVE_CUDA
+#endif // PARSEC_HAVE_DEV_CUDA_SUPPORT
 
 namespace ttg_parsec {
   namespace detail {
@@ -28,25 +29,29 @@ namespace ttg_parsec {
       /* TODO: check whether the device is current */
 
       auto flow_flags = PARSEC_FLOW_ACCESS_RW;
-      bool pushout = false;
       if constexpr (std::is_const_v<view_type>) {
         flow_flags = PARSEC_FLOW_ACCESS_READ;
       } else if constexpr (ttg_parsec::is_devicescratch_v<view_type>) {
+        if (view.scope() == ttg::scope::Allocate) {
+          flow_flags = PARSEC_FLOW_ACCESS_WRITE;
+        }
+#if 0
         switch(view.scope()) {
           case ttg::scope::Allocate:
-            flow_flags = PARSEC_FLOW_ACCESS_NONE;
+            flow_flags = PARSEC_FLOW_ACCESS_WRITE;
             break;
           case ttg::scope::SyncIn:
             flow_flags = PARSEC_FLOW_ACCESS_READ;
             break;
         }
+#endif // 0
       }
       assert(nullptr != detail::parsec_ttg_caller->dev_ptr);
       parsec_gpu_task_t *gpu_task = detail::parsec_ttg_caller->dev_ptr->gpu_task;
       parsec_flow_t *flows = detail::parsec_ttg_caller->dev_ptr->flows;
 
-      std::cout << "register_device_memory task " << detail::parsec_ttg_caller << " data " << I << " "
-                << data << " size " << data->nb_elts << std::endl;
+      //std::cout << "register_device_memory task " << detail::parsec_ttg_caller << " data " << I << " "
+      //          << data << " size " << data->nb_elts << std::endl;
 
       /* build the flow */
       /* TODO: reuse the flows of the task class? How can we control the sync direction then? */
@@ -59,11 +64,6 @@ namespace ttg_parsec {
       gpu_task->flow_nb_elts[I] = data->nb_elts; // size in bytes
       gpu_task->flow[I] = &flows[I];
 
-      if (pushout) {
-        std::cout << "PUSHOUT " << I << std::endl;
-        gpu_task->pushout |= 1<<I;
-      }
-
       /* set the input data copy, parsec will take care of the transfer
        * and the buffer will look at the parsec_data_t for the current pointer */
       //detail::parsec_ttg_caller->parsec_task.data[I].data_in = data->device_copies[data->owner_device];
@@ -113,22 +113,23 @@ namespace ttg_parsec {
 
       /* get_parsec_data is overloaded for buffer and devicescratch */
       parsec_data_t* data = detail::get_parsec_data(view);
-      /* find the data copy and mark it as pushout */
-      int i = 0;
       parsec_gpu_task_t *gpu_task = detail::parsec_ttg_caller->dev_ptr->gpu_task;
       parsec_gpu_exec_stream_t *stream = detail::parsec_ttg_caller->dev_ptr->stream;
       /* enqueue the transfer into the compute stream to come back once the compute and transfer are complete */
 
-#if defined(TTG_HAVE_CUDART) && defined(PARSEC_HAVE_CUDA)
+#if defined(TTG_HAVE_CUDART) && defined(PARSEC_HAVE_DEV_CUDA_SUPPORT)
+      std::cout << "cudaMemcpyAsync of " << data->nb_elts << "B" << std::endl;
       parsec_cuda_exec_stream_t *cuda_stream = (parsec_cuda_exec_stream_t *)stream;
       cudaMemcpyAsync(data->device_copies[0]->device_private,
                       data->device_copies[data->owner_device]->device_private,
                       data->nb_elts, cudaMemcpyDeviceToHost, cuda_stream->cuda_stream);
 #else
       static_assert(DeviceAvail, "No device implementation detected!");
-#endif // defined(PARSEC_HAVE_CUDA)
+#endif // defined(PARSEC_HAVE_DEV_CUDA_SUPPORT)
 
 #if 0
+      /* find the data copy and mark it as pushout */
+      int i = 0;
       while (detail::parsec_ttg_caller->parsec_task.data[i].data_in != nullptr) {
         if (detail::parsec_ttg_caller->parsec_task.data[i].data_in == data->device_copies[0]) {
           gpu_task->pushout |= 1<<i;
@@ -170,9 +171,8 @@ namespace ttg_parsec {
 
         /* get_parsec_data is overloaded for buffer and devicescratch */
         parsec_data_t* data = detail::get_parsec_data(view);
-
-        data->device_copies[0]->version++;
-        data->owner_device = 0;
+        data->device_copies[0]->version = data->device_copies[data->owner_device]->version;
+        parsec_data_transfer_ownership_to_copy(data, 0, PARSEC_FLOW_ACCESS_READ);
       }
 
       if constexpr (sizeof...(Is) > 0) {
@@ -186,6 +186,29 @@ namespace ttg_parsec {
     detail::post_device_out(b, std::index_sequence_for<Buffer...>{});
   }
 
+  namespace detail {
+
+    template<typename Value>
+    void send_inc_version(Value&& value) {
+      if constexpr(!std::is_const_v<Value>) {
+        assert(nullptr != detail::parsec_ttg_caller);
+        /* increment the version of the copy, needed for potential pushout */
+        parsec_task_t *parsec_task = &detail::parsec_ttg_caller->parsec_task;
+        int i = 0;
+        bool found = false;
+        while (parsec_task->data[i].data_in != nullptr) {
+          parsec_data_copy_t *copy = parsec_task->data[i].data_in;
+          if (copy->device_private == &value) {
+            copy->version++;
+            found = true;
+            break;
+          }
+        }
+        assert(found);
+      }
+    }
+
+  } // namespace detail
 
 } // namespace ttg_parsec
 
diff --git a/ttg/ttg/parsec/devicescratch.h b/ttg/ttg/parsec/devicescratch.h
index 1c0487cdf..25ca2e05a 100644
--- a/ttg/ttg/parsec/devicescratch.h
+++ b/ttg/ttg/parsec/devicescratch.h
@@ -8,7 +8,9 @@
 #include <parsec.h>
 #include <parsec/data_internal.h>
 #include <parsec/mca/device/device.h>
+#include <parsec/mca/device/device_gpu.h>
 #include <ttg/devicescope.h>
+#include "ttg/parsec/task.h"
 
 namespace ttg_parsec {
 
@@ -36,25 +38,29 @@ struct devicescratch {
 private:
 
   parsec_data_t* m_data = nullptr;
-  parsec_data_copy_t m_data_copy;
   ttg::scope m_scope;
 
-  void create_host_copy(element_type *ptr, std::size_t count) {
-    /* TODO: is the construction call necessary? */
-    /* TODO: handle the scope */
-    PARSEC_OBJ_CONSTRUCT(&m_data_copy, parsec_data_copy_t);
-    m_data_copy.device_index    = 0;
-    //m_data_copy.original        = &m_data;
-    //m_data_copy.older           = NULL;
-    m_data_copy.flags           = PARSEC_DATA_FLAG_PARSEC_MANAGED;
-    m_data_copy.dtt             = parsec_datatype_int8_t;
-    m_data_copy.version         = 1;
-    m_data_copy.device_private  = ptr;
-    m_data_copy.coherency_state = PARSEC_DATA_COHERENCY_SHARED;
-
-    m_data->nb_elts              = count * sizeof(element_type);
-    m_data->owner_device         = 0;
-    parsec_data_copy_attach(m_data, &m_data_copy, 0);
+  static parsec_data_t* create_parsec_data(void *ptr, size_t count) {
+
+    parsec_data_t *data = parsec_data_create_with_type(nullptr, 0, ptr,
+                                                       sizeof(element_type)*count,
+                                                       parsec_datatype_int8_t);
+    data->device_copies[0]->flags |= PARSEC_DATA_FLAG_PARSEC_MANAGED;
+    data->device_copies[0]->coherency_state = PARSEC_DATA_COHERENCY_SHARED;
+    return data;
+  }
+
+  void remove_from_flow() {
+    /* remove the scratch from the gpu-task flow */
+    assert(nullptr != detail::parsec_ttg_caller);
+    parsec_task_t *parsec_task = &detail::parsec_ttg_caller->parsec_task;
+    parsec_flow_t *flows = detail::parsec_ttg_caller->dev_ptr->flows;
+    for (int i = 0; i < MAX_PARAM_COUNT; ++i) {
+      if (nullptr != parsec_task->data[i].data_in && parsec_task->data[i].data_in->original == m_data) {
+        flows[i].flow_flags = PARSEC_FLOW_ACCESS_NONE; // disable this flow
+        break;
+      }
+    }
   }
 
   friend parsec_data_t* detail::get_parsec_data<T>(const ttg_parsec::devicescratch<T>&);
@@ -65,9 +71,13 @@ struct devicescratch {
    * The memory pointed to by ptr must be accessible during
    * the life-time of the devicescratch. */
   devicescratch(element_type* ptr, ttg::scope scope = ttg::scope::SyncIn, std::size_t count = 1)
-  : m_data(parsec_data_new())
-  , m_scope(scope) {
-    create_host_copy(ptr, count);
+  : m_data(create_parsec_data(ptr, count))
+  , m_scope(scope)
+  {
+    if (ttg::scope::SyncIn == scope) {
+      /* increment the version to force the first initial transfer */
+      m_data->device_copies[0]->version = 1;
+    }
   }
 
   /* don't allow moving */
@@ -83,8 +93,12 @@ struct devicescratch {
   devicescratch& operator=(const devicescratch& db) = delete;
 
   ~devicescratch() {
-    PARSEC_OBJ_DESTRUCT(&m_data_copy);
-    parsec_data_destroy(m_data);
+    /* remove data from flow */
+    remove_from_flow();
+    if (nullptr != m_data) {
+      parsec_data_destroy(m_data);
+    }
+    //parsec_data_destroy(m_data);
     m_data = nullptr;
   }
 
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index c6e01c2f2..2d0d64ff6 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -67,11 +67,6 @@
 #include <mpi-ext.h>
 
 
-/* TODO: remove once we use PaRSEC master */
-#ifndef PARSEC_HAVE_CUDA
-#define PARSEC_HAVE_CUDA 1
-#endif // PARSEC_HAVE_CUDA
-
 #include <parsec.h>
 #include <parsec/class/parsec_hash_table.h>
 #include <parsec/data_internal.h>
@@ -95,21 +90,17 @@
 #include <cstdlib>
 #include <cstring>
 
+#if defined(TTG_PARSEC_DEBUG_TRACK_DATA_COPIES)
+#include <unordered_set>
+#endif
+
 #include "ttg/parsec/ttg_data_copy.h"
 #include "ttg/parsec/thread_local.h"
 #include "ttg/parsec/ptr.h"
 #include "ttg/parsec/task.h"
 
-/* This is missing in the parsec_comm_engine.h interface... But we are discussing
-   if this execution stream should be exposed. Workaround this for now. */
-extern "C" parsec_execution_stream_t parsec_comm_es;
-
 #undef TTG_PARSEC_DEBUG_TRACK_DATA_COPIES
 
-#if defined(TTG_PARSEC_DEBUG_TRACK_DATA_COPIES)
-#include <unordered_set>
-#endif
-
 /* PaRSEC function declarations */
 extern "C" {
 void parsec_taskpool_termination_detected(parsec_taskpool_t *tp);
@@ -377,9 +368,9 @@ namespace ttg_parsec {
         ttg::detail::deregister_world(*this);
         destroy_tpool();
         if (own_ctx) {
-          unregister_parsec_tags(&parsec_comm_engine_cb_idx);
+          unregister_parsec_tags(nullptr);
         } else {
-          parsec_context_at_fini(unregister_parsec_tags, &parsec_comm_engine_cb_idx);
+          parsec_context_at_fini(unregister_parsec_tags, nullptr);
         }
 #if defined(PARSEC_PROF_TRACE)
         if(nullptr != profiling_array) {
@@ -1226,33 +1217,51 @@ namespace ttg_parsec {
 
       task->dev_ptr->stream = gpu_stream;
 
-      std::cout << "device_static_submit task " << task << std::endl;
+      //std::cout << "device_static_submit task " << task << std::endl;
 
       // get the promise which contains the views
       auto dev_data = dev_task.promise();
 
       /* we should still be waiting for the transfer to complete */
-      assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_TRANSFER);
+      assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_TRANSFER ||
+             dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_KERNEL);
 
       /* Here we call back into the coroutine again after the transfers have completed */
       static_op<Space>(&task->parsec_task);
 
-      /* Get a new handle for the promise*/
-      dev_task = ttg::device_task_handle_type::from_address(task->suspended_task_address);
-      dev_data = dev_task.promise();
-
-      assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_KERNEL ||
-             dev_data.state() == ttg::TTG_DEVICE_CORO_COMPLETE);
-
-      /* we will come back into this function once the kernel and transfers are */
-      int rc = PARSEC_HOOK_RETURN_AGAIN;
-      if (ttg::TTG_DEVICE_CORO_COMPLETE == dev_data.state()) {
-        /* the task started sending so we won't come back here */
-        rc = PARSEC_HOOK_RETURN_DONE;
+      /* we will come back into this function once the kernel and transfers are done */
+      int rc = PARSEC_HOOK_RETURN_DONE;
+      if (nullptr != task->suspended_task_address) {
+        /* Get a new handle for the promise*/
+        dev_task = ttg::device_task_handle_type::from_address(task->suspended_task_address);
+        dev_data = dev_task.promise();
+
+        assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_KERNEL ||
+               dev_data.state() == ttg::TTG_DEVICE_CORO_SENDOUT     ||
+               dev_data.state() == ttg::TTG_DEVICE_CORO_COMPLETE);
+
+        if (ttg::TTG_DEVICE_CORO_SENDOUT == dev_data.state() ||
+            ttg::TTG_DEVICE_CORO_COMPLETE == dev_data.state()) {
+          /* the task started sending so we won't come back here */
+          //std::cout << "device_static_submit task " << task << " complete" << std::endl;
+        } else {
+          //std::cout << "device_static_submit task " << task << " return-again" << std::endl;
+          rc = PARSEC_HOOK_RETURN_AGAIN;
+        }
+      } else {
+        /* the task is done so we won't come back here */
+        //std::cout << "device_static_submit task " << task << " complete" << std::endl;
       }
       return rc;
     }
 
+    static int
+    static_cuda_stage_in(parsec_gpu_task_t        *gtask,
+                         uint32_t                  flow_mask,
+                         parsec_gpu_exec_stream_t *gpu_stream) {
+      return parsec_default_cuda_stage_in(gtask, flow_mask, gpu_stream);
+    }
+
     template <ttg::ExecutionSpace Space>
     static parsec_hook_return_t device_static_op(parsec_task_t* parsec_task) {
       static_assert(derived_has_cuda_op());
@@ -1263,8 +1272,7 @@ namespace ttg_parsec {
       task_t *task = (task_t*)parsec_task;
       parsec_execution_stream_s *es = task->tt->world.impl().execution_stream();
 
-      std::cout << "device_static_op: task " << parsec_task << std::endl;
-
+      //std::cout << "device_static_op: task " << parsec_task << std::endl;
 
       /* set up a device task */
       parsec_gpu_task_t *gpu_task;
@@ -1298,104 +1306,23 @@ namespace ttg_parsec {
       /* TODO: is this the right place to set the mask? */
       task->parsec_task.chore_mask = PARSEC_DEV_ALL;
       /* get a device and come back if we need another one */
-      dev_index = parsec_get_best_device(parsec_task, ratio);
+      int64_t task_load = 1;
+      dev_index = parsec_get_best_device(parsec_task, &task_load);
       assert(dev_index >= 0);
       if (dev_index < 2) {
           return PARSEC_HOOK_RETURN_NEXT; /* Fall back */
       }
 
-
-#if 0
-      // manage the gpu_task flows
-
-      // set the input flows
-      uint8_t i = 0;
-      /* TODO: need to free flows at the end of the task lifetime */
-      parsec_flow_t* flows = new parsec_flow_t[MAX_PARAM_COUNT];
-      for (auto& view : dev_data) {
-        void *host_obj = view.host_obj();
-
-        /* iterate over all viewspans of this view (i.e., all memory ranges in this view) */
-        for (auto& view_span : view) {
-          gpu_task->flow_nb_elts[i] = view_span.size(); // size in bytes
-          gpu_task->flow[i] = &flows[i];
-          parsec_flow_t flow = {.name = nullptr,
-                                .sym_type = PARSEC_SYM_INOUT,
-                                .flow_flags = view_span.is_sync_out() ? PARSEC_FLOW_ACCESS_RW : PARSEC_FLOW_ACCESS_READ,
-                                .flow_index = i,
-                                .flow_datatype_mask = ~0 };
-          std::cout << "view_span.is_sync_out() " << view_span.is_sync_out() << std::endl;
-          *(parsec_flow_t*)gpu_task->flow[i] = flow; // why are flows constant?!
-
-          parsec_data_copy_t* copy = nullptr;
-          ttg_parsec::detail::ttg_data_copy_t* obj_copy = nullptr;
-          int input_obj_idx = -1;
-          /* try to find the view in the task and allocate a new copy if needed */
-          for (int i = 0; nullptr == copy && i < numins; ++i) {
-            ttg_parsec::detail::ttg_data_copy_t* obj_copy = task->copies[i];
-            if (obj_copy->get_ptr() == host_obj) {
-              for (auto& dev_copy : *obj_copy) {
-                if (view_span.data() == dev_copy->device_private) {
-                  copy = dev_copy;
-                  input_obj_idx = i;
-                  break;
-                }
-              }
-            }
-          }
-
-          /* push all data back out, EXCEPT if the host object is a const input for the task
-           * TODO [JS]: if PaRSEC lets us send from the device we can avoid pushing out here
-           */
-          if (input_obj_idx == -1 || !input_is_const[input_obj_idx]) {
-            gpu_task->pushout |= 1<<i;
-          }
-
-          /* no copy found, create a new copy */
-          if (nullptr == copy) {
-            parsec_data_t *data = parsec_data_new();
-            copy = parsec_data_copy_new(data, 0, parsec_datatype_int8_t, PARSEC_DATA_FLAG_PARSEC_MANAGED);
-            copy->device_private = view_span.data();
-            copy->version = 1; // this version is valid
-            data->nb_elts = view_span.size();
-            data->owner_device = 0;
-            copy->coherency_state = PARSEC_DATA_COHERENCY_SHARED;
-            std::cout << "copy " << copy << " device_private " << copy->device_private << std::endl;
-          }
-
-          if (obj_copy != nullptr) {
-            /* add the */
-            obj_copy->add_device_copy(copy);
-          }
-          /* register the copy with the task */
-          task->parsec_task.data[i].data_in = copy;
-          task->parsec_task.data[i].source_repo_entry = NULL;
-
-          ++i;
-        }
-      }
-
-      /* mark all other flows as ignored */
-      for (; i < MAX_PARAM_COUNT; ++i) {
-        gpu_task->flow_nb_elts[i] = 0;
-        flows[i].flow_flags = PARSEC_FLOW_ACCESS_NONE;
-        flows[i].flow_index = i;
-        gpu_task->flow[i] = &flows[i];
-        task->parsec_task.data[i].data_in = nullptr;
-        task->parsec_task.data[i].source_repo_entry = NULL;
-      }
-#endif // 0
-
       parsec_device_module_t *device = parsec_mca_device_get(dev_index);
       assert(NULL != device);
       switch(device->type) {
 
-#if defined(PARSEC_HAVE_CUDA)
+#if defined(PARSEC_HAVE_DEV_CUDA_SUPPORT)
         case PARSEC_DEV_CUDA:
           if constexpr (Space == ttg::ExecutionSpace::CUDA) {
             /* TODO: we need custom staging functions because PaRSEC looks at the
              *       task-class to determine the number of flows. */
-            gpu_task->stage_in  = parsec_default_cuda_stage_in;
+            gpu_task->stage_in  = static_cuda_stage_in;
             gpu_task->stage_out = parsec_default_cuda_stage_out;
             return parsec_cuda_kernel_scheduler(es, gpu_task, dev_index);
           }
@@ -1745,7 +1672,7 @@ namespace ttg_parsec {
             copy = detail::create_new_datacopy(decvalueT{});
             /* unpack the object, potentially discovering iovecs */
             pos = unpack(*static_cast<decvalueT *>(copy->get_ptr()), msg->bytes, pos);
-            std::cout << "num_iovecs " << num_iovecs << " distance " << std::distance(copy->iovec_begin(), copy->iovec_end()) << std::endl;
+            //std::cout << "num_iovecs " << num_iovecs << " distance " << std::distance(copy->iovec_begin(), copy->iovec_end()) << std::endl;
             assert(std::distance(copy->iovec_begin(), copy->iovec_end()) == num_iovecs);
           }
 
@@ -1799,7 +1726,7 @@ namespace ttg_parsec {
                                             iov.num_bytes, &lreg, &lreg_size);
                     world.impl().increment_inflight_msg();
                     /* TODO: PaRSEC should treat the remote callback as a tag, not a function pointer! */
-                    std::cout << "set_arg_from_msg: get rreg " << rreg << " remote " << remote << std::endl;
+                    //std::cout << "set_arg_from_msg: get rreg " << rreg << " remote " << remote << std::endl;
                     parsec_ce.get(&parsec_ce, lreg, 0, rreg, 0, iov.num_bytes, remote,
                                   &detail::get_complete_cb<ActivationT>, activation,
                                   /*world.impl().parsec_ttg_rma_tag()*/
@@ -2180,6 +2107,9 @@ namespace ttg_parsec {
     template <std::size_t i, typename Key, typename Value>
     void set_arg_impl(const Key &key, Value &&value, detail::ttg_data_copy_t *copy_in = nullptr) {
       int owner;
+      using decvalueT = std::decay_t<Value>;
+      using norefvalueT = std::remove_reference_t<Value>;
+      norefvalueT *value_ptr = &value;
 
 #if defined(PARSEC_PROF_TRACE) && defined(PARSEC_TTG_PROFILE_BACKEND)
       if(world.impl().profiling()) {
@@ -2212,17 +2142,18 @@ namespace ttg_parsec {
       int num_iovecs = 0;
       std::unique_ptr<msg_t> msg = std::make_unique<msg_t>(get_instance_id(), world_impl.taskpool()->taskpool_id,
                                                            msg_header_t::MSG_SET_ARG, i, world_impl.rank(), 1);
-      using decvalueT = std::decay_t<Value>;
 
       if constexpr (!ttg::meta::is_void_v<decvalueT>) {
 
         detail::ttg_data_copy_t *copy = copy_in;
         /* make sure we have a data copy to register with */
         if (nullptr == copy) {
-          copy = detail::find_copy_in_task(detail::parsec_ttg_caller, &value);
+          copy = detail::find_copy_in_task(detail::parsec_ttg_caller, value_ptr);
           if (nullptr == copy) {
             // We need to create a copy for this data, as it does not exist yet.
             copy = detail::create_new_datacopy(std::forward<Value>(value));
+            // use the new value from here on out
+            value_ptr = static_cast<norefvalueT*>(copy->get_ptr());
           }
         }
 
@@ -2255,7 +2186,7 @@ namespace ttg_parsec {
             pos += sizeof(lreg_size_i);
             std::memcpy(msg->bytes + pos, lreg, lreg_size);
             pos += lreg_size;
-            std::cout << "set_arg_impl lreg " << lreg << std::endl;
+            //std::cout << "set_arg_impl lreg " << lreg << std::endl;
             /* TODO: can we avoid the extra indirection of going through std::function? */
             std::function<void(void)> *fn = new std::function<void(void)>([=]() mutable {
               /* shared_ptr of value and registration captured by value so resetting
@@ -2271,19 +2202,19 @@ namespace ttg_parsec {
 
         if constexpr (ttg::has_split_metadata<std::decay_t<Value>>::value) {
           ttg::SplitMetadataDescriptor<decvalueT> descr;
-          auto iovs = descr.get_data(*const_cast<decvalueT *>(&value));
+          auto iovs = descr.get_data(*const_cast<decvalueT *>(value_ptr));
           num_iovecs = std::distance(std::begin(iovs), std::end(iovs));
           /* pack the metadata */
-          auto metadata = descr.get_metadata(value);
+          auto metadata = descr.get_metadata(value_ptr);
           size_t metadata_size = sizeof(metadata);
           pos = pack(metadata, msg->bytes, pos);
           handle_iovec_fn(iovs);
         } else if constexpr (!ttg::has_split_metadata<std::decay_t<Value>>::value) {
           /* serialize the object */
-          std::cout << "PRE pack num_iovecs " << std::distance(copy->iovec_begin(), copy->iovec_end()) << std::endl;
-          pos = pack(value, msg->bytes, pos, copy);
+          //std::cout << "PRE pack num_iovecs " << std::distance(copy->iovec_begin(), copy->iovec_end()) << std::endl;
+          pos = pack(*value_ptr, msg->bytes, pos, copy);
           num_iovecs = std::distance(copy->iovec_begin(), copy->iovec_end());
-          std::cout << "POST pack num_iovecs " << num_iovecs << std::endl;
+          //std::cout << "POST pack num_iovecs " << num_iovecs << std::endl;
           /* handle any iovecs contained in it */
           handle_iovec_fn(copy->iovec_span());
           copy->iovec_reset();
@@ -2304,7 +2235,7 @@ namespace ttg_parsec {
       parsec_taskpool_t *tp = world_impl.taskpool();
       tp->tdm.module->outgoing_message_start(tp, owner, NULL);
       tp->tdm.module->outgoing_message_pack(tp, owner, NULL, NULL, 0);
-      std::cout << "set_arg_impl send_am owner " << owner << " sender " << msg->tt_id.sender << std::endl;
+      //std::cout << "set_arg_impl send_am owner " << owner << " sender " << msg->tt_id.sender << std::endl;
       parsec_ce.send_am(&parsec_ce, world_impl.parsec_ttg_tag(), owner, static_cast<void *>(msg.get()),
                         sizeof(msg_header_t) + pos);
 #if defined(PARSEC_PROF_TRACE) && defined(PARSEC_TTG_PROFILE_BACKEND)
@@ -2314,7 +2245,7 @@ namespace ttg_parsec {
 #endif
 #if defined(PARSEC_PROF_GRAPHER)
       if(NULL != detail::parsec_ttg_caller && !detail::parsec_ttg_caller->dummy()) {
-        int orig_index = detail::find_index_of_copy_in_task(detail::parsec_ttg_caller, &value);
+        int orig_index = detail::find_index_of_copy_in_task(detail::parsec_ttg_caller, value_ptr);
         char orig_str[32];
         char dest_str[32];
         if(orig_index >= 0) {
@@ -2407,10 +2338,10 @@ namespace ttg_parsec {
                                             std::shared_ptr<void>{lreg, [](void *ptr) {
                                                                     parsec_ce_mem_reg_handle_t memreg =
                                                                         (parsec_ce_mem_reg_handle_t)ptr;
-                                                                    std::cout << "broadcast_arg memunreg lreg " << memreg << std::endl;
+                                                                    //std::cout << "broadcast_arg memunreg lreg " << memreg << std::endl;
                                                                     parsec_ce.mem_unregister(&memreg);
                                                                   }}));
-            std::cout << "broadcast_arg memreg lreg " << lreg << std::endl;
+            //std::cout << "broadcast_arg memreg lreg " << lreg << std::endl;
           }
         };
 
@@ -2477,7 +2408,7 @@ namespace ttg_parsec {
             pos += sizeof(lreg_size);
             std::memcpy(msg->bytes + pos, lreg_ptr.get(), lreg_size);
             pos += lreg_size;
-            std::cout << "broadcast_arg lreg_ptr " << lreg_ptr.get() << std::endl;
+            //std::cout << "broadcast_arg lreg_ptr " << lreg_ptr.get() << std::endl;
             /* mark another reader on the copy */
             copy = detail::register_data_copy<valueT>(copy, nullptr, true);
             /* create a function that will be invoked upon RMA completion at the target */
@@ -2506,7 +2437,7 @@ namespace ttg_parsec {
 
           tp->tdm.module->outgoing_message_start(tp, owner, NULL);
           tp->tdm.module->outgoing_message_pack(tp, owner, NULL, NULL, 0);
-          std::cout << "broadcast_arg send_am owner " << owner << std::endl;
+          //std::cout << "broadcast_arg send_am owner " << owner << std::endl;
           parsec_ce.send_am(&parsec_ce, world_impl.parsec_ttg_tag(), owner, static_cast<void *>(msg.get()),
                             sizeof(msg_header_t) + pos);
         }
@@ -3082,11 +3013,15 @@ namespace ttg_parsec {
         auto dev_data = dev_task.promise();
 
         /* for now make sure we're waiting for the kernel to complete and the coro hasn't skipped this step */
-        assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_KERNEL);
-
-        /* the kernel has completed, resume the coroutine once again to get to the send stage */
-        /* TODO: how can we get the execution space here? */
-        static_op<ttg::ExecutionSpace::CUDA>(parsec_task);
+        assert(dev_data.state() == ttg::TTG_DEVICE_CORO_SENDOUT);
+
+        /* execute the sends we stored */
+        if (dev_data.state() == ttg::TTG_DEVICE_CORO_SENDOUT) {
+          /* set the current task, needed inside the sends */
+          detail::parsec_ttg_caller = task;
+          dev_data.do_sends();
+          detail::parsec_ttg_caller = nullptr;
+        }
 
         /* the coroutine should have completed and we cannot access the promise anymore */
         task->suspended_task_address = nullptr;
@@ -3273,6 +3208,18 @@ namespace ttg_parsec {
         free((void*)self.name);
         self.name = nullptr;
       }
+
+      for (int i = 0; i < MAX_PARAM_COUNT; i++) {
+        parsec_flow_t *flow = (parsec_flow_t *)(self.in[i]);
+        free(flow->name);
+        delete flow;
+        *((parsec_flow_t **)&(self.in[i])) = nullptr;
+
+        flow = (parsec_flow_t *)(self.out[i]);
+        free(flow->name);
+        delete flow;
+        *((parsec_flow_t **)&(self.out[i])) = nullptr;
+      }
       release();
     }
 
@@ -3595,8 +3542,12 @@ struct ttg::detail::value_copy_handler<ttg::Runtime::PaRSEC> {
       value_ptr = reinterpret_cast<value_type *>(copy->get_ptr());
       copy_to_remove = copy;
     } else {
-      /* this copy won't be modified anymore so mark it as read-only */
-      copy->reset_readers();
+      if constexpr (std::is_rvalue_reference_v<decltype(value)>) {
+        /* this copy won't be modified anymore so mark it as read-only */
+        copy->reset_readers();
+      }
+      /* the value was potentially changed, so increment version */
+      copy->inc_current_version();
     }
     return std::move(*value_ptr);
   }
diff --git a/ttg/ttg/parsec/ttg_data_copy.h b/ttg/ttg/parsec/ttg_data_copy.h
index e97184b28..9440a4692 100644
--- a/ttg/ttg/parsec/ttg_data_copy.h
+++ b/ttg/ttg/parsec/ttg_data_copy.h
@@ -176,6 +176,28 @@ namespace ttg_parsec {
         return m_refs.load(std::memory_order_relaxed);
       }
 
+      /* increment the version of the current copy */
+      void inc_current_version() {
+        if (m_num_dev_data == 1) {
+          //std::cout << "data-copy " << this << " inc version " << m_single_dev_data << std::endl;
+          assert(m_single_dev_data->device_copies[m_single_dev_data->owner_device] != nullptr);
+          m_single_dev_data->device_copies[m_single_dev_data->owner_device]->version++;
+          //std::cout << "Incrementing version of data " << m_single_dev_data << " copy "
+          //          << m_single_dev_data->device_copies[m_single_dev_data->owner_device]
+          //          << " on device " << (int)m_single_dev_data->owner_device << " to "
+          //          << m_single_dev_data->device_copies[m_single_dev_data->owner_device]->version << std::endl;
+        } else if (m_num_dev_data > 1) {
+          std::for_each(m_dev_data.begin(), m_dev_data.end(),
+                        [](parsec_data_t *data){
+                           assert(data->device_copies[data->owner_device] != nullptr);
+                           data->device_copies[data->owner_device]->version++;
+                           //std::cout << "Incrementing version of copy "
+                           //          << " on device " << (int)data->owner_device << data->device_copies[data->owner_device] << " to "
+                           //          << data->device_copies[data->owner_device]->version << std::endl;
+                        });
+        }
+      }
+
       /* manage device copies owned by this object
        * we only touch the vector if we have more than one copies to track
        * and otherwise use the single-element member.
@@ -183,7 +205,7 @@ namespace ttg_parsec {
       using iterator = parsec_data_t**;
 
       void add_device_data(parsec_data_t* data) {
-        // TODO: properly release again!
+        //std::cout << "data-copy " << this << " add data " << data << std::endl;
         PARSEC_OBJ_RETAIN(data);
         switch (m_num_dev_data) {
           case 0:
@@ -202,7 +224,10 @@ namespace ttg_parsec {
       }
 
       void remove_device_data(parsec_data_t* data) {
+        //std::cout << "data-copy " << this << " remove data " << data << std::endl;
+        assert(m_num_dev_data >= 1);
         if (m_num_dev_data == 1) {
+          assert(m_single_dev_data == data);
           m_single_dev_data = nullptr;
         } else if (m_num_dev_data > 1) {
           auto it = std::find(m_dev_data.begin(), m_dev_data.end(), data);
@@ -211,6 +236,12 @@ namespace ttg_parsec {
           }
         }
         --m_num_dev_data;
+        /* make single-entry if needed */
+        if (m_num_dev_data == 1) {
+          m_single_dev_data = m_dev_data[0];
+          m_dev_data.clear();
+        }
+        PARSEC_OBJ_RELEASE(data);
       }
 
       int num_dev_data() const {
@@ -275,8 +306,8 @@ namespace ttg_parsec {
       std::vector<ttg::iovec> m_iovecs;
 
       std::vector<parsec_data_t*> m_dev_data;   //< used if there are multiple device copies
-                                                  //  that belong to this object
-      parsec_data_t *m_single_dev_data;           //< used if there is a single device copy
+                                                //  that belong to this object
+      parsec_data_t *m_single_dev_data;         //< used if there is a single device copy
       int m_num_dev_data = 0;                   //< number of device copies
     };
 
diff --git a/ttg/ttg/view.h b/ttg/ttg/view.h
index 1ea95fe7e..5ab00e961 100644
--- a/ttg/ttg/view.h
+++ b/ttg/ttg/view.h
@@ -232,6 +232,7 @@ namespace ttg {
     TTG_DEVICE_CORO_INIT,
     TTG_DEVICE_CORO_WAIT_TRANSFER,
     TTG_DEVICE_CORO_WAIT_KERNEL,
+    TTG_DEVICE_CORO_SENDOUT,
     TTG_DEVICE_CORO_COMPLETE
   };
 
@@ -383,20 +384,6 @@ namespace ttg {
     };
   } // namespace detail
 
-
-  namespace detail {
-    /* TODO: is this still needed? */
-    template<typename... Ts>
-    struct await_t {
-      std::tuple<Ts&...> ties;
-    };
-  }
-
-  template<typename... Args>
-  inline auto make_await(Args&&... args) {
-    return detail::await_t{std::tie(std::forward<Args>(args)...)};
-  }
-
   namespace detail {
     template<typename... Ts>
     struct to_device_t {
@@ -405,22 +392,11 @@ namespace ttg {
   } // namespace detail
 
   template<typename... Args>
+  [[nodiscard]]
   inline auto to_device(Args&&... args) {
     return detail::to_device_t{std::tie(std::forward<Args>(args)...)};
   }
 
-  namespace detail {
-    template<typename... Ts>
-    struct to_host_t {
-      std::tuple<Ts&...> ties;
-    };
-  } // namespace detail
-
-  template<typename... Args>
-  inline auto to_host(Args&&... args) {
-    return detail::to_host_t{std::tie(std::forward<Args>(args)...)};
-  }
-
   namespace detail {
     template<typename... Ts>
     struct wait_kernel_t {
@@ -444,6 +420,7 @@ namespace ttg {
   /* Wait for kernel to complete and provided ttg::buffer and ttg::devicescratch
    * to be transferred back to host */
   template<typename... Buffers>
+  [[nodiscard]]
   inline auto wait_kernel(Buffers&&... args) {
     static_assert(((ttg::detail::is_buffer_v<std::decay_t<Buffers>>
                     ||ttg::detail::is_devicescratch_v<std::decay_t<Buffers>>)&&...),
@@ -701,6 +678,7 @@ namespace ttg {
     /* overload with explicit terminals and keylist passed by const reference */
     template <size_t I, size_t... Is, typename rangeT, typename valueT, typename... out_keysT, typename... out_valuesT,
               ttg::Runtime Runtime = ttg::ttg_runtime>
+    [[nodiscard]]
     inline detail::send_t broadcast(rangeT &&keylist,
                                     valueT &&value,
                                     const std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
@@ -722,6 +700,7 @@ namespace ttg {
     }
 
     template<typename... Args, ttg::Runtime Runtime = ttg::ttg_runtime>
+    [[nodiscard]]
     std::vector<device::detail::send_t> forward(Args&&... args) {
       // TODO: check the cost of this!
       return std::vector{std::forward<Args>(args)...};
@@ -858,14 +837,6 @@ namespace ttg {
       return {};
     }
 
-    template<typename... Ts>
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(detail::await_t<Ts...>&& a) {
-      bool need_transfer = !(TTG_IMPL_NS::register_device_memory(a.ties));
-      /* TODO: are we allowed to not suspend here and launch the kernel directly? */
-      m_state = TTG_DEVICE_CORO_WAIT_TRANSFER;
-      return {};
-    }
-
     template<typename... Ts>
     TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(detail::to_device_t<Ts...>&& a) {
       bool need_transfer = !(TTG_IMPL_NS::register_device_memory(a.ties));
@@ -886,14 +857,14 @@ namespace ttg {
 
     TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(std::vector<device::detail::send_t>&& v) {
       m_sends = std::forward<std::vector<device::detail::send_t>>(v);
-      m_state = TTG_DEVICE_CORO_COMPLETE;
+      m_state = TTG_DEVICE_CORO_SENDOUT;
       return {};
     }
 
 #if 0
     void return_value(std::vector<device::detail::send_t>&& sends) {
       m_sends = std::forward<std::vector<device::detail::send_t>>(sends);
-      m_state = TTG_DEVICE_CORO_COMPLETE;
+      m_state = TTG_DEVICE_CORO_SENDOUT;
     }
 #endif // 0
 

From 19adc69745aaa5ebf0e046054eee4ee99eb21899 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 26 Jun 2023 02:13:09 +0000
Subject: [PATCH 081/259] CUDA version of BSPMM

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/CMakeLists.txt        |    6 +
 examples/spmm/cuda_gemm.cc     |   29 +
 examples/spmm/cuda_gemm.h      |   24 +
 examples/spmm/spmm_cuda.cc     | 1837 ++++++++++++++++++++++++++++++++
 ttg/ttg/device/send.h          |   26 -
 ttg/ttg/device/task.h          |  642 +++++++++++
 ttg/ttg/parsec/buffer.h        |   34 +-
 ttg/ttg/parsec/devicefunc.h    |   68 +-
 ttg/ttg/parsec/devicescratch.h |   11 +-
 ttg/ttg/parsec/task.h          |   56 +-
 ttg/ttg/parsec/ttg.h           |  206 +++-
 ttg/ttg/parsec/ttg_data_copy.h |   34 +-
 ttg/ttg/view.h                 |  526 +--------
 13 files changed, 2815 insertions(+), 684 deletions(-)
 create mode 100644 examples/spmm/cuda_gemm.cc
 create mode 100644 examples/spmm/cuda_gemm.h
 create mode 100644 examples/spmm/spmm_cuda.cc
 delete mode 100644 ttg/ttg/device/send.h
 create mode 100644 ttg/ttg/device/task.h

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 2ca2e3214..56895a057 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -12,6 +12,12 @@ if (TARGET eigen3)
     if (TARGET BTAS::BTAS)
         # since only need to use matrices, limit BTAS_TARGET_MAX_INDEX_RANK to 2
         add_ttg_executable(bspmm spmm/spmm.cc LINK_LIBRARIES eigen3 BTAS Boost::boost COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2)
+        if (TARGET CUDA::cublas)
+          add_ttg_executable(bspmm-cuda spmm/spmm_cuda.cc spmm/cuda_gemm.cc
+                             LINK_LIBRARIES eigen3 BTAS Boost::boost CUDA::cublas
+                             COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2
+                             RUNTIMES "parsec")
+        endif (TARGET CUDA::cublas)
     endif (TARGET BTAS::BTAS)
 endif(TARGET eigen3)
 
diff --git a/examples/spmm/cuda_gemm.cc b/examples/spmm/cuda_gemm.cc
new file mode 100644
index 000000000..68987f64f
--- /dev/null
+++ b/examples/spmm/cuda_gemm.cc
@@ -0,0 +1,29 @@
+
+#include <cublas.h>
+
+
+void my_cublas_init_because_cublas_is_stupid() {
+    cublasInit();
+}
+
+void my_cublas_shutdown_because_cublas_is_stupid() {
+    cublasShutdown();
+}
+
+void my_cublas_dgemm_because_cublas_is_stupid(
+    char transa,
+    char transb,
+    int m,
+    int n,
+    int k,
+    double alpha,
+    const double* A,
+    int lda,
+    const double* B,
+    int ldb,
+    double beta,
+    double* C,
+    int ldc)
+{
+  cublasDgemm(transa, transb, m, n, k, alpha, A, lda, B, ldb, beta, C, ldc);
+}
diff --git a/examples/spmm/cuda_gemm.h b/examples/spmm/cuda_gemm.h
new file mode 100644
index 000000000..cc6aba6c8
--- /dev/null
+++ b/examples/spmm/cuda_gemm.h
@@ -0,0 +1,24 @@
+
+
+/* We need wrap cublas v1 functions because some header (blaspp)
+ * includes cublas_v2 and cublas is braindead and overwrites the v1
+ * signatures. What a clusterfuck. */
+
+void my_cublas_init_because_cublas_is_stupid();
+
+void my_cublas_shutdown_because_cublas_is_stupid();
+
+void my_cublas_dgemm_because_cublas_is_stupid(
+    char transa,
+    char transb,
+    int m,
+    int n,
+    int k,
+    double alpha,
+    const double* A,
+    int lda,
+    const double* B,
+    int ldb,
+    double beta,
+    double* C,
+    int ldc);
\ No newline at end of file
diff --git a/examples/spmm/spmm_cuda.cc b/examples/spmm/spmm_cuda.cc
new file mode 100644
index 000000000..aac469a34
--- /dev/null
+++ b/examples/spmm/spmm_cuda.cc
@@ -0,0 +1,1837 @@
+
+#include <Eigen/SparseCore>
+#include <algorithm>
+#include <array>
+#include <chrono>
+#include <iostream>
+#include <random>
+#include <thread>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#if __has_include(<btas/features.h>)
+#pragma message("C Preprocessor got here!")
+#include <btas/features.h>
+#ifdef BTAS_IS_USABLE
+#include <btas/btas.h>
+#include <btas/optimize/contract.h>
+#include <btas/util/mohndle.h>
+#else
+#warning "found btas/features.h but Boost.Iterators is missing, hence BTAS is unusable ... add -I/path/to/boost"
+#endif
+#endif
+
+#include <sys/time.h>
+#include <boost/graph/rmat_graph_generator.hpp>
+#if !defined(BLOCK_SPARSE_GEMM)
+#include <boost/graph/directed_graph.hpp>
+#include <boost/random/linear_congruential.hpp>
+#include <unsupported/Eigen/SparseExtra>
+#endif
+
+#include "ttg.h"
+
+using namespace ttg;
+
+#include "ttg/util/future.h"
+
+#include "ttg/util/multiindex.h"
+
+#include "ttg/util/bug.h"
+
+#include "ttg/serialization/std/pair.h"
+
+#include "cuda_gemm.h"
+
+#ifdef CUBLAS_V2_H_
+#define CUBLAS_V2_H_ 1
+#endif
+
+#if defined(BLOCK_SPARSE_GEMM) && defined(BTAS_IS_USABLE)
+
+template <typename _T, class _Range, class _Storage>
+struct DeviceTensor : public btas::Tensor<_T, _Range, _Storage> {
+  using tensor_type = btas::Tensor<_T, _Range, _Storage>;
+  tensor_type t;
+  ttg::buffer<_T> b; // does not own the host buffer
+
+  using value_type = tensor_type::value_type;
+  using size_type = tensor_type::size_type;
+  using storage_type = tensor_type::storage_type;
+  using range_type = tensor_type::range_type;
+
+
+   public:
+    DeviceTensor() = default;
+    ~DeviceTensor() = default;
+
+    /// constructor with index extent
+    template <typename... _args>
+    explicit DeviceTensor(const size_type& first, const _args&... rest)
+    : tensor_type(first, rest...)
+    , b(this->size() ? this->data() : nullptr, this->size())
+    { }
+
+    /// construct from \c range, allocate data, but not initialized
+    template <typename Range>
+    explicit DeviceTensor(const Range& range, typename std::enable_if<btas::is_boxrange<Range>::value>::type* = 0)
+    : tensor_type(range)
+    , b(this->size() ? this->data() : nullptr, this->size())
+    { }
+
+    /// construct from \c range object, set all elements to \c v
+    template <typename Range>
+    DeviceTensor(const Range& range, value_type v, typename std::enable_if<btas::is_boxrange<Range>::value>::type* = 0)
+    : tensor_type(range)
+    , b(this->size() ? this->data() : nullptr, this->size())
+    { }
+
+    /// construct from \c range object, copy elements from \c vec
+    template <typename Range, typename U>
+    DeviceTensor(const Range& range, U* vec, typename std::enable_if<btas::is_boxrange<Range>::value>::type* = 0)
+    : tensor_type(range, vec)
+    , b(this->size() ? this->data() : nullptr, this->size())
+    { }
+
+    /// construct from \c range and \c storage
+    template <typename Range, typename Storage>
+    DeviceTensor(const Range& range, const Storage& storage,
+           typename std::enable_if<btas::is_boxrange<Range>::value & not std::is_same<Range, range_type>::value &
+                                   not std::is_same<Storage, storage_type>::value>::type* = 0)
+    : tensor_type(range, storage)
+    , b(this->size() ? this->data() : nullptr, this->size())
+    { }
+
+    /// copy-copy-construct from \c range and \c storage
+    DeviceTensor(const range_type& range, const storage_type& storage)
+    : tensor_type(range, storage)
+    , b(this->size() ? this->data() : nullptr, this->size())
+    { }
+
+    /// copy-move-construct from \c range and \c storage
+    DeviceTensor(const range_type& range, storage_type&& storage)
+    : tensor_type(range, std::forward<storage_type>(storage))
+    , b(this->size() ? this->data() : nullptr, this->size())
+    { }
+
+    /// move-construct from \c range and \c storage
+    DeviceTensor(range_type&& range, storage_type&& storage)
+    : tensor_type(std::forward<range_type>(range), std::forward<storage_type>(storage))
+    , b(this->size() ? this->data() : nullptr, this->size())
+    { }
+
+    /// Construct an evaluated tensor
+
+    /// This constructor will allocate memory for \c range.area() elements. Each element
+    /// will be initialized as:
+    /// \code
+    ///   for(auto&& idx: range)
+    ///     (*this)[idx] = op(*(it++));
+    /// \endcode
+    /// \tparam Range An input Range type.
+    /// \tparam InIter An input iterator type.
+    /// \tparam Op A unary operation type
+    /// \param range the input range type
+    /// \param first An input iterator for the argument
+    /// \param op The unary operation to be applied to the argument data
+    template <typename Range, typename InIter, typename Op>
+    DeviceTensor(const Range& range, InIter it, const Op& op,
+           typename std::enable_if<btas::is_boxrange<Range>::value>::type* = 0)
+    : tensor_type(range, it, op)
+    , b(this->size() ? this->data() : nullptr, this->size())
+    { }
+
+    /// copy constructor
+    /// It will accept Tensors and TensorViews
+    template <class _Tensor, class = typename std::enable_if<btas::is_boxtensor<_Tensor>::value>::type>
+    DeviceTensor(const _Tensor& x)
+    : tensor_type(x)
+    , b(this->size() ? this->data() : nullptr, this->size())
+    { }
+
+    /// copy constructor: devicebuf cannot be copied, so deleted
+    DeviceTensor(const DeviceTensor& x)
+    : tensor_type(x)
+    , b(this->size() ? this->data() : nullptr, this->size())
+    { }
+
+    /// move constructor
+    DeviceTensor(tensor_type&& x)
+    : tensor_type(std::forward<tensor_type>(x))
+    , b(this->size() ? this->data() : nullptr, this->size())
+    { }
+
+    DeviceTensor(DeviceTensor&& x)
+    : tensor_type(std::forward<tensor_type>(x))
+    , b(std::move(x.b))
+    { }
+
+    /// copy assignment operator
+    template <class _Tensor, class = typename std::enable_if<
+                                 btas::is_boxtensor<_Tensor>::value &&
+                                 not std::is_same<typename _Tensor::storage_type, storage_type>::value>::type>
+    DeviceTensor& operator=(const _Tensor& x) {
+      tensor_type::operator=(x);
+      b.reset(this->size() ? this->data() : nullptr, this->size());
+      return *this;
+    }
+
+    /// copy assignment operator
+    template <class _Tensor, class = typename std::enable_if<btas::is_boxtensor<_Tensor>::value>::type,
+              class = typename std::enable_if<
+                  std::is_same<typename _Tensor::storage_type, storage_type>::value>::type>
+    DeviceTensor& operator=(const _Tensor& x) {
+      tensor_type::operator=(x);
+      b.reset(this->size() ? this->data() : nullptr, this->size());
+      return *this;
+    }
+
+    /// copy assignment: devicebuf cannot be copied, deleted
+    DeviceTensor& operator=(const DeviceTensor& x) {
+      tensor_type::operator=(x);
+      b.reset(this->size() ? this->data() : nullptr, this->size());
+      return *this;
+    }
+
+    /// move assignment operator
+    DeviceTensor& operator=(DeviceTensor&& x) {
+      tensor_type::operator=(std::forward<tensor_type>(x));
+      std::swap(x.b, b);
+      return *this;
+    }
+
+};
+
+using blk_t = DeviceTensor<double, btas::DEFAULT::range,
+                           btas::mohndle<btas::varray<double>,
+                                         btas::Handle::shared_ptr>>;
+
+
+
+//inline blk_t operator*(const blk_t &A, const blk_t &B) {
+//  blk_t::tensor_type c;
+//  btas::contract(1.0, A, {1, 2}, B, {2, 3}, 0.0, c, {1, 3});
+//  return blk_t(std::move(c));
+//}
+
+/* TODO: call CUDA gemm here */
+static void cuda_gemm(blk_t &C, const blk_t &A, const blk_t &B) {
+  using array = btas::DEFAULT::index<int>;
+  array aA = array{1, 2};
+  array aB = array{2, 3};
+  array aC = array{1, 3};
+  bool conjgA = false;
+  bool conjgB = false;
+  double alpha = 1.0;
+  double beta  = 1.0;
+  const bool notransA = aA.front() == aC.front();
+  const bool notransB = aB.front() != aC.back();
+  const auto   cA     = notransA ? 'N' : (conjgA ? 'C' : 'T');
+  const size_t condim = notransA ? A.extent(1) : A.extent(0);
+  const auto   cB     = notransB ? 'N' : (conjgB ? 'C' : 'T');
+  // make sure all memory is on the device
+  // TODO: A and B are read-only so the owner device will be 0. How to fix?
+  //assert(A.b.get_current_device() != 0);
+  //assert(B.b.get_current_device() != 0);
+  assert(C.b.get_current_device() != 0);
+  // NOTE: A and B are swapped due to the ColMajor layout, see btas/generic/gemm_impl.h
+  my_cublas_dgemm_because_cublas_is_stupid(
+              cB, cA, C.extent(0), C.extent(1), condim, alpha,
+              &*B.b.device_ptr_on(C.b.get_current_device()), B.extent(0),
+              &*A.b.device_ptr_on(C.b.get_current_device()), A.extent(0), beta,
+              &*C.b.current_device_ptr(), C.extent(0));
+  //gemm_impl<true>::call(blas::Layout::ColMajor, cA, cB, C.extent(0), C.extent(1), condim,
+  //                      alpha, &*A.begin(), A.extent(0), &*B.begin(), B.extent(0), beta, &*C.begin(), C.extent(0));
+}
+
+//using blk_t = btas::Tensor<double, btas::DEFAULT::range, btas::mohndle<btas::varray<double>, btas::Handle::shared_ptr>>;
+
+#if defined(TTG_USE_PARSEC)
+namespace ttg {
+  template <>
+  struct SplitMetadataDescriptor<blk_t> {
+    // TODO: this is a quick and dirty approach.
+    //   - blk_t could have any number of dimensions, this code only works for 2 dim blocks
+    //   - we use Blk{} to send a control flow in some tasks below, these blocks have only
+    //     1 dimension (of size 0), to code this, we set the second dimension to 0 in our
+    //     quick and dirty linearization, then have a case when we create the object
+    //   - when we create the object with the metadata, we use a constructor that initializes
+    //     the data to 0, which is useless: the data could be left uninitialized
+    static auto get_metadata(const blk_t &b) {
+      std::pair<int, int> dim{0, 0};
+      if (!b.empty()) {
+        assert(b.range().extent().size() == 2);
+        std::get<0>(dim) = (int)b.range().extent(0);
+        std::get<1>(dim) = (int)b.range().extent(1);
+      }
+      return dim;
+    }
+    static auto get_data(blk_t &b) {
+      if (!b.empty())
+        return boost::container::small_vector<iovec, 1>(1, iovec{b.size() * sizeof(double), b.data()});
+      else
+        return boost::container::small_vector<iovec, 1>{};
+    }
+    static auto create_from_metadata(const std::pair<int, int> &meta) {
+      if (meta != std::pair{0, 0})
+        return blk_t(btas::Range(std::get<0>(meta), std::get<1>(meta)), 0.0);
+      else
+        return blk_t{};
+    }
+  };
+}  // namespace ttg
+#endif /* TTG_USE_PARSEC */
+
+// declare btas::Tensor serializable by Boost
+#include "ttg/serialization/backends/boost.h"
+namespace ttg::detail {
+  // BTAS defines all of its Boost serializers in boost::serialization namespace ... as explained in
+  // ttg/serialization/boost.h such functions are not detectable via SFINAE, so must explicitly define serialization
+  // traits here
+  template <typename Archive>
+  inline static constexpr bool is_boost_serializable_v<Archive, blk_t> = is_boost_archive_v<Archive>;
+  template <typename Archive>
+  inline static constexpr bool is_boost_serializable_v<Archive, const blk_t> = is_boost_archive_v<Archive>;
+}  // namespace ttg::detail
+
+#else
+using blk_t = double;
+#endif
+template <typename T = blk_t>
+using SpMatrix = Eigen::SparseMatrix<T>;
+template <typename T = blk_t>
+using SpMatrixTriplet = Eigen::Triplet<T>;  // {row,col,value}
+
+#if defined(BLOCK_SPARSE_GEMM) && defined(BTAS_IS_USABLE)
+
+#if __has_include(<madness/world/archive.h>)
+
+#include <madness/world/archive.h>
+
+#endif  // __has_include(<madness/world/archive.h>)
+
+namespace btas {
+  template <typename T_, class Range_, class Store_>
+  inline btas::Tensor<T_, Range_, Store_> operator*(const btas::Tensor<T_, Range_, Store_> &A,
+                                                    const btas::Tensor<T_, Range_, Store_> &B) {
+    btas::Tensor<T_, Range_, Store_> C;
+    btas::contract(1.0, A, {1, 2}, B, {2, 3}, 0.0, C, {1, 3});
+    return C;
+  }
+
+  template <typename T_, class Range_, class Store_>
+  btas::Tensor<T_, Range_, Store_> gemm(btas::Tensor<T_, Range_, Store_> &&C, const btas::Tensor<T_, Range_, Store_> &A,
+                                        const btas::Tensor<T_, Range_, Store_> &B) {
+    using array = btas::DEFAULT::index<int>;
+    if (C.empty()) {
+      C = btas::Tensor<T_, Range_, Store_>(btas::Range(A.range().extent(0), B.range().extent(1)), 0.0);
+    }
+    btas::contract_222(1.0, A, array{1, 2}, B, array{2, 3}, 1.0, C, array{1, 3}, false, false);
+    return std::move(C);
+  }
+}  // namespace btas
+#endif  // BTAS_IS_USABLE
+double gemm(double C, double A, double B) { return C + A * B; }
+
+// template <typename _Scalar, int _Options, typename _StorageIndex>
+// struct colmajor_layout;
+// template <typename _Scalar, typename _StorageIndex>
+// struct colmajor_layout<_Scalar, Eigen::ColMajor, _StorageIndex> : public std::true_type {};
+// template <typename _Scalar, typename _StorageIndex>
+// struct colmajor_layout<_Scalar, Eigen::RowMajor, _StorageIndex> : public std::false_type {};
+
+template <std::size_t Rank>
+using Key = MultiIndex<Rank>;
+
+/// maps {i,j} to rank within first (R=0) layer of the 3-d process grid
+inline int ij2rank(int i, int j, int P, int Q) {
+  std::vector<int> vec;
+  int p = (i % P);
+  int q = (j % Q);
+  int rank = (q * P) + p;
+  return rank;
+}
+
+/// maps {i,j,k} to rank within a 3-d process grid
+inline int ijk2rank(int i, int j, int k, int P, int Q, int R) {
+  std::vector<int> vec;
+  int p = (i % P);
+  int q = (j % Q);
+  int l = (k % R);
+  int rank = (l * P * Q) + (q * P) + p;
+  return rank;
+}
+
+// flow data from an existing SpMatrix on rank 0
+template <typename Blk = blk_t, typename Keymap = std::function<int(const Key<2> &)>>
+class Read_SpMatrix : public TT<Key<2>, std::tuple<Out<Key<2>, Blk>>, Read_SpMatrix<Blk, Keymap>, ttg::typelist<void>> {
+ public:
+  using baseT = typename Read_SpMatrix::ttT;
+  Read_SpMatrix(const char *label, const SpMatrix<Blk> &matrix, Edge<Key<2>> &ctl, Edge<Key<2>, Blk> &out,
+                Keymap &ij_keymap)
+      : baseT(edges(ctl), edges(out), std::string("read_spmatrix(") + label + ")", {"ctl"}, {std::string(label) + "ij"},
+              ij_keymap)
+      , matrix_(matrix) {}
+
+  void op(const Key<2> &, std::tuple<Out<Key<2>, Blk>> &out) {
+    auto rank = ttg::default_execution_context().rank();
+    for (int k = 0; k < matrix_.outerSize(); ++k) {
+      for (typename SpMatrix<Blk>::InnerIterator it(matrix_, k); it; ++it) {
+        if (rank == this->get_keymap()(Key<2>(std::initializer_list<long>({it.row(), it.col()}))))
+          ::send<0>(Key<2>(std::initializer_list<long>({it.row(), it.col()})), it.value(), out);
+      }
+    }
+  }
+
+ private:
+  const SpMatrix<Blk> &matrix_;
+};
+
+// flow (move?) data into an existing SpMatrix on rank 0
+template <typename Blk = blk_t>
+class Write_SpMatrix : public TT<Key<2>, std::tuple<>, Write_SpMatrix<Blk>, ttg::typelist<Blk>> {
+ public:
+  using baseT = typename Write_SpMatrix::ttT;
+
+  template <typename Keymap2>
+  Write_SpMatrix(SpMatrix<Blk> &matrix, Edge<Key<2>, Blk> &in, Keymap2 &&ij_keymap)
+      : baseT(edges(in), edges(), "write_spmatrix", {"Cij"}, {}, ij_keymap), matrix_(matrix) {}
+
+  void op(const Key<2> &key, typename baseT::input_refs_tuple_type &&elem, std::tuple<> &) {
+    std::lock_guard<std::mutex> lock(mtx_);
+    ttg::trace("rank =", default_execution_context().rank(),
+               "/ thread_id =", reinterpret_cast<std::uintptr_t>(pthread_self()), "spmm.cc Write_SpMatrix wrote {",
+               key[0], ",", key[1], "} = ", baseT::template get<0>(elem), " in ", static_cast<void *>(&matrix_),
+               " with mutex @", static_cast<void *>(&mtx_), " for object @", static_cast<void *>(this));
+    values_.emplace_back(key[0], key[1], baseT::template get<0>(elem));
+  }
+
+  /// grab completion status as a future<void>
+  /// \note cannot be called once this is executable
+  const std::shared_future<void> &status() const {
+    assert(!this->is_executable());
+    if (!completion_status_) {  // if not done yet, register completion work with the world
+      auto promise = std::make_shared<std::promise<void>>();
+      completion_status_ = std::make_shared<std::shared_future<void>>(promise->get_future());
+      ttg_register_status(this->get_world(), std::move(promise));
+      ttg_register_callback(this->get_world(),
+                            [this]() { this->matrix_.setFromTriplets(this->values_.begin(), this->values_.end()); });
+    } else {  // if done already, commit the result
+      this->matrix_.setFromTriplets(this->values_.begin(), this->values_.end());
+    }
+    return *completion_status_;
+  }
+
+ private:
+  std::mutex mtx_;
+  SpMatrix<Blk> &matrix_;
+  std::vector<SpMatrixTriplet<Blk>> values_;
+  mutable std::shared_ptr<std::shared_future<void>> completion_status_;
+};
+
+/// sparse mm via 2.5D SUMMA
+
+/// @tparam KeyMap2 maps {i,j} to processor
+/// @tparam KeyMap3 maps {i,j,k} to processor
+template <typename Keymap2 = std::function<int(const Key<2> &)>, typename Keymap3 = std::function<int(const Key<3> &)>,
+          typename Blk = blk_t>
+class SpMM25D {
+ public:
+  /// @param ij_keymap maps {i,j} to process, specifies distribution of tiles of A, B, and C
+  /// @param ijk_keymap maps {i,j,k} to process, controls distribution of tasks performing C[i][j] += A[i][k]*B[k][j]
+  /// @param R the number of "layers" in the 3-D process grid
+  SpMM25D(Edge<Key<2>, Blk> &a, Edge<Key<2>, Blk> &b, Edge<Key<2>, Blk> &c, const SpMatrix<Blk> &a_mat,
+          const SpMatrix<Blk> &b_mat, const std::vector<std::vector<long>> &a_rowidx_to_colidx,
+          const std::vector<std::vector<long>> &a_colidx_to_rowidx,
+          const std::vector<std::vector<long>> &b_rowidx_to_colidx,
+          const std::vector<std::vector<long>> &b_colidx_to_rowidx, const std::vector<int> &mTiles,
+          const std::vector<int> &nTiles, const std::vector<int> &kTiles, Keymap2 ij_keymap, Keymap3 ijk_keymap, long R)
+      : a_rowidx_to_colidx_(a_rowidx_to_colidx)
+      , b_colidx_to_rowidx_(b_colidx_to_rowidx)
+      , a_colidx_to_rowidx_(a_colidx_to_rowidx)
+      , b_rowidx_to_colidx_(b_rowidx_to_colidx)
+      , ij_keymap_(std::move(ij_keymap))
+      , ijk_keymap_(std::move(ijk_keymap)) {
+    bcast_a_ = std::make_unique<BcastA>(a, local_a_ijk_, b_rowidx_to_colidx_, ij_keymap_, ijk_keymap_);
+    local_bcast_a_ = std::make_unique<LocalBcastA>(local_a_ijk_, a_ijk_, b_rowidx_to_colidx_, ijk_keymap_);
+    bcast_b_ = std::make_unique<BcastB>(b, local_b_ijk_, a_colidx_to_rowidx_, ij_keymap_, ijk_keymap_);
+    local_bcast_b_ = std::make_unique<LocalBcastB>(local_b_ijk_, b_ijk_, a_colidx_to_rowidx_, ijk_keymap_);
+    multiplyadd_ = std::make_unique<MultiplyAdd>(a_ijk_, b_ijk_, c_ijk_, c_ij_p_, a_rowidx_to_colidx_,
+                                                 b_colidx_to_rowidx_, mTiles, nTiles, ijk_keymap_);
+    reduce_c_ = std::make_unique<ReduceC>(c_ij_p_, c, ij_keymap_);
+    reduce_c_->template set_input_reducer<0>([](Blk &c_ij, const Blk &c_ij_p) { c_ij = c_ij + c_ij_p; });
+    // compute how many contributions each C[i][j] should expect ... MultiplyAdd already does this, but need a way to
+    // send message from each process p to the process owning C[i][j] to expect a contribution from it for now replicate
+    // this logic ...
+    // TODO: do this in MultiplyAdd (need to allreduce this info so that everyone has it)
+    // N.B. only need to set stream size on the rank that will accumulate the C[i][j] contribution
+    const auto my_rank = ttg::default_execution_context().rank();
+    for (auto i = 0ul; i != a_rowidx_to_colidx_.size(); ++i) {
+      if (a_rowidx_to_colidx_[i].empty()) continue;
+      for (auto j = 0ul; j != b_colidx_to_rowidx_.size(); ++j) {
+        if (b_colidx_to_rowidx_[j].empty()) continue;
+
+        if (ij_keymap_(Key<2>{i, j}) == my_rank) {
+          decltype(i) k;
+          bool have_k;
+          std::tie(k, have_k) = multiplyadd_->compute_first_k(i, j);
+          std::vector<bool> c_ij_procmask(R, false);
+          if (have_k) {
+            const auto pR = k % R;  // k values are distributed round-robin among the layers of the 3-D grid
+            assert(pR < c_ij_procmask.size());
+            c_ij_procmask[pR] = true;
+            while (have_k) {
+              std::tie(k, have_k) = multiplyadd_->compute_next_k(i, j, k);
+              if (have_k) {
+                const auto pR = k % R;
+                assert(pR < c_ij_procmask.size());
+                c_ij_procmask[pR] = true;
+              }
+            }
+          }
+          const auto c_ij_nprocs = std::count_if(c_ij_procmask.begin(), c_ij_procmask.end(), [](bool b) { return b; });
+          if (c_ij_nprocs > 0) reduce_c_->template set_argstream_size<0>(Key<2>{i, j}, c_ij_nprocs);
+        }
+      }
+    }
+
+    TTGUNUSED(bcast_a_);
+    TTGUNUSED(bcast_b_);
+    TTGUNUSED(multiplyadd_);
+    TTGUNUSED(reduce_c_);
+  }
+
+  /// Locally broadcast `A[i][k]` assigned to this processor `p` to matmul tasks `{i,j,k}` for all `j` such that
+  /// `B[k][j]` exists AND `k` contribution to `C[i][j]` is assigned to this processor
+  class LocalBcastA : public TT<Key<3>, std::tuple<Out<Key<3>, Blk>>, LocalBcastA, ttg::typelist<Blk>> {
+   public:
+    using baseT = typename LocalBcastA::ttT;
+
+    LocalBcastA(Edge<Key<3>, Blk> &a, Edge<Key<3>, Blk> &a_ijk,
+                const std::vector<std::vector<long>> &b_rowidx_to_colidx, const Keymap3 &ijk_keymap)
+        : baseT(edges(a), edges(a_ijk), "SpMM25D::local_bcast_a", {"a_ikp"}, {"a_ijk"},
+                [](const Key<3> &ikp) { return ikp[2]; })
+        , b_rowidx_to_colidx_(b_rowidx_to_colidx)
+        , ijk_keymap_(ijk_keymap) {}
+
+    void op(const Key<3> &ikp, typename baseT::input_refs_tuple_type &&a_ik, std::tuple<Out<Key<3>, Blk>> &a_ijk) {
+      const auto i = ikp[0];
+      const auto k = ikp[1];
+      const auto p = ikp[2];
+
+      auto world = default_execution_context();
+      assert(p == world.rank());
+      ttg::trace("LocalBcastA(", i, ", ", k, ", ", p, ")");
+      if (k >= b_rowidx_to_colidx_.size()) return;
+      // local broadcast a_ik to all {i,j,k} such that b_kj exists
+      std::vector<Key<3>> ijk_keys;
+      for (auto &j : b_rowidx_to_colidx_[k]) {
+        if (ijk_keymap_(Key<3>({i, j, k})) == world.rank()) {
+          ttg::trace("Broadcasting A[", i, "][", k, "] on proc ", p, " to j=", j);
+          ijk_keys.emplace_back(Key<3>({i, j, k}));
+        }
+      }
+      ::broadcast<0>(ijk_keys, std::move(baseT::template get<0>(a_ik)), a_ijk);
+    }
+
+   private:
+    const std::vector<std::vector<long>> &b_rowidx_to_colidx_;
+    const Keymap3 &ijk_keymap_;
+  };  // class LocalBcastA
+
+  /// broadcast `A[i][k]` to all processors which will contain at least one `C[i][j]` such that `B[k][j]` exists
+  class BcastA : public TT<Key<2>, std::tuple<Out<Key<3>, Blk>>, BcastA, ttg::typelist<Blk>> {
+   public:
+    using baseT = typename BcastA::ttT;
+
+    BcastA(Edge<Key<2>, Blk> &a_ik, Edge<Key<3>, Blk> &a_ikp, const std::vector<std::vector<long>> &b_rowidx_to_colidx,
+           const Keymap2 &ij_keymap, const Keymap3 &ijk_keymap)
+        : baseT(edges(a_ik), edges(a_ikp), "SpMM25D::bcast_a", {"a_ik"}, {"a_ikp"}, ij_keymap)
+        , b_rowidx_to_colidx_(b_rowidx_to_colidx)
+        , ijk_keymap_(ijk_keymap) {}
+
+    void op(const Key<2> &ik, typename baseT::input_refs_tuple_type &&a_ik, std::tuple<Out<Key<3>, Blk>> &a_ikp) {
+      const auto i = ik[0];
+      const auto k = ik[1];
+      ttg::trace("BcastA(", i, ", ", k, ")");
+      std::vector<Key<3>> ikp_keys;
+
+      if (k >= b_rowidx_to_colidx_.size()) return;
+      auto world = default_execution_context();
+      std::vector<bool> procmap(world.size());
+      for (auto &j : b_rowidx_to_colidx_[k]) {
+        const long p = ijk_keymap_(Key<3>(
+            {i, j, k}));  // N.B. in 2.5D SUMMA different k contributions to C[i][j] are computed on different nodes
+        if (!procmap[p]) {
+          ttg::trace("Broadcasting A[", i, "][", k, "] to proc ", p);
+          ikp_keys.emplace_back(Key<3>({i, k, p}));
+          procmap[p] = true;
+        }
+      }
+      ::broadcast<0>(ikp_keys, std::move(baseT::template get<0>(a_ik)), a_ikp);
+    }
+
+   private:
+    const std::vector<std::vector<long>> &b_rowidx_to_colidx_;
+    const Keymap3 &ijk_keymap_;
+  };  // class BcastA
+
+  /// Locally broadcast `B[k][j]` assigned to this processor `p` to matmul tasks `{i,j,k}` for all `k` such that
+  /// `A[i][k]` exists AND `k` contribution to `C[i][j]` is assigned to this processor
+  class LocalBcastB : public TT<Key<3>, std::tuple<Out<Key<3>, Blk>>, LocalBcastB, ttg::typelist<Blk>> {
+   public:
+    using baseT = typename LocalBcastB::ttT;
+
+    LocalBcastB(Edge<Key<3>, Blk> &b_kjp, Edge<Key<3>, Blk> &b_ijk,
+                const std::vector<std::vector<long>> &a_colidx_to_rowidx, const Keymap3 &ijk_keymap)
+        : baseT(edges(b_kjp), edges(b_ijk), "SpMM25D::local_bcast_b", {"b_kjp"}, {"b_ijk"},
+                [](const Key<3> &kjp) { return kjp[2]; })
+        , a_colidx_to_rowidx_(a_colidx_to_rowidx)
+        , ijk_keymap_(ijk_keymap) {}
+
+    void op(const Key<3> &kjp, typename baseT::input_refs_tuple_type &&b_kj, std::tuple<Out<Key<3>, Blk>> &b_ijk) {
+      const auto k = kjp[0];
+      const auto j = kjp[1];
+      const auto p = kjp[2];
+      auto world = default_execution_context();
+      assert(p == world.rank());
+      ttg::trace("BcastB(", k, ", ", j, ", ", p, ")");
+      if (k >= a_colidx_to_rowidx_.size()) return;
+      // broadcast b_kj to all ijk for which c_ij is on this processor and a_ik exists
+      std::vector<Key<3>> ijk_keys;
+      for (auto &i : a_colidx_to_rowidx_[k]) {
+        if (ijk_keymap_(Key<3>({i, j, k})) == world.rank()) {
+          ttg::trace("Broadcasting B[", k, "][", j, "] on proc ", p, " to i=", i);
+          ijk_keys.emplace_back(Key<3>({i, j, k}));
+        }
+      }
+      ::broadcast<0>(ijk_keys, std::move(baseT::template get<0>(b_kj)), b_ijk);
+    }
+
+   private:
+    const std::vector<std::vector<long>> &a_colidx_to_rowidx_;
+    const Keymap3 &ijk_keymap_;
+  };  // class LocalBcastB
+
+  /// broadcast `B[k][j]` to all processors which will contain at least one `C[i][j]` such that `A[i][k]` exists
+  class BcastB : public TT<Key<2>, std::tuple<Out<Key<3>, Blk>>, BcastB, ttg::typelist<Blk>> {
+   public:
+    using baseT = typename BcastB::ttT;
+
+    BcastB(Edge<Key<2>, Blk> &b_kj, Edge<Key<3>, Blk> &b_kjp, const std::vector<std::vector<long>> &a_colidx_to_rowidx,
+           const Keymap2 &ij_keymap, const Keymap3 &ijk_keymap)
+        : baseT(edges(b_kj), edges(b_kjp), "SpMM25D::bcast_b", {"b_kj"}, {"b_kjp"}, ij_keymap)
+        , a_colidx_to_rowidx_(a_colidx_to_rowidx)
+        , ijk_keymap_(ijk_keymap) {}
+
+    void op(const Key<2> &kj, typename baseT::input_refs_tuple_type &&b_kj, std::tuple<Out<Key<3>, Blk>> &b_kjp) {
+      const auto k = kj[0];
+      const auto j = kj[1];
+      // broadcast b_kj to all processors which will contain at least one c_ij such that a_ik exists
+      std::vector<Key<3>> kjp_keys;
+      ttg::trace("BcastB(", k, ", ", j, ")");
+      if (k >= a_colidx_to_rowidx_.size()) return;
+      auto world = default_execution_context();
+      std::vector<bool> procmap(world.size());
+      for (auto &i : a_colidx_to_rowidx_[k]) {
+        long p = ijk_keymap_(Key<3>({i, j, k}));
+        if (!procmap[p]) {
+          ttg::trace("Broadcasting B[", k, "][", j, "] to proc ", p);
+          kjp_keys.emplace_back(Key<3>({k, j, p}));
+          procmap[p] = true;
+        }
+      }
+      ::broadcast<0>(kjp_keys, std::move(baseT::template get<0>(b_kj)), b_kjp);
+    }
+
+   private:
+    const std::vector<std::vector<long>> &a_colidx_to_rowidx_;
+    const Keymap3 &ijk_keymap_;
+  };  // class BcastB
+
+  /// multiply task has 3 input flows: a_ijk, b_ijk, and c_ijk, c_ijk contains the running total for this kayer of the
+  /// 3-D process grid only
+  class MultiplyAdd : public TT<Key<3>, std::tuple<Out<Key<2>, Blk>, Out<Key<3>, Blk>>, MultiplyAdd,
+                                ttg::typelist<const Blk, const Blk, Blk>> {
+   public:
+    using baseT = typename MultiplyAdd::ttT;
+
+    static constexpr bool have_cuda_op = true;
+
+    MultiplyAdd(Edge<Key<3>, Blk> &a_ijk, Edge<Key<3>, Blk> &b_ijk, Edge<Key<3>, Blk> &c_ijk, Edge<Key<2>, Blk> &c,
+                const std::vector<std::vector<long>> &a_rowidx_to_colidx,
+                const std::vector<std::vector<long>> &b_colidx_to_rowidx, const std::vector<int> &mTiles,
+                const std::vector<int> &nTiles, const Keymap3 &ijk_keymap)
+        : baseT(edges(a_ijk, b_ijk, c_ijk), edges(c, c_ijk), "SpMM25D::MultiplyAdd", {"a_ijk", "b_ijk", "c_ijk"},
+                {"c_ij", "c_ijk"}, ijk_keymap)
+        , a_rowidx_to_colidx_(a_rowidx_to_colidx)
+        , b_colidx_to_rowidx_(b_colidx_to_rowidx) {
+      this->set_priomap([this](const Key<3> &ijk) { return this->prio(ijk); });  // map a key to an integral priority value
+
+      // for each {i,j} determine first k that contributes AND belongs to this node,
+      // initialize input {i,j,first_k} flow to 0
+      for (auto i = 0ul; i != a_rowidx_to_colidx_.size(); ++i) {
+        if (a_rowidx_to_colidx_[i].empty()) continue;
+        for (auto j = 0ul; j != b_colidx_to_rowidx_.size(); ++j) {
+          if (b_colidx_to_rowidx_[j].empty()) continue;
+
+          const auto p = ttg::default_execution_context().rank();
+          decltype(i) k;
+          bool have_k;
+          std::tie(k, have_k) = compute_first_k(i, j, p);
+          if (have_k) {
+            ttg::trace("Initializing C[", i, "][", j, "] on process ", p, " to zero");
+#if BLOCK_SPARSE_GEMM
+            Blk zero(btas::Range(mTiles[i], nTiles[j]), 0.0);
+#else
+            Blk zero{0.0};
+#endif
+            this->template in<2>()->send(Key<3>({i, j, k}), zero);
+          } else {
+            if (tracing() && a_rowidx_to_colidx_.size() * b_colidx_to_rowidx_.size() < 400)
+              ttg::print("C[", i, "][", j, "] is empty");
+          }
+        }
+      }
+    }
+
+    ttg::device_task op(const Key<3> &ijk, typename baseT::input_refs_tuple_type &&_ijk,
+            std::tuple<Out<Key<2>, Blk>, Out<Key<3>, Blk>> &result) {
+      const auto i = ijk[0];
+      const auto j = ijk[1];
+      const auto k = ijk[2];  // k==l same because 000 will always be on layer 0, 001 will be accessed on layer 1
+      const auto p = ttg::default_execution_context().rank();
+      long next_k;
+      bool have_next_k;
+
+      const blk_t& A = baseT::template get<0>(_ijk);
+      const blk_t& B = baseT::template get<1>(_ijk);
+      blk_t& C = baseT::template get<2>(_ijk);
+
+      if (C.empty()) {
+        C = blk_t(btas::Range(A.range().extent(0), B.range().extent(1)), 0.0);
+      }
+
+      /* pull all buffers onto the device */
+      co_await ttg::to_device(A.b, B.b, C.b);
+
+      /* everything is on the device, call the gemm */
+      cuda_gemm(C, A, B);
+
+      /* compute next k while the kernel is running */
+      std::tie(next_k, have_next_k) = compute_next_k(i, j, k, p);
+      ttg::trace("Rank ", ttg::default_execution_context().rank(),
+                 " :"
+                 " C[",
+                 i, "][", j, "]  += A[", i, "][", k, "] by B[", k, "][", j, "],  next_k? ",
+                 (have_next_k ? std::to_string(next_k) : "does not exist"));
+
+      /* wait for the kernel to complete */
+
+      co_await ttg::wait_kernel();
+
+
+      // compute the contrib, pass the running total to the next flow, if needed
+      // otherwise write to the result flow
+      if (have_next_k) {
+        co_await ttg::device::forward(ttg::device::send<1>(
+                                                Key<3>({i, j, next_k}),
+                                                std::move(C),
+                                                result));
+      } else {  // done with all local contributions to C[i][j], reduce with others on the process to which C[i][j]
+                // belongs
+        co_await ttg::device::forward(ttg::device::send<0>(
+                                                Key<2>({i, j}),
+                                                std::move(C),
+                                                result));
+      }
+    }
+
+   private:
+    const std::vector<std::vector<long>> &a_rowidx_to_colidx_;
+    const std::vector<std::vector<long>> &b_colidx_to_rowidx_;
+
+    /* Compute the length of the remaining sequence on that tile */
+    int32_t prio(const Key<3> &key) const {
+      const auto i = key[0];
+      const auto j = key[1];
+      const auto k = key[2];
+      int32_t len = -1;  // will be incremented at least once
+      long next_k = k;
+      bool have_next_k;
+      do {
+        std::tie(next_k, have_next_k) = compute_next_k(i, j, next_k);  // here I know how many 'k' I have with same ij
+        ++len;
+      } while (have_next_k);
+      return len;
+    }
+
+   public:  // to be able to reuse this logic in SpMM25D
+    // given {i,j} return first k such that A[i][k] and B[k][j] exist
+    std::tuple<long, bool> compute_first_k(long i, long j) const {
+      const auto &a_k_range = a_rowidx_to_colidx_.at(i);
+      auto a_iter = a_k_range.begin();
+      auto a_iter_fence = a_k_range.end();
+      if (a_iter == a_iter_fence) return std::make_tuple(-1, false);
+      const auto &b_k_range = b_colidx_to_rowidx_.at(j);
+      auto b_iter = b_k_range.begin();
+      auto b_iter_fence = b_k_range.end();
+      if (b_iter == b_iter_fence) return std::make_tuple(-1, false);
+
+      {
+        auto a_colidx = *a_iter;  // pointing to next kth element
+        auto b_rowidx = *b_iter;
+        while (a_colidx != b_rowidx) {
+          if (a_colidx < b_rowidx) {
+            ++a_iter;
+            if (a_iter == a_iter_fence) return std::make_tuple(-1, false);
+            a_colidx = *a_iter;
+          } else {
+            ++b_iter;
+            if (b_iter == b_iter_fence) return std::make_tuple(-1, false);
+            b_rowidx = *b_iter;
+          }
+        }
+        return std::make_tuple(a_colidx, true);  // returned true for kth element exist and also returns next k since
+                                                 // a_colidx points to ++a_iter,  if not reaches to fence
+      }
+      assert(false);
+    }
+
+    // given {i,j,k} such that A[i][k] and B[k][j] exist
+    // return next k such that this condition holds
+    std::tuple<long, bool> compute_next_k(long i, long j, long k) const {
+      const auto &a_k_range = a_rowidx_to_colidx_.at(i);
+      auto a_iter_fence = a_k_range.end();
+      auto a_iter = std::find(a_k_range.begin(), a_iter_fence, k);
+      assert(a_iter != a_iter_fence);
+      const auto &b_k_range = b_colidx_to_rowidx_.at(j);
+      auto b_iter_fence = b_k_range.end();
+      auto b_iter = std::find(b_k_range.begin(), b_iter_fence, k);
+      assert(b_iter != b_iter_fence);
+      while (a_iter != a_iter_fence && b_iter != b_iter_fence) {
+        ++a_iter;
+        ++b_iter;
+        if (a_iter == a_iter_fence || b_iter == b_iter_fence) return std::make_tuple(-1, false);
+        auto a_colidx = *a_iter;
+        auto b_rowidx = *b_iter;
+        while (a_colidx != b_rowidx) {
+          if (a_colidx < b_rowidx) {
+            ++a_iter;
+            if (a_iter == a_iter_fence) return std::make_tuple(-1, false);
+            a_colidx = *a_iter;
+          } else {
+            ++b_iter;
+            if (b_iter == b_iter_fence) return std::make_tuple(-1, false);
+            b_rowidx = *b_iter;
+          }
+        }
+        return std::make_tuple(a_colidx, true);
+      }
+      ttg::abort();  // unreachable
+      return std::make_tuple(0, false);
+    }
+
+    // given {i,j} return first k such that A[i][k] and B[k][j] exist AND ijk_keymap_(i,j,k) == p
+    std::tuple<long, bool> compute_first_k(long i, long j, long p) const {
+      long first_k = 0;
+      bool have_k = false;
+      std::tie(first_k, have_k) = compute_first_k(i, j);
+      while (have_k) {
+        if (this->get_keymap()(Key<3>{i, j, first_k}) == p)
+          return {first_k, true};
+        else
+          std::tie(first_k, have_k) = compute_next_k(i, j, first_k);
+      }
+      return {0, false};
+    }
+
+    // given {i,j,k} such that A[i][k] and B[k][j] exist
+    // return next k such that this condition holds AND ijk_keymap_(i,j,k) == p
+    std::tuple<long, bool> compute_next_k(long i, long j, long k, long p) const {
+      long next_k = 0;
+      bool have_k = false;
+      std::tie(next_k, have_k) = compute_next_k(i, j, k);
+      while (have_k) {
+        if (this->get_keymap()(Key<3>{i, j, next_k}) == p)
+          return {next_k, true};
+        else
+          std::tie(next_k, have_k) = compute_next_k(i, j, next_k);
+      }
+      return {0, false};
+    }
+
+  };  // MultiplyAdd
+
+  /// reduces contributions to `C[i][j]` produced on different layers of the 3-d process grid
+  class ReduceC : public TT<Key<2>, std::tuple<Out<Key<2>, Blk>>, ReduceC, ttg::typelist<Blk>> {
+   public:
+    using baseT = typename ReduceC::ttT;
+
+    ReduceC(Edge<Key<2>, Blk> &c_ij_p, Edge<Key<2>, Blk> &c_ij, const Keymap2 &ij_keymap)
+        : baseT(edges(c_ij_p), edges(c_ij), "SpMM25D::reduce_c", {"c_ij(p)"}, {"c_ij"}, ij_keymap) {}
+
+    void op(const Key<2> &ij, typename baseT::input_refs_tuple_type &&c_ij_p, std::tuple<Out<Key<2>, Blk>> &c_ij) {
+      ttg::trace("ReduceC(", ij[0], ", ", ij[1], ")");
+      ::send<0>(ij, std::move(baseT::template get<0>(c_ij_p)), c_ij);
+    }
+  };  // class ReduceC
+
+ private:
+  Edge<Key<3>, Blk> a_ijk_;
+  Edge<Key<3>, Blk> local_a_ijk_;
+  Edge<Key<3>, Blk> b_ijk_;
+  Edge<Key<3>, Blk> local_b_ijk_;
+  Edge<Key<3>, Blk> c_ijk_;
+  Edge<Key<2>, Blk> c_ij_p_;
+  const std::vector<std::vector<long>> &a_rowidx_to_colidx_;
+  const std::vector<std::vector<long>> &b_colidx_to_rowidx_;
+  const std::vector<std::vector<long>> &a_colidx_to_rowidx_;
+  const std::vector<std::vector<long>> &b_rowidx_to_colidx_;
+  std::unique_ptr<BcastA> bcast_a_;
+  std::unique_ptr<LocalBcastA> local_bcast_a_;
+  std::unique_ptr<BcastB> bcast_b_;
+  std::unique_ptr<LocalBcastB> local_bcast_b_;
+  std::unique_ptr<MultiplyAdd> multiplyadd_;
+  std::unique_ptr<ReduceC> reduce_c_;
+  Keymap2 ij_keymap_;
+  Keymap3 ijk_keymap_;
+};
+
+class Control : public TT<void, std::tuple<Out<Key<2>>>, Control> {
+  using baseT = typename Control::ttT;
+  int P;
+  int Q;
+
+ public:
+  explicit Control(Edge<Key<2>> &ctl) : baseT(edges(), edges(ctl), "Control", {}, {"ctl"}), P(0), Q(0) {}
+
+  void op(std::tuple<Out<Key<2>>> &out) const {
+    for (int p = 0; p < P; p++) {
+      for (int q = 0; q < Q; q++) {
+        ttg::trace("Control: start computing on process {", p, ", ", q, "}");
+        ::sendk<0>(Key<2>{p, q}, out);
+      }
+    }
+  }
+
+  void start(const int _p, const int _q) {
+    P = _p;
+    Q = _q;
+    invoke();
+  }
+};
+
+#ifdef BTAS_IS_USABLE
+template <typename T_, class Range_, class Store_>
+std::tuple<T_, T_> norms(const btas::Tensor<T_, Range_, Store_> &t) {
+  T_ norm_2_square = 0.0;
+  T_ norm_inf = 0.0;
+  for (auto k : t) {
+    norm_2_square += k * k;
+    norm_inf = std::max(norm_inf, std::abs(k));
+  }
+  return std::make_tuple(norm_2_square, norm_inf);
+}
+#endif
+
+std::tuple<double, double> norms(double t) { return std::make_tuple(t * t, std::abs(t)); }
+
+template <typename Blk = blk_t>
+std::tuple<double, double> norms(const SpMatrix<Blk> &A) {
+  double norm_2_square = 0.0;
+  double norm_inf = 0.0;
+  for (int i = 0; i < A.outerSize(); ++i) {
+    for (typename SpMatrix<Blk>::InnerIterator it(A, i); it; ++it) {
+      //  cout << 1+it.row() << "\t"; // row index
+      //  cout << 1+it.col() << "\t"; // col index (here it is equal to k)
+      //  cout << it.value() << endl;
+      auto& elem = it.value();
+      double elem_norm_2_square, elem_norm_inf;
+      std::tie(elem_norm_2_square, elem_norm_inf) = norms(elem);
+      norm_2_square += elem_norm_2_square;
+      norm_inf = std::max(norm_inf, elem_norm_inf);
+    }
+  }
+  return std::make_tuple(norm_2_square, norm_inf);
+}
+
+char *getCmdOption(char **begin, char **end, const std::string &option) {
+  static char *empty = (char *)"";
+  char **itr = std::find(begin, end, option);
+  if (itr != end && ++itr != end) return *itr;
+  return empty;
+}
+
+bool cmdOptionExists(char **begin, char **end, const std::string &option) {
+  return std::find(begin, end, option) != end;
+}
+
+int cmdOptionIndex(char **begin, char **end, const std::string &option) {
+  char **itr = std::find(begin, end, option);
+  if (itr != end) return (int)(itr - begin);
+  return -1;
+}
+
+static int parseOption(std::string &option, int default_value) {
+  size_t pos;
+  std::string token;
+  int N = default_value;
+  if (option.length() == 0) return N;
+  pos = option.find(':');
+  if (pos == std::string::npos) {
+    pos = option.length();
+  }
+  token = option.substr(0, pos);
+  N = std::stoi(token);
+  option.erase(0, pos + 1);
+  return N;
+}
+
+static long parseOption(std::string &option, long default_value) {
+  size_t pos;
+  std::string token;
+  long N = default_value;
+  if (option.length() == 0) return N;
+  pos = option.find(':');
+  if (pos == std::string::npos) {
+    pos = option.length();
+  }
+  token = option.substr(0, pos);
+  N = std::stol(token);
+  option.erase(0, pos + 1);
+  return N;
+}
+
+static double parseOption(std::string &option, double default_value = 0.25) {
+  size_t pos;
+  std::string token;
+  double N = default_value;
+  if (option.length() == 0) return N;
+  pos = option.find(':');
+  if (pos == std::string::npos) {
+    pos = option.length();
+  }
+  token = option.substr(0, pos);
+  N = std::stod(token);
+  option.erase(0, pos + 1);
+  return N;
+}
+
+#if !defined(BLOCK_SPARSE_GEMM)
+static void initSpMatrixMarket(const std::function<int(const Key<2> &)> &keymap, const char *filename, SpMatrix<> &A,
+                               SpMatrix<> &B, SpMatrix<> &C, int &M, int &N, int &K) {
+  std::vector<int> sizes;
+  // We load the entire matrix on each rank, but we only use the local part for the GEMM
+  // loadMarket() is the eigan fuction to load matrix from a file
+  if (!loadMarket(A, filename)) {
+    std::cerr << "Failed to load " << filename << ", bailing out..." << std::endl;
+    ttg::ttg_abort();
+  }
+  if (0 == ttg::default_execution_context().rank()) {
+    std::cout << "##MatrixMarket file " << filename << " -- " << A.rows() << " x " << A.cols() << " -- " << A.nonZeros()
+              << " nnz (density: " << (float)A.nonZeros() / (float)A.rows() / (float)A.cols() << ")" << std::endl;
+  }
+  if (A.rows() != A.cols()) {
+    B = A.transpose();
+  } else {
+    B = A;
+  }
+
+  C.resize(A.rows(), B.cols());
+  M = (int)A.rows();
+  N = (int)C.cols();
+  K = (int)A.cols();
+}
+
+static void initSpRmat(const std::function<int(const Key<2> &)> &keymap, const char *opt, SpMatrix<> &A, SpMatrix<> &B,
+                       SpMatrix<> &C, int &M, int &N, int &K, unsigned long seed) {
+  int E;
+  double a = 0.25, b = 0.25, c = 0.25, d = 0.25;
+  size_t nnz = 0;
+
+  if (nullptr == opt) {
+    std::cerr << "Usage: -rmat <#nodes>[:<#edges>[:<a>[:<b>:[<c>[:<d>]]]]]" << std::endl;
+    exit(1);
+  }
+  std::string token;
+  std::string option = std::string(opt);
+  N = parseOption(option, -1);
+  K = N;
+  M = N;
+
+  // We build the entire sparse matrix on each rank, but use only the local part
+  // on a given rank, according to keymap
+  A.resize(N, N);
+
+  E = parseOption(option, (int)(0.01 * N * N));
+  a = parseOption(option, a);
+  b = parseOption(option, b);
+  c = parseOption(option, c);
+  d = parseOption(option, d);
+
+  if (ttg::default_execution_context().rank() == 0) {
+    std::cout << "#R-MAT: " << N << " nodes, " << E << " edges, a/b/c/d = " << a << "/" << b << "/" << c << "/" << d
+              << std::endl;
+  }
+
+  boost::minstd_rand gen(seed);
+  boost::rmat_iterator<boost::minstd_rand, boost::directed_graph<>> rmat_it(gen, N, E, a, b, c, d);
+
+  using triplet_t = Eigen::Triplet<blk_t>;
+  std::vector<triplet_t> A_elements;
+  for (int i = 0; i < N; i++) {
+    nnz++;
+    A_elements.emplace_back(i, i, 1.0);
+  }
+  for (int i = 0; i < E; i++) {
+    auto x = *rmat_it++;
+    if (x.first != x.second) {
+      A_elements.emplace_back(x.first, x.second, 1.0);
+      nnz++;
+    }
+  }
+  A.setFromTriplets(A_elements.begin(), A_elements.end());
+
+  B = A;
+  C.resize(N, N);
+
+  if (ttg::default_execution_context().rank() == 0) {
+    std::cout << "#R-MAT: " << E << " nonzero elements, density: " << (double)nnz / (double)N / (double)N << std::endl;
+  }
+}
+
+static void initSpHardCoded(const std::function<int(const Key<2> &)> &keymap, SpMatrix<> &A, SpMatrix<> &B,
+                            SpMatrix<> &C, int &m, int &n, int &k) {
+  m = 2;
+  n = 3;
+  k = 4;
+
+  std::cout << "#HardCoded A, B, C" << std::endl;
+  A.resize(m, k);
+  B.resize(k, n);
+  C.resize(m, n);
+  // We initialize the same matrices on all the ranks, but we will use only the local part
+  // following the keymap
+  using triplet_t = Eigen::Triplet<blk_t>;
+  std::vector<triplet_t> A_elements;
+  A_elements.emplace_back(0, 1, 12.3);
+  A_elements.emplace_back(0, 2, 10.7);
+  A_elements.emplace_back(0, 3, -2.3);
+  A_elements.emplace_back(1, 0, -0.3);
+  A_elements.emplace_back(1, 2, 1.2);
+  A.setFromTriplets(A_elements.begin(), A_elements.end());
+
+  std::vector<triplet_t> B_elements;
+  B_elements.emplace_back(0, 0, 12.3);
+  B_elements.emplace_back(1, 0, 10.7);
+  B_elements.emplace_back(3, 0, -2.3);
+  B_elements.emplace_back(1, 1, -0.3);
+  B_elements.emplace_back(1, 2, 1.2);
+  B_elements.emplace_back(2, 2, 7.2);
+  B_elements.emplace_back(3, 2, 0.2);
+  B.setFromTriplets(B_elements.begin(), B_elements.end());
+}
+
+#else
+static void initBlSpHardCoded(const std::function<int(const Key<2> &)> &keymap, SpMatrix<> &A, SpMatrix<> &B,
+                              SpMatrix<> &C, SpMatrix<> &Aref, SpMatrix<> &Bref, bool buildRefs,
+                              std::vector<int> &mTiles, std::vector<int> &nTiles, std::vector<int> &kTiles,
+                              std::vector<std::vector<long>> &a_rowidx_to_colidx,
+                              std::vector<std::vector<long>> &a_colidx_to_rowidx,
+                              std::vector<std::vector<long>> &b_rowidx_to_colidx,
+                              std::vector<std::vector<long>> &b_colidx_to_rowidx, int &m, int &n, int &k) {
+  m = 2;
+  n = 3;
+  k = 4;
+
+  std::cout << "#HardCoded A, B, C" << std::endl;
+  A.resize(m, k);
+  B.resize(k, n);
+  C.resize(m, n);
+  if (buildRefs) {
+    Aref.resize(m, k);
+    Bref.resize(k, n);
+  }
+
+  for (int mt = 0; mt < m; mt++) mTiles.push_back(128);
+  for (int nt = 0; nt < n; nt++) nTiles.push_back(196);
+  for (int kt = 0; kt < k; kt++) kTiles.push_back(256);
+
+  int rank = ttg::default_execution_context().rank();
+
+  using triplet_t = Eigen::Triplet<blk_t>;
+  std::vector<triplet_t> A_elements;
+  std::vector<triplet_t> Aref_elements;
+#if defined(BTAS_IS_USABLE)
+  if (keymap({0, 1}) == rank) {
+    A_elements.emplace_back(0, 1, blk_t(btas::Range(128, 256), 12.3));
+  }
+  if (keymap({0, 2}) == rank) {
+    A_elements.emplace_back(0, 2, blk_t(btas::Range(128, 256), 10.7));
+  }
+  if (keymap({0, 3}) == rank) {
+    A_elements.emplace_back(0, 3, blk_t(btas::Range(128, 256), -2.3));
+  }
+  if (keymap({1, 0}) == rank) {
+    A_elements.emplace_back(1, 0, blk_t(btas::Range(128, 256), -0.3));
+  }
+  if (keymap({1, 2}) == rank) {
+    A_elements.emplace_back(1, 2, blk_t(btas::Range(128, 256), 1.2));
+  }
+  if (buildRefs && rank == 0) {
+    Aref_elements.emplace_back(0, 1, blk_t(btas::Range(128, 256), 12.3));
+    Aref_elements.emplace_back(0, 2, blk_t(btas::Range(128, 256), 10.7));
+    Aref_elements.emplace_back(0, 3, blk_t(btas::Range(128, 256), -2.3));
+    Aref_elements.emplace_back(1, 0, blk_t(btas::Range(128, 256), -0.3));
+    Aref_elements.emplace_back(1, 2, blk_t(btas::Range(128, 256), 1.2));
+  }
+#else
+  if ((buildRefs && rank == 0) || keymap({0, 1}) == rank) {
+    A_elements.emplace_back(0, 1, 12.3);
+  }
+  if ((buildRefs && rank == 0) || keymap({0, 2}) == rank) {
+    A_elements.emplace_back(0, 2, 10.7);
+  }
+  if ((buildRefs && rank == 0) || keymap({0, 3}) == rank) {
+    A_elements.emplace_back(0, 3, -2.3);
+  }
+  if ((buildRefs && rank == 0) || keymap({1, 0}) == rank) {
+    A_elements.emplace_back(1, 0, -0.3);
+  }
+  if ((buildRefs && rank == 0) || keymap({1, 2}) == rank) {
+    A_elements.emplace_back(1, 2, .2);
+  }
+  if (buildRefs && rank == 0) {
+    Aref_elements.emplace_back(0, 1, 12.3);
+    Aref_elements.emplace_back(0, 2, 10.7);
+    Aref_elements.emplace_back(0, 3, -2.3);
+    Aref_elements.emplace_back(1, 0, -0.3);
+    Aref_elements.emplace_back(1, 2, .2);
+  }
+#endif
+  a_rowidx_to_colidx.resize(2);
+  a_rowidx_to_colidx[0].emplace_back(1);  // A[0][1]
+  a_rowidx_to_colidx[0].emplace_back(2);  // A[0][2]
+  a_rowidx_to_colidx[0].emplace_back(3);  // A[0][3]
+  a_rowidx_to_colidx[1].emplace_back(0);  // A[1][0]
+  a_rowidx_to_colidx[1].emplace_back(2);  // A[1][2]
+
+  a_colidx_to_rowidx.resize(4);
+  a_colidx_to_rowidx[0].emplace_back(1);  // A[1][0]
+  a_colidx_to_rowidx[1].emplace_back(0);  // A[0][1]
+  a_colidx_to_rowidx[2].emplace_back(0);  // A[0][2]
+  a_colidx_to_rowidx[2].emplace_back(1);  // A[1][2]
+  a_colidx_to_rowidx[3].emplace_back(0);  // A[0][3]
+
+  A.setFromTriplets(A_elements.begin(), A_elements.end());
+  std::cout << "A_elements.begin()" << A_elements.begin() << "A_elements.end()" << A_elements.end() << "\n";
+
+  if (buildRefs && 0 == rank) {
+    Aref.setFromTriplets(Aref_elements.begin(), Aref_elements.end());
+  }
+
+  std::vector<triplet_t> B_elements;
+  std::vector<triplet_t> Bref_elements;
+#if defined(BTAS_IS_USABLE)
+  if (keymap({0, 0}) == rank) {
+    B_elements.emplace_back(0, 0, blk_t(btas::Range(256, 196), 12.3));
+  }
+  if (keymap({1, 0}) == rank) {
+    B_elements.emplace_back(1, 0, blk_t(btas::Range(256, 196), 10.7));
+  }
+  if (keymap({3, 0}) == rank) {
+    B_elements.emplace_back(3, 0, blk_t(btas::Range(256, 196), -2.3));
+  }
+  if (keymap({1, 1}) == rank) {
+    B_elements.emplace_back(1, 1, blk_t(btas::Range(256, 196), -0.3));
+  }
+  if (keymap({1, 2}) == rank) {
+    B_elements.emplace_back(1, 2, blk_t(btas::Range(256, 196), 1.2));
+  }
+  if (keymap({2, 2}) == rank) {
+    B_elements.emplace_back(2, 2, blk_t(btas::Range(256, 196), 7.2));
+  }
+  if (keymap({3, 2}) == rank) {
+    B_elements.emplace_back(3, 2, blk_t(btas::Range(256, 196), 0.2));
+  }
+  if (buildRefs && rank == 0) {
+    Bref_elements.emplace_back(0, 0, blk_t(btas::Range(256, 196), 12.3));
+    Bref_elements.emplace_back(1, 0, blk_t(btas::Range(256, 196), 10.7));
+    Bref_elements.emplace_back(3, 0, blk_t(btas::Range(256, 196), -2.3));
+    Bref_elements.emplace_back(1, 1, blk_t(btas::Range(256, 196), -0.3));
+    Bref_elements.emplace_back(1, 2, blk_t(btas::Range(256, 196), 1.2));
+    Bref_elements.emplace_back(2, 2, blk_t(btas::Range(256, 196), 7.2));
+    Bref_elements.emplace_back(3, 2, blk_t(btas::Range(256, 196), 0.2));
+  }
+#else
+  if (keymap({0, 0}) == rank) {
+    B_elements.emplace_back(0, 0, 12.3);
+  }
+  if (keymap({1, 0}) == rank) {
+    B_elements.emplace_back(1, 0, 10.7);
+  }
+  if (keymap({3, 0}) == rank) {
+    B_elements.emplace_back(3, 0, -2.3);
+  }
+  if (keymap({1, 1}) == rank) {
+    B_elements.emplace_back(1, 1, -0.3);
+  }
+  if (keymap({1, 2}) == rank) {
+    B_elements.emplace_back(1, 2, 1.2);
+  }
+  if (keymap({2, 2}) == rank) {
+    B_elements.emplace_back(2, 2, 7.2);
+  }
+  if (keymap({3, 2}) == rank) {
+    B_elements.emplace_back(3, 2, 0.2);
+  }
+#endif
+  b_rowidx_to_colidx.resize(4);
+  b_rowidx_to_colidx[0].emplace_back(0);  // B[0][0]
+  b_rowidx_to_colidx[1].emplace_back(0);  // B[1][0]
+  b_rowidx_to_colidx[1].emplace_back(1);  // B[1][1]
+  b_rowidx_to_colidx[1].emplace_back(2);  // B[1][2]
+  b_rowidx_to_colidx[2].emplace_back(2);  // B[2][2]
+  b_rowidx_to_colidx[3].emplace_back(0);  // B[3][0]
+  b_rowidx_to_colidx[3].emplace_back(2);  // B[3][2]
+
+  b_colidx_to_rowidx.resize(3);
+  b_colidx_to_rowidx[0].emplace_back(0);  // B[0][0]
+  b_colidx_to_rowidx[0].emplace_back(1);  // B[1][0]
+  b_colidx_to_rowidx[0].emplace_back(3);  // B[3][0]
+  b_colidx_to_rowidx[1].emplace_back(1);  // B[1][1]
+  b_colidx_to_rowidx[2].emplace_back(1);  // B[1][2]
+  b_colidx_to_rowidx[2].emplace_back(2);  // B[2][2]
+  b_colidx_to_rowidx[2].emplace_back(3);  // A[3][2]
+
+  B.setFromTriplets(B_elements.begin(), B_elements.end());
+  if (buildRefs && 0 == rank) {
+    Bref.setFromTriplets(Bref_elements.begin(), Bref_elements.end());
+  }
+}
+
+#if defined(BTAS_IS_USABLE)
+static void initBlSpRandom(const std::function<int(const Key<2> &)> &keymap, size_t M, size_t N, size_t K, int minTs,
+                           int maxTs, double avgDensity, SpMatrix<> &A, SpMatrix<> &B, SpMatrix<> &Aref,
+                           SpMatrix<> &Bref, bool buildRefs, std::vector<int> &mTiles, std::vector<int> &nTiles,
+                           std::vector<int> &kTiles, std::vector<std::vector<long>> &a_rowidx_to_colidx,
+                           std::vector<std::vector<long>> &a_colidx_to_rowidx,
+                           std::vector<std::vector<long>> &b_rowidx_to_colidx,
+                           std::vector<std::vector<long>> &b_colidx_to_rowidx, double &average_tile_size,
+                           double &Adensity, double &Bdensity, unsigned int seed) {
+  int rank = ttg::default_execution_context().rank();
+
+  int ts;
+  std::mt19937 gen(seed);
+  std::mt19937 genv(seed + 1);
+
+  std::uniform_int_distribution<> dist(minTs, maxTs);  // randomly pick any value in the range minTs, maxTs
+  using triplet_t = Eigen::Triplet<blk_t>;
+  std::vector<triplet_t> A_elements;
+  std::vector<triplet_t> B_elements;
+  std::vector<triplet_t> Aref_elements;
+  std::vector<triplet_t> Bref_elements;
+
+  for (int m = 0; m < M; m += ts) {
+    ts = dist(gen);
+    if (ts > M - m) ts = M - m;
+    mTiles.push_back(ts);
+  }
+  for (int n = 0; n < N; n += ts) {
+    ts = dist(gen);
+    if (ts > N - n) ts = N - n;
+    nTiles.push_back(ts);
+  }
+  for (int k = 0; k < K; k += ts) {
+    ts = dist(gen);
+    if (ts > K - k) ts = K - k;
+    kTiles.push_back(ts);
+  }
+
+  A.resize(mTiles.size(), kTiles.size());
+  B.resize(kTiles.size(), nTiles.size());
+  if (buildRefs) {
+    Aref.resize(mTiles.size(), kTiles.size());
+    Bref.resize(kTiles.size(), nTiles.size());
+  }
+
+  std::uniform_int_distribution<> mDist(0, mTiles.size() - 1);
+  std::uniform_int_distribution<> nDist(0, nTiles.size() - 1);
+  std::uniform_int_distribution<> kDist(0, kTiles.size() - 1);
+  std::uniform_real_distribution<> vDist(-1.0, 1.0);
+
+  size_t filling = 0;
+  size_t avg_nb = 0;
+  int avg_nb_nb = 0;
+
+  struct tuple_hash : public std::unary_function<std::tuple<int, int>, std::size_t> {
+    std::size_t operator()(const std::tuple<int, int> &k) const {
+      return static_cast<size_t>(std::get<0>(k)) | (static_cast<size_t>(std::get<1>(k)) << 32);
+    }
+  };
+
+  std::unordered_set<std::tuple<int, int>, tuple_hash> fills;
+
+  fills.clear();
+  while ((double)filling / (double)(M * K) < avgDensity) {
+    int mt = mDist(gen);
+    int kt = kDist(gen);
+
+    if (fills.find({mt, kt}) != fills.end()) continue;
+    fills.insert({mt, kt});
+
+    if (mt >= a_rowidx_to_colidx.size()) a_rowidx_to_colidx.resize(mt + 1);
+    a_rowidx_to_colidx[mt].emplace_back(kt);
+    if (kt >= a_colidx_to_rowidx.size()) a_colidx_to_rowidx.resize(kt + 1);
+    a_colidx_to_rowidx[kt].emplace_back(mt);
+
+    filling += mTiles[mt] * kTiles[kt];
+    avg_nb += mTiles[mt] * kTiles[kt];
+    avg_nb_nb++;
+    double value = vDist(genv);
+    if (0 == rank && buildRefs) Aref_elements.emplace_back(mt, kt, blk_t(btas::Range(mTiles[mt], kTiles[kt]), value));
+    if (rank != keymap({mt, kt})) continue;
+    A_elements.emplace_back(mt, kt, blk_t(btas::Range(mTiles[mt], kTiles[kt]), value));
+  }
+  for (auto &row : a_rowidx_to_colidx) {
+    std::sort(row.begin(), row.end());
+  }
+  for (auto &col : a_colidx_to_rowidx) {
+    std::sort(col.begin(), col.end());
+  }
+  A.setFromTriplets(A_elements.begin(), A_elements.end());
+  Adensity = (double)filling / (double)(M * K);
+  if (0 == rank && buildRefs) Aref.setFromTriplets(Aref_elements.begin(), Aref_elements.end());
+
+  filling = 0;
+  fills.clear();
+  while ((double)filling / (double)(K * N) < avgDensity) {
+    int nt = nDist(gen);
+    int kt = kDist(gen);
+
+    if (fills.find({kt, nt}) != fills.end()) continue;
+    fills.insert({kt, nt});
+
+    if (kt >= b_rowidx_to_colidx.size()) b_rowidx_to_colidx.resize(kt + 1);
+    b_rowidx_to_colidx[kt].emplace_back(nt);
+    if (nt >= b_colidx_to_rowidx.size()) b_colidx_to_rowidx.resize(nt + 1);
+    b_colidx_to_rowidx[nt].emplace_back(kt);
+
+    filling += kTiles[kt] * nTiles[nt];
+    avg_nb += kTiles[kt] * nTiles[nt];
+    avg_nb_nb++;
+    double value = vDist(genv);
+    if (0 == rank && buildRefs) Bref_elements.emplace_back(kt, nt, blk_t(btas::Range(kTiles[kt], nTiles[nt]), value));
+    if (rank != keymap({kt, nt})) continue;
+    B_elements.emplace_back(kt, nt, blk_t(btas::Range(kTiles[kt], nTiles[nt]), value));
+  }
+  for (auto &row : b_rowidx_to_colidx) {
+    std::sort(row.begin(), row.end());
+  }
+  for (auto &col : b_colidx_to_rowidx) {
+    std::sort(col.begin(), col.end());
+  }
+  B.setFromTriplets(B_elements.begin(), B_elements.end());
+  Bdensity = (double)filling / (double)(K * N);
+  if (0 == rank && buildRefs) Bref.setFromTriplets(Bref_elements.begin(), Bref_elements.end());
+  fills.clear();
+
+  average_tile_size = (double)avg_nb / avg_nb_nb;
+}
+#endif
+
+#endif
+
+static void timed_measurement(SpMatrix<> &A, SpMatrix<> &B, const std::function<int(const Key<2> &)> &ij_keymap,
+                              const std::function<int(const Key<3> &)> &ijk_keymap, const std::string &tiling_type,
+                              double gflops, double avg_nb, double Adensity, double Bdensity,
+                              const std::vector<std::vector<long>> &a_rowidx_to_colidx,
+                              const std::vector<std::vector<long>> &a_colidx_to_rowidx,
+                              const std::vector<std::vector<long>> &b_rowidx_to_colidx,
+                              const std::vector<std::vector<long>> &b_colidx_to_rowidx, std::vector<int> &mTiles,
+                              std::vector<int> &nTiles, std::vector<int> &kTiles, int M, int N, int K, int minTs,
+                              int maxTs, int P, int Q, int R) {
+  int MT = (int)A.rows();
+  int NT = (int)B.cols();
+  int KT = (int)A.cols();
+  assert(KT == B.rows());
+
+  SpMatrix<> C;
+  C.resize(MT, NT);
+
+  // flow graph needs to exist on every node
+  Edge<Key<2>> ctl("control");
+  Control control(ctl);
+  Edge<Key<2>, blk_t> eA, eB;
+  Edge<Key<2>, blk_t> eC;
+
+  Read_SpMatrix a("A", A, ctl, eA, ij_keymap);
+  Read_SpMatrix b("B", B, ctl, eB, ij_keymap);
+  Write_SpMatrix<> c(C, eC, ij_keymap);
+  auto &c_status = c.status();
+  assert(!has_value(c_status));
+  //  SpMM25D a_times_b(world, eA, eB, eC, A, B);
+  SpMM25D<> a_times_b(eA, eB, eC, A, B, a_rowidx_to_colidx, a_colidx_to_rowidx, b_rowidx_to_colidx, b_colidx_to_rowidx,
+                      mTiles, nTiles, kTiles, ij_keymap, ijk_keymap, R);
+  TTGUNUSED(a);
+  TTGUNUSED(b);
+  TTGUNUSED(a_times_b);
+
+  auto connected = make_graph_executable(&control);
+  assert(connected);
+  TTGUNUSED(connected);
+
+  struct timeval start {
+    0
+  }, end{0}, diff{0};
+  gettimeofday(&start, nullptr);
+  // ready, go! need only 1 kick, so must be done by 1 thread only
+  if (ttg::default_execution_context().rank() == 0) control.start(P, Q);
+  fence();
+  gettimeofday(&end, nullptr);
+  timersub(&end, &start, &diff);
+  double tc = (double)diff.tv_sec + (double)diff.tv_usec / 1e6;
+#if defined(TTG_USE_MADNESS)
+  std::string rt("MAD");
+#elif defined(TTG_USE_PARSEC)
+  std::string rt("PARSEC");
+#else
+  std::string rt("Unkown???");
+#endif
+  if (ttg::default_execution_context().rank() == 0) {
+    std::cout << "TTG-" << rt << " PxQxR=   " << P << " " << Q << " " << R << " 1 average_NB= " << avg_nb << " M= " << M
+              << " N= " << N << " K= " << K << " t= " << minTs << " T=" << maxTs << " Tiling= " << tiling_type
+              << " A_density= " << Adensity << " B_density= " << Bdensity << " gflops= " << gflops << " seconds= " << tc
+              << " gflops/s= " << gflops / tc << std::endl;
+  }
+}
+
+#if !defined(BLOCK_SPARSE_GEMM)
+static void make_rowidx_to_colidx_from_eigen(const SpMatrix<> &mat, std::vector<std::vector<long>> &r2c) {
+  for (int k = 0; k < mat.outerSize(); ++k) {  // cols, if col-major, rows otherwise
+    for (typename SpMatrix<blk_t>::InnerIterator it(mat, k); it; ++it) {
+      const long row = it.row();
+      const long col = it.col();
+      if (row >= r2c.size()) r2c.resize(row + 1);
+      r2c[row].push_back(col);
+    }
+  }
+  // Sort each vector of column indices, as we pushed them in an arbitrary order
+  for (auto &row : r2c) {
+    std::sort(row.begin(), row.end());
+  }
+}
+
+static void make_colidx_to_rowidx_from_eigen(const SpMatrix<> &mat, std::vector<std::vector<long>> &c2r) {
+  for (int k = 0; k < mat.outerSize(); ++k) {  // cols, if col-major, rows otherwise
+    for (typename SpMatrix<blk_t>::InnerIterator it(mat, k); it; ++it) {
+      const long row = it.row();
+      const long col = it.col();
+
+      if (col >= c2r.size()) c2r.resize(col + 1);
+      c2r[col].push_back(row);
+    }
+    // Sort each vector of row indices, as we pushed them in an arbitrary order
+    for (auto &col : c2r) {
+      std::sort(col.begin(), col.end());
+    }
+  }
+}
+#endif
+
+static double compute_gflops(const std::vector<std::vector<long>> &a_r2c, const std::vector<std::vector<long>> &b_r2c,
+                             const std::vector<int> &mTiles, const std::vector<int> &nTiles,
+                             const std::vector<int> &kTiles) {
+  unsigned long flops = 0;
+  for (auto i = 0; i < a_r2c.size(); i++) {
+    for (auto kk = 0; kk < a_r2c[i].size(); kk++) {
+      auto k = a_r2c[i][kk];
+      if (k > b_r2c.size()) continue;
+      for (auto jj = 0; jj < b_r2c[k].size(); jj++) {
+        auto j = b_r2c[k][jj];
+        flops += static_cast<long>(mTiles[i]) * nTiles[j] * kTiles[k];
+      }
+    }
+  }
+  return 2.0 * (double)flops / 1e9;
+}
+
+int main(int argc, char **argv) {
+  bool timing;
+  double gflops;
+
+  my_cublas_init_because_cublas_is_stupid();
+
+  // warm up silicon by calling gemm a few times
+#ifdef BTAS_IS_USABLE
+  for (int i = 0; i < 20; i++) {
+    using baseT = typename btas::Tensor<double>;
+    btas::Tensor<double, btas::Range, std::vector<double>> At(30, 30);
+    btas::Tensor<double, btas::Range, std::vector<double>> Bt(30, 30);
+    btas::Tensor<double, btas::Range, std::vector<double>> Ct(30, 30);
+    At.fill(1.0);
+    Bt.fill(2.0);
+    Ct.fill(3.0);
+    btas::gemm(std::move(Ct), Bt, At);
+  }
+#endif  // BTAS_IS_USABLE
+
+  int cores = -1;
+  std::string nbCoreStr(getCmdOption(argv, argv + argc, "-c"));
+  cores = parseOption(nbCoreStr, cores);
+
+  if (int dashdash = cmdOptionIndex(argv, argv + argc, "--") > -1) {
+    initialize(argc - dashdash, argv + dashdash, cores);
+  } else {
+    initialize(1, argv, cores);
+  }
+
+  std::string debugStr(getCmdOption(argv, argv + argc, "-d"));
+  auto debug = (unsigned int)parseOption(debugStr, 0);
+
+  if (debug & (1 << 1)) {
+    using ttg::Debugger;
+    auto debugger = std::make_shared<Debugger>();
+    Debugger::set_default_debugger(debugger);
+    debugger->set_exec(argv[0]);
+    debugger->set_prefix(ttg::default_execution_context().rank());
+    // debugger->set_cmd("lldb_xterm");
+    debugger->set_cmd("gdb_xterm");
+  }
+
+  int mpi_size = ttg::default_execution_context().size();
+  int mpi_rank = ttg::default_execution_context().rank();
+  int best_pqc = mpi_size;
+  int P, Q, R;
+  for (int c = 1; c <= (int)cbrt(mpi_size); c++) {
+    for (int p = 1; p <= (int)sqrt(mpi_size / c); p++) {
+      if ((mpi_size % (p * c)) == 0) {
+        int q = mpi_size / (p * c);
+        if (abs(c - p - q) <= best_pqc) {
+          best_pqc = abs(c - p - q);
+          P = p;
+          Q = q;
+          R = c;
+        }
+      }
+    }
+    // ttg::launch_lldb(ttg::default_execution_context().rank(), argv[0]);
+
+    {
+      if (debug & (1 << 0)) {
+        ttg::trace_on();
+        TTBase::set_trace_all(true);
+      }
+
+      SpMatrix<> A, B, C, Aref, Bref;
+      std::string tiling_type;
+      int M = 0, N = 0, K = 0;
+      int minTs = 0, maxTs = 0;
+
+      double avg_nb = nan("undefined");
+      double Adensity = nan("undefined");
+      double Bdensity = nan("undefined");
+
+      std::string PStr(getCmdOption(argv, argv + argc, "-P"));
+      P = parseOption(PStr, P);
+      std::string QStr(getCmdOption(argv, argv + argc, "-Q"));
+      Q = parseOption(QStr, Q);
+      // to make code behave like 2D summa if R not given
+      std::string RStr(getCmdOption(argv, argv + argc, "-R"));
+      R = parseOption(RStr, 1);
+
+      if (P * Q * R != mpi_size) {
+        if (!cmdOptionExists(argv, argv + argc, "-Q") && (mpi_size % (P * R) == 0))
+          Q = mpi_size / (P * R);
+        else if (!cmdOptionExists(argv, argv + argc, "-P") && (mpi_size % (Q * R)) == 0)
+          P = mpi_size / (Q * R);
+        else if (!cmdOptionExists(argv, argv + argc, "-R") && (mpi_size % (Q * P)) == 0)
+          R = mpi_size / (Q * P);
+        else {
+          if (0 == mpi_rank) {
+            std::cerr << P << "x" << Q << "x" << R << " is not a valid process grid -- bailing out" << std::endl;
+            MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE);
+          }
+        }
+      }
+
+      auto ij_keymap = [P, Q](const Key<2> &ij) {
+        int i = (int)ij[0];
+        int j = (int)ij[1];
+        int r = ij2rank(i, j, P, Q);
+        return r;
+      };
+
+      auto ijk_keymap = [P, Q, R](const Key<3> &ijk) {
+        int i = (int)ijk[0];
+        int j = (int)ijk[1];
+        int k = (int)ijk[2];
+        int r = ijk2rank(i, j, k, P, Q, R);
+        return r;
+      };
+
+      std::string seedStr(getCmdOption(argv, argv + argc, "-s"));
+      unsigned int seed = parseOption(seedStr, 0);
+      if (seed == 0) {
+        std::random_device rd;
+        seed = rd();
+        if (0 == ttg::default_execution_context().rank()) std::cerr << "#Random seeded with " << seed << std::endl;
+      }
+      ttg_broadcast(ttg::default_execution_context(), seed, 0);
+
+      std::vector<int> mTiles;
+      std::vector<int> nTiles;
+      std::vector<int> kTiles;
+      std::vector<std::vector<long>> a_rowidx_to_colidx;
+      std::vector<std::vector<long>> a_colidx_to_rowidx;
+      std::vector<std::vector<long>> b_rowidx_to_colidx;
+      std::vector<std::vector<long>> b_colidx_to_rowidx;
+
+      std::string checkStr(getCmdOption(argv, argv + argc, "-x"));
+      int check = parseOption(checkStr, !(argc >= 2));
+      timing = (check == 0);
+
+#if !defined(BLOCK_SPARSE_GEMM)
+      if (cmdOptionExists(argv, argv + argc, "-mm")) {
+        char *filename = getCmdOption(argv, argv + argc, "-mm");
+        tiling_type = filename;
+        initSpMatrixMarket(ij_keymap, filename, A, B, C, M, N, K);
+      } else if (cmdOptionExists(argv, argv + argc, "-rmat")) {
+        char *opt = getCmdOption(argv, argv + argc, "-rmat");
+        tiling_type = "RandomSparseMatrix";
+        initSpRmat(ij_keymap, opt, A, B, C, M, N, K, seed);
+      } else {
+        tiling_type = "HardCodedSparseMatrix";
+        initSpHardCoded(ij_keymap, A, B, C, M, N, K);
+      }
+
+      if (check) {
+        // We don't generate the sparse matrices in distributed, so Aref and Bref can
+        // just point to the same matrix, or be a local copy.
+        Aref = A;
+        Bref = B;
+      }
+
+      // We still need to build the metadata from the  matrices.
+      make_rowidx_to_colidx_from_eigen(A, a_rowidx_to_colidx);
+      make_colidx_to_rowidx_from_eigen(A, a_colidx_to_rowidx);
+      make_rowidx_to_colidx_from_eigen(B, b_rowidx_to_colidx);
+      make_colidx_to_rowidx_from_eigen(B, b_colidx_to_rowidx);
+      // This is only needed to compute the flops
+      for (int mt = 0; mt < M; mt++) mTiles.emplace_back(1);
+      for (int nt = 0; nt < N; nt++) nTiles.emplace_back(1);
+      for (int kt = 0; kt < K; kt++) kTiles.emplace_back(1);
+#else
+      if (argc >= 2) {
+        std::string Mstr(getCmdOption(argv, argv + argc, "-M"));
+        M = parseOption(Mstr, 1200);
+        std::string Nstr(getCmdOption(argv, argv + argc, "-N"));
+        N = parseOption(Nstr, 1200);
+        std::string Kstr(getCmdOption(argv, argv + argc, "-K"));
+        K = parseOption(Kstr, 1200);
+        std::string minTsStr(getCmdOption(argv, argv + argc, "-t"));
+        minTs = parseOption(minTsStr, 32);
+        std::string maxTsStr(getCmdOption(argv, argv + argc, "-T"));
+        maxTs = parseOption(maxTsStr, 256);
+        std::string avgStr(getCmdOption(argv, argv + argc, "-a"));
+        double avg = parseOption(avgStr, 0.3);
+        timing = (check == 0);
+        tiling_type = "RandomIrregularTiling";
+        initBlSpRandom(ij_keymap, M, N, K, minTs, maxTs, avg, A, B, Aref, Bref, check, mTiles, nTiles, kTiles,
+                       a_rowidx_to_colidx, a_colidx_to_rowidx, b_rowidx_to_colidx, b_colidx_to_rowidx, avg_nb, Adensity,
+                       Bdensity, seed);
+
+        C.resize(mTiles.size(), nTiles.size());
+      } else {
+        tiling_type = "HardCodedBlockSparseMatrix";
+        initBlSpHardCoded(ij_keymap, A, B, C, Aref, Bref, true, mTiles, nTiles, kTiles, a_rowidx_to_colidx,
+                          a_colidx_to_rowidx, b_rowidx_to_colidx, b_colidx_to_rowidx, M, N, K);
+      }
+#endif  // !defined(BLOCK_SPARSE_GEMM)
+
+      gflops = compute_gflops(a_rowidx_to_colidx, b_rowidx_to_colidx, mTiles, nTiles, kTiles);
+
+      std::string nbrunStr(getCmdOption(argv, argv + argc, "-n"));
+      int nb_runs = parseOption(nbrunStr, 1);
+
+      if (timing) {
+        // Start up engine
+        execute();
+        for (int nrun = 0; nrun < nb_runs; nrun++) {
+          timed_measurement(A, B, ij_keymap, ijk_keymap, tiling_type, gflops, avg_nb, Adensity, Bdensity,
+                            a_rowidx_to_colidx, a_colidx_to_rowidx, b_rowidx_to_colidx, b_colidx_to_rowidx, mTiles,
+                            nTiles, kTiles, M, N, K, minTs, maxTs, P, Q, R);
+        }
+      } else {
+        // flow graph needs to exist on every node
+        // N.B. to validate C we need it on node 0!
+        auto keymap_write = [](const Key<2> &key) { return 0; };
+        Edge<Key<2>> ctl("control");
+        Control control(ctl);
+        Edge<Key<2>, blk_t> eA, eB, eC;
+        Read_SpMatrix a("A", A, ctl, eA, ij_keymap);
+        Read_SpMatrix b("B", B, ctl, eB, ij_keymap);
+        Write_SpMatrix<> c(C, eC, keymap_write);
+        auto &c_status = c.status();
+        assert(!has_value(c_status));
+        //  SpMM25D a_times_b(world, eA, eB, eC, A, B);
+        SpMM25D<> a_times_b(eA, eB, eC, A, B, a_rowidx_to_colidx, a_colidx_to_rowidx, b_rowidx_to_colidx,
+                            b_colidx_to_rowidx, mTiles, nTiles, kTiles, ij_keymap, ijk_keymap, R);
+        TTGUNUSED(a_times_b);
+        // calling the Dot constructor with 'true' argument disables the type
+        if (default_execution_context().rank() == 0) std::cout << Dot{/*disable_type=*/true}(&control) << std::endl;
+
+        // ready to run!
+        auto connected = make_graph_executable(&control);
+        assert(connected);
+        TTGUNUSED(connected);
+
+        // ready, go! need only 1 kick, so must be done by 1 thread only
+        if (ttg::default_execution_context().rank() == 0) control.start(P, Q);
+
+        execute();
+        fence();
+
+        // validate C=A*B against the reference output
+        assert(has_value(c_status));
+        if (ttg::default_execution_context().rank() == 0) {
+          SpMatrix<> Cref = Aref * Bref;
+
+          double norm_2_square, norm_inf;
+          std::tie(norm_2_square, norm_inf) = norms<blk_t>(Cref - C);
+          std::cout << "||Cref - C||_2      = " << std::sqrt(norm_2_square) << std::endl;
+          std::cout << "||Cref - C||_\\infty = " << norm_inf << std::endl;
+          if (norm_inf > 1e-9) {
+            std::cout << "Cref:\n" << Cref << std::endl;
+            std::cout << "C:\n" << C << std::endl;
+            ttg_abort();
+          }
+        }
+
+        // validate Acopy=A against the reference output
+        //      assert(has_value(copy_status));
+        //      if (ttg::default_execution_context().rank() == 0) {
+        //        double norm_2_square, norm_inf;
+        //        std::tie(norm_2_square, norm_inf) = norms<blk_t>(Acopy - A);
+        //        std::cout << "||Acopy - A||_2      = " << std::sqrt(norm_2_square) << std::endl;
+        //        std::cout << "||Acopy - A||_\\infty = " << norm_inf << std::endl;
+        //        if (::ttg::tracing()) {
+        //          std::cout << "Acopy (" << static_cast<void *>(&Acopy) << "):\n" << Acopy << std::endl;
+        //          std::cout << "A (" << static_cast<void *>(&A) << "):\n" << A << std::endl;
+        //        }
+        //        if (norm_inf != 0) {
+        //          ttg_abort();
+        //        }
+        //      }
+      }
+    }
+
+    ttg_finalize();
+
+    my_cublas_shutdown_because_cublas_is_stupid();
+
+    return 0;
+  }
+}
diff --git a/ttg/ttg/device/send.h b/ttg/ttg/device/send.h
deleted file mode 100644
index d125c87f2..000000000
--- a/ttg/ttg/device/send.h
+++ /dev/null
@@ -1,26 +0,0 @@
-#ifndef TTG_DEVICE_SEND_H
-#define TTG_DEVICE_SEND_H
-
-
-namespace ttg {
-
-  namespace detail {
-
-    /* structure holding references to the key and value, awaitable */
-    template<typename Key, typename Value>
-    struct await_send {
-        const Key& m_key;
-        Value& m_value;
-
-        await_send(const Key& key, Value& val)
-        : m_key(key)
-        , m_value(val)
-        { }
-    };
-
-  } // namespace ttg
-
-} // namespace ttg
-
-
-#endif // TTG_DEVICE_SEND_H
\ No newline at end of file
diff --git a/ttg/ttg/device/task.h b/ttg/ttg/device/task.h
new file mode 100644
index 000000000..25bee3af3
--- /dev/null
+++ b/ttg/ttg/device/task.h
@@ -0,0 +1,642 @@
+#ifndef TTG_DEVICE_TASK_H
+#define TTG_DEVICE_TASK_H
+
+#include <array>
+#include <type_traits>
+#include <span>
+
+#include "ttg/fwd.h"
+#include "ttg/impl_selector.h"
+#include "ttg/ptr.h"
+
+namespace ttg {
+
+
+  /* yielded when waiting on a kernel to complete */
+  struct device_op_wait_kernel
+  { };
+
+  enum ttg_device_coro_state {
+    TTG_DEVICE_CORO_STATE_NONE,
+    TTG_DEVICE_CORO_INIT,
+    TTG_DEVICE_CORO_WAIT_TRANSFER,
+    TTG_DEVICE_CORO_WAIT_KERNEL,
+    TTG_DEVICE_CORO_SENDOUT,
+    TTG_DEVICE_CORO_COMPLETE
+  };
+
+
+  namespace detail {
+    template<typename... Ts>
+    struct to_device_t {
+      std::tuple<Ts&...> ties;
+    };
+  } // namespace detail
+
+  template<typename... Args>
+  [[nodiscard]]
+  inline auto to_device(Args&&... args) {
+    return detail::to_device_t{std::tie(std::forward<Args>(args)...)};
+  }
+
+  namespace detail {
+    template<typename... Ts>
+    struct wait_kernel_t {
+      std::tuple<Ts&...> ties;
+
+      /* always suspend */
+      constexpr bool await_ready() const noexcept { return false; }
+
+      /* always suspend */
+      constexpr void await_suspend( TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<> ) const noexcept {}
+
+      void await_resume() noexcept {
+        if constexpr (sizeof...(Ts) > 0) {
+          /* hook to allow the backend to handle the data after pushout */
+          TTG_IMPL_NS::post_device_out(ties);
+        }
+      }
+    };
+  } // namespace detail
+
+  /* Wait for kernel to complete and provided ttg::buffer and ttg::devicescratch
+   * to be transferred back to host */
+  template<typename... Buffers>
+  [[nodiscard]]
+  inline auto wait_kernel(Buffers&&... args) {
+    static_assert(((ttg::detail::is_buffer_v<std::decay_t<Buffers>>
+                    ||ttg::detail::is_devicescratch_v<std::decay_t<Buffers>>)&&...),
+                  "Only ttg::buffer and ttg::devicescratch can be waited on!");
+    return detail::wait_kernel_t<std::remove_reference_t<Buffers>...>{std::tie(std::forward<Buffers>(args)...)};
+  }
+
+
+  /* TODO: move all device code into ttg::device */
+  namespace device {
+
+    /******************************
+     * Send/Broadcast handling
+     * We pass the value returned by the backend's copy handler into a coroutine
+     * and execute the first part (prepare), before suspending it.
+     * The second part (send/broadcast) is executed after the task completed.
+     ******************************/
+
+    namespace detail {
+      struct send_coro_promise_type;
+
+      using send_coro_handle_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<send_coro_promise_type>;
+
+      /// task that can be resumed after some events occur
+      struct send_coro_state : public send_coro_handle_type {
+        using base_type = send_coro_handle_type;
+
+        /// these are members mandated by the promise_type concept
+        ///@{
+
+        using promise_type = send_coro_promise_type;
+
+        ///@}
+
+        send_coro_state(base_type base) : base_type(std::move(base)) {}
+
+        base_type& handle() { return *this; }
+
+        /// @return true if ready to resume
+        inline bool ready() {
+          return true;
+        }
+
+        /// @return true if task completed and can be destroyed
+        inline bool completed();
+      };
+
+      struct send_coro_promise_type {
+
+        /* do not suspend the coroutine on first invocation, we want to run
+        * the coroutine immediately and suspend only once.
+        */
+        TTG_CXX_COROUTINE_NAMESPACE::suspend_never initial_suspend() {
+          return {};
+        }
+
+        /* we don't suspend the coroutine at the end.
+        * it can be destroyed once the send/broadcast is done
+        */
+        TTG_CXX_COROUTINE_NAMESPACE::suspend_never final_suspend() noexcept {
+          return {};
+        }
+
+        send_coro_state get_return_object() { return send_coro_state{send_coro_handle_type::from_promise(*this)}; }
+
+        /* the send coros only have an empty co_await */
+        TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(ttg::Void) {
+          return {};
+        }
+
+        void unhandled_exception() {
+
+        }
+
+      };
+
+      template<typename Key, typename Value, ttg::Runtime Runtime = ttg::ttg_runtime>
+      inline send_coro_state send_coro (const Key& key, Value&& value, ttg::Out<Key, Value> &t,
+                                        ttg::detail::value_copy_handler<Runtime>& ch) {
+        ttg::detail::value_copy_handler<Runtime> copy_handler = std::move(ch); // destroyed at the end of the coro
+        Key k = key;
+        t.prepare_send(k, std::forward<Value>(value));
+        co_await ttg::Void{}; // we'll come back once the task is done
+        t.send(k, std::forward<Value>(value));
+      };
+
+      template<typename Value, ttg::Runtime Runtime = ttg::ttg_runtime>
+      inline send_coro_state sendv_coro (Value&& value, ttg::Out<void, Value> &t,
+                                         ttg::detail::value_copy_handler<Runtime>& ch) {
+        ttg::detail::value_copy_handler<Runtime> copy_handler = std::move(ch); // destroyed at the end of the coro
+        t.prepare_send(std::forward<Value>(value));
+        co_await ttg::Void{}; // we'll come back once the task is done
+        t.sendv(std::forward<Value>(value));
+      };
+    } // namespace detail
+
+
+    /* functionality to deal with sending/broadcasting */
+    namespace detail {
+      struct send_t {
+        send_coro_state coro;
+      };
+    } // namespace detail
+
+    template <size_t i, typename keyT, typename valueT, typename... out_keysT, typename... out_valuesT,
+              ttg::Runtime Runtime = ttg::ttg_runtime>
+    inline detail::send_t send(const keyT &key, valueT &&value,
+                                        std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
+      ttg::detail::value_copy_handler<Runtime> copy_handler;
+      return detail::send_t{detail::send_coro(key, copy_handler(std::forward<valueT>(value)), std::get<i>(t), copy_handler)};
+    }
+
+    template <size_t i, typename valueT, typename... out_keysT, typename... out_valuesT,
+              ttg::Runtime Runtime = ttg::ttg_runtime>
+    inline detail::send_t sendv(
+        valueT &&value, std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
+      ttg::detail::value_copy_handler<Runtime> copy_handler;
+      return detail::send_t{detail::sendv_coro(copy_handler(std::forward<valueT>(value)), std::get<i>(t), copy_handler)};
+    }
+
+
+    // clang-format off
+    /// \brief Sends a task id and a value to the template tasks attached to the output terminal of this template task
+    /// \param[in] i Identifies which output terminal of this template task to select for sending
+    /// \param[in] key: the id of the task(s) receiving the value
+    /// \param[in] value: the value to send to the receiving task(s)
+    // clang-format on
+    template <typename keyT, typename valueT, ttg::Runtime Runtime = ttg::ttg_runtime>
+    inline detail::send_t send(size_t i, const keyT &key, valueT &&value) {
+      ttg::detail::value_copy_handler<Runtime> copy_handler;
+      auto *terminal_ptr = ttg::detail::get_out_terminal<keyT, valueT>(i, "ttg::device::send(i, key, value)");
+      return detail::send_t{detail::send_coro(key, copy_handler(std::forward<valueT>(value)), *terminal_ptr, copy_handler)};
+    }
+
+    // clang-format off
+    /// \brief Sends a task id and a value to the template tasks attached to the output terminal of this template task
+    /// \note this is provided to support `send<i>` with and without explicitly-passed terminal tuple
+    /// \tparam <i> Identifies which output terminal of this template task to select for sending
+    /// \param[in] key: the id of the task(s) receiving the value
+    /// \param[in] value: the value to send to the receiving task(s)
+    // clang-format on
+    template <size_t i, typename keyT, typename valueT>
+    inline auto send(const keyT &key, valueT &&value) {
+      return send(i, key, std::forward<valueT>(value));
+    }
+
+
+
+    namespace detail {
+
+      template<typename T, typename Enabler = void>
+      struct broadcast_keylist_trait {
+        using type = T;
+      };
+
+      /* overload for iterable types that extracts the type of the first element */
+      template<typename T>
+      struct broadcast_keylist_trait<T, std::enable_if_t<ttg::meta::is_iterable_v<T>>> {
+        using key_type = decltype(*std::begin(std::get<0>(std::declval<T>())));
+      };
+
+      template <size_t KeyId, size_t I, size_t... Is, typename... RangesT, typename valueT,
+                typename... out_keysT, typename... out_valuesT>
+      inline void prepare_broadcast(const std::tuple<RangesT...> &keylists, valueT &&value,
+                                    std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
+        std::get<I>(t)->prepare_send(std::get<KeyId>(keylists), std::forward<valueT>(value));
+        if constexpr (sizeof...(Is) > 0) {
+          detail::prepare_broadcast<KeyId+1, Is...>(keylists, std::forward<valueT>(value), t);
+        }
+      }
+
+      template <size_t KeyId, size_t I, size_t... Is, typename... RangesT, typename valueT,
+                typename... out_keysT, typename... out_valuesT>
+      inline void prepare_broadcast(const std::tuple<RangesT...> &keylists, valueT &&value) {
+        using key_t = typename broadcast_keylist_trait<
+                        std::tuple_element_t<KeyId, std::tuple<std::remove_reference_t<RangesT>...>>
+                      >::key_type;
+        auto *terminal_ptr = ttg::detail::get_out_terminal<key_t, valueT>(I, "ttg::device::broadcast(keylists, value)");
+        terminal_ptr->prepare_send(std::get<KeyId>(keylists), value);
+        if constexpr (sizeof...(Is) > 0) {
+          detail::prepare_broadcast<KeyId+1, Is...>(keylists, std::forward<valueT>(value));
+        }
+      }
+
+      template <size_t KeyId, size_t I, size_t... Is, typename... RangesT, typename valueT,
+                typename... out_keysT, typename... out_valuesT>
+      inline void broadcast(const std::tuple<RangesT...> &keylists, valueT &&value,
+                                    std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
+        std::get<I>(t)->broadcast(std::get<KeyId>(keylists), std::forward<valueT>(value));
+        if constexpr (sizeof...(Is) > 0) {
+          detail::broadcast<KeyId+1, Is...>(keylists, std::forward<valueT>(value), t);
+        }
+      }
+
+      template <size_t KeyId, size_t I, size_t... Is, typename... RangesT, typename valueT,
+                typename... out_keysT, typename... out_valuesT>
+      inline void broadcast(const std::tuple<RangesT...> &keylists, valueT &&value) {
+        using key_t = typename broadcast_keylist_trait<
+                        std::tuple_element_t<KeyId, std::tuple<std::remove_reference_t<RangesT>...>>
+                      >::key_type;
+        auto *terminal_ptr = ttg::detail::get_out_terminal<key_t, valueT>(I, "ttg::device::broadcast(keylists, value)");
+        terminal_ptr->broadcast(std::get<KeyId>(keylists), value);
+        if constexpr (sizeof...(Is) > 0) {
+          detail::broadcast<KeyId+1, Is...>(keylists, std::forward<valueT>(value));
+        }
+      }
+
+      /* overload with explicit terminals */
+      template <size_t I, size_t... Is, typename RangesT, typename valueT,
+                typename... out_keysT, typename... out_valuesT,
+                ttg::Runtime Runtime = ttg::ttg_runtime>
+      inline send_coro_state
+      broadcast_coro(RangesT &&keylists, valueT &&value,
+                     std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t,
+                     ttg::detail::value_copy_handler<Runtime>&& ch) {
+        ttg::detail::value_copy_handler<Runtime> copy_handler = std::move(ch); // destroyed at the end of the coro
+        RangesT kl = std::forward<RangesT>(keylists); // capture the keylist(s)
+        if constexpr (ttg::meta::is_tuple_v<RangesT>) {
+          // treat as tuple
+          prepare_broadcast<0, I, Is...>(kl, std::forward<std::decay_t<decltype(value)>>(value), t);
+          co_await ttg::Void{}; // we'll come back once the task is done
+          broadcast<0, I, Is...>(kl, std::forward<std::decay_t<decltype(value)>>(value), t);
+        } else if constexpr (!ttg::meta::is_tuple_v<RangesT>) {
+          // create a tie to the captured keylist
+          prepare_broadcast<0, I, Is...>(std::tie(kl), std::forward<std::decay_t<decltype(value)>>(value), t);
+          co_await ttg::Void{}; // we'll come back once the task is done
+          broadcast<0, I, Is...>(std::tie(kl), std::forward<std::decay_t<decltype(value)>>(value), t);
+        }
+      }
+
+      /* overload with implicit terminals */
+      template <size_t I, size_t... Is, typename RangesT, typename valueT,
+                ttg::Runtime Runtime = ttg::ttg_runtime>
+      inline send_coro_state
+      broadcast_coro(RangesT &&keylists, valueT &&value,
+                     ttg::detail::value_copy_handler<Runtime>&& ch) {
+        ttg::detail::value_copy_handler<Runtime> copy_handler = std::move(ch); // destroyed at the end of the coro
+        RangesT kl = std::forward<RangesT>(keylists); // capture the keylist(s)
+        if constexpr (ttg::meta::is_tuple_v<RangesT>) {
+          // treat as tuple
+          static_assert(sizeof...(Is)+1 == std::tuple_size_v<RangesT>,
+                        "Size of keylist tuple must match the number of output terminals");
+          prepare_broadcast<0, I, Is...>(kl, std::forward<std::decay_t<decltype(value)>>(value));
+          co_await ttg::Void{}; // we'll come back once the task is done
+          broadcast<0, I, Is...>(kl, std::forward<std::decay_t<decltype(value)>>(value));
+        } else if constexpr (!ttg::meta::is_tuple_v<RangesT>) {
+          // create a tie to the captured keylist
+          prepare_broadcast<0, I, Is...>(std::tie(kl), std::forward<std::decay_t<decltype(value)>>(value));
+          co_await ttg::Void{}; // we'll come back once the task is done
+          broadcast<0, I, Is...>(std::tie(kl), std::forward<std::decay_t<decltype(value)>>(value));
+        }
+      }
+    }  // namespace detail
+
+    /* overload with explicit terminals and keylist passed by const reference */
+    template <size_t I, size_t... Is, typename rangeT, typename valueT, typename... out_keysT, typename... out_valuesT,
+              ttg::Runtime Runtime = ttg::ttg_runtime>
+    [[nodiscard]]
+    inline detail::send_t broadcast(rangeT &&keylist,
+                                    valueT &&value,
+                                    const std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
+      ttg::detail::value_copy_handler<Runtime> copy_handler;
+      return detail::send_t{
+              broadcast_coro<0, I, Is...>(std::forward<rangeT>(keylist),
+                                          copy_handler(std::forward<valueT>(value)),
+                                          t, std::move(copy_handler))};
+    }
+
+    /* overload with implicit terminals and keylist passed by const reference */
+    template <size_t i, typename rangeT, typename valueT,
+              ttg::Runtime Runtime = ttg::ttg_runtime>
+    inline detail::send_t broadcast(rangeT &&keylist, valueT &&value) {
+      ttg::detail::value_copy_handler<Runtime> copy_handler;
+      return detail::send_t{broadcast_coro<0, i>(std::tie(keylist), copy_handler(std::forward<valueT>(value)),
+                                                 std::move(copy_handler))};
+    }
+
+    template<typename... Args, ttg::Runtime Runtime = ttg::ttg_runtime>
+    [[nodiscard]]
+    std::vector<device::detail::send_t> forward(Args&&... args) {
+      // TODO: check the cost of this!
+      return std::vector{std::forward<Args>(args)...};
+    }
+
+  } // namespace device
+
+  /*******************************************
+   * Device task promise and coroutine handle
+   *******************************************/
+
+
+  struct device_task_promise_type;
+
+  using device_task_handle_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<device_task_promise_type>;
+
+  /// task that can be resumed after some events occur
+  struct device_task : public device_task_handle_type {
+    using base_type = device_task_handle_type;
+
+    /// these are members mandated by the promise_type concept
+    ///@{
+
+    using promise_type = device_task_promise_type;
+
+    ///@}
+
+    device_task(base_type base) : base_type(std::move(base)) {}
+
+    base_type& handle() { return *this; }
+
+    /// @return true if ready to resume
+    inline bool ready() {
+      return true;
+    }
+
+    /// @return true if task completed and can be destroyed
+    inline bool completed();
+  };
+
+  /* The promise type that stores the views provided by the
+   * application task coroutine on the first co_yield. It subsequently
+   * tracks the state of the task when it moves from waiting for transfers
+   * to waiting for the submitted kernel to complete. */
+  struct device_task_promise_type {
+
+    /* do not suspend the coroutine on first invocation, we want to run
+     * the coroutine immediately and suspend when we get the device transfers.
+     */
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_never initial_suspend() {
+      m_state = TTG_DEVICE_CORO_INIT;
+      return {};
+    }
+
+    /* suspend the coroutine at the end of the execution
+     * so we can access the promise.
+     * TODO: necessary? maybe we can save one suspend here
+     */
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_always final_suspend() noexcept {
+      m_state = TTG_DEVICE_CORO_COMPLETE;
+      return {};
+    }
+
+#if 0
+    /* waiting for transfers to complete should always suspend
+     * TODO: as an optimization, we could check here if all data
+     *       is already available and avoid suspending...
+     */
+    template<typename... Views>
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_always yield_value(std::tuple<Views&...> &views) {
+      /* gather all the views (host object + view spans, type-punned) into a vector */
+      constexpr static std::size_t view_count = std::tuple_size_v<std::tuple<Views...>>;
+      std::cout << "yield_value: views" << std::endl;
+      m_spans.clear(); // in case we ever come back here
+      m_spans.reserve(view_count);
+      if constexpr(view_count > 0) {
+        auto unpack_lambda = [&](Views&... view){
+                                ((m_spans.push_back(device_obj_view(&view.get_host_object(),
+                                                                    view.view_spans()))),
+                                  ...);
+                              };
+        std::apply(unpack_lambda, views);
+      }
+      m_state = TTG_DEVICE_CORO_WAIT_TRANSFER;
+      return {};
+    }
+
+    /* convenience-function to yield a single view */
+    template<typename HostT, typename... DeviceViewTs>
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_always yield_value(View<HostT, DeviceViewTs...> &view) {
+      auto tmp_tuple = std::tie(view);
+      return yield_value(tmp_tuple);
+    }
+
+    /* convenience-function to yield a single view */
+    template<typename HostT, typename... DeviceViewTs>
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_always yield_value(PersistentView<HostT, DeviceViewTs...> &view) {
+      auto tmp_tuple = std::tie(view);
+      return yield_value(tmp_tuple);
+    }
+#endif // 0
+
+    /* waiting for the kernel to complete should always suspend */
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_always yield_value(device_op_wait_kernel) {
+      std::cout << "yield_value: device_op_wait_kernel" << std::endl;
+      m_state = TTG_DEVICE_CORO_WAIT_KERNEL;
+      return {};
+    }
+
+    /* Allow co_await on a tuple */
+    template<typename... Views>
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(std::tuple<Views&...> &views) {
+      return yield_value(views);
+    }
+
+#if 0
+    /* convenience-function to await a single view */
+    template<typename HostT, typename... DeviceViewTs>
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(View<HostT, DeviceViewTs...> &view) {
+      auto tmp_tuple = std::tie(view);
+      return yield_value(tmp_tuple);
+    }
+
+    /* convenience-function to await a single view */
+    template<typename HostT, typename... DeviceViewTs>
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(PersistentView<HostT, DeviceViewTs...> &view) {
+      auto tmp_tuple = std::tie(view);
+      return yield_value(tmp_tuple);
+    }
+#endif // 0
+
+    /* co_await for the kernel to complete should always suspend */
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(device_op_wait_kernel) {
+      std::cout << "yield_value: device_op_wait_kernel" << std::endl;
+      m_state = TTG_DEVICE_CORO_WAIT_KERNEL;
+      return {};
+    }
+
+    template<typename... Ts>
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(detail::to_device_t<Ts...>&& a) {
+      bool need_transfer = !(TTG_IMPL_NS::register_device_memory(a.ties));
+      /* TODO: are we allowed to not suspend here and launch the kernel directly? */
+      m_state = TTG_DEVICE_CORO_WAIT_TRANSFER;
+      return {};
+    }
+
+    template<typename... Ts>
+    auto await_transform(detail::wait_kernel_t<Ts...>&& a) {
+      std::cout << "yield_value: wait_kernel_t" << std::endl;
+      if constexpr (sizeof...(Ts) > 0) {
+        TTG_IMPL_NS::mark_device_out(a.ties);
+      }
+      m_state = TTG_DEVICE_CORO_WAIT_KERNEL;
+      return a;
+    }
+
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(std::vector<device::detail::send_t>&& v) {
+      m_sends = std::forward<std::vector<device::detail::send_t>>(v);
+      m_state = TTG_DEVICE_CORO_SENDOUT;
+      return {};
+    }
+
+#if 0
+    void return_value(std::vector<device::detail::send_t>&& sends) {
+      m_sends = std::forward<std::vector<device::detail::send_t>>(sends);
+      m_state = TTG_DEVICE_CORO_SENDOUT;
+    }
+#endif // 0
+
+    void return_void() {
+      m_state = TTG_DEVICE_CORO_COMPLETE;
+    }
+
+    bool complete() const {
+      return m_state == TTG_DEVICE_CORO_COMPLETE;
+    }
+
+    device_task get_return_object() { return device_task{device_task_handle_type::from_promise(*this)}; }
+
+    void unhandled_exception() { }
+
+    //using iterator = std::vector<device_obj_view>::iterator;
+
+    /* execute all pending send and broadcast operations */
+    void do_sends() {
+      for (auto& send : m_sends) {
+        send.coro();
+      }
+      m_sends.clear();
+    }
+
+    auto state() {
+      return m_state;
+    }
+
+  private:
+    std::vector<device::detail::send_t> m_sends;
+    ttg_device_coro_state m_state = TTG_DEVICE_CORO_STATE_NONE;
+
+  };
+
+  bool device_task::completed() { return base_type::promise().state() == TTG_DEVICE_CORO_COMPLETE; }
+
+  struct device_wait_kernel
+  { };
+
+
+  /**************************
+   * Device reduction coros *
+   **************************/
+
+  struct device_reducer_promise_type;
+
+  using device_reducer_handle_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<device_reducer_promise_type>;
+
+  /// task that can be resumed after some events occur
+  struct device_reducer : public device_reducer_handle_type {
+    using base_type = device_reducer_handle_type;
+
+    /// these are members mandated by the promise_type concept
+    ///@{
+
+    using promise_type = device_reducer_promise_type;
+
+    ///@}
+
+    device_reducer(base_type base) : base_type(std::move(base)) {}
+
+    base_type& handle() { return *this; }
+
+    /// @return true if ready to resume
+    inline bool ready() {
+      return true;
+    }
+
+    /// @return true if task completed and can be destroyed
+    inline bool completed();
+  };
+
+
+  /* The promise type that stores the views provided by the
+   * application task coroutine on the first co_yield. It subsequently
+   * tracks the state of the task when it moves from waiting for transfers
+   * to waiting for the submitted kernel to complete. */
+  struct device_reducer_promise_type {
+
+    /* do not suspend the coroutine on first invocation, we want to run
+     * the coroutine immediately and suspend when we get the device transfers.
+     */
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_never initial_suspend() {
+      m_state = TTG_DEVICE_CORO_INIT;
+      return {};
+    }
+
+    /* suspend the coroutine at the end of the execution
+     * so we can access the promise.
+     * TODO: necessary? maybe we can save one suspend here
+     */
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_always final_suspend() noexcept {
+      m_state = TTG_DEVICE_CORO_COMPLETE;
+      return {};
+    }
+
+    template<typename... Ts>
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(detail::to_device_t<Ts...>&& a) {
+      bool need_transfer = !(TTG_IMPL_NS::register_device_memory(a.ties));
+      /* TODO: are we allowed to not suspend here and launch the kernel directly? */
+      m_state = TTG_DEVICE_CORO_WAIT_TRANSFER;
+      return {};
+    }
+
+    void return_void() {
+      m_state = TTG_DEVICE_CORO_COMPLETE;
+    }
+
+    bool complete() const {
+      return m_state == TTG_DEVICE_CORO_COMPLETE;
+    }
+
+    device_reducer get_return_object() { return device_reducer{device_reducer_handle_type::from_promise(*this)}; }
+
+    void unhandled_exception() { }
+
+    auto state() {
+      return m_state;
+    }
+
+
+  private:
+    ttg_device_coro_state m_state = TTG_DEVICE_CORO_STATE_NONE;
+
+  };
+
+  bool device_reducer::completed() { return base_type::promise().state() == TTG_DEVICE_CORO_COMPLETE; }
+
+}  // namespace ttg
+
+#endif // TTG_DEVICE_TASK_H
\ No newline at end of file
diff --git a/ttg/ttg/parsec/buffer.h b/ttg/ttg/parsec/buffer.h
index 401b1bc70..c1d5172f7 100644
--- a/ttg/ttg/parsec/buffer.h
+++ b/ttg/ttg/parsec/buffer.h
@@ -71,7 +71,9 @@ struct buffer {
   }
 
   static parsec_data_t* create_parsec_data(void *ptr, size_t count) {
-    assert(ptr != nullptr);
+    if (ptr == nullptr) {
+
+    }
     parsec_data_t *data = parsec_data_create_with_type(nullptr, 0, ptr,
                                                        sizeof(element_type)*count,
                                                        parsec_datatype_int8_t);
@@ -106,17 +108,28 @@ struct buffer {
   , m_data(create_parsec_data(m_host_data.get(), count), &delete_parsec_data)
   , m_count(count)
   , m_ttg_copy(detail::ttg_data_copy_container())
-  { }
+  {
+    if (nullptr != m_ttg_copy) {
+      /* register with the new ttg_copy */
+      m_ttg_copy->add_device_data(m_data.get());
+    }
+  }
 
   /* Constructing a buffer using application-managed memory.
    * The memory pointed to by ptr must be accessible during
    * the life-time of the buffer. */
   buffer(element_type* ptr, std::size_t count = 1)
   : m_host_data(ptr, &delete_non_owned)
-  , m_data(create_parsec_data(ptr, count), &delete_parsec_data)
+  , m_data(nullptr == ptr ? nullptr : create_parsec_data(ptr, count),
+           nullptr == ptr ? &delete_null_parsec_data : &delete_parsec_data)
   , m_count(count)
   , m_ttg_copy(detail::ttg_data_copy_container())
-  { }
+  {
+    if (nullptr != m_ttg_copy) {
+      /* register with the new ttg_copy */
+      m_ttg_copy->add_device_data(m_data.get());
+    }
+  }
 
   ~buffer() {
     unpin(); // make sure the copies are not pinned
@@ -168,6 +181,7 @@ struct buffer {
     //          << " parsec-data " << m_data.get()
     //          << std::endl;
     /* don't update the ttg_copy, we keep the connection */
+    return *this;
   }
 
   /* explicitly disable copying of buffers
@@ -312,7 +326,7 @@ struct buffer {
       m_count = 0;
     } else {
       m_host_data = host_data_ptr(ptr, &delete_non_owned);
-      m_data = parsec_data_ptr(create_parsec_data(m_host_data.get()), &delete_parsec_data);
+      m_data = parsec_data_ptr(create_parsec_data(m_host_data.get(), count), &delete_parsec_data);
       if (nullptr != m_ttg_copy) {
         m_ttg_copy->add_device_data(m_data.get());
       }
@@ -372,7 +386,8 @@ struct buffer {
       std::size_t s = size();
       ar& s;
       assert(m_ttg_copy != nullptr); // only tracked objects allowed
-      m_ttg_copy->iovec_add(ttg::iovec{s*sizeof(T), current_device_ptr()});
+      /* TODO: move to device pointer once we are sure we can send from the device */
+      m_ttg_copy->iovec_add(ttg::iovec{s*sizeof(T), host_ptr()});
     } else {
       std::size_t s;
       ar & s;
@@ -380,7 +395,8 @@ struct buffer {
       /* initialize internal pointers and then reset */
       reset(s);
       assert(m_ttg_copy != nullptr); // only tracked objects allowed
-      m_ttg_copy->iovec_add(ttg::iovec{s*sizeof(T), current_device_ptr()});
+      /* TODO: move to device pointer once we are sure we can send from the device */
+      m_ttg_copy->iovec_add(ttg::iovec{s*sizeof(T), host_ptr()});
     }
   }
 #endif // TTG_SERIALIZATION_SUPPORTS_MADNESS
@@ -396,6 +412,10 @@ template<typename T>
 struct is_buffer<buffer<T>> : std::true_type
 { };
 
+template<typename T>
+struct is_buffer<const buffer<T>> : std::true_type
+{ };
+
 template<typename T>
 constexpr static const bool is_buffer_v = is_buffer<T>::value;
 
diff --git a/ttg/ttg/parsec/devicefunc.h b/ttg/ttg/parsec/devicefunc.h
index 9509f5dcf..22229f0e7 100644
--- a/ttg/ttg/parsec/devicefunc.h
+++ b/ttg/ttg/parsec/devicefunc.h
@@ -21,6 +21,11 @@ namespace ttg_parsec {
                     "PaRSEC only supports MAX_PARAM_COUNT device input/outputs. "
                     "Increase MAX_PARAM_COUNT and recompile PaRSEC/TTG.");
       using view_type = std::remove_reference_t<std::tuple_element_t<I, std::tuple<Views&...>>>;
+      parsec_ttg_task_base_t *caller = detail::parsec_ttg_caller;
+      assert(nullptr != caller->dev_ptr);
+      parsec_gpu_task_t *gpu_task = caller->dev_ptr->gpu_task;
+      parsec_flow_t *flows = caller->dev_ptr->flows;
+
       auto& view = std::get<I>(views);
       bool is_current = false;
       static_assert(ttg::is_buffer_v<view_type> || ttg_parsec::is_devicescratch_v<view_type>);
@@ -28,27 +33,17 @@ namespace ttg_parsec {
       parsec_data_t* data = detail::get_parsec_data(view);
       /* TODO: check whether the device is current */
 
-      auto flow_flags = PARSEC_FLOW_ACCESS_RW;
+      auto access = PARSEC_FLOW_ACCESS_RW;
       if constexpr (std::is_const_v<view_type>) {
-        flow_flags = PARSEC_FLOW_ACCESS_READ;
+        // keep the flow at RW if it was RW to make sure we pull the data back out eventually
+        if (flows[I].flow_flags != PARSEC_FLOW_ACCESS_RW) {
+          access = PARSEC_FLOW_ACCESS_READ;
+        }
       } else if constexpr (ttg_parsec::is_devicescratch_v<view_type>) {
         if (view.scope() == ttg::scope::Allocate) {
-          flow_flags = PARSEC_FLOW_ACCESS_WRITE;
+          access = PARSEC_FLOW_ACCESS_WRITE;
         }
-#if 0
-        switch(view.scope()) {
-          case ttg::scope::Allocate:
-            flow_flags = PARSEC_FLOW_ACCESS_WRITE;
-            break;
-          case ttg::scope::SyncIn:
-            flow_flags = PARSEC_FLOW_ACCESS_READ;
-            break;
-        }
-#endif // 0
       }
-      assert(nullptr != detail::parsec_ttg_caller->dev_ptr);
-      parsec_gpu_task_t *gpu_task = detail::parsec_ttg_caller->dev_ptr->gpu_task;
-      parsec_flow_t *flows = detail::parsec_ttg_caller->dev_ptr->flows;
 
       //std::cout << "register_device_memory task " << detail::parsec_ttg_caller << " data " << I << " "
       //          << data << " size " << data->nb_elts << std::endl;
@@ -57,7 +52,7 @@ namespace ttg_parsec {
       /* TODO: reuse the flows of the task class? How can we control the sync direction then? */
       flows[I] = parsec_flow_t{.name = nullptr,
                                .sym_type = PARSEC_SYM_INOUT,
-                               .flow_flags = static_cast<uint8_t>(flow_flags),
+                               .flow_flags = static_cast<uint8_t>(access),
                                .flow_index = I,
                                .flow_datatype_mask = ~0 };
 
@@ -67,8 +62,8 @@ namespace ttg_parsec {
       /* set the input data copy, parsec will take care of the transfer
        * and the buffer will look at the parsec_data_t for the current pointer */
       //detail::parsec_ttg_caller->parsec_task.data[I].data_in = data->device_copies[data->owner_device];
-      detail::parsec_ttg_caller->parsec_task.data[I].data_in = data->device_copies[0];
-      detail::parsec_ttg_caller->parsec_task.data[I].source_repo_entry = NULL;
+      caller->parsec_task.data[I].data_in = data->device_copies[0];
+      caller->parsec_task.data[I].source_repo_entry = NULL;
 
       if constexpr (sizeof...(Is) > 0) {
         is_current |= register_device_memory(views, std::index_sequence<Is...>{});
@@ -127,17 +122,6 @@ namespace ttg_parsec {
       static_assert(DeviceAvail, "No device implementation detected!");
 #endif // defined(PARSEC_HAVE_DEV_CUDA_SUPPORT)
 
-#if 0
-      /* find the data copy and mark it as pushout */
-      int i = 0;
-      while (detail::parsec_ttg_caller->parsec_task.data[i].data_in != nullptr) {
-        if (detail::parsec_ttg_caller->parsec_task.data[i].data_in == data->device_copies[0]) {
-          gpu_task->pushout |= 1<<i;
-          break;
-        }
-        ++i;
-      }
-#endif // 0
       if constexpr (sizeof...(Is) > 0) {
         // recursion
         mark_device_out(views, std::index_sequence<Is...>{});
@@ -186,30 +170,6 @@ namespace ttg_parsec {
     detail::post_device_out(b, std::index_sequence_for<Buffer...>{});
   }
 
-  namespace detail {
-
-    template<typename Value>
-    void send_inc_version(Value&& value) {
-      if constexpr(!std::is_const_v<Value>) {
-        assert(nullptr != detail::parsec_ttg_caller);
-        /* increment the version of the copy, needed for potential pushout */
-        parsec_task_t *parsec_task = &detail::parsec_ttg_caller->parsec_task;
-        int i = 0;
-        bool found = false;
-        while (parsec_task->data[i].data_in != nullptr) {
-          parsec_data_copy_t *copy = parsec_task->data[i].data_in;
-          if (copy->device_private == &value) {
-            copy->version++;
-            found = true;
-            break;
-          }
-        }
-        assert(found);
-      }
-    }
-
-  } // namespace detail
-
 } // namespace ttg_parsec
 
 #endif // TTG_PARSEC_DEVICEFUNC_H
\ No newline at end of file
diff --git a/ttg/ttg/parsec/devicescratch.h b/ttg/ttg/parsec/devicescratch.h
index 25ca2e05a..9e36757b5 100644
--- a/ttg/ttg/parsec/devicescratch.h
+++ b/ttg/ttg/parsec/devicescratch.h
@@ -94,9 +94,12 @@ struct devicescratch {
 
   ~devicescratch() {
     /* remove data from flow */
-    remove_from_flow();
+    //remove_from_flow();
     if (nullptr != m_data) {
-      parsec_data_destroy(m_data);
+      //parsec_data_destroy(m_data);
+      //parsec_data_copy_detach(m_data, parsec_data_get_copy(m_data, 0), 0);
+      //auto *copy = parsec_data_get_copy(m_data, 0);
+      //PARSEC_OBJ_RELEASE(copy);
     }
     //parsec_data_destroy(m_data);
     m_data = nullptr;
@@ -138,6 +141,10 @@ template<typename T>
 struct is_devicescratch<devicescratch<T>> : std::true_type
 { };
 
+template<typename T>
+struct is_devicescratch<const devicescratch<T>> : std::true_type
+{ };
+
 template<typename T>
 constexpr static const bool is_devicescratch_v = is_devicescratch<T>::value;
 
diff --git a/ttg/ttg/parsec/task.h b/ttg/ttg/parsec/task.h
index 3636e903a..f95c53683 100644
--- a/ttg/ttg/parsec/task.h
+++ b/ttg/ttg/parsec/task.h
@@ -39,6 +39,40 @@ namespace ttg_parsec {
       }
     };
 
+    enum class ttg_parsec_data_flags : uint8_t {
+      NONE           = 0,
+      SINGLE_READER  = 1 << 0,
+      MULTIPLE_READER   = 1 << 1,
+      SINGLE_WRITER  = 1 << 2,
+      MULTIPLE_WRITER   = 1 << 3,
+      MARKED_PUSHOUT = 1 << 4
+    };
+
+    inline
+    ttg_parsec_data_flags operator|(ttg_parsec_data_flags lhs, ttg_parsec_data_flags rhs) {
+        using flags_type = std::underlying_type<ttg_parsec_data_flags>::type;
+        return ttg_parsec_data_flags(static_cast<flags_type>(lhs) | static_cast<flags_type>(rhs));
+    }
+
+    inline
+    ttg_parsec_data_flags operator|=(ttg_parsec_data_flags lhs, ttg_parsec_data_flags rhs) {
+        using flags_type = std::underlying_type<ttg_parsec_data_flags>::type;
+        return ttg_parsec_data_flags(static_cast<flags_type>(lhs) | static_cast<flags_type>(rhs));
+    }
+
+    inline
+    uint8_t operator&(ttg_parsec_data_flags lhs, ttg_parsec_data_flags rhs) {
+        using flags_type = std::underlying_type<ttg_parsec_data_flags>::type;
+        return static_cast<flags_type>(lhs) & static_cast<flags_type>(rhs);
+    }
+
+    inline
+    bool operator!(ttg_parsec_data_flags lhs) {
+        using flags_type = std::underlying_type<ttg_parsec_data_flags>::type;
+        return lhs == ttg_parsec_data_flags::NONE;
+    }
+
+
     typedef parsec_hook_return_t (*parsec_static_op_t)(void *);  // static_op will be cast to this type
 
     struct parsec_ttg_task_base_t {
@@ -47,8 +81,6 @@ namespace ttg_parsec {
       int32_t data_count = 0;      //< number of data elements in the copies array
       ttg_data_copy_t **copies;    //< pointer to the fixed copies array of the derived task
       parsec_hash_table_item_t tt_ht_item = {};
-      parsec_static_op_t function_template_class_ptr[ttg::runtime_traits<ttg::Runtime::PaRSEC>::num_execution_spaces] =
-          {nullptr};
 
       typedef struct {
         std::size_t goal;
@@ -65,7 +97,7 @@ namespace ttg_parsec {
       bool remove_from_hash = true;
       bool is_dummy = false;
       bool defer_writer = TTG_PARSEC_DEFER_WRITER; // whether to defer writer instead of creating a new copy
-
+      ttg_parsec_data_flags data_flags; // HACKY: flags set by prepare_send and reset by the copy_handler
 
       /*
       virtual void release_task() = 0;
@@ -164,6 +196,15 @@ namespace ttg_parsec {
         tt->release_task(task);
       }
 
+      template<ttg::ExecutionSpace Space>
+      parsec_hook_return_t invoke_op() {
+        if constexpr (Space == ttg::ExecutionSpace::Host) {
+          return TT::template static_op<Space>(&this->parsec_task);
+        } else {
+          return TT::template device_static_op<Space>(&this->parsec_task);
+        }
+      }
+
       parsec_key_t pkey() { return reinterpret_cast<parsec_key_t>(&key); }
     };
 
@@ -199,6 +240,15 @@ namespace ttg_parsec {
         tt->release_task(task);
       }
 
+      template<ttg::ExecutionSpace Space>
+      parsec_hook_return_t invoke_op() {
+        if constexpr (Space == ttg::ExecutionSpace::Host) {
+          return TT::template static_op<Space>(&this->parsec_task);
+        } else {
+          return TT::template device_static_op<Space>(&this->parsec_task);
+        }
+      }
+
       parsec_key_t pkey() { return 0; }
     };
 
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 2d0d64ff6..994bf3afc 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -33,10 +33,11 @@
 #include "ttg/util/print.h"
 #include "ttg/util/trace.h"
 #include "ttg/util/typelist.h"
+#include "ttg/device/task.h"
 
 #include "ttg/serialization/data_descriptor.h"
 
-#include "ttg/view.h"
+//#include "ttg/view.h"
 
 #include "ttg/parsec/fwd.h"
 
@@ -107,8 +108,6 @@ void parsec_taskpool_termination_detected(parsec_taskpool_t *tp);
 int parsec_add_fetch_runtime_task(parsec_taskpool_t *tp, int tasks);
 }
 
-#include "ttg/view.h"
-
 namespace ttg_parsec {
   typedef void (*static_set_arg_fct_type)(void *, size_t, ttg::TTBase *);
   typedef std::pair<static_set_arg_fct_type, ttg::TTBase *> static_set_arg_fct_call_t;
@@ -661,16 +660,47 @@ namespace ttg_parsec {
       return copy;
     }
 
+#if 0
+    template <std::size_t... IS, typename Key = keyT>
+    void invoke_pull_terminals(std::index_sequence<IS...>, const Key &key, detail::parsec_ttg_task_base_t *task) {
+      int junk[] = {0, (invoke_pull_terminal<IS>(
+                            std::get<IS>(input_terminals), key, task),
+                        0)...};
+      junk[0]++;
+    }
+#endif // 0
+
+    template<typename TT, std::size_t I>
+    inline void transfer_ownership_impl(ttg_data_copy_t *copy, int device) {
+      if constexpr(!std::is_const_v<std::tuple_element_t<I, typename TT::input_args_type>>) {
+        copy->transfer_ownership(PARSEC_FLOW_ACCESS_RW, device);
+      }
+    }
+
+    template<typename TT, std::size_t... Is>
+    inline void transfer_ownership(parsec_ttg_task_t<TT> *me, int device, std::index_sequence<Is...>) {
+      /* transfer ownership of each data */
+      int junk[] = {0, (transfer_ownership_impl<TT, Is>(me->copies[Is], device), 0)...};
+      junk[0]++;
+    }
+
+    template<typename TT>
     inline parsec_hook_return_t hook(struct parsec_execution_stream_s *es, parsec_task_t *parsec_task) {
-      parsec_ttg_task_base_t *me = (parsec_ttg_task_base_t *)parsec_task;
-      me->function_template_class_ptr[static_cast<std::size_t>(ttg::ExecutionSpace::Host)](parsec_task);
-      return PARSEC_HOOK_RETURN_DONE;
+      parsec_ttg_task_t<TT> *me = (parsec_ttg_task_t<TT> *)parsec_task;
+      if constexpr(std::tuple_size_v<typename TT::input_values_tuple_type> > 0) {
+        transfer_ownership<TT>(me, 0, std::index_sequence_for<typename TT::input_values_tuple_type>{});
+      }
+      return me->template invoke_op<ttg::ExecutionSpace::Host>();
     }
 
+    template<typename TT>
     inline parsec_hook_return_t hook_cuda(struct parsec_execution_stream_s *es, parsec_task_t *parsec_task) {
-      parsec_ttg_task_base_t *me = (parsec_ttg_task_base_t *)parsec_task;
-      auto ret = me->function_template_class_ptr[static_cast<std::size_t>(ttg::ExecutionSpace::CUDA)](parsec_task);
-      return ret;
+      if constexpr(TT::derived_has_cuda_op()) {
+        parsec_ttg_task_t<TT> *me = (parsec_ttg_task_t<TT> *)parsec_task;
+        return me->template invoke_op<ttg::ExecutionSpace::CUDA>();
+      } else {
+        assert(TT::derived_has_cuda_op());
+      }
     }
 
     static parsec_key_fn_t parsec_tasks_hash_fcts = {.key_equal = parsec_hash_table_generic_64bits_key_equal,
@@ -1869,12 +1899,6 @@ namespace ttg_parsec {
         newtask = new (taskobj) task_t(mempool, &this->self, world_impl.taskpool(), this, priority);
       }
 
-      newtask->function_template_class_ptr[static_cast<std::size_t>(ttg::ExecutionSpace::Host)] =
-          reinterpret_cast<detail::parsec_static_op_t>(&TT::static_op<ttg::ExecutionSpace::Host>);
-      if constexpr (derived_has_cuda_op())
-        newtask->function_template_class_ptr[static_cast<std::size_t>(ttg::ExecutionSpace::CUDA)] =
-            reinterpret_cast<detail::parsec_static_op_t>(&TT::device_static_op<ttg::ExecutionSpace::CUDA>);
-
       for (int i = 0; i < static_stream_goal.size(); ++i) {
         newtask->stream[i].goal = static_stream_goal[i];
       }
@@ -1965,6 +1989,16 @@ namespace ttg_parsec {
 #endif
       }
 
+      /* transfer ownership changes: readers, coherency state, ownership
+       * version of CPU copy must be changed when used inside a CPU task
+       * PTG increases version: complete_execution (if W)
+       * PTG calls transfer_ownership at beginning of CPU task (hook) if RW access
+       * DSL never modifies GPU data copy
+       * Pushout happens at the end of GPU task if successor can be CPU?
+       */
+
+      //parsec_data_transfer_ownership_to_copy(data, 0, PARSEC_FLOW_ACCESS_RW);
+
       if (reducer) {  // is this a streaming input? reduce the received value
         // N.B. Right now reductions are done eagerly, without spawning tasks
         //      this means we must lock
@@ -2205,7 +2239,7 @@ namespace ttg_parsec {
           auto iovs = descr.get_data(*const_cast<decvalueT *>(value_ptr));
           num_iovecs = std::distance(std::begin(iovs), std::end(iovs));
           /* pack the metadata */
-          auto metadata = descr.get_metadata(value_ptr);
+          auto metadata = descr.get_metadata(*const_cast<decvalueT *>(value_ptr));
           size_t metadata_size = sizeof(metadata);
           pos = pack(metadata, msg->bytes, pos);
           handle_iovec_fn(iovs);
@@ -2713,8 +2747,7 @@ namespace ttg_parsec {
       assert(detail::parsec_ttg_caller->dev_ptr && detail::parsec_ttg_caller->dev_ptr->gpu_task);
       parsec_gpu_task_t *gpu_task = detail::parsec_ttg_caller->dev_ptr->gpu_task;
       for (auto data : *copy) {
-        if (data->owner_device != 0 &&
-            data->device_copies[0]->version < data->device_copies[data->owner_device]->version) {
+        if (data->owner_device != 0) {
           /* find the flow */
           int flowidx = 0;
           while (flowidx < MAX_PARAM_COUNT &&
@@ -2731,61 +2764,119 @@ namespace ttg_parsec {
           if (gpu_task->flow[flowidx]->flow_flags == PARSEC_FLOW_ACCESS_NONE) {
             /* no flow found, add one and mark it pushout */
             detail::parsec_ttg_caller->parsec_task.data[flowidx].data_in = data->device_copies[0];
-            ((parsec_flow_t *)gpu_task->flow[flowidx])->flow_flags |= PARSEC_FLOW_ACCESS_RW;
             gpu_task->flow_nb_elts[flowidx] = data->nb_elts;
           }
+          /* need to mark the flow RW to make PaRSEC happy */
+          ((parsec_flow_t *)gpu_task->flow[flowidx])->flow_flags |= PARSEC_FLOW_ACCESS_RW;
           gpu_task->pushout |= 1<<flowidx;
         }
       }
     }
 
-    template <std::size_t i, typename Key, typename Value>
-    std::enable_if_t<!ttg::meta::is_void_v<Key> && !std::is_void_v<std::decay_t<Value>>,
+
+    /* check whether a data needs to be pushed out */
+    template <std::size_t i, typename Value, typename RemoteCheckFn>
+    std::enable_if_t<!std::is_void_v<std::decay_t<Value>>,
                      void>
-    prepare_send(const ttg::span<const Key> &keylist, const Value &value) {
+    do_prepare_send(const Value &value, RemoteCheckFn&& remote_check) {
       using valueT = std::tuple_element_t<i, input_values_full_tuple_type>;
+      static constexpr const bool value_is_const = std::is_const_v<valueT>;
 
       /* get the copy */
       detail::ttg_data_copy_t *copy;
       copy = detail::find_copy_in_task(detail::parsec_ttg_caller, &value);
-      assert(nullptr != copy);
 
-      /* check if there are non-local successors if it's a cuda task */
-      bool have_remote = false;
-      if constexpr (derived_has_cuda_op()) {
-        auto world = ttg_default_execution_context();
-        int rank = world.rank();
-        uint64_t pos = 0;
-        bool have_remote = keylist.end() != std::find_if(keylist.begin(), keylist.end(),
-                                                        [&](const Key &key) { return keymap(key) != rank; });
+      /* if there is no copy we don't need to prepare anything */
+      if (nullptr == copy) {
+        return;
       }
-      if (!derived_has_cuda_op() || have_remote) {
+
+      detail::parsec_ttg_task_base_t *caller = detail::parsec_ttg_caller;
+      bool need_pushout = false;
+
+      if (caller->data_flags & detail::ttg_parsec_data_flags::MARKED_PUSHOUT) {
+        /* already marked pushout, skip the rest */
+        return;
+      }
+
+      /* TODO: remove this once we support reductions on the GPU */
+      auto &reducer = std::get<i>(input_reducers);
+      if (reducer) {
         /* non-gpu task, check if we need to push back to the host */
         copy_mark_pushout(copy);
+        caller->data_flags |= detail::ttg_parsec_data_flags::MARKED_PUSHOUT;
+        return;
       }
-    }
 
-    template <std::size_t i, typename Key, typename Value>
-    std::enable_if_t<ttg::meta::is_void_v<Key> && !std::is_void_v<std::decay_t<Value>>,
-                     void>
-    prepare_send(const Value &value) {
-      using valueT = std::tuple_element_t<i, input_values_full_tuple_type>;
+      if constexpr (value_is_const) {
+        /* check for multiple readers */
+        if (caller->data_flags & detail::ttg_parsec_data_flags::SINGLE_READER) {
+          caller->data_flags |= detail::ttg_parsec_data_flags::MULTIPLE_READER;
+        }
 
-      /* get the copy */
-      detail::ttg_data_copy_t *copy;
-      copy = detail::find_copy_in_task(detail::parsec_ttg_caller, &value);
-      assert(nullptr != copy);
+        if (caller->data_flags & detail::ttg_parsec_data_flags::SINGLE_WRITER) {
+          /* there is a writer already, we will need to create a copy */
+          need_pushout = true;
+        }
+
+        caller->data_flags |= detail::ttg_parsec_data_flags::SINGLE_READER;
+      } else {
+        if (caller->data_flags & detail::ttg_parsec_data_flags::SINGLE_WRITER) {
+          caller->data_flags |= detail::ttg_parsec_data_flags::MULTIPLE_WRITER;
+          need_pushout = true;
+        } else {
+          if (caller->data_flags & detail::ttg_parsec_data_flags::SINGLE_READER) {
+            /* there are readers, we will need to create a copy */
+            need_pushout = true;
+          }
+          caller->data_flags |= detail::ttg_parsec_data_flags::SINGLE_WRITER;
+        }
+      }
+
+      if constexpr (!derived_has_cuda_op()) {
+        need_pushout = true;
+      }
 
       /* check if there are non-local successors if it's a cuda task */
-      auto world = ttg_default_execution_context();
-      int rank = world.rank();
-      bool have_remote = (keymap() != rank);
-      if (!derived_has_cuda_op() || have_remote) {
+      if (!need_pushout) {
+        need_pushout = remote_check();
+      }
+
+      if (need_pushout) {
         /* non-gpu task, check if we need to push back to the host */
         copy_mark_pushout(copy);
+        caller->data_flags |= detail::ttg_parsec_data_flags::MARKED_PUSHOUT;
       }
     }
 
+    /* check whether a data needs to be pushed out */
+    template <std::size_t i, typename Key, typename Value>
+    std::enable_if_t<!ttg::meta::is_void_v<Key> && !std::is_void_v<std::decay_t<Value>>,
+                     void>
+    prepare_send(const ttg::span<const Key> &keylist, const Value &value) {
+      auto remote_check = [&](){
+          auto world = ttg_default_execution_context();
+          int rank = world.rank();
+          uint64_t pos = 0;
+          bool remote = keylist.end() != std::find_if(keylist.begin(), keylist.end(),
+                                                      [&](const Key &key) { return keymap(key) != rank; });
+          return remote;
+        };
+      do_prepare_send<i>(value, remote_check);
+    }
+
+    template <std::size_t i, typename Key, typename Value>
+    std::enable_if_t<ttg::meta::is_void_v<Key> && !std::is_void_v<std::decay_t<Value>>,
+                     void>
+    prepare_send(const Value &value) {
+      auto remote_check = [&](){
+          auto world = ttg_default_execution_context();
+          int rank = world.rank();
+          return (keymap() != rank);
+        };
+      do_prepare_send<i>(value, remote_check);
+    }
+
    private:
     // Copy/assign/move forbidden ... we could make it work using
     // PIMPL for this base class.  However, this instance of the base
@@ -3109,10 +3200,10 @@ namespace ttg_parsec {
         self.incarnations = (__parsec_chore_t *)malloc(3 * sizeof(__parsec_chore_t));
         ((__parsec_chore_t *)self.incarnations)[0].type = PARSEC_DEV_CUDA;
         ((__parsec_chore_t *)self.incarnations)[0].evaluate = NULL;
-        ((__parsec_chore_t *)self.incarnations)[0].hook = detail::hook_cuda;
+        ((__parsec_chore_t *)self.incarnations)[0].hook = &detail::hook_cuda<TT>;
         ((__parsec_chore_t *)self.incarnations)[1].type = PARSEC_DEV_CPU;
         ((__parsec_chore_t *)self.incarnations)[1].evaluate = NULL;
-        ((__parsec_chore_t *)self.incarnations)[1].hook = detail::hook;
+        ((__parsec_chore_t *)self.incarnations)[1].hook = &detail::hook<TT>;
         ((__parsec_chore_t *)self.incarnations)[2].type = PARSEC_DEV_NONE;
         ((__parsec_chore_t *)self.incarnations)[2].evaluate = NULL;
         ((__parsec_chore_t *)self.incarnations)[2].hook = NULL;
@@ -3120,7 +3211,7 @@ namespace ttg_parsec {
         self.incarnations = (__parsec_chore_t *)malloc(2 * sizeof(__parsec_chore_t));
         ((__parsec_chore_t *)self.incarnations)[0].type = PARSEC_DEV_CPU;
         ((__parsec_chore_t *)self.incarnations)[0].evaluate = NULL;
-        ((__parsec_chore_t *)self.incarnations)[0].hook = detail::hook;
+        ((__parsec_chore_t *)self.incarnations)[0].hook = &detail::hook<TT>;
         ((__parsec_chore_t *)self.incarnations)[1].type = PARSEC_DEV_NONE;
         ((__parsec_chore_t *)self.incarnations)[1].evaluate = NULL;
         ((__parsec_chore_t *)self.incarnations)[1].hook = NULL;
@@ -3524,12 +3615,14 @@ struct ttg::detail::value_copy_handler<ttg::Runtime::PaRSEC> {
     static_assert(std::is_rvalue_reference_v<decltype(value)> ||
                   std::is_copy_constructible_v<std::decay_t<Value>>,
                   "Data sent without being moved must be copy-constructible!");
-    if (nullptr == ttg_parsec::detail::parsec_ttg_caller) {
+
+    auto caller = ttg_parsec::detail::parsec_ttg_caller;
+    if (nullptr == caller) {
       ttg::print("ERROR: ttg_send or ttg_broadcast called outside of a task!\n");
     }
     using value_type = std::remove_reference_t<Value>;
     ttg_parsec::detail::ttg_data_copy_t *copy;
-    copy = ttg_parsec::detail::find_copy_in_task(ttg_parsec::detail::parsec_ttg_caller, &value);
+    copy = ttg_parsec::detail::find_copy_in_task(caller, &value);
     value_type *value_ptr = &value;
     if (nullptr == copy) {
       /**
@@ -3537,7 +3630,7 @@ struct ttg::detail::value_copy_handler<ttg::Runtime::PaRSEC> {
        * depending on Value, this uses either the copy or move constructor
        */
       copy = ttg_parsec::detail::create_new_datacopy(std::forward<Value>(value));
-      bool inserted = ttg_parsec::detail::add_copy_to_task(copy, ttg_parsec::detail::parsec_ttg_caller);
+      bool inserted = ttg_parsec::detail::add_copy_to_task(copy, caller);
       assert(inserted);
       value_ptr = reinterpret_cast<value_type *>(copy->get_ptr());
       copy_to_remove = copy;
@@ -3549,6 +3642,7 @@ struct ttg::detail::value_copy_handler<ttg::Runtime::PaRSEC> {
       /* the value was potentially changed, so increment version */
       copy->inc_current_version();
     }
+    caller->data_flags = ttg_parsec::detail::ttg_parsec_data_flags::NONE;
     return std::move(*value_ptr);
   }
 
@@ -3556,11 +3650,12 @@ struct ttg::detail::value_copy_handler<ttg::Runtime::PaRSEC> {
   inline const Value &operator()(const Value &value) {
     static_assert(std::is_copy_constructible_v<std::decay_t<Value>>,
                   "Data sent without being moved must be copy-constructible!");
-    if (nullptr == ttg_parsec::detail::parsec_ttg_caller) {
+    auto caller = ttg_parsec::detail::parsec_ttg_caller;
+    if (nullptr == caller) {
       ttg::print("ERROR: ttg_send or ttg_broadcast called outside of a task!\n");
     }
     ttg_parsec::detail::ttg_data_copy_t *copy;
-    copy = ttg_parsec::detail::find_copy_in_task(ttg_parsec::detail::parsec_ttg_caller, &value);
+    copy = ttg_parsec::detail::find_copy_in_task(caller, &value);
     const Value *value_ptr = &value;
     if (nullptr == copy) {
       /**
@@ -3568,11 +3663,12 @@ struct ttg::detail::value_copy_handler<ttg::Runtime::PaRSEC> {
        * depending on Value, this uses either the copy or move constructor
        */
       copy = ttg_parsec::detail::create_new_datacopy(value);
-      bool inserted = ttg_parsec::detail::add_copy_to_task(copy, ttg_parsec::detail::parsec_ttg_caller);
+      bool inserted = ttg_parsec::detail::add_copy_to_task(copy, caller);
       assert(inserted);
       value_ptr = reinterpret_cast<Value *>(copy->get_ptr());
       copy_to_remove = copy;
     }
+    caller->data_flags = ttg_parsec::detail::ttg_parsec_data_flags::NONE;
     return *value_ptr;
   }
 
diff --git a/ttg/ttg/parsec/ttg_data_copy.h b/ttg/ttg/parsec/ttg_data_copy.h
index 9440a4692..865395be3 100644
--- a/ttg/ttg/parsec/ttg_data_copy.h
+++ b/ttg/ttg/parsec/ttg_data_copy.h
@@ -51,7 +51,6 @@ namespace ttg_parsec {
 
       ttg_data_copy_t(ttg_data_copy_t&& c)
       : ttg_data_copy_container_setter(this)
-      , m_ptr(c.m_ptr)
       , m_next_task(c.m_next_task)
       , m_readers(c.m_readers)
       , m_refs(c.m_refs.load(std::memory_order_relaxed))
@@ -66,8 +65,6 @@ namespace ttg_parsec {
 
       ttg_data_copy_t& operator=(ttg_data_copy_t&& c)
       {
-        m_ptr = c.m_ptr;
-        c.m_ptr = nullptr;
         m_next_task = c.m_next_task;
         c.m_next_task = nullptr;
         m_readers = c.m_readers;
@@ -148,9 +145,8 @@ namespace ttg_parsec {
         return m_readers;
       }
 
-      void *get_ptr() const {
-        return m_ptr;
-      }
+      /* Returns the pointer to the user data wrapped by the the copy object */
+      virtual void* get_ptr() = 0;
 
       parsec_task_t* get_next_task() const {
         return m_next_task;
@@ -180,8 +176,8 @@ namespace ttg_parsec {
       void inc_current_version() {
         if (m_num_dev_data == 1) {
           //std::cout << "data-copy " << this << " inc version " << m_single_dev_data << std::endl;
-          assert(m_single_dev_data->device_copies[m_single_dev_data->owner_device] != nullptr);
-          m_single_dev_data->device_copies[m_single_dev_data->owner_device]->version++;
+          assert(m_single_dev_data->device_copies[0] != nullptr);
+          m_single_dev_data->device_copies[0]->version++;
           //std::cout << "Incrementing version of data " << m_single_dev_data << " copy "
           //          << m_single_dev_data->device_copies[m_single_dev_data->owner_device]
           //          << " on device " << (int)m_single_dev_data->owner_device << " to "
@@ -189,8 +185,8 @@ namespace ttg_parsec {
         } else if (m_num_dev_data > 1) {
           std::for_each(m_dev_data.begin(), m_dev_data.end(),
                         [](parsec_data_t *data){
-                           assert(data->device_copies[data->owner_device] != nullptr);
-                           data->device_copies[data->owner_device]->version++;
+                           assert(data->device_copies[0] != nullptr);
+                           data->device_copies[0]->version++;
                            //std::cout << "Incrementing version of copy "
                            //          << " on device " << (int)data->owner_device << data->device_copies[data->owner_device] << " to "
                            //          << data->device_copies[data->owner_device]->version << std::endl;
@@ -198,6 +194,18 @@ namespace ttg_parsec {
         }
       }
 
+      void transfer_ownership(int access, int device = 0) {
+
+        if (m_num_dev_data == 1) {
+          parsec_data_transfer_ownership_to_copy(m_single_dev_data, device, access);
+        } else if (m_num_dev_data > 1) {
+          std::for_each(m_dev_data.begin(), m_dev_data.end(),
+                        [&](parsec_data_t *data){
+                          parsec_data_transfer_ownership_to_copy(data, device, access);
+                        });
+        }
+      }
+
       /* manage device copies owned by this object
        * we only touch the vector if we have more than one copies to track
        * and otherwise use the single-element member.
@@ -298,7 +306,6 @@ namespace ttg_parsec {
       int64_t uid;
 #endif
     protected:
-      void          *m_ptr;
       parsec_task_t *m_next_task = nullptr;
       int32_t        m_readers  = 1;
       std::atomic<int32_t>  m_refs = 1; // number of entities referencing this copy (TTGs, external)
@@ -327,7 +334,6 @@ namespace ttg_parsec {
       : ttg_data_copy_t()
       , m_value(std::forward<T>(value))
       {
-        this->m_ptr = const_cast<value_type*>(&m_value);
         /* reset the container tracker */
         ttg_data_copy_container() = nullptr;
       }
@@ -374,6 +380,10 @@ namespace ttg_parsec {
 
       /* will destruct the value */
       virtual ~ttg_data_value_copy_t() = default;
+
+      virtual void* get_ptr() override final {
+        return &m_value;
+      }
     };
 
   } // namespace detail
diff --git a/ttg/ttg/view.h b/ttg/ttg/view.h
index 5ab00e961..6bb99a633 100644
--- a/ttg/ttg/view.h
+++ b/ttg/ttg/view.h
@@ -384,530 +384,6 @@ namespace ttg {
     };
   } // namespace detail
 
-  namespace detail {
-    template<typename... Ts>
-    struct to_device_t {
-      std::tuple<Ts&...> ties;
-    };
-  } // namespace detail
-
-  template<typename... Args>
-  [[nodiscard]]
-  inline auto to_device(Args&&... args) {
-    return detail::to_device_t{std::tie(std::forward<Args>(args)...)};
-  }
-
-  namespace detail {
-    template<typename... Ts>
-    struct wait_kernel_t {
-      std::tuple<Ts&...> ties;
-
-      /* always suspend */
-      constexpr bool await_ready() const noexcept { return false; }
-
-      /* always suspend */
-      constexpr void await_suspend( TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<> ) const noexcept {}
-
-      void await_resume() noexcept {
-        if constexpr (sizeof...(Ts) > 0) {
-          /* hook to allow the backend to handle the data after pushout */
-          TTG_IMPL_NS::post_device_out(ties);
-        }
-      }
-    };
-  } // namespace detail
-
-  /* Wait for kernel to complete and provided ttg::buffer and ttg::devicescratch
-   * to be transferred back to host */
-  template<typename... Buffers>
-  [[nodiscard]]
-  inline auto wait_kernel(Buffers&&... args) {
-    static_assert(((ttg::detail::is_buffer_v<std::decay_t<Buffers>>
-                    ||ttg::detail::is_devicescratch_v<std::decay_t<Buffers>>)&&...),
-                  "Only ttg::buffer and ttg::devicescratch can be waited on!");
-    return detail::wait_kernel_t<std::remove_reference_t<Buffers>...>{std::tie(std::forward<Buffers>(args)...)};
-  }
-
-
-  /* TODO: move all device code into ttg::device */
-  namespace device {
-
-    /******************************
-     * Send/Broadcast handling
-     * We pass the value returned by the backend's copy handler into a coroutine
-     * and execute the first part (prepare), before suspending it.
-     * The second part (send/broadcast) is executed after the task completed.
-     ******************************/
-
-    namespace detail {
-      struct send_coro_promise_type;
-
-      using send_coro_handle_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<send_coro_promise_type>;
-
-      /// task that can be resumed after some events occur
-      struct send_coro_state : public send_coro_handle_type {
-        using base_type = send_coro_handle_type;
-
-        /// these are members mandated by the promise_type concept
-        ///@{
-
-        using promise_type = send_coro_promise_type;
-
-        ///@}
-
-        send_coro_state(base_type base) : base_type(std::move(base)) {}
-
-        base_type& handle() { return *this; }
-
-        /// @return true if ready to resume
-        inline bool ready() {
-          return true;
-        }
-
-        /// @return true if task completed and can be destroyed
-        inline bool completed();
-      };
-
-      struct send_coro_promise_type {
-
-        /* do not suspend the coroutine on first invocation, we want to run
-        * the coroutine immediately and suspend only once.
-        */
-        TTG_CXX_COROUTINE_NAMESPACE::suspend_never initial_suspend() {
-          return {};
-        }
-
-        /* we don't suspend the coroutine at the end.
-        * it can be destroyed once the send/broadcast is done
-        */
-        TTG_CXX_COROUTINE_NAMESPACE::suspend_never final_suspend() noexcept {
-          return {};
-        }
-
-        send_coro_state get_return_object() { return send_coro_state{send_coro_handle_type::from_promise(*this)}; }
-
-        /* the send coros only have an empty co_await */
-        TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(ttg::Void) {
-          return {};
-        }
-
-        void unhandled_exception() {
-
-        }
-
-      };
-
-      template<typename Key, typename Value, ttg::Runtime Runtime = ttg::ttg_runtime>
-      inline send_coro_state send_coro (const Key& key, Value&& value, ttg::Out<Key, Value> &t,
-                                        ttg::detail::value_copy_handler<Runtime>& ch) {
-        ttg::detail::value_copy_handler<Runtime> copy_handler = std::move(ch); // destroyed at the end of the coro
-        Key k = key;
-        t.prepare_send(k, std::forward<Value>(value));
-        co_await ttg::Void{}; // we'll come back once the task is done
-        t.send(k, std::forward<Value>(value));
-      };
-
-      template<typename Value, ttg::Runtime Runtime = ttg::ttg_runtime>
-      inline send_coro_state sendv_coro (Value&& value, ttg::Out<void, Value> &t,
-                                         ttg::detail::value_copy_handler<Runtime>& ch) {
-        ttg::detail::value_copy_handler<Runtime> copy_handler = std::move(ch); // destroyed at the end of the coro
-        t.prepare_send(std::forward<Value>(value));
-        co_await ttg::Void{}; // we'll come back once the task is done
-        t.sendv(std::forward<Value>(value));
-      };
-    } // namespace detail
-
-
-    /* functionality to deal with sending/broadcasting */
-    namespace detail {
-      struct send_t {
-        send_coro_state coro;
-      };
-    } // namespace detail
-
-    template <size_t i, typename keyT, typename valueT, typename... out_keysT, typename... out_valuesT,
-              ttg::Runtime Runtime = ttg::ttg_runtime>
-    inline detail::send_t send(const keyT &key, valueT &&value,
-                                        std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
-      ttg::detail::value_copy_handler<Runtime> copy_handler;
-      return detail::send_t{detail::send_coro(key, copy_handler(std::forward<valueT>(value)), std::get<i>(t), copy_handler)};
-    }
-
-    template <size_t i, typename valueT, typename... out_keysT, typename... out_valuesT,
-              ttg::Runtime Runtime = ttg::ttg_runtime>
-    inline detail::send_t sendv(
-        valueT &&value, std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
-      ttg::detail::value_copy_handler<Runtime> copy_handler;
-      return detail::send_t{detail::sendv_coro(copy_handler(std::forward<valueT>(value)), std::get<i>(t), copy_handler)};
-    }
-
-
-    // clang-format off
-    /// \brief Sends a task id and a value to the template tasks attached to the output terminal of this template task
-    /// \param[in] i Identifies which output terminal of this template task to select for sending
-    /// \param[in] key: the id of the task(s) receiving the value
-    /// \param[in] value: the value to send to the receiving task(s)
-    // clang-format on
-    template <typename keyT, typename valueT, ttg::Runtime Runtime = ttg::ttg_runtime>
-    inline detail::send_t send(size_t i, const keyT &key, valueT &&value) {
-      ttg::detail::value_copy_handler<Runtime> copy_handler;
-      auto *terminal_ptr = ttg::detail::get_out_terminal<keyT, valueT>(i, "ttg::device::send(i, key, value)");
-      return detail::send_t{detail::send_coro(key, copy_handler(std::forward<valueT>(value)), *terminal_ptr, copy_handler)};
-    }
-
-    // clang-format off
-    /// \brief Sends a task id and a value to the template tasks attached to the output terminal of this template task
-    /// \note this is provided to support `send<i>` with and without explicitly-passed terminal tuple
-    /// \tparam <i> Identifies which output terminal of this template task to select for sending
-    /// \param[in] key: the id of the task(s) receiving the value
-    /// \param[in] value: the value to send to the receiving task(s)
-    // clang-format on
-    template <size_t i, typename keyT, typename valueT>
-    inline auto send(const keyT &key, valueT &&value) {
-      return send(i, key, std::forward<valueT>(value));
-    }
-
-
-
-    namespace detail {
-
-      template<typename T, typename Enabler = void>
-      struct broadcast_keylist_trait {
-        using type = T;
-      };
-
-      /* overload for iterable types that extracts the type of the first element */
-      template<typename T>
-      struct broadcast_keylist_trait<T, std::enable_if_t<ttg::meta::is_iterable_v<T>>> {
-        using key_type = decltype(*std::begin(std::get<0>(std::declval<T>())));
-      };
-
-      template <size_t KeyId, size_t I, size_t... Is, typename... RangesT, typename valueT,
-                typename... out_keysT, typename... out_valuesT>
-      inline void prepare_broadcast(const std::tuple<RangesT...> &keylists, valueT &&value,
-                                    std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
-        std::get<I>(t)->prepare_send(std::get<KeyId>(keylists), std::forward<valueT>(value));
-        if constexpr (sizeof...(Is) > 0) {
-          detail::prepare_broadcast<KeyId+1, Is...>(keylists, std::forward<valueT>(value), t);
-        }
-      }
-
-      template <size_t KeyId, size_t I, size_t... Is, typename... RangesT, typename valueT,
-                typename... out_keysT, typename... out_valuesT>
-      inline void prepare_broadcast(const std::tuple<RangesT...> &keylists, valueT &&value) {
-        using key_t = typename broadcast_keylist_trait<
-                        std::tuple_element_t<KeyId, std::tuple<std::remove_reference_t<RangesT>...>>
-                      >::key_type;
-        auto *terminal_ptr = ttg::detail::get_out_terminal<key_t, valueT>(I, "ttg::device::broadcast(keylists, value)");
-        terminal_ptr->prepare_send(std::get<KeyId>(keylists), value);
-        if constexpr (sizeof...(Is) > 0) {
-          detail::prepare_broadcast<KeyId+1, Is...>(keylists, std::forward<valueT>(value));
-        }
-      }
-
-      template <size_t KeyId, size_t I, size_t... Is, typename... RangesT, typename valueT,
-                typename... out_keysT, typename... out_valuesT>
-      inline void broadcast(const std::tuple<RangesT...> &keylists, valueT &&value,
-                                    std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
-        std::get<I>(t)->broadcast(std::get<KeyId>(keylists), std::forward<valueT>(value));
-        if constexpr (sizeof...(Is) > 0) {
-          detail::broadcast<KeyId+1, Is...>(keylists, std::forward<valueT>(value), t);
-        }
-      }
-
-      template <size_t KeyId, size_t I, size_t... Is, typename... RangesT, typename valueT,
-                typename... out_keysT, typename... out_valuesT>
-      inline void broadcast(const std::tuple<RangesT...> &keylists, valueT &&value) {
-        using key_t = typename broadcast_keylist_trait<
-                        std::tuple_element_t<KeyId, std::tuple<std::remove_reference_t<RangesT>...>>
-                      >::key_type;
-        auto *terminal_ptr = ttg::detail::get_out_terminal<key_t, valueT>(I, "ttg::device::broadcast(keylists, value)");
-        terminal_ptr->broadcast(std::get<KeyId>(keylists), value);
-        if constexpr (sizeof...(Is) > 0) {
-          detail::broadcast<KeyId+1, Is...>(keylists, std::forward<valueT>(value));
-        }
-      }
-
-      /* overload with explicit terminals */
-      template <size_t I, size_t... Is, typename RangesT, typename valueT,
-                typename... out_keysT, typename... out_valuesT,
-                ttg::Runtime Runtime = ttg::ttg_runtime>
-      inline send_coro_state
-      broadcast_coro(RangesT &&keylists, valueT &&value,
-                     std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t,
-                     ttg::detail::value_copy_handler<Runtime>&& ch) {
-        ttg::detail::value_copy_handler<Runtime> copy_handler = std::move(ch); // destroyed at the end of the coro
-        RangesT kl = std::forward<RangesT>(keylists); // capture the keylist(s)
-        if constexpr (ttg::meta::is_tuple_v<RangesT>) {
-          // treat as tuple
-          prepare_broadcast<0, I, Is...>(kl, std::forward<std::decay_t<decltype(value)>>(value), t);
-          co_await ttg::Void{}; // we'll come back once the task is done
-          broadcast<0, I, Is...>(kl, std::forward<std::decay_t<decltype(value)>>(value), t);
-        } else if constexpr (!ttg::meta::is_tuple_v<RangesT>) {
-          // create a tie to the captured keylist
-          prepare_broadcast<0, I, Is...>(std::tie(kl), std::forward<std::decay_t<decltype(value)>>(value), t);
-          co_await ttg::Void{}; // we'll come back once the task is done
-          broadcast<0, I, Is...>(std::tie(kl), std::forward<std::decay_t<decltype(value)>>(value), t);
-        }
-      }
-
-      /* overload with implicit terminals */
-      template <size_t I, size_t... Is, typename RangesT, typename valueT,
-                ttg::Runtime Runtime = ttg::ttg_runtime>
-      inline send_coro_state
-      broadcast_coro(RangesT &&keylists, valueT &&value,
-                     ttg::detail::value_copy_handler<Runtime>&& ch) {
-        ttg::detail::value_copy_handler<Runtime> copy_handler = std::move(ch); // destroyed at the end of the coro
-        RangesT kl = std::forward<RangesT>(keylists); // capture the keylist(s)
-        if constexpr (ttg::meta::is_tuple_v<RangesT>) {
-          // treat as tuple
-          static_assert(sizeof...(Is)+1 == std::tuple_size_v<RangesT>,
-                        "Size of keylist tuple must match the number of output terminals");
-          prepare_broadcast<0, I, Is...>(kl, std::forward<std::decay_t<decltype(value)>>(value));
-          co_await ttg::Void{}; // we'll come back once the task is done
-          broadcast<0, I, Is...>(kl, std::forward<std::decay_t<decltype(value)>>(value));
-        } else if constexpr (!ttg::meta::is_tuple_v<RangesT>) {
-          // create a tie to the captured keylist
-          prepare_broadcast<0, I, Is...>(std::tie(kl), std::forward<std::decay_t<decltype(value)>>(value));
-          co_await ttg::Void{}; // we'll come back once the task is done
-          broadcast<0, I, Is...>(std::tie(kl), std::forward<std::decay_t<decltype(value)>>(value));
-        }
-      }
-    }  // namespace detail
-
-    /* overload with explicit terminals and keylist passed by const reference */
-    template <size_t I, size_t... Is, typename rangeT, typename valueT, typename... out_keysT, typename... out_valuesT,
-              ttg::Runtime Runtime = ttg::ttg_runtime>
-    [[nodiscard]]
-    inline detail::send_t broadcast(rangeT &&keylist,
-                                    valueT &&value,
-                                    const std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
-      ttg::detail::value_copy_handler<Runtime> copy_handler;
-      return detail::send_t{
-              broadcast_coro<0, I, Is...>(std::forward<rangeT>(keylist),
-                                          copy_handler(std::forward<valueT>(value)),
-                                          t, std::move(copy_handler))};
-
-    }
-
-    /* overload with implicit terminals and keylist passed by const reference */
-    template <size_t i, typename rangeT, typename valueT,
-              ttg::Runtime Runtime = ttg::ttg_runtime>
-    inline detail::send_t broadcast(rangeT &&keylist, valueT &&value) {
-      ttg::detail::value_copy_handler<Runtime> copy_handler;
-      return detail::send_t{broadcast_coro<0, i>(std::tie(keylist), copy_handler(std::forward<valueT>(value)),
-                                                 std::move(copy_handler))};
-    }
-
-    template<typename... Args, ttg::Runtime Runtime = ttg::ttg_runtime>
-    [[nodiscard]]
-    std::vector<device::detail::send_t> forward(Args&&... args) {
-      // TODO: check the cost of this!
-      return std::vector{std::forward<Args>(args)...};
-    }
-
-  } // namespace device
-
-  /*******************************************
-   * Device task promise and coroutine handle
-   *******************************************/
-
-
-  struct device_task_promise_type;
-
-  using device_task_handle_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<device_task_promise_type>;
-
-  /// task that can be resumed after some events occur
-  struct device_task : public device_task_handle_type {
-    using base_type = device_task_handle_type;
-
-    /// these are members mandated by the promise_type concept
-    ///@{
-
-    using promise_type = device_task_promise_type;
-
-    ///@}
-
-    device_task(base_type base) : base_type(std::move(base)) {}
-
-    base_type& handle() { return *this; }
-
-    /// @return true if ready to resume
-    inline bool ready() {
-      return true;
-    }
-
-    /// @return true if task completed and can be destroyed
-    inline bool completed();
-  };
-
-  /* The promise type that stores the views provided by the
-   * application task coroutine on the first co_yield. It subsequently
-   * tracks the state of the task when it moves from waiting for transfers
-   * to waiting for the submitted kernel to complete. */
-  struct device_task_promise_type {
-
-    /* do not suspend the coroutine on first invocation, we want to run
-     * the coroutine immediately and suspend when we get the device transfers.
-     */
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_never initial_suspend() {
-      m_state = TTG_DEVICE_CORO_INIT;
-      return {};
-    }
-
-    /* suspend the coroutine at the end of the execution
-     * so we can access the promise.
-     * TODO: necessary? maybe we can save one suspend here
-     */
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_always final_suspend() noexcept {
-      m_state = TTG_DEVICE_CORO_COMPLETE;
-      return {};
-    }
-
-#if 0
-    /* waiting for transfers to complete should always suspend
-     * TODO: as an optimization, we could check here if all data
-     *       is already available and avoid suspending...
-     */
-    template<typename... Views>
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_always yield_value(std::tuple<Views&...> &views) {
-      /* gather all the views (host object + view spans, type-punned) into a vector */
-      constexpr static std::size_t view_count = std::tuple_size_v<std::tuple<Views...>>;
-      std::cout << "yield_value: views" << std::endl;
-      m_spans.clear(); // in case we ever come back here
-      m_spans.reserve(view_count);
-      if constexpr(view_count > 0) {
-        auto unpack_lambda = [&](Views&... view){
-                                ((m_spans.push_back(device_obj_view(&view.get_host_object(),
-                                                                    view.view_spans()))),
-                                  ...);
-                              };
-        std::apply(unpack_lambda, views);
-      }
-      m_state = TTG_DEVICE_CORO_WAIT_TRANSFER;
-      return {};
-    }
-#endif // 0
-
-    /* convenience-function to yield a single view */
-    template<typename HostT, typename... DeviceViewTs>
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_always yield_value(View<HostT, DeviceViewTs...> &view) {
-      auto tmp_tuple = std::tie(view);
-      return yield_value(tmp_tuple);
-    }
-
-    /* convenience-function to yield a single view */
-    template<typename HostT, typename... DeviceViewTs>
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_always yield_value(PersistentView<HostT, DeviceViewTs...> &view) {
-      auto tmp_tuple = std::tie(view);
-      return yield_value(tmp_tuple);
-    }
-
-    /* waiting for the kernel to complete should always suspend */
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_always yield_value(device_op_wait_kernel) {
-      std::cout << "yield_value: device_op_wait_kernel" << std::endl;
-      m_state = TTG_DEVICE_CORO_WAIT_KERNEL;
-      return {};
-    }
-
-    /* Allow co_await on a tuple */
-    template<typename... Views>
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(std::tuple<Views&...> &views) {
-      return yield_value(views);
-    }
-
-    /* convenience-function to await a single view */
-    template<typename HostT, typename... DeviceViewTs>
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(View<HostT, DeviceViewTs...> &view) {
-      auto tmp_tuple = std::tie(view);
-      return yield_value(tmp_tuple);
-    }
-
-    /* convenience-function to await a single view */
-    template<typename HostT, typename... DeviceViewTs>
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(PersistentView<HostT, DeviceViewTs...> &view) {
-      auto tmp_tuple = std::tie(view);
-      return yield_value(tmp_tuple);
-    }
-
-    /* co_await for the kernel to complete should always suspend */
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(device_op_wait_kernel) {
-      std::cout << "yield_value: device_op_wait_kernel" << std::endl;
-      m_state = TTG_DEVICE_CORO_WAIT_KERNEL;
-      return {};
-    }
-
-    template<typename... Ts>
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(detail::to_device_t<Ts...>&& a) {
-      bool need_transfer = !(TTG_IMPL_NS::register_device_memory(a.ties));
-      /* TODO: are we allowed to not suspend here and launch the kernel directly? */
-      m_state = TTG_DEVICE_CORO_WAIT_TRANSFER;
-      return {};
-    }
-
-    template<typename... Ts>
-    auto await_transform(detail::wait_kernel_t<Ts...>&& a) {
-      std::cout << "yield_value: wait_kernel_t" << std::endl;
-      if constexpr (sizeof...(Ts) > 0) {
-        TTG_IMPL_NS::mark_device_out(a.ties);
-      }
-      m_state = TTG_DEVICE_CORO_WAIT_KERNEL;
-      return a;
-    }
-
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(std::vector<device::detail::send_t>&& v) {
-      m_sends = std::forward<std::vector<device::detail::send_t>>(v);
-      m_state = TTG_DEVICE_CORO_SENDOUT;
-      return {};
-    }
-
-#if 0
-    void return_value(std::vector<device::detail::send_t>&& sends) {
-      m_sends = std::forward<std::vector<device::detail::send_t>>(sends);
-      m_state = TTG_DEVICE_CORO_SENDOUT;
-    }
-#endif // 0
-
-    void return_void() {
-      m_state = TTG_DEVICE_CORO_COMPLETE;
-    }
-
-    bool complete() const {
-      return m_state == TTG_DEVICE_CORO_COMPLETE;
-    }
-
-    device_task get_return_object() { return device_task{device_task_handle_type::from_promise(*this)}; }
-
-    void unhandled_exception() {
-
-    }
-
-    using iterator = std::vector<device_obj_view>::iterator;
-
-    /* execute all pending send and broadcast operations */
-    void do_sends() {
-      for (auto& send : m_sends) {
-        send.coro();
-      }
-      m_sends.clear();
-    }
-
-    auto state() {
-      return m_state;
-    }
-
-  private:
-    std::vector<device::detail::send_t> m_sends;
-    ttg_device_coro_state m_state = TTG_DEVICE_CORO_STATE_NONE;
-
-  };
-
-  bool device_task::completed() { return base_type::promise().state() == TTG_DEVICE_CORO_COMPLETE; }
-
-  struct device_wait_kernel
-  { };
-
-}  // namespace ttg
-
+} // namespace ttg
 
 #endif // TTG_VIEW_H

From 26341dc8d200b17864aff42f0656ca3040c01483 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 26 Jun 2023 15:42:45 +0000
Subject: [PATCH 082/259] Call cublasDgemm with the right arguments

---
 examples/spmm/spmm_cuda.cc | 26 ++++++--------------------
 ttg/CMakeLists.txt         |  1 +
 ttg/ttg/device/task.h      |  2 +-
 ttg/ttg/parsec/ttg.h       |  6 +++++-
 4 files changed, 13 insertions(+), 22 deletions(-)

diff --git a/examples/spmm/spmm_cuda.cc b/examples/spmm/spmm_cuda.cc
index aac469a34..d05816ab7 100644
--- a/examples/spmm/spmm_cuda.cc
+++ b/examples/spmm/spmm_cuda.cc
@@ -217,32 +217,19 @@ using blk_t = DeviceTensor<double, btas::DEFAULT::range,
 
 /* TODO: call CUDA gemm here */
 static void cuda_gemm(blk_t &C, const blk_t &A, const blk_t &B) {
-  using array = btas::DEFAULT::index<int>;
-  array aA = array{1, 2};
-  array aB = array{2, 3};
-  array aC = array{1, 3};
-  bool conjgA = false;
-  bool conjgB = false;
   double alpha = 1.0;
   double beta  = 1.0;
-  const bool notransA = aA.front() == aC.front();
-  const bool notransB = aB.front() != aC.back();
-  const auto   cA     = notransA ? 'N' : (conjgA ? 'C' : 'T');
-  const size_t condim = notransA ? A.extent(1) : A.extent(0);
-  const auto   cB     = notransB ? 'N' : (conjgB ? 'C' : 'T');
   // make sure all memory is on the device
   // TODO: A and B are read-only so the owner device will be 0. How to fix?
   //assert(A.b.get_current_device() != 0);
   //assert(B.b.get_current_device() != 0);
-  assert(C.b.get_current_device() != 0);
-  // NOTE: A and B are swapped due to the ColMajor layout, see btas/generic/gemm_impl.h
+  int device = C.b.get_current_device();
+  assert(device != 0);
   my_cublas_dgemm_because_cublas_is_stupid(
-              cB, cA, C.extent(0), C.extent(1), condim, alpha,
-              &*B.b.device_ptr_on(C.b.get_current_device()), B.extent(0),
-              &*A.b.device_ptr_on(C.b.get_current_device()), A.extent(0), beta,
-              &*C.b.current_device_ptr(), C.extent(0));
-  //gemm_impl<true>::call(blas::Layout::ColMajor, cA, cB, C.extent(0), C.extent(1), condim,
-  //                      alpha, &*A.begin(), A.extent(0), &*B.begin(), B.extent(0), beta, &*C.begin(), C.extent(0));
+              'N', 'N', C.extent(0), C.extent(1), A.extent(1), alpha,
+              A.b.device_ptr_on(device), A.extent(0),
+              B.b.device_ptr_on(device), B.extent(0), beta,
+              C.b.current_device_ptr(), C.extent(0));
 }
 
 //using blk_t = btas::Tensor<double, btas::DEFAULT::range, btas::mohndle<btas::varray<double>, btas::Handle::shared_ptr>>;
@@ -728,7 +715,6 @@ class SpMM25D {
                  (have_next_k ? std::to_string(next_k) : "does not exist"));
 
       /* wait for the kernel to complete */
-
       co_await ttg::wait_kernel();
 
 
diff --git a/ttg/CMakeLists.txt b/ttg/CMakeLists.txt
index 4f94ede42..a949cb961 100644
--- a/ttg/CMakeLists.txt
+++ b/ttg/CMakeLists.txt
@@ -64,6 +64,7 @@ set(ttg-impl-headers
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/make_tt.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/make_device_tt.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/view.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/ttg/device/task.h
         ${CMAKE_CURRENT_BINARY_DIR}/ttg/config.h
     )
 set(ttg-headers
diff --git a/ttg/ttg/device/task.h b/ttg/ttg/device/task.h
index 25bee3af3..d5382de0c 100644
--- a/ttg/ttg/device/task.h
+++ b/ttg/ttg/device/task.h
@@ -490,7 +490,7 @@ namespace ttg {
 
     template<typename... Ts>
     auto await_transform(detail::wait_kernel_t<Ts...>&& a) {
-      std::cout << "yield_value: wait_kernel_t" << std::endl;
+      //std::cout << "yield_value: wait_kernel_t" << std::endl;
       if constexpr (sizeof...(Ts) > 0) {
         TTG_IMPL_NS::mark_device_out(a.ties);
       }
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 994bf3afc..5dd51f572 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -3643,7 +3643,11 @@ struct ttg::detail::value_copy_handler<ttg::Runtime::PaRSEC> {
       copy->inc_current_version();
     }
     caller->data_flags = ttg_parsec::detail::ttg_parsec_data_flags::NONE;
-    return std::move(*value_ptr);
+    if constexpr (std::is_rvalue_reference_v<decltype(value)>) {
+      return std::move(*value_ptr);
+    } else {
+      return *value_ptr;
+    }
   }
 
   template <typename Value>

From 836274266c1b96c985093052481dabe9464243bc Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 6 Jul 2023 19:46:40 +0000
Subject: [PATCH 083/259] SPMM: make CUDA version work

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 CMakeLists.txt                   |   1 +
 examples/spmm/spmm_cuda.cc       | 109 ++++++++++++++++++++-----------
 ttg/CMakeLists.txt               |   9 ++-
 ttg/ttg/config.in.h              |   2 +
 ttg/ttg/device/cublas_helper.cpp |  27 ++++++++
 ttg/ttg/device/cublas_helper.h   |  17 +++++
 ttg/ttg/device/task.h            |   4 +-
 ttg/ttg/parsec/buffer.h          |  36 +++++++---
 ttg/ttg/parsec/devicefunc.h      |   5 +-
 ttg/ttg/parsec/parsec-ext.h      |   7 ++
 ttg/ttg/parsec/ttg.h             |  65 +++++++++++++++---
 ttg/ttg/parsec/ttg_data_copy.h   |  37 ++++++-----
 ttg/ttg/parsec/ttvalue.h         | 101 ++++++++++++++++++++++++++++
 13 files changed, 343 insertions(+), 77 deletions(-)
 create mode 100644 ttg/ttg/device/cublas_helper.cpp
 create mode 100644 ttg/ttg/device/cublas_helper.h
 create mode 100644 ttg/ttg/parsec/parsec-ext.h
 create mode 100644 ttg/ttg/parsec/ttvalue.h

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7245600de..4f0048c2c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,4 +1,5 @@
 cmake_minimum_required (VERSION 3.14) # supports FetchContent_MakeAvailable
+cmake_policy(SET CMP0104 OLD)
 
 # Set TTG version =======================================================
 
diff --git a/examples/spmm/spmm_cuda.cc b/examples/spmm/spmm_cuda.cc
index d05816ab7..fa2e95fda 100644
--- a/examples/spmm/spmm_cuda.cc
+++ b/examples/spmm/spmm_cuda.cc
@@ -51,9 +51,10 @@ using namespace ttg;
 #if defined(BLOCK_SPARSE_GEMM) && defined(BTAS_IS_USABLE)
 
 template <typename _T, class _Range, class _Storage>
-struct DeviceTensor : public btas::Tensor<_T, _Range, _Storage> {
+struct DeviceTensor : public ttg::TTValue<DeviceTensor<_T, _Range, _Storage>>
+                    , public btas::Tensor<_T, _Range, _Storage> {
   using tensor_type = btas::Tensor<_T, _Range, _Storage>;
-  tensor_type t;
+  using ttvalue_type = ttg::TTValue<DeviceTensor<_T, _Range, _Storage>>;
   ttg::buffer<_T> b; // does not own the host buffer
 
   using value_type = tensor_type::value_type;
@@ -69,28 +70,32 @@ struct DeviceTensor : public btas::Tensor<_T, _Range, _Storage> {
     /// constructor with index extent
     template <typename... _args>
     explicit DeviceTensor(const size_type& first, const _args&... rest)
-    : tensor_type(first, rest...)
+    : ttvalue_type()
+    , tensor_type(first, rest...)
     , b(this->size() ? this->data() : nullptr, this->size())
     { }
 
     /// construct from \c range, allocate data, but not initialized
     template <typename Range>
     explicit DeviceTensor(const Range& range, typename std::enable_if<btas::is_boxrange<Range>::value>::type* = 0)
-    : tensor_type(range)
+    : ttvalue_type()
+    , tensor_type(range)
     , b(this->size() ? this->data() : nullptr, this->size())
     { }
 
     /// construct from \c range object, set all elements to \c v
     template <typename Range>
     DeviceTensor(const Range& range, value_type v, typename std::enable_if<btas::is_boxrange<Range>::value>::type* = 0)
-    : tensor_type(range)
+    : ttvalue_type()
+    , tensor_type(range)
     , b(this->size() ? this->data() : nullptr, this->size())
     { }
 
     /// construct from \c range object, copy elements from \c vec
     template <typename Range, typename U>
     DeviceTensor(const Range& range, U* vec, typename std::enable_if<btas::is_boxrange<Range>::value>::type* = 0)
-    : tensor_type(range, vec)
+    : ttvalue_type()
+    , tensor_type(range, vec)
     , b(this->size() ? this->data() : nullptr, this->size())
     { }
 
@@ -99,25 +104,29 @@ struct DeviceTensor : public btas::Tensor<_T, _Range, _Storage> {
     DeviceTensor(const Range& range, const Storage& storage,
            typename std::enable_if<btas::is_boxrange<Range>::value & not std::is_same<Range, range_type>::value &
                                    not std::is_same<Storage, storage_type>::value>::type* = 0)
-    : tensor_type(range, storage)
+    : ttvalue_type()
+    , tensor_type(range, storage)
     , b(this->size() ? this->data() : nullptr, this->size())
     { }
 
     /// copy-copy-construct from \c range and \c storage
     DeviceTensor(const range_type& range, const storage_type& storage)
-    : tensor_type(range, storage)
+    : ttvalue_type()
+    , tensor_type(range, storage)
     , b(this->size() ? this->data() : nullptr, this->size())
     { }
 
     /// copy-move-construct from \c range and \c storage
     DeviceTensor(const range_type& range, storage_type&& storage)
-    : tensor_type(range, std::forward<storage_type>(storage))
+    : ttvalue_type()
+    , tensor_type(range, std::forward<storage_type>(storage))
     , b(this->size() ? this->data() : nullptr, this->size())
     { }
 
     /// move-construct from \c range and \c storage
     DeviceTensor(range_type&& range, storage_type&& storage)
-    : tensor_type(std::forward<range_type>(range), std::forward<storage_type>(storage))
+    : ttvalue_type()
+    , tensor_type(std::forward<range_type>(range), std::forward<storage_type>(storage))
     , b(this->size() ? this->data() : nullptr, this->size())
     { }
 
@@ -138,42 +147,56 @@ struct DeviceTensor : public btas::Tensor<_T, _Range, _Storage> {
     template <typename Range, typename InIter, typename Op>
     DeviceTensor(const Range& range, InIter it, const Op& op,
            typename std::enable_if<btas::is_boxrange<Range>::value>::type* = 0)
-    : tensor_type(range, it, op)
+    : ttvalue_type()
+    , tensor_type(range, it, op)
     , b(this->size() ? this->data() : nullptr, this->size())
     { }
 
     /// copy constructor
     /// It will accept Tensors and TensorViews
     template <class _Tensor, class = typename std::enable_if<btas::is_boxtensor<_Tensor>::value>::type>
-    DeviceTensor(const _Tensor& x)
-    : tensor_type(x)
+    DeviceTensor(const _Tensor& x) noexcept
+    : ttvalue_type()
+    , tensor_type(x.clone())
     , b(this->size() ? this->data() : nullptr, this->size())
-    { }
+    {
+      //std::cout << "DeviceTensor tensor_type copy ctor" << std::endl;
+    }
 
     /// copy constructor: devicebuf cannot be copied, so deleted
-    DeviceTensor(const DeviceTensor& x)
-    : tensor_type(x)
+    DeviceTensor(const DeviceTensor& x) noexcept
+    : ttvalue_type(x)
+    , tensor_type(x.clone())
     , b(this->size() ? this->data() : nullptr, this->size())
-    { }
+    {
+      //std::cout << "DeviceTensor copy ctor" << std::endl;
+    }
 
     /// move constructor
-    DeviceTensor(tensor_type&& x)
-    : tensor_type(std::forward<tensor_type>(x))
+    DeviceTensor(tensor_type&& x) noexcept
+    : ttvalue_type()
+    , tensor_type(std::move(x))
     , b(this->size() ? this->data() : nullptr, this->size())
-    { }
+    {
+      //std::cout << "DeviceTensor tensor_type move ctor" << std::endl;
+    }
 
-    DeviceTensor(DeviceTensor&& x)
-    : tensor_type(std::forward<tensor_type>(x))
+    DeviceTensor(DeviceTensor&& x) noexcept
+    : ttvalue_type(std::move(x))
+    , tensor_type(std::move(x))
     , b(std::move(x.b))
-    { }
+    {
+      //std::cout << "DeviceTensor move ctor" << std::endl;
+    }
 
     /// copy assignment operator
     template <class _Tensor, class = typename std::enable_if<
                                  btas::is_boxtensor<_Tensor>::value &&
                                  not std::is_same<typename _Tensor::storage_type, storage_type>::value>::type>
-    DeviceTensor& operator=(const _Tensor& x) {
-      tensor_type::operator=(x);
+    DeviceTensor& operator=(const _Tensor& x) noexcept {
+      tensor_type::operator=(x.clone());
       b.reset(this->size() ? this->data() : nullptr, this->size());
+      //std::cout << "DeviceTensor tensor_type copy operator" << std::endl;
       return *this;
     }
 
@@ -181,26 +204,36 @@ struct DeviceTensor : public btas::Tensor<_T, _Range, _Storage> {
     template <class _Tensor, class = typename std::enable_if<btas::is_boxtensor<_Tensor>::value>::type,
               class = typename std::enable_if<
                   std::is_same<typename _Tensor::storage_type, storage_type>::value>::type>
-    DeviceTensor& operator=(const _Tensor& x) {
-      tensor_type::operator=(x);
+    DeviceTensor& operator=(const _Tensor& x) noexcept {
+      tensor_type::operator=(x.clone());
       b.reset(this->size() ? this->data() : nullptr, this->size());
+      //std::cout << "DeviceTensor tensor_type copy operator" << std::endl;
       return *this;
     }
 
     /// copy assignment: devicebuf cannot be copied, deleted
-    DeviceTensor& operator=(const DeviceTensor& x) {
-      tensor_type::operator=(x);
+    DeviceTensor& operator=(const DeviceTensor& x) noexcept {
+      ttvalue_type::operator=(x);
+      tensor_type::operator=(x.clone());
       b.reset(this->size() ? this->data() : nullptr, this->size());
+      //std::cout << "DeviceTensor copy operator" << std::endl;
       return *this;
     }
 
     /// move assignment operator
-    DeviceTensor& operator=(DeviceTensor&& x) {
-      tensor_type::operator=(std::forward<tensor_type>(x));
+    DeviceTensor& operator=(DeviceTensor&& x) noexcept {
+      ttvalue_type::operator=(std::move(x));
+      tensor_type::operator=(std::move(x));
       std::swap(x.b, b);
+      //std::cout << "DeviceTensor move ctor" << std::endl;
       return *this;
     }
 
+    using tensor_type::begin;
+    using tensor_type::cbegin;
+    using tensor_type::end;
+    using tensor_type::cend;
+
 };
 
 using blk_t = DeviceTensor<double, btas::DEFAULT::range,
@@ -366,7 +399,7 @@ class Read_SpMatrix : public TT<Key<2>, std::tuple<Out<Key<2>, Blk>>, Read_SpMat
     for (int k = 0; k < matrix_.outerSize(); ++k) {
       for (typename SpMatrix<Blk>::InnerIterator it(matrix_, k); it; ++it) {
         if (rank == this->get_keymap()(Key<2>(std::initializer_list<long>({it.row(), it.col()}))))
-          ::send<0>(Key<2>(std::initializer_list<long>({it.row(), it.col()})), it.value(), out);
+          ::send<0>(Key<2>(std::initializer_list<long>({it.row(), it.col()})), ttg::persistent(it.value()), out);
       }
     }
   }
@@ -386,12 +419,14 @@ class Write_SpMatrix : public TT<Key<2>, std::tuple<>, Write_SpMatrix<Blk>, ttg:
       : baseT(edges(in), edges(), "write_spmatrix", {"Cij"}, {}, ij_keymap), matrix_(matrix) {}
 
   void op(const Key<2> &key, typename baseT::input_refs_tuple_type &&elem, std::tuple<> &) {
+#if 0
     std::lock_guard<std::mutex> lock(mtx_);
     ttg::trace("rank =", default_execution_context().rank(),
                "/ thread_id =", reinterpret_cast<std::uintptr_t>(pthread_self()), "spmm.cc Write_SpMatrix wrote {",
                key[0], ",", key[1], "} = ", baseT::template get<0>(elem), " in ", static_cast<void *>(&matrix_),
                " with mutex @", static_cast<void *>(&mtx_), " for object @", static_cast<void *>(this));
-    values_.emplace_back(key[0], key[1], baseT::template get<0>(elem));
+    values_.emplace_back(key[0], key[1], std::move(baseT::template get<0>(elem)));
+#endif // 0
   }
 
   /// grab completion status as a future<void>
@@ -491,7 +526,7 @@ class SpMM25D {
 
   /// Locally broadcast `A[i][k]` assigned to this processor `p` to matmul tasks `{i,j,k}` for all `j` such that
   /// `B[k][j]` exists AND `k` contribution to `C[i][j]` is assigned to this processor
-  class LocalBcastA : public TT<Key<3>, std::tuple<Out<Key<3>, Blk>>, LocalBcastA, ttg::typelist<Blk>> {
+  class LocalBcastA : public TT<Key<3>, std::tuple<Out<Key<3>, Blk>>, LocalBcastA, ttg::typelist<const Blk>> {
    public:
     using baseT = typename LocalBcastA::ttT;
 
@@ -528,7 +563,7 @@ class SpMM25D {
   };  // class LocalBcastA
 
   /// broadcast `A[i][k]` to all processors which will contain at least one `C[i][j]` such that `B[k][j]` exists
-  class BcastA : public TT<Key<2>, std::tuple<Out<Key<3>, Blk>>, BcastA, ttg::typelist<Blk>> {
+  class BcastA : public TT<Key<2>, std::tuple<Out<Key<3>, Blk>>, BcastA, ttg::typelist<const Blk>> {
    public:
     using baseT = typename BcastA::ttT;
 
@@ -566,7 +601,7 @@ class SpMM25D {
 
   /// Locally broadcast `B[k][j]` assigned to this processor `p` to matmul tasks `{i,j,k}` for all `k` such that
   /// `A[i][k]` exists AND `k` contribution to `C[i][j]` is assigned to this processor
-  class LocalBcastB : public TT<Key<3>, std::tuple<Out<Key<3>, Blk>>, LocalBcastB, ttg::typelist<Blk>> {
+  class LocalBcastB : public TT<Key<3>, std::tuple<Out<Key<3>, Blk>>, LocalBcastB, ttg::typelist<const Blk>> {
    public:
     using baseT = typename LocalBcastB::ttT;
 
@@ -602,7 +637,7 @@ class SpMM25D {
   };  // class LocalBcastB
 
   /// broadcast `B[k][j]` to all processors which will contain at least one `C[i][j]` such that `A[i][k]` exists
-  class BcastB : public TT<Key<2>, std::tuple<Out<Key<3>, Blk>>, BcastB, ttg::typelist<Blk>> {
+  class BcastB : public TT<Key<2>, std::tuple<Out<Key<3>, Blk>>, BcastB, ttg::typelist<const Blk>> {
    public:
     using baseT = typename BcastB::ttT;
 
diff --git a/ttg/CMakeLists.txt b/ttg/CMakeLists.txt
index a949cb961..67277a83a 100644
--- a/ttg/CMakeLists.txt
+++ b/ttg/CMakeLists.txt
@@ -10,6 +10,7 @@ configure_file(
 set(ttg-util-headers
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/backtrace.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/bug.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/cublas_helper.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/demangle.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/diagnose.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/dot.h
@@ -67,6 +68,10 @@ set(ttg-impl-headers
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/device/task.h
         ${CMAKE_CURRENT_BINARY_DIR}/ttg/config.h
     )
+set(ttg_device_headers
+        ${CMAKE_CURRENT_SOURCE_DIR}/ttg/device/cublas_helper.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/ttg/device/task.h
+    )
 set(ttg-headers
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg.h
     )
@@ -76,6 +81,7 @@ set(ttg-sources
         ${ttg-base-headers}
         ${ttg-util-headers}
         ${ttg-external-headers}
+        ${CMAKE_CURRENT_SOURCE_DIR}/ttg/device/cublas_helper.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/backtrace.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/bug.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/env.cpp
@@ -112,7 +118,6 @@ if (TARGET std::coroutine)
 endif ()
 if (TARGET CUDA::cudart)
   list(APPEND ttg-deps CUDA::cudart)
-  list(APPEND ttg-defs "TTG_HAVE_CUDART=1")
 endif (TARGET CUDA::cudart)
 
 set(ttg-public-headers ${ttg-headers};${ttg-impl-headers};${ttg-base-headers};${ttg-util-headers})
@@ -217,10 +222,12 @@ if (TARGET PaRSEC::parsec)
           ${CMAKE_CURRENT_SOURCE_DIR}/ttg/parsec/fwd.h
           ${CMAKE_CURRENT_SOURCE_DIR}/ttg/parsec/import.h
           ${CMAKE_CURRENT_SOURCE_DIR}/ttg/parsec/ptr.h
+          ${CMAKE_CURRENT_SOURCE_DIR}/ttg/parsec/parsec-ext.h
           ${CMAKE_CURRENT_SOURCE_DIR}/ttg/parsec/task.h
           ${CMAKE_CURRENT_SOURCE_DIR}/ttg/parsec/thread_local.h
           ${CMAKE_CURRENT_SOURCE_DIR}/ttg/parsec/ttg.h
           ${CMAKE_CURRENT_SOURCE_DIR}/ttg/parsec/ttg_data_copy.h
+          ${CMAKE_CURRENT_SOURCE_DIR}/ttg/parsec/ttvalue.h
           )
   find_package(MPI)
   set(ttg-parsec-deps "ttg;MPI::MPI_CXX;PaRSEC::parsec")
diff --git a/ttg/ttg/config.in.h b/ttg/ttg/config.in.h
index 8f208c611..36b3adced 100644
--- a/ttg/ttg/config.in.h
+++ b/ttg/ttg/config.in.h
@@ -13,4 +13,6 @@
 
 #cmakedefine TTG_HAVE_CUDA
 
+#cmakedefine TTG_HAVE_CUDART
+
 #endif  // TTG_CONFIG_IN_H
diff --git a/ttg/ttg/device/cublas_helper.cpp b/ttg/ttg/device/cublas_helper.cpp
new file mode 100644
index 000000000..0f77c2a5f
--- /dev/null
+++ b/ttg/ttg/device/cublas_helper.cpp
@@ -0,0 +1,27 @@
+#ifdef TTG_HAVE_CUDART
+#include <cublas.h>
+#endif // TTG_HAVE_CUDART
+
+#include <exception>
+#include <stdexcept>
+
+#include "ttg/config.h"
+#include "ttg/device/cublas_helper.h"
+
+namespace ttg::detail {
+
+/* shim wrapper to work around the fact that cublas
+ * deliberately breaks its API depending on the order
+ * in which header are included */
+void cublas_set_kernel_stream(cudaStream_t stream) {
+#ifdef TTG_HAVE_CUDART
+    cublasStatus_t status = cublasSetKernelStream(stream);
+    if (CUBLAS_STATUS_SUCCESS != status) {
+        throw std::runtime_error("cublasSetKernelStream failed");
+    }
+#else
+    throw std::runtime_error("Support for cublas missing during installation!");
+#endif // TTG_HAVE_CUDART
+}
+
+} // namespace
\ No newline at end of file
diff --git a/ttg/ttg/device/cublas_helper.h b/ttg/ttg/device/cublas_helper.h
new file mode 100644
index 000000000..51c13601c
--- /dev/null
+++ b/ttg/ttg/device/cublas_helper.h
@@ -0,0 +1,17 @@
+#ifndef TTG_DEVICE_CUBLAS_HELPER_H
+#define TTG_DEVICE_CUBLAS_HELPER_H
+
+#include "ttg/config.h"
+
+#ifdef TTG_HAVE_CUDART
+#include <cublas.h>
+
+namespace ttg::detail {
+
+void cublas_set_kernel_stream(cudaStream_t stream);
+
+} // namespace ttg::detail
+#endif // TTG_HAVE_CUDART
+
+
+#endif // TTG_DEVICE_CUBLAS_HELPER_H
\ No newline at end of file
diff --git a/ttg/ttg/device/task.h b/ttg/ttg/device/task.h
index d5382de0c..b811c4484 100644
--- a/ttg/ttg/device/task.h
+++ b/ttg/ttg/device/task.h
@@ -140,7 +140,7 @@ namespace ttg {
       };
 
       template<typename Key, typename Value, ttg::Runtime Runtime = ttg::ttg_runtime>
-      inline send_coro_state send_coro (const Key& key, Value&& value, ttg::Out<Key, Value> &t,
+      inline send_coro_state send_coro (const Key& key, Value&& value, ttg::Out<Key, std::decay_t<Value>> &t,
                                         ttg::detail::value_copy_handler<Runtime>& ch) {
         ttg::detail::value_copy_handler<Runtime> copy_handler = std::move(ch); // destroyed at the end of the coro
         Key k = key;
@@ -150,7 +150,7 @@ namespace ttg {
       };
 
       template<typename Value, ttg::Runtime Runtime = ttg::ttg_runtime>
-      inline send_coro_state sendv_coro (Value&& value, ttg::Out<void, Value> &t,
+      inline send_coro_state sendv_coro (Value&& value, ttg::Out<void, std::decay_t<Value>> &t,
                                          ttg::detail::value_copy_handler<Runtime>& ch) {
         ttg::detail::value_copy_handler<Runtime> copy_handler = std::move(ch); // destroyed at the end of the coro
         t.prepare_send(std::forward<Value>(value));
diff --git a/ttg/ttg/parsec/buffer.h b/ttg/ttg/parsec/buffer.h
index c1d5172f7..f65b34f65 100644
--- a/ttg/ttg/parsec/buffer.h
+++ b/ttg/ttg/parsec/buffer.h
@@ -1,17 +1,19 @@
 #ifndef TTG_PARSEC_BUFFER_H
 #define TTG_PARSEC_BUFFER_H
 
-// TODO: replace with short vector
-#define TTG_PARSEC_MAX_NUM_DEVICES 4
-
 #include <array>
 #include <vector>
 #include <parsec.h>
 #include <parsec/data_internal.h>
 #include <parsec/mca/device/device.h>
 #include "ttg/parsec/ttg_data_copy.h"
+#include "ttg/parsec/parsec-ext.h"
 #include "ttg/util/iovec.h"
 
+#if defined(PARSEC_HAVE_DEV_CUDA_SUPPORT)
+#include <cuda_runtime.h>
+#endif // PARSEC_HAVE_DEV_CUDA_SUPPORT
+
 namespace ttg_parsec {
 
 
@@ -61,8 +63,15 @@ struct buffer {
   }
 
   static void delete_parsec_data(parsec_data_t *data) {
-    //std::cout << "delete parsec_data " << data << std::endl;
-    //assert(0);
+#if defined(PARSEC_HAVE_DEV_CUDA_SUPPORT)
+    if (data->device_copies[0]->flags & TTG_PARSEC_DATA_FLAG_REGISTERED) {
+      // register the memory for faster access
+      cudaError_t status;
+      status = cudaHostUnregister(data->device_copies[0]->device_private);
+      assert(cudaSuccess == status);
+      data->device_copies[0]->flags ^= TTG_PARSEC_DATA_FLAG_REGISTERED;
+    }
+#endif // PARSEC_HAVE_DEV_CUDA_SUPPORT
     parsec_data_destroy(data);
   }
 
@@ -71,9 +80,6 @@ struct buffer {
   }
 
   static parsec_data_t* create_parsec_data(void *ptr, size_t count) {
-    if (ptr == nullptr) {
-
-    }
     parsec_data_t *data = parsec_data_create_with_type(nullptr, 0, ptr,
                                                        sizeof(element_type)*count,
                                                        parsec_datatype_int8_t);
@@ -83,6 +89,10 @@ struct buffer {
     return data;
   }
 
+  static void register_memory(void *ptr, size_t count) {
+
+  }
+
   void reset() {
     if (m_data) {
       if (nullptr != m_ttg_copy) {
@@ -100,7 +110,7 @@ struct buffer {
   /* The device ID of the CPU. */
   static constexpr int cpu_device = 0;
 
-  buffer() : buffer(1)
+  buffer() : buffer(nullptr, 0)
   { }
 
   buffer(std::size_t count)
@@ -109,6 +119,7 @@ struct buffer {
   , m_count(count)
   , m_ttg_copy(detail::ttg_data_copy_container())
   {
+    //std::cout << "buffer " << this << " ctor ttg_copy " << m_ttg_copy << std::endl;
     if (nullptr != m_ttg_copy) {
       /* register with the new ttg_copy */
       m_ttg_copy->add_device_data(m_data.get());
@@ -125,7 +136,8 @@ struct buffer {
   , m_count(count)
   , m_ttg_copy(detail::ttg_data_copy_container())
   {
-    if (nullptr != m_ttg_copy) {
+    //std::cout << "buffer " << this << " ctor ttg_copy " << m_ttg_copy << std::endl;
+    if (nullptr != m_ttg_copy && m_data) {
       /* register with the new ttg_copy */
       m_ttg_copy->add_device_data(m_data.get());
     }
@@ -146,6 +158,7 @@ struct buffer {
   , m_count(db.m_count)
   , m_ttg_copy(detail::ttg_data_copy_container())
   {
+    //std::cout << "buffer " << this << " other " << &db << " mv ctor ttg_copy " << m_ttg_copy << std::endl;
     db.m_count = 0;
 
     //std::cout << "buffer::move-ctor from " << &db << " ttg-copy " << db.m_ttg_copy
@@ -153,6 +166,7 @@ struct buffer {
     //          << " parsec-data " << m_data.get()
     //          << std::endl;
 
+#if 0 // This is erroneous: the ttg copy will move its content over by itself
     if (nullptr != db.m_ttg_copy) {
       /* remove from old ttg copy */
       db.m_ttg_copy->remove_device_data(m_data.get());
@@ -163,6 +177,7 @@ struct buffer {
       /* register with the new ttg_copy */
       m_ttg_copy->add_device_data(m_data.get());
     }
+#endif // 0
   }
 
   /* explicitly disable copying of buffers
@@ -176,6 +191,7 @@ struct buffer {
     m_host_data = std::move(db.m_host_data);
     m_count = db.m_count;
     db.m_count = 0;
+    //std::cout << "buffer " << this << " other " << &db << " mv op ttg_copy " << m_ttg_copy << std::endl;
     //std::cout << "buffer::move-assign from " << &db << " ttg-copy " << db.m_ttg_copy
     //          << " to " << this << " ttg-copy " << m_ttg_copy
     //          << " parsec-data " << m_data.get()
diff --git a/ttg/ttg/parsec/devicefunc.h b/ttg/ttg/parsec/devicefunc.h
index 22229f0e7..773218e4d 100644
--- a/ttg/ttg/parsec/devicefunc.h
+++ b/ttg/ttg/parsec/devicefunc.h
@@ -36,9 +36,9 @@ namespace ttg_parsec {
       auto access = PARSEC_FLOW_ACCESS_RW;
       if constexpr (std::is_const_v<view_type>) {
         // keep the flow at RW if it was RW to make sure we pull the data back out eventually
-        if (flows[I].flow_flags != PARSEC_FLOW_ACCESS_RW) {
+        //if (flows[I].flow_flags != PARSEC_FLOW_ACCESS_RW) {
           access = PARSEC_FLOW_ACCESS_READ;
-        }
+        //}
       } else if constexpr (ttg_parsec::is_devicescratch_v<view_type>) {
         if (view.scope() == ttg::scope::Allocate) {
           access = PARSEC_FLOW_ACCESS_WRITE;
@@ -62,6 +62,7 @@ namespace ttg_parsec {
       /* set the input data copy, parsec will take care of the transfer
        * and the buffer will look at the parsec_data_t for the current pointer */
       //detail::parsec_ttg_caller->parsec_task.data[I].data_in = data->device_copies[data->owner_device];
+      assert(nullptr != data->device_copies[0]->original);
       caller->parsec_task.data[I].data_in = data->device_copies[0];
       caller->parsec_task.data[I].source_repo_entry = NULL;
 
diff --git a/ttg/ttg/parsec/parsec-ext.h b/ttg/ttg/parsec/parsec-ext.h
new file mode 100644
index 000000000..a7e5e5222
--- /dev/null
+++ b/ttg/ttg/parsec/parsec-ext.h
@@ -0,0 +1,7 @@
+#ifndef TTG_PARSEC_EXT_H
+#define TTG_PARSEC_EXT_H
+
+/* HACK: we need this flag on a data copy to indicate whether it has been registered */
+#define TTG_PARSEC_DATA_FLAG_REGISTERED        ((parsec_data_flag_t)1<<2)
+
+#endif // TTG_PARSEC_EXT_H
\ No newline at end of file
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 5dd51f572..3c48b8bcb 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -45,6 +45,7 @@
 #include "ttg/parsec/devicescratch.h"
 #include "ttg/parsec/thread_local.h"
 #include "ttg/parsec/devicefunc.h"
+#include "ttg/parsec/ttvalue.h"
 
 #include <algorithm>
 #include <array>
@@ -99,6 +100,8 @@
 #include "ttg/parsec/thread_local.h"
 #include "ttg/parsec/ptr.h"
 #include "ttg/parsec/task.h"
+#include "ttg/device/cublas_helper.h"
+#include "ttg/parsec/parsec-ext.h"
 
 #undef TTG_PARSEC_DEBUG_TRACK_DATA_COPIES
 
@@ -633,8 +636,10 @@ namespace ttg_parsec {
     inline ttg_data_copy_t *create_new_datacopy(Value &&value) {
       using value_type = std::decay_t<Value>;
       ttg_data_copy_t *copy;
-      if constexpr (std::is_rvalue_reference_v<decltype(value)> ||
-                    std::is_copy_constructible_v<std::decay_t<Value>>) {
+      if constexpr (std::is_base_of_v<ttg::TTValue<value_type>, value_type>) {
+        copy = new value_type(std::forward<Value>(value));
+      } else if constexpr (std::is_rvalue_reference_v<decltype(value)> ||
+                           std::is_copy_constructible_v<std::decay_t<Value>>) {
         copy = new ttg_data_value_copy_t<value_type>(std::forward<Value>(value));
       } else {
         throw std::logic_error("Trying to copy-construct data that is not copy-constructible!");
@@ -1256,6 +1261,11 @@ namespace ttg_parsec {
       assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_TRANSFER ||
              dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_KERNEL);
 
+#if defined(PARSEC_HAVE_DEV_CUDA_SUPPORT)
+      parsec_cuda_exec_stream_t *cuda_stream = (parsec_cuda_exec_stream_t *)gpu_stream;
+      ttg::detail::cublas_set_kernel_stream(cuda_stream->cuda_stream);
+#endif // PARSEC_HAVE_DEV_CUDA_SUPPORT
+
       /* Here we call back into the coroutine again after the transfers have completed */
       static_op<Space>(&task->parsec_task);
 
@@ -1289,6 +1299,22 @@ namespace ttg_parsec {
     static_cuda_stage_in(parsec_gpu_task_t        *gtask,
                          uint32_t                  flow_mask,
                          parsec_gpu_exec_stream_t *gpu_stream) {
+      /* register any memory that hasn't been registered yet */
+      for (int i = 0; i < MAX_PARAM_COUNT; ++i) {
+        if (flow_mask & (1<<i)) {
+          task_t *task = (task_t*)gtask->ec;
+          parsec_data_copy_t *copy = task->parsec_task.data[i].data_in;
+          if (0 == (copy->flags & TTG_PARSEC_DATA_FLAG_REGISTERED)) {
+#if defined(PARSEC_HAVE_DEV_CUDA_SUPPORT)
+            // register host memory for faster device access
+            cudaError_t status;
+            status = cudaHostRegister(copy->device_private, gtask->flow_nb_elts[i], cudaHostRegisterPortable);
+            assert(cudaSuccess == status);
+#endif // PARSEC_HAVE_DEV_CUDA_SUPPORT
+            copy->flags |= TTG_PARSEC_DATA_FLAG_REGISTERED;
+          }
+        }
+      }
       return parsec_default_cuda_stage_in(gtask, flow_mask, gpu_stream);
     }
 
@@ -3586,6 +3612,7 @@ template <>
 struct ttg::detail::value_copy_handler<ttg::Runtime::PaRSEC> {
  private:
   ttg_parsec::detail::ttg_data_copy_t *copy_to_remove = nullptr;
+  bool do_release = true;
 
  public:
   value_copy_handler() = default;
@@ -3606,19 +3633,21 @@ struct ttg::detail::value_copy_handler<ttg::Runtime::PaRSEC> {
   ~value_copy_handler() {
     if (nullptr != copy_to_remove) {
       ttg_parsec::detail::remove_data_copy(copy_to_remove, ttg_parsec::detail::parsec_ttg_caller);
-      ttg_parsec::detail::release_data_copy(copy_to_remove);
+      if (do_release) {
+        ttg_parsec::detail::release_data_copy(copy_to_remove);
+      }
     }
   }
 
   template <typename Value>
-  inline Value &&operator()(Value &&value) {
+  inline std::add_lvalue_reference_t<Value> operator()(Value &&value) {
     static_assert(std::is_rvalue_reference_v<decltype(value)> ||
                   std::is_copy_constructible_v<std::decay_t<Value>>,
                   "Data sent without being moved must be copy-constructible!");
 
     auto caller = ttg_parsec::detail::parsec_ttg_caller;
     if (nullptr == caller) {
-      ttg::print("ERROR: ttg_send or ttg_broadcast called outside of a task!\n");
+      ttg::print("ERROR: ttg::send or ttg::broadcast called outside of a task!\n");
     }
     using value_type = std::remove_reference_t<Value>;
     ttg_parsec::detail::ttg_data_copy_t *copy;
@@ -3643,11 +3672,27 @@ struct ttg::detail::value_copy_handler<ttg::Runtime::PaRSEC> {
       copy->inc_current_version();
     }
     caller->data_flags = ttg_parsec::detail::ttg_parsec_data_flags::NONE;
-    if constexpr (std::is_rvalue_reference_v<decltype(value)>) {
-      return std::move(*value_ptr);
-    } else {
-      return *value_ptr;
+    return *value_ptr;
+  }
+
+  template<typename Value>
+  inline std::add_lvalue_reference_t<Value> operator()(ttg::detail::persistent_value_ref<Value> vref) {
+    auto caller = ttg_parsec::detail::parsec_ttg_caller;
+    if (nullptr == caller) {
+      ttg::print("ERROR: ttg::send or ttg::broadcast called outside of a task!\n");
+    }
+    ttg_parsec::detail::ttg_data_copy_t *copy;
+    copy = ttg_parsec::detail::find_copy_in_task(caller, &vref.value_ref);
+    if (nullptr == copy) {
+      // no need to create a new copy since it's derived from the copy already
+      copy = const_cast<ttg_parsec::detail::ttg_data_copy_t *>(static_cast<const ttg_parsec::detail::ttg_data_copy_t *>(&vref.value_ref));
+      bool inserted = ttg_parsec::detail::add_copy_to_task(copy, caller);
+      assert(inserted);
+      copy_to_remove = copy; // we want to remove the copy from the task once done sending
+      do_release = false; // we don't release the copy since we didn't allocate it
+      copy->add_ref(); // add a reference so that TTG does not attempt to delete this object
     }
+    return vref.value_ref;
   }
 
   template <typename Value>
@@ -3656,7 +3701,7 @@ struct ttg::detail::value_copy_handler<ttg::Runtime::PaRSEC> {
                   "Data sent without being moved must be copy-constructible!");
     auto caller = ttg_parsec::detail::parsec_ttg_caller;
     if (nullptr == caller) {
-      ttg::print("ERROR: ttg_send or ttg_broadcast called outside of a task!\n");
+      ttg::print("ERROR: ttg::send or ttg::broadcast called outside of a task!\n");
     }
     ttg_parsec::detail::ttg_data_copy_t *copy;
     copy = ttg_parsec::detail::find_copy_in_task(caller, &value);
diff --git a/ttg/ttg/parsec/ttg_data_copy.h b/ttg/ttg/parsec/ttg_data_copy.h
index 865395be3..dcc3e33a2 100644
--- a/ttg/ttg/parsec/ttg_data_copy.h
+++ b/ttg/ttg/parsec/ttg_data_copy.h
@@ -18,7 +18,11 @@ namespace ttg_parsec {
 
   namespace detail {
 
-    template<typename T>
+    // fwd-decl
+    struct ttg_data_copy_t;
+
+    /* templated to break cyclic dependency with ttg_data_copy_container */
+    template<typename T = ttg_data_copy_t>
     struct ttg_data_copy_container_setter {
       ttg_data_copy_container_setter(T* ptr) {
         /* set the container ptr here, will be reset in the the ttg_data_value_copy_t ctor */
@@ -31,18 +35,16 @@ namespace ttg_parsec {
      * readers or writers but merely prevent the object from being
      * destroyed once no readers/writers exist.
      */
-    struct ttg_data_copy_t : private ttg_data_copy_container_setter<ttg_data_copy_t> {
+    struct ttg_data_copy_t {
 
       /* special value assigned to parsec_data_copy_t::readers to mark the copy as
       * mutable, i.e., a task will modify it */
       static constexpr int mutable_tag = std::numeric_limits<int>::min();
 
       ttg_data_copy_t()
-      : ttg_data_copy_container_setter(this)
       { }
 
       ttg_data_copy_t(const ttg_data_copy_t& c)
-      : ttg_data_copy_container_setter(this)
       {
         /* we allow copying but do not copy any data over from the original
          * device copies will have to be allocated again
@@ -50,8 +52,7 @@ namespace ttg_parsec {
       }
 
       ttg_data_copy_t(ttg_data_copy_t&& c)
-      : ttg_data_copy_container_setter(this)
-      , m_next_task(c.m_next_task)
+      : m_next_task(c.m_next_task)
       , m_readers(c.m_readers)
       , m_refs(c.m_refs.load(std::memory_order_relaxed))
       , m_dev_data(std::move(c.m_dev_data))
@@ -76,8 +77,6 @@ namespace ttg_parsec {
         c.m_single_dev_data = nullptr;
         m_num_dev_data = c.m_num_dev_data;
         c.m_num_dev_data = 0;
-        /* set the container ptr here, will be reset in the the ttg_data_value_copy_t ctor */
-        ttg_data_copy_container() = this;
         return *this;
       }
 
@@ -86,8 +85,6 @@ namespace ttg_parsec {
          * device copies will have to be allocated again
          * and it's a new object to reference */
 
-        /* set the container ptr here, will be reset in the the ttg_data_value_copy_t ctor */
-        ttg_data_copy_container() = this;
         return *this;
       }
 
@@ -122,7 +119,9 @@ namespace ttg_parsec {
       * Reset the number of readers to read-only with a single reader.
       */
       void reset_readers() {
-        m_readers = 1;
+        if (mutable_tag == m_readers) {
+          m_readers = 1;
+        }
       }
 
       /* Decrement the reader counter and return previous value.
@@ -325,13 +324,15 @@ namespace ttg_parsec {
     * the destructor of ttg_data_copy_t base class is called.
     */
     template<typename ValueT>
-    struct ttg_data_value_copy_t final : public ttg_data_copy_t {
+    struct ttg_data_value_copy_t final : private ttg_data_copy_container_setter<ttg_data_copy_t>
+                                       , public ttg_data_copy_t {
       using value_type = ValueT;
       value_type m_value;
 
       template<typename T>
       ttg_data_value_copy_t(T&& value)
-      : ttg_data_copy_t()
+      : ttg_data_copy_container_setter(this)
+      , ttg_data_copy_t()
       , m_value(std::forward<T>(value))
       {
         /* reset the container tracker */
@@ -340,7 +341,8 @@ namespace ttg_parsec {
 
       ttg_data_value_copy_t(ttg_data_value_copy_t&& c)
         noexcept(std::is_nothrow_move_constructible_v<value_type>)
-      : ttg_data_copy_t(std::move(c))
+      : ttg_data_copy_container_setter(this)
+      , ttg_data_copy_t(std::move(c))
       , m_value(std::move(c.m_value))
       {
         /* reset the container tracker */
@@ -349,7 +351,8 @@ namespace ttg_parsec {
 
       ttg_data_value_copy_t(const ttg_data_value_copy_t& c)
         noexcept(std::is_nothrow_copy_constructible_v<value_type>)
-      : ttg_data_copy_t(c)
+      : ttg_data_copy_container_setter(this)
+      , ttg_data_copy_t(c)
       , m_value(c.m_value)
       {
         /* reset the container tracker */
@@ -359,6 +362,8 @@ namespace ttg_parsec {
       ttg_data_value_copy_t& operator=(ttg_data_value_copy_t&& c)
         noexcept(std::is_nothrow_move_assignable_v<value_type>)
       {
+        /* set the container ptr here, will be reset in the the ttg_data_value_copy_t ctor */
+        ttg_data_copy_container() = this;
         ttg_data_copy_t::operator=(std::move(c));
         m_value = std::move(c.m_value);
         /* reset the container tracker */
@@ -368,6 +373,8 @@ namespace ttg_parsec {
       ttg_data_value_copy_t& operator=(const ttg_data_value_copy_t& c)
         noexcept(std::is_nothrow_copy_assignable_v<value_type>)
       {
+        /* set the container ptr here, will be reset in the the ttg_data_value_copy_t ctor */
+        ttg_data_copy_container() = this;
         ttg_data_copy_t::operator=(c);
         m_value = c.m_value;
         /* reset the container tracker */
diff --git a/ttg/ttg/parsec/ttvalue.h b/ttg/ttg/parsec/ttvalue.h
new file mode 100644
index 000000000..6ed31c9fc
--- /dev/null
+++ b/ttg/ttg/parsec/ttvalue.h
@@ -0,0 +1,101 @@
+#ifndef TTG_PARSEC_TTVALUE_H
+#define TTG_PARSEC_TTVALUE_H
+
+#include <type_traits>
+
+#include "ttg/parsec/ttg_data_copy.h"
+
+namespace ttg {
+
+  /**
+   * Base class for data to moved into, through, and out of
+   * a task graph. By inheriting from this base class,
+   * TTG is able to easily track the data and avoid some
+   * of the copies otherwise necessary.
+   */
+  template<typename DerivedT>
+  struct TTValue : private ttg_parsec::detail::ttg_data_copy_container_setter<ttg_parsec::detail::ttg_data_copy_t>
+                 , public ttg_parsec::detail::ttg_data_copy_t {
+
+    using derived_type = std::decay_t<DerivedT>;
+
+    /* Constructor called with a pointer to the derived class object */
+    TTValue()
+    : ttg_data_copy_container_setter(this)
+    , ttg_data_copy_t()
+    { }
+
+    /* default copy ctor */
+    TTValue(const TTValue& v)
+    : ttg_data_copy_container_setter(this)
+    , ttg_data_copy_t(v)
+    { }
+
+    /* default move ctor */
+    TTValue(TTValue&& v)
+    : ttg_data_copy_container_setter(this)
+    , ttg_data_copy_t(std::move(v))
+    { }
+
+    /* virtual mark destructor */
+    virtual ~TTValue() = default;
+
+    /* default copy operator */
+    TTValue& operator=(const TTValue& v) {
+      ttg_parsec::detail::ttg_data_copy_container() = this;
+      ttg_data_copy_t::operator=(v);
+      return *this;
+    }
+
+    /* default move operator */
+    TTValue& operator=(TTValue&& v) {
+      ttg_parsec::detail::ttg_data_copy_container() = this;
+      ttg_data_copy_t::operator=(std::move(v));
+      return *this;
+    }
+
+    virtual void* get_ptr() override final {
+      return static_cast<DerivedT*>(this);
+    }
+
+    derived_type& get_derived() {
+        return *static_cast<DerivedT*>(this);
+    }
+
+    const derived_type& get_derived() const {
+        return *static_cast<DerivedT*>(this);
+    }
+  };
+
+  namespace detail {
+
+    template<typename T, bool Enabler = false>
+    struct is_ttvalue_base : std::false_type {};
+
+    template<typename T>
+    struct is_ttvalue_base<T, std::is_base_of_v<TTValue<std::decay_t<T>>, std::decay_t<T>>>
+    : std::true_type
+    { };
+
+    template<typename T>
+    static constexpr const bool is_ttvalue_base_v = is_ttvalue_base<T>::value;
+
+    template<typename ValueT>
+    struct persistent_value_ref {
+      using reference_type = ValueT;
+      using value_type = std::decay_t<ValueT>;
+      using lvalue_reference_type = std::add_lvalue_reference_t<std::remove_reference_t<ValueT>>;
+      lvalue_reference_type value_ref;
+    };
+  } // namespace detail
+
+  template<typename ValueT>
+  inline auto persistent(ValueT&& value) {
+    static_assert(std::is_base_of_v<TTValue<std::decay_t<ValueT>>, std::decay_t<ValueT>>,
+                  "ttg::persistent can only be used on types derived from ttg::TTValue");
+    return detail::persistent_value_ref<ValueT>{value};
+  }
+
+} // namespace ttg
+
+#endif // TTG_PARSEC_TTVALUE_H
\ No newline at end of file

From 705f06e8f50b6aaca840271542da7fb9adf9aac8 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 26 May 2023 14:22:51 -0400
Subject: [PATCH 084/259] TTG parsec data copy: add back-pointer to self

This is needed to work-around missing support for offsetof in virtual
classes. Instead we store the pointer to the derived class inside the
non-virtual base class. This allows us to store data copies in parsec
lifo or lists and get the virtual derived object back.

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/ttg_data_copy.h | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/ttg/ttg/parsec/ttg_data_copy.h b/ttg/ttg/parsec/ttg_data_copy.h
index dcc3e33a2..98d952777 100644
--- a/ttg/ttg/parsec/ttg_data_copy.h
+++ b/ttg/ttg/parsec/ttg_data_copy.h
@@ -30,18 +30,30 @@ namespace ttg_parsec {
       }
     };
 
+    /* special type: stores a pointer to the ttg_data_copy_t. This is necessary
+     * because ttg_data_copy_t has virtual functions so we cannot cast from parsec_data_copy_t
+     * to ttg_data_copy_t (offsetof is not supported for virtual classes).
+     * The self pointer is a back-pointer to the ttg_data_copy_t. */
+    struct ttg_data_copy_self_t : public parsec_data_copy_t {
+      ttg_data_copy_t *self;
+      ttg_data_copy_self_t(ttg_data_copy_t* dc)
+      : self(dc)
+      { }
+    };
+
     /* Non-owning copy-tracking wrapper, accounting for N readers or 1 writer.
      * Also counts external references, which are not treated as
      * readers or writers but merely prevent the object from being
      * destroyed once no readers/writers exist.
      */
-    struct ttg_data_copy_t {
+    struct ttg_data_copy_t : public ttg_data_copy_self_t {
 
       /* special value assigned to parsec_data_copy_t::readers to mark the copy as
       * mutable, i.e., a task will modify it */
       static constexpr int mutable_tag = std::numeric_limits<int>::min();
 
       ttg_data_copy_t()
+      : ttg_data_copy_self_t(this)
       { }
 
       ttg_data_copy_t(const ttg_data_copy_t& c)
@@ -52,7 +64,8 @@ namespace ttg_parsec {
       }
 
       ttg_data_copy_t(ttg_data_copy_t&& c)
-      : m_next_task(c.m_next_task)
+      : ttg_data_copy_self_t(this)
+      , m_next_task(c.m_next_task)
       , m_readers(c.m_readers)
       , m_refs(c.m_refs.load(std::memory_order_relaxed))
       , m_dev_data(std::move(c.m_dev_data))
@@ -397,4 +410,4 @@ namespace ttg_parsec {
 
 } // namespace ttg_parsec
 
-#endif // TTG_DATA_COPY_H
\ No newline at end of file
+#endif // TTG_DATA_COPY_H

From 1e775cbdc8d6659b7bc3822cda1a7ebc060b835e Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 26 May 2023 14:25:03 -0400
Subject: [PATCH 085/259] PaRSEC backend: Restore priorities

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/ttg.h | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 3c48b8bcb..5a2c05f50 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -708,10 +708,6 @@ namespace ttg_parsec {
       }
     }
 
-    static parsec_key_fn_t parsec_tasks_hash_fcts = {.key_equal = parsec_hash_table_generic_64bits_key_equal,
-                                                     .key_print = parsec_hash_table_generic_64bits_key_print,
-                                                     .key_hash = parsec_hash_table_generic_64bits_key_hash};
-
     template <typename KeyT, typename ActivationCallbackT>
     class rma_delayed_activate {
       std::vector<KeyT> _keylist;
@@ -1916,11 +1912,11 @@ namespace ttg_parsec {
       char *taskobj = (char *)parsec_thread_mempool_allocate(mempool);
       int32_t priority = 0;
       if constexpr (!keyT_is_Void) {
-        //priority = priomap(key);
+        priority = priomap(key);
         /* placement-new the task */
         newtask = new (taskobj) task_t(key, mempool, &this->self, world_impl.taskpool(), this, priority);
       } else {
-        //priority = priomap();
+        priority = priomap();
         /* placement-new the task */
         newtask = new (taskobj) task_t(mempool, &this->self, world_impl.taskpool(), this, priority);
       }

From 3a762023ab60161f37397d154d21c7bad766942e Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 26 May 2023 14:28:56 -0400
Subject: [PATCH 086/259] PaRSEC backend: offload reductions to dedicated task

When we encounter a streaming terminal we schedule a dedicated
reducer task. Subsequent inputs are put into a LIFO. Once the
reducer task runs it will process all available inputs for that
terminal but will not block waiting for all inputs. In the
worst case, we create N tasks (one for each input), likely less.

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 tests/unit/fibonacci.cc |   3 +-
 ttg/ttg/parsec/task.h   |  33 +++-
 ttg/ttg/parsec/ttg.h    | 322 +++++++++++++++++++++++++++++++++++-----
 3 files changed, 317 insertions(+), 41 deletions(-)

diff --git a/tests/unit/fibonacci.cc b/tests/unit/fibonacci.cc
index 6434aaf55..574661b33 100644
--- a/tests/unit/fibonacci.cc
+++ b/tests/unit/fibonacci.cc
@@ -49,8 +49,9 @@ TEST_CASE("Fibonacci", "[fib][core]") {
               const auto F_n_plus_2 = F_n_plus_1 + F_n;
               ttg::sendv<1>(F_n_plus_1, outs);
               ttg::send<0>(F_n_plus_2, F_n_plus_1, outs);
-            } else
+            } else {
               ttg::finalize<1>(outs);
+            }
           },
           ttg::edges(F2F), ttg::edges(F2F, F2P));
       auto print_op = ttg::make_tt(
diff --git a/ttg/ttg/parsec/task.h b/ttg/ttg/parsec/task.h
index f95c53683..c1791d0ad 100644
--- a/ttg/ttg/parsec/task.h
+++ b/ttg/ttg/parsec/task.h
@@ -82,11 +82,36 @@ namespace ttg_parsec {
       ttg_data_copy_t **copies;    //< pointer to the fixed copies array of the derived task
       parsec_hash_table_item_t tt_ht_item = {};
 
-      typedef struct {
+      struct stream_info_t {
         std::size_t goal;
         std::size_t size;
-      } size_goal_t;
+        parsec_lifo_t reduce_copies;
+        std::atomic<std::size_t> reduce_count;
+      };
+
+    protected:
+      template<std::size_t i = 0, typename TT>
+      void init_stream_info_impl(TT *tt, std::array<stream_info_t, TT::numins>& streams) {
+        if constexpr (TT::numins > i) {
+          if (std::get<i>(tt->input_reducers)) {
+            streams[i].goal = tt->static_stream_goal[i];
+            streams[i].size = 0;
+            PARSEC_OBJ_CONSTRUCT(&streams[i].reduce_copies, parsec_lifo_t);
+            streams[i].reduce_count.store(0, std::memory_order_relaxed);
+          }
+          /* recursion */
+          if constexpr((i + 1) < TT::numins) {
+            init_stream_info_impl<i+1>(tt, streams);
+          }
+        }
+      }
+
+      template<typename TT>
+      void init_stream_info(TT *tt, std::array<stream_info_t, TT::numins>& streams) {
+        init_stream_info_impl<0>(tt, streams);
+      }
 
+    public:
       typedef void (release_task_fn)(parsec_ttg_task_base_t*);
       /* Poor-mans virtual function
        * We cannot use virtual inheritance or private visibility because we
@@ -188,6 +213,8 @@ namespace ttg_parsec {
         uint64_t hv = ttg::hash<std::decay_t<decltype(key)>>{}(key);
         *(uintptr_t*)&(parsec_task.locals[0]) = hv;
         *(uintptr_t*)&(parsec_task.locals[2]) = reinterpret_cast<uintptr_t>(&this->key);
+
+        init_stream_info(tt, streams);
       }
 
       static void release_task(parsec_ttg_task_base_t* task_base) {
@@ -232,6 +259,8 @@ namespace ttg_parsec {
                                    &release_task, tt_ptr->m_defer_writer)
           , tt(tt_ptr) {
         tt_ht_item.key = pkey();
+
+        init_stream_info(tt, streams);
       }
 
       static void release_task(parsec_ttg_task_base_t* task_base) {
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 5a2c05f50..30183a22c 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1089,6 +1089,7 @@ namespace ttg_parsec {
    private:
     using task_t = detail::parsec_ttg_task_t<ttT>;
 
+    friend detail::parsec_ttg_task_base_t;
     friend task_t;
 
     /* the offset of the key placed after the task structure in the memory from mempool */
@@ -1146,7 +1147,8 @@ namespace ttg_parsec {
     // For now use same type for unary/streaming input terminals, and stream reducers assigned at runtime
     ttg::meta::detail::input_reducers_t<actual_input_tuple_type>
         input_reducers;  //!< Reducers for the input terminals (empty = expect single value)
-    std::array<std::size_t, numins> static_stream_goal;
+    std::array<parsec_task_class_t*, numins> inpute_reducers_taskclass = { nullptr };
+    std::array<std::size_t, numins> static_stream_goal = { std::numeric_limits<std::size_t>::max() };
     int num_pullins = 0;
 
     bool m_defer_writer = TTG_PARSEC_DEFER_WRITER;
@@ -1548,6 +1550,95 @@ namespace ttg_parsec {
         return PARSEC_HOOK_RETURN_DONE;
     }
 
+    struct reducer_task_t {
+      parsec_task_t parsec_task;
+      task_t *task;
+
+      reducer_task_t(task_t* task, parsec_thread_mempool_t *mempool,
+                     parsec_task_class_t *task_class, parsec_taskpool_t *taskpool,
+                     int32_t priority)
+      : task(task)
+      {
+        PARSEC_LIST_ITEM_SINGLETON(&parsec_task.super);
+        parsec_task.mempool_owner = mempool;
+        parsec_task.task_class = task_class;
+        parsec_task.status = PARSEC_TASK_STATUS_HOOK;
+        parsec_task.taskpool = taskpool;
+        parsec_task.priority = priority;
+        parsec_task.chore_mask = 1<<0;
+      }
+    };
+
+    template <std::size_t i>
+    static parsec_hook_return_t static_reducer_op(parsec_execution_stream_s *es, parsec_task_t *parsec_task) {
+      using rtask_t = reducer_task_t;
+      using value_t = std::tuple_element_t<i, actual_input_tuple_type>;
+      constexpr const bool val_is_void = ttg::meta::is_void_v<value_t>;
+      rtask_t *rtask = (rtask_t*)parsec_task;
+      task_t *task = rtask->task;
+      ttT *baseobj = task->tt;
+      derivedT *obj = static_cast<derivedT *>(baseobj);
+
+      auto& reducer = std::get<i>(baseobj->input_reducers);
+
+      assert(parsec_ttg_caller == NULL);
+      parsec_ttg_caller = static_cast<detail::parsec_ttg_task_base_t*>(task);
+
+      if (obj->tracing()) {
+        if constexpr (!ttg::meta::is_void_v<keyT>)
+          ttg::trace(obj->get_world().rank(), ":", obj->get_name(), " : ", task->key, ": reducer executing");
+        else
+          ttg::trace(obj->get_world().rank(), ":", obj->get_name(), " : reducer executing");
+      }
+
+      /* the copy to reduce into */
+      detail::ttg_data_copy_t *target_copy;
+      target_copy = static_cast<detail::ttg_data_copy_t *>(task->parsec_task.data[i].data_in);
+      assert(val_is_void || nullptr != target_copy);
+      /* once we hit 0 we have to stop since another thread might enqueue a new reduction task */
+      std::size_t c;
+      do {
+        if constexpr(!val_is_void) {
+          /* the copies to reduce out of */
+          detail::ttg_data_copy_t *source_copy;
+          parsec_list_item_t *item;
+          item = parsec_lifo_pop(&task->streams[i].reduce_copies);
+          if (nullptr == item) {
+            break; // maybe someone is changing the goal right now
+          }
+          source_copy = ((detail::ttg_data_copy_self_t *)(item))->self;
+          reducer(*reinterpret_cast<std::decay_t<value_t> *>(target_copy->device_private),
+                  *reinterpret_cast<std::decay_t<value_t> *>(source_copy->device_private));
+          detail::release_data_copy(source_copy);
+        } else if constexpr(val_is_void) {
+          reducer(); // invoke control reducer
+        }
+        // there is only one task working on this stream, so no need to be atomic here
+        task->streams[i].size++;
+      } while ((c = task->streams[i].reduce_count.fetch_sub(1, std::memory_order_acq_rel)) > 1);
+
+      /* finalize_argstream sets goal to 1, so size may be larger than goal */
+      bool complete = (task->streams[i].size >= task->streams[i].goal);
+
+      if (complete && c == 1) {
+        /* task is still in the hash table, have release_task remove it */
+        task->remove_from_hash = true;
+        task->release_task(task);
+      }
+
+      parsec_ttg_caller = NULL;
+
+      if (obj->tracing()) {
+        if constexpr (!ttg::meta::is_void_v<keyT>)
+          ttg::trace(obj->get_world().rank(), ":", obj->get_name(), " : ", task->key, ": done executing");
+        else
+          ttg::trace(obj->get_world().rank(), ":", obj->get_name(), " : done executing");
+      }
+
+      return PARSEC_HOOK_RETURN_DONE;
+    }
+
+
    protected:
     template <typename T>
     uint64_t unpack(T &obj, void *_bytes, uint64_t pos) {
@@ -1929,6 +2020,33 @@ namespace ttg_parsec {
       return newtask;
     }
 
+
+    template <std::size_t i>
+    reducer_task_t *create_new_reducer_task(task_t *task) {
+      /* make sure we can reuse the existing memory pool and don't have to create a new one */
+      static_assert(sizeof(task_t) >= sizeof(reducer_task_t));
+      constexpr const bool keyT_is_Void = ttg::meta::is_void_v<keyT>;
+      auto &world_impl = world.impl();
+      reducer_task_t *newtask;
+      parsec_thread_mempool_t *mempool = get_task_mempool();
+      char *taskobj = (char *)parsec_thread_mempool_allocate(mempool);
+      // use the priority of the task we stream into
+      int32_t priority = 0;
+      if constexpr (!keyT_is_Void) {
+        priority = priomap(task->key);
+        ttg::trace(world.rank(), ":", get_name(), " : ", task->key, ": creating reducer task");
+      } else {
+        priority = priomap();
+        ttg::trace(world.rank(), ":", get_name(), ": creating reducer task");
+      }
+      /* placement-new the task */
+      newtask = new (taskobj) reducer_task_t(task, mempool, inpute_reducers_taskclass[i],
+                                                    world_impl.taskpool(), priority);
+
+      return newtask;
+    }
+
+
     // Used to set the i'th argument
     template <std::size_t i, typename Key, typename Value>
     void set_arg_local_impl(const Key &key, Value &&value, detail::ttg_data_copy_t *copy_in = nullptr,
@@ -1954,7 +2072,7 @@ namespace ttg_parsec {
       task_t *task;
       auto &world_impl = world.impl();
       auto &reducer = std::get<i>(input_reducers);
-      bool release = true;
+      bool release = false;
       bool remove_from_hash = true;
       bool discover_task = true;
       bool get_pull_data = false;
@@ -2024,7 +2142,20 @@ namespace ttg_parsec {
       if (reducer) {  // is this a streaming input? reduce the received value
         // N.B. Right now reductions are done eagerly, without spawning tasks
         //      this means we must lock
-        parsec_hash_table_lock_bucket(&tasks_table, hk);
+        //parsec_hash_table_lock_bucket(&tasks_table, hk);
+
+        auto submit_reducer_task = [&](auto *task){
+          /* check if we need to create a task */
+          std::size_t c = task->streams[i].reduce_count.fetch_add(1, std::memory_order_release);
+          if (0 == c) {
+            /* we are responsible for creating the reduction task */
+            reducer_task_t *reduce_task;
+            reduce_task = create_new_reducer_task<i>(task);
+            parsec_task_t *vp_task_rings[1] = { &reduce_task->parsec_task };
+            parsec_execution_stream_t *es = world_impl.execution_stream();
+            __parsec_schedule_vp(es, vp_task_rings, 0);
+          }
+        };
 
         if constexpr (!ttg::meta::is_void_v<valueT>) {  // for data values
           // have a value already? if not, set, otherwise reduce
@@ -2034,20 +2165,36 @@ namespace ttg_parsec {
             /* For now, we always create a copy because we cannot rely on the task_release
              * mechanism (it would release the task, not the reduction value). */
             copy = detail::create_new_datacopy(std::forward<Value>(value));
-            task->copies[i] = copy;
+            task->parsec_task.data[i].data_in = copy;
+            task->streams[i].size++;
+            if (task->streams[i].size == task->streams[i].goal) {
+              release = true;
+            }
           } else {
-            reducer(*reinterpret_cast<std::decay_t<valueT> *>(copy->get_ptr()), value);
+            if (nullptr != parsec_ttg_caller) {
+              copy = detail::find_copy_in_task(parsec_ttg_caller, &value);
+            }
+            if (nullptr != copy) {
+              /* retain the data copy */
+              copy = detail::register_data_copy<valueT>(copy, task, input_is_const);
+            } else {
+              /* create a new copy */
+              copy = detail::create_new_datacopy(std::forward<Value>(value));
+            }
+            /* enqueue the data copy to be reduced */
+            parsec_lifo_push(&task->streams[i].reduce_copies, &copy->super);
+            submit_reducer_task(task);
+            //reducer(*reinterpret_cast<std::decay_t<valueT> *>(copy->device_private), value);
           }
         } else {
-          reducer();  // even if this was a control input, must execute the reducer for possible side effects
-        }
-        task->stream[i].size++;
-        release = (task->stream[i].size == task->stream[i].goal);
-        if (release) {
-          parsec_hash_table_nolock_remove(&tasks_table, hk);
-          remove_from_hash = false;
+          /* submit reducer for void values to handle side effects */
+          submit_reducer_task(task);
         }
-        parsec_hash_table_unlock_bucket(&tasks_table, hk);
+        //if (release) {
+        //  parsec_hash_table_nolock_remove(&tasks_table, hk);
+        //  remove_from_hash = false;
+        //}
+        //parsec_hash_table_unlock_bucket(&tasks_table, hk);
       } else {
         /* whether the task needs to be deferred or not */
         if constexpr (!valueT_is_Void) {
@@ -2073,6 +2220,8 @@ namespace ttg_parsec {
            * make a copy of the original data */
           release = (copy->get_next_task() != &task->parsec_task);
           task->copies[i] = copy;
+        } else {
+          release = true;
         }
       }
       task->remove_from_hash = remove_from_hash;
@@ -2557,7 +2706,7 @@ namespace ttg_parsec {
       this->trace(world.rank(), ":", get_name(), ": setting global stream size for terminal ", i);
 
       // Check if stream is already bounded
-      if (static_stream_goal[i] > 0) {
+      if (static_stream_goal[i] > std::numeric_limits<std::size_t>::max()) {
         ttg::print_error(world.rank(), ":", get_name(), " : error stream is already bounded : ", i);
         throw std::runtime_error("TT::set_static_argstream_size called for a bounded stream");
       }
@@ -2608,16 +2757,22 @@ namespace ttg_parsec {
 #endif
           }
         }
+        parsec_hash_table_unlock_bucket(&tasks_table, hk);
 
         // TODO: Unfriendly implementation, cannot check if stream is already bounded
         // TODO: Unfriendly implementation, cannot check if stream has been finalized already
 
         // commit changes
-        task->stream[i].goal = size;
-        bool release = (task->stream[i].size == task->stream[i].goal);
-        parsec_hash_table_unlock_bucket(&tasks_table, hk);
-
-        if (release) release_task(task);
+        // 1) "lock" the stream by incrementing the reduce_count
+        // 2) set the goal
+        // 3) "unlock" the stream
+        // only one thread will see the reduce_count be zero and the goal match the size
+        task->streams[i].reduce_count.fetch_add(1, std::memory_order_acquire);
+        task->streams[i].goal = size;
+        auto c = task->streams[i].reduce_count.fetch_sub(1, std::memory_order_release);
+        if (1 == c && (task->streams[i].size >= size)) {
+          release_task(task);
+        }
       }
     }
 
@@ -2661,16 +2816,22 @@ namespace ttg_parsec {
 #endif
           }
         }
+        parsec_hash_table_unlock_bucket(&tasks_table, hk);
 
         // TODO: Unfriendly implementation, cannot check if stream is already bounded
         // TODO: Unfriendly implementation, cannot check if stream has been finalized already
 
         // commit changes
-        task->stream[i].goal = size;
-        bool release = (task->stream[i].size == task->stream[i].goal);
-        parsec_hash_table_unlock_bucket(&tasks_table, hk);
-
-        if (release) release_task(task);
+        // 1) "lock" the stream by incrementing the reduce_count
+        // 2) set the goal
+        // 3) "unlock" the stream
+        // only one thread will see the reduce_count be zero and the goal match the size
+        task->streams[i].reduce_count.fetch_add(1, std::memory_order_acquire);
+        task->streams[i].goal = size;
+        auto c = task->streams[i].reduce_count.fetch_sub(1, std::memory_order_release);
+        if (1 == c && (task->streams[i].size >= size)) {
+          release_task(task);
+        }
       }
     }
 
@@ -2702,8 +2863,8 @@ namespace ttg_parsec {
 
         auto hk = reinterpret_cast<parsec_key_t>(&key);
         task_t *task = nullptr;
-        parsec_hash_table_lock_bucket(&tasks_table, hk);
-        if (nullptr == (task = (task_t *)parsec_hash_table_nolock_find(&tasks_table, hk))) {
+        //parsec_hash_table_lock_bucket(&tasks_table, hk);
+        if (nullptr == (task = (task_t *)parsec_hash_table_find(&tasks_table, hk))) {
           ttg::print_error(world.rank(), ":", get_name(), ":", key,
                            " : error finalize called on stream that never received an input data: ", i);
           throw std::runtime_error("TT::finalize called on stream that never received an input data");
@@ -2713,10 +2874,16 @@ namespace ttg_parsec {
         // TODO: Unfriendly implementation, cannot check if stream has been finalized already
 
         // commit changes
-        task->stream[i].size = 1;
-        parsec_hash_table_unlock_bucket(&tasks_table, hk);
-
-        release_task(task);
+        // 1) "lock" the stream by incrementing the reduce_count
+        // 2) set the goal
+        // 3) "unlock" the stream
+        // only one thread will see the reduce_count be zero and the goal match the size
+        task->streams[i].reduce_count.fetch_add(1, std::memory_order_acquire);
+        task->streams[i].goal = 1;
+        auto c = task->streams[i].reduce_count.fetch_sub(1, std::memory_order_release);
+        if (1 == c && (task->streams[i].size >= 1)) {
+          release_task(task);
+        }
       }
     }
 
@@ -2746,8 +2913,7 @@ namespace ttg_parsec {
 
         auto hk = static_cast<parsec_key_t>(0);
         task_t *task = nullptr;
-        parsec_hash_table_lock_bucket(&tasks_table, hk);
-        if (nullptr == (task = (task_t *)parsec_hash_table_nolock_find(&tasks_table, hk))) {
+        if (nullptr == (task = (task_t *)parsec_hash_table_find(&tasks_table, hk))) {
           ttg::print_error(world.rank(), ":", get_name(),
                            " : error finalize called on stream that never received an input data: ", i);
           throw std::runtime_error("TT::finalize called on stream that never received an input data");
@@ -2757,10 +2923,16 @@ namespace ttg_parsec {
         // TODO: Unfriendly implementation, cannot check if stream has been finalized already
 
         // commit changes
-        task->stream[i].size = 1;
-        parsec_hash_table_unlock_bucket(&tasks_table, hk);
-
-        release_task(task);
+        // 1) "lock" the stream by incrementing the reduce_count
+        // 2) set the goal
+        // 3) "unlock" the stream
+        // only one thread will see the reduce_count be zero and the goal match the size
+        task->streams[i].reduce_count.fetch_add(1, std::memory_order_acquire);
+        task->streams[i].goal = 1;
+        auto c = task->streams[i].reduce_count.fetch_sub(1, std::memory_order_release);
+        if (1 == c && (task->streams[i].size >= 1)) {
+          release_task(task);
+        }
       }
     }
 
@@ -3161,8 +3333,7 @@ namespace ttg_parsec {
         , keymap(std::is_same<keymapT, ttg::detail::default_keymap<keyT>>::value
                      ? decltype(keymap)(ttg::detail::default_keymap<keyT>(world))
                      : decltype(keymap)(std::forward<keymapT>(keymap_)))
-        , priomap(decltype(keymap)(std::forward<priomapT>(priomap_)))
-        , static_stream_goal() {
+        , priomap(decltype(keymap)(std::forward<priomapT>(priomap_))) {
       // Cannot call these in base constructor since terminals not yet constructed
       if (innames.size() != numinedges) throw std::logic_error("ttg_parsec::TT: #input names != #input terminals");
       if (outnames.size() != numouts) throw std::logic_error("ttg_parsec::TT: #output names != #output terminals");
@@ -3333,6 +3504,13 @@ namespace ttg_parsec {
         delete flow;
         *((parsec_flow_t **)&(self.out[i])) = nullptr;
       }
+
+      for (std::size_t i = 0; i < numins; ++i) {
+        if (inpute_reducers_taskclass[i] != nullptr) {
+          std::free(inpute_reducers_taskclass[i]);
+          inpute_reducers_taskclass[i] = nullptr;
+        }
+      }
       release();
     }
 
@@ -3384,6 +3562,74 @@ namespace ttg_parsec {
     void set_input_reducer(Reducer &&reducer) {
       ttg::trace(world.rank(), ":", get_name(), " : setting reducer for terminal ", i);
       std::get<i>(input_reducers) = reducer;
+
+      parsec_task_class_t *tc = inpute_reducers_taskclass[i];
+      if (nullptr == tc) {
+        tc = (parsec_task_class_t *)std::calloc(1, sizeof(*tc));
+        inpute_reducers_taskclass[i] = tc;
+
+        tc->name = strdup((get_name() + std::string(" reducer ") + std::to_string(i)).c_str());
+        tc->task_class_id = get_instance_id();
+        tc->nb_parameters = 0;
+        tc->nb_locals = 0;
+        tc->nb_flows = numflows;
+
+        auto &world_impl = world.impl();
+
+        if( world_impl.profiling() ) {
+          // first two ints are used to store the hash of the key.
+          tc->nb_parameters = (sizeof(void*)+sizeof(int)-1)/sizeof(int);
+          // seconds two ints are used to store a pointer to the key of the task.
+          tc->nb_locals     = self.nb_parameters + (sizeof(void*)+sizeof(int)-1)/sizeof(int);
+
+          // If we have parameters and locals, we need to define the corresponding dereference arrays
+          tc->params[0] = &detail::parsec_taskclass_param0;
+          tc->params[1] = &detail::parsec_taskclass_param1;
+
+          tc->locals[0] = &detail::parsec_taskclass_param0;
+          tc->locals[1] = &detail::parsec_taskclass_param1;
+          tc->locals[2] = &detail::parsec_taskclass_param2;
+          tc->locals[3] = &detail::parsec_taskclass_param3;
+        }
+        tc->make_key = make_key;
+        tc->key_functions = &tasks_hash_fcts;
+        tc->task_snprintf = parsec_ttg_task_snprintf;
+
+#if defined(PARSEC_PROF_TRACE)
+        tc->profile_info = &parsec_ttg_task_info;
+#endif
+
+        world_impl.taskpool()->nb_task_classes = std::max(world_impl.taskpool()->nb_task_classes, static_cast<decltype(world_impl.taskpool()->nb_task_classes)>(self.task_class_id+1));
+
+#if 0
+        // FIXME: currently only support reduction on the host
+        if constexpr (derived_has_cuda_op()) {
+          self.incarnations = (__parsec_chore_t *)malloc(3 * sizeof(__parsec_chore_t));
+          ((__parsec_chore_t *)self.incarnations)[0].type = PARSEC_DEV_CUDA;
+          ((__parsec_chore_t *)self.incarnations)[0].evaluate = NULL;
+          ((__parsec_chore_t *)self.incarnations)[0].hook = detail::hook_cuda;
+          ((__parsec_chore_t *)self.incarnations)[1].type = PARSEC_DEV_CPU;
+          ((__parsec_chore_t *)self.incarnations)[1].evaluate = NULL;
+          ((__parsec_chore_t *)self.incarnations)[1].hook = detail::hook;
+          ((__parsec_chore_t *)self.incarnations)[2].type = PARSEC_DEV_NONE;
+          ((__parsec_chore_t *)self.incarnations)[2].evaluate = NULL;
+          ((__parsec_chore_t *)self.incarnations)[2].hook = NULL;
+        } else
+#endif // 0
+        {
+          tc->incarnations = (__parsec_chore_t *)malloc(2 * sizeof(__parsec_chore_t));
+          ((__parsec_chore_t *)tc->incarnations)[0].type = PARSEC_DEV_CPU;
+          ((__parsec_chore_t *)tc->incarnations)[0].evaluate = NULL;
+          ((__parsec_chore_t *)tc->incarnations)[0].hook = &static_reducer_op<i>;
+          ((__parsec_chore_t *)tc->incarnations)[1].type = PARSEC_DEV_NONE;
+          ((__parsec_chore_t *)tc->incarnations)[1].evaluate = NULL;
+          ((__parsec_chore_t *)tc->incarnations)[1].hook = NULL;
+        }
+
+        /* the reduction task does not alter the termination detection because the target task will execute */
+        tc->release_task = &parsec_release_task_to_mempool;
+        tc->complete_execution = NULL;
+      }
     }
 
     /// define the reducer function to be called when additional inputs are

From 1bdf2f3e0233ac8e829b49e84ef9fcf58b1d08c5 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Tue, 6 Jun 2023 15:43:17 -0400
Subject: [PATCH 087/259] PaRSEC: replace thread-local es with
 parsec_my_execution_stream()

The thread-local variable was a hack that is not needed anymore, and
apparently didn't work properly.

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>

From 2a1158d98170647c037bd75a725a316e1b52c388 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Tue, 6 Jun 2023 15:45:15 -0400
Subject: [PATCH 088/259] PaRSEC: Fix check for bounded stream in
 static_argstream_size

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/ttg.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 30183a22c..d4a647f2a 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -2706,7 +2706,7 @@ namespace ttg_parsec {
       this->trace(world.rank(), ":", get_name(), ": setting global stream size for terminal ", i);
 
       // Check if stream is already bounded
-      if (static_stream_goal[i] > std::numeric_limits<std::size_t>::max()) {
+      if (static_stream_goal[i] < std::numeric_limits<std::size_t>::max()) {
         ttg::print_error(world.rank(), ":", get_name(), " : error stream is already bounded : ", i);
         throw std::runtime_error("TT::set_static_argstream_size called for a bounded stream");
       }

From e329fd5f2eb0a26b8d5eaec52d089fc92bffe770 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Tue, 6 Jun 2023 16:11:25 -0400
Subject: [PATCH 089/259] Add stream unit test

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 tests/unit/CMakeLists.txt |  2 +-
 tests/unit/streams.cc     | 74 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 75 insertions(+), 1 deletion(-)
 create mode 100644 tests/unit/streams.cc

diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt
index 67c6e6143..ac48e70f5 100644
--- a/tests/unit/CMakeLists.txt
+++ b/tests/unit/CMakeLists.txt
@@ -6,6 +6,7 @@ set(ut_src
         #ranges.cc
         #tt.cc
         unit_main.cpp
+		streams.cc
     )
 set(ut_libs Catch2::Catch2)
 #if (TARGET std::coroutine)
@@ -18,7 +19,6 @@ set(ut_libs Catch2::Catch2)
 #endif()
 add_ttg_executable(core-unittests-ttg "${ut_src}" LINK_LIBRARIES "${ut_libs}" COMPILE_DEFINITIONS "-DCATCH_CONFIG_NO_POSIX_SIGNALS=1" )
 
-
 # serialization test: probes serialization via all supported serialization methods (MADNESS, Boost::serialization, cereal) that are available
 add_executable(serialization "serialization.cc;unit_main.cpp")
 target_link_libraries(serialization "Catch2::Catch2;ttg-serialization")
diff --git a/tests/unit/streams.cc b/tests/unit/streams.cc
new file mode 100644
index 000000000..bef3a2837
--- /dev/null
+++ b/tests/unit/streams.cc
@@ -0,0 +1,74 @@
+#include <catch2/catch.hpp>
+#include <ctime>
+
+#include "ttg.h"
+
+#include "ttg/serialization/std/pair.h"
+#include "ttg/util/hash/std/pair.h"
+
+
+
+TEST_CASE("streams", "[streams][core]") {
+  // in distributed memory we must count how many messages the reducer will receive
+  SECTION("concurrent-stream-size") {
+    ttg::Edge<int, int> I2O;
+    ttg::Edge<int, int> O2S;
+    const auto nranks = ttg::default_execution_context().size();
+
+    constexpr std::size_t N = 10000;
+    constexpr std::size_t SLICE = 500;
+    constexpr const timespec ts = { .tv_sec = 0, .tv_nsec = 10000 };
+    constexpr int VALUE = 1;
+    std::atomic<std::size_t> reduce_ops = 0;
+
+    auto op = ttg::make_tt(
+        [&](const int &n, int&& i,
+           std::tuple<ttg::Out<int, int>> &outs) {
+          int key = n/SLICE;
+          nanosleep(&ts, nullptr);
+          if (n < N-1) {
+            ttg::send<0>(key, std::forward<int>(i), outs);
+            //ttg::print("sent to sink ", key);
+          } else {
+            // set the size of the last reducer
+            if (N%SLICE > 0) {
+              ttg::set_size<0>(key, N%SLICE, outs);
+              std::cout << "set_size key " << key << " size " << N%SLICE << std::endl;
+            }
+            // forward the value
+            ttg::send<0>(key, std::forward<int>(i), outs);
+            //ttg::print("finalized last sink ", key);
+          }
+        },
+        ttg::edges(I2O), ttg::edges(O2S));
+
+    auto sink_op = ttg::make_tt(
+        [&](const int key, const int &value) {
+          std::cout << "sink " << key << std::endl;
+          if (!(value == SLICE || key == (N/SLICE))) {
+            std::cout << "SINK ERROR: key " << key << " value " << value << " SLICE " << SLICE << " N " << N << std::endl;
+          }
+          CHECK((value == SLICE || key == (N/SLICE)));
+          reduce_ops++;
+        },
+        ttg::edges(O2S), ttg::edges());
+
+    op->set_keymap([=](const auto &key) { return nranks - 1; });
+    op->set_trace_instance(true);
+    sink_op->set_input_reducer<0>([&](int &a, const int &b) {
+      a += 1; // we count invocations
+      CHECK(b == VALUE);
+      reduce_ops++;
+    }, SLICE);
+    make_graph_executable(op);
+    ttg::ttg_fence(ttg::default_execution_context());
+    if (ttg::default_execution_context().rank() == 0) {
+      for (std::size_t i = 0; i < N; ++i) {
+        op->invoke(i, VALUE);
+      }
+    }
+    auto &sink_op_real = *sink_op;
+    ttg::ttg_fence(ttg::default_execution_context());
+    CHECK(reduce_ops == N);
+  }
+}  // TEST_CASE("streams")
\ No newline at end of file

From 552f3125d972160ac6478e286f20b1d6d75991bf Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Tue, 6 Jun 2023 16:14:33 -0400
Subject: [PATCH 090/259] PaRSEC: misc fixes to the reducer offload task

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/ttg.h | 39 ++++++++++++++++++++++++++-------------
 1 file changed, 26 insertions(+), 13 deletions(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index d4a647f2a..e4aad2768 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1596,7 +1596,9 @@ namespace ttg_parsec {
       target_copy = static_cast<detail::ttg_data_copy_t *>(task->parsec_task.data[i].data_in);
       assert(val_is_void || nullptr != target_copy);
       /* once we hit 0 we have to stop since another thread might enqueue a new reduction task */
-      std::size_t c;
+      std::size_t c = 0;
+      std::size_t size = 0;
+      assert(task->streams[i].reduce_count > 0);
       do {
         if constexpr(!val_is_void) {
           /* the copies to reduce out of */
@@ -1604,7 +1606,8 @@ namespace ttg_parsec {
           parsec_list_item_t *item;
           item = parsec_lifo_pop(&task->streams[i].reduce_copies);
           if (nullptr == item) {
-            break; // maybe someone is changing the goal right now
+            // maybe someone is changing the goal right now
+            break;
           }
           source_copy = ((detail::ttg_data_copy_self_t *)(item))->self;
           reducer(*reinterpret_cast<std::decay_t<value_t> *>(target_copy->device_private),
@@ -1614,13 +1617,16 @@ namespace ttg_parsec {
           reducer(); // invoke control reducer
         }
         // there is only one task working on this stream, so no need to be atomic here
-        task->streams[i].size++;
-      } while ((c = task->streams[i].reduce_count.fetch_sub(1, std::memory_order_acq_rel)) > 1);
+        size = ++task->streams[i].size;
+        //std::cout << "static_reducer_op key " << task->key << " size " << size << " of " << task->streams[i].goal << std::endl;
+      } while ((c = (task->streams[i].reduce_count.fetch_sub(1, std::memory_order_acq_rel)-1)) > 0);
+      //} while ((c = (--task->streams[i].reduce_count)) > 0);
 
       /* finalize_argstream sets goal to 1, so size may be larger than goal */
-      bool complete = (task->streams[i].size >= task->streams[i].goal);
+      bool complete = (size >= task->streams[i].goal);
 
-      if (complete && c == 1) {
+
+      if (complete && c == 0) {
         /* task is still in the hash table, have release_task remove it */
         task->remove_from_hash = true;
         task->release_task(task);
@@ -2095,7 +2101,10 @@ namespace ttg_parsec {
           parsec_hash_table_nolock_remove(&tasks_table, hk);
           remove_from_hash = false;
         }
-        parsec_hash_table_unlock_bucket(&tasks_table, hk);
+        /* if we have a reducer, we need to hold on to the lock for just a little longer */
+        if (!reducer) {
+          parsec_hash_table_unlock_bucket(&tasks_table, hk);
+        }
       } else {
         task = create_new_task(key);
         world_impl.increment_created();
@@ -2140,14 +2149,13 @@ namespace ttg_parsec {
       //parsec_data_transfer_ownership_to_copy(data, 0, PARSEC_FLOW_ACCESS_RW);
 
       if (reducer) {  // is this a streaming input? reduce the received value
-        // N.B. Right now reductions are done eagerly, without spawning tasks
-        //      this means we must lock
-        //parsec_hash_table_lock_bucket(&tasks_table, hk);
-
         auto submit_reducer_task = [&](auto *task){
           /* check if we need to create a task */
           std::size_t c = task->streams[i].reduce_count.fetch_add(1, std::memory_order_release);
+          //std::size_t c = task->streams[i].reduce_count++;
           if (0 == c) {
+            //std::cout << "submit reducer task for task " << task->key
+            //          << " size " << task->streams[i].size << " of " << task->streams[i].goal << std::endl;
             /* we are responsible for creating the reduction task */
             reducer_task_t *reduce_task;
             reduce_task = create_new_reducer_task<i>(task);
@@ -2170,13 +2178,17 @@ namespace ttg_parsec {
             if (task->streams[i].size == task->streams[i].goal) {
               release = true;
             }
+            /* now we can unlock the bucket */
+            parsec_hash_table_unlock_bucket(&tasks_table, hk);
           } else {
+            /* unlock the bucket, the lock is not needed anymore */
+            parsec_hash_table_unlock_bucket(&tasks_table, hk);
             if (nullptr != parsec_ttg_caller) {
               copy = detail::find_copy_in_task(parsec_ttg_caller, &value);
             }
             if (nullptr != copy) {
               /* retain the data copy */
-              copy = detail::register_data_copy<valueT>(copy, task, input_is_const);
+              copy = detail::register_data_copy<valueT>(copy, task, true);
             } else {
               /* create a new copy */
               copy = detail::create_new_datacopy(std::forward<Value>(value));
@@ -2184,9 +2196,10 @@ namespace ttg_parsec {
             /* enqueue the data copy to be reduced */
             parsec_lifo_push(&task->streams[i].reduce_copies, &copy->super);
             submit_reducer_task(task);
-            //reducer(*reinterpret_cast<std::decay_t<valueT> *>(copy->device_private), value);
           }
         } else {
+          /* unlock the bucket, the lock is not needed anymore */
+          parsec_hash_table_unlock_bucket(&tasks_table, hk);
           /* submit reducer for void values to handle side effects */
           submit_reducer_task(task);
         }

From 717167abe4eb2cabd652cd8c3a7c1cd57cc6141b Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 9 Jun 2023 10:35:33 -0400
Subject: [PATCH 091/259] PaRSEC: fix copy handling for streaming terminals

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/ttg.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index e4aad2768..c718951ad 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -2172,8 +2172,7 @@ namespace ttg_parsec {
             using decay_valueT = std::decay_t<valueT>;
             /* For now, we always create a copy because we cannot rely on the task_release
              * mechanism (it would release the task, not the reduction value). */
-            copy = detail::create_new_datacopy(std::forward<Value>(value));
-            task->parsec_task.data[i].data_in = copy;
+            task->parsec_task.data[i].data_in = detail::create_new_datacopy(std::forward<Value>(value));
             task->streams[i].size++;
             if (task->streams[i].size == task->streams[i].goal) {
               release = true;
@@ -2181,6 +2180,7 @@ namespace ttg_parsec {
             /* now we can unlock the bucket */
             parsec_hash_table_unlock_bucket(&tasks_table, hk);
           } else {
+            detail::ttg_data_copy_t *copy = nullptr;
             /* unlock the bucket, the lock is not needed anymore */
             parsec_hash_table_unlock_bucket(&tasks_table, hk);
             if (nullptr != parsec_ttg_caller) {
@@ -2713,7 +2713,7 @@ namespace ttg_parsec {
     /// \param size positive integer that specifies the default stream size
     template <std::size_t i>
     void set_static_argstream_size(std::size_t size) {
-      assert(std::get<i>(input_reducers) && "TT::set_argstream_size called on nonstreaming input terminal");
+      assert(std::get<i>(input_reducers) && "TT::set_static_argstream_size called on nonstreaming input terminal");
       assert(size > 0 && "TT::set_static_argstream_size(key,size) called with size=0");
 
       this->trace(world.rank(), ":", get_name(), ": setting global stream size for terminal ", i);

From 97b3a20be1a562b95011d1b2f704d4d103f1350e Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 6 Jul 2023 19:30:23 -0400
Subject: [PATCH 092/259] PaRSEC: Allow reducer tasks to be deferred to avoid
 copying stream inputs

In the PaRSEC backend, reducer tasks will be treated as any other
task and be deferred if there are readers on their first input.
All other inputs are read-only and will be handled that way.
This avoids extra copies if there are no other writers on these copies.

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 tests/unit/streams.cc          |   3 +-
 ttg/CMakeLists.txt             |   4 +-
 ttg/ttg/parsec/task.h          |  34 ++++++-
 ttg/ttg/parsec/ttg.h           | 179 ++++++++++++++++-----------------
 ttg/ttg/parsec/ttg_data_copy.h |   8 +-
 5 files changed, 128 insertions(+), 100 deletions(-)

diff --git a/tests/unit/streams.cc b/tests/unit/streams.cc
index bef3a2837..108c8610b 100644
--- a/tests/unit/streams.cc
+++ b/tests/unit/streams.cc
@@ -60,6 +60,7 @@ TEST_CASE("streams", "[streams][core]") {
       CHECK(b == VALUE);
       reduce_ops++;
     }, SLICE);
+
     make_graph_executable(op);
     ttg::ttg_fence(ttg::default_execution_context());
     if (ttg::default_execution_context().rank() == 0) {
@@ -67,7 +68,7 @@ TEST_CASE("streams", "[streams][core]") {
         op->invoke(i, VALUE);
       }
     }
-    auto &sink_op_real = *sink_op;
+
     ttg::ttg_fence(ttg::default_execution_context());
     CHECK(reduce_ops == N);
   }
diff --git a/ttg/CMakeLists.txt b/ttg/CMakeLists.txt
index 67277a83a..12f1df749 100644
--- a/ttg/CMakeLists.txt
+++ b/ttg/CMakeLists.txt
@@ -10,7 +10,6 @@ configure_file(
 set(ttg-util-headers
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/backtrace.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/bug.h
-        ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/cublas_helper.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/demangle.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/diagnose.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/dot.h
@@ -80,6 +79,7 @@ set(ttg-sources
         ${ttg-impl-headers}
         ${ttg-base-headers}
         ${ttg-util-headers}
+        ${ttg_device_headers}
         ${ttg-external-headers}
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/device/cublas_helper.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/backtrace.cpp
@@ -120,7 +120,7 @@ if (TARGET CUDA::cudart)
   list(APPEND ttg-deps CUDA::cudart)
 endif (TARGET CUDA::cudart)
 
-set(ttg-public-headers ${ttg-headers};${ttg-impl-headers};${ttg-base-headers};${ttg-util-headers})
+set(ttg-public-headers ${ttg-headers};${ttg-impl-headers};${ttg-base-headers};${ttg-util-headers};${ttg_device_headers})
 if (NOT TTG_IGNORE_BUNDLED_EXTERNALS)
   list(APPEND ttg-sources ${ttg-external-headers})
   list(APPEND ttg-public-headers ${ttg-external-headers})
diff --git a/ttg/ttg/parsec/task.h b/ttg/ttg/parsec/task.h
index c1791d0ad..c5edc148f 100644
--- a/ttg/ttg/parsec/task.h
+++ b/ttg/ttg/parsec/task.h
@@ -184,7 +184,7 @@ namespace ttg_parsec {
                                                                       : (num_streams+1);
       TT* tt;
       key_type key;
-      size_goal_t stream[num_streams] = {};
+      stream_info_t streams[num_streams] = {};
 #ifdef TTG_HAS_COROUTINE
       void* suspended_task_address = nullptr;  // if not null the function is suspended
 #endif
@@ -239,7 +239,7 @@ namespace ttg_parsec {
     struct parsec_ttg_task_t<TT, true> : public parsec_ttg_task_base_t {
       static constexpr size_t num_streams = TT::numins;
       TT* tt;
-      size_goal_t stream[num_streams] = {};
+      stream_info_t streams[num_streams] = {};
 #ifdef TTG_HAS_COROUTINE
       void* suspended_task_address = nullptr;  // if not null the function is suspended
 #endif
@@ -282,6 +282,36 @@ namespace ttg_parsec {
     };
 
 
+    /**
+     * Reducer task representing one or more stream reductions.
+     * A reducer task may be deferred on its first input (the object into which
+     * all other inputs are folded). Once that input becomes available the task
+     * is submitted and reduces all available inputs. Additional reducer tasks may
+     * be submitted until all required inputs have been processed.
+     */
+    struct reducer_task_t : public parsec_ttg_task_base_t {
+      parsec_ttg_task_base_t *parent_task;
+      bool is_first;
+
+      reducer_task_t(parsec_ttg_task_base_t* task, parsec_thread_mempool_t *mempool,
+                     parsec_task_class_t *task_class, parsec_taskpool_t *taskpool,
+                     int32_t priority, bool is_first)
+      : parsec_ttg_task_base_t(mempool, task_class, taskpool, priority,
+                               0, nullptr, nullptr,
+                               &release_task,
+                               true /* deferred until other readers have completed */)
+      , parent_task(task)
+      , is_first(is_first)
+      { }
+
+      static void release_task(parsec_ttg_task_base_t* task_base) {
+        /* reducer tasks have one mutable input so the task can be submitted on the first release */
+        parsec_task_t *vp_task_rings[1] = { &task_base->parsec_task };
+        parsec_execution_stream_t *es = parsec_my_execution_stream();
+        __parsec_schedule_vp(es, vp_task_rings, 0);
+      }
+    };
+
   } // namespace detail
 
 } // namespace ttg_parsec
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index c718951ad..8b9c12f4c 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1550,93 +1550,90 @@ namespace ttg_parsec {
         return PARSEC_HOOK_RETURN_DONE;
     }
 
-    struct reducer_task_t {
-      parsec_task_t parsec_task;
-      task_t *task;
-
-      reducer_task_t(task_t* task, parsec_thread_mempool_t *mempool,
-                     parsec_task_class_t *task_class, parsec_taskpool_t *taskpool,
-                     int32_t priority)
-      : task(task)
-      {
-        PARSEC_LIST_ITEM_SINGLETON(&parsec_task.super);
-        parsec_task.mempool_owner = mempool;
-        parsec_task.task_class = task_class;
-        parsec_task.status = PARSEC_TASK_STATUS_HOOK;
-        parsec_task.taskpool = taskpool;
-        parsec_task.priority = priority;
-        parsec_task.chore_mask = 1<<0;
-      }
-    };
-
     template <std::size_t i>
     static parsec_hook_return_t static_reducer_op(parsec_execution_stream_s *es, parsec_task_t *parsec_task) {
-      using rtask_t = reducer_task_t;
+      using rtask_t = detail::reducer_task_t;
       using value_t = std::tuple_element_t<i, actual_input_tuple_type>;
       constexpr const bool val_is_void = ttg::meta::is_void_v<value_t>;
       rtask_t *rtask = (rtask_t*)parsec_task;
-      task_t *task = rtask->task;
-      ttT *baseobj = task->tt;
+      task_t *parent_task = static_cast<task_t*>(rtask->parent_task);
+      ttT *baseobj = parent_task->tt;
       derivedT *obj = static_cast<derivedT *>(baseobj);
 
       auto& reducer = std::get<i>(baseobj->input_reducers);
 
-      assert(parsec_ttg_caller == NULL);
-      parsec_ttg_caller = static_cast<detail::parsec_ttg_task_base_t*>(task);
-
       if (obj->tracing()) {
         if constexpr (!ttg::meta::is_void_v<keyT>)
-          ttg::trace(obj->get_world().rank(), ":", obj->get_name(), " : ", task->key, ": reducer executing");
+          ttg::trace(obj->get_world().rank(), ":", obj->get_name(), " : ", parent_task->key, ": reducer executing");
         else
           ttg::trace(obj->get_world().rank(), ":", obj->get_name(), " : reducer executing");
       }
 
       /* the copy to reduce into */
       detail::ttg_data_copy_t *target_copy;
-      target_copy = static_cast<detail::ttg_data_copy_t *>(task->parsec_task.data[i].data_in);
+      target_copy = static_cast<detail::ttg_data_copy_t *>(parent_task->parsec_task.data[i].data_in);
       assert(val_is_void || nullptr != target_copy);
       /* once we hit 0 we have to stop since another thread might enqueue a new reduction task */
       std::size_t c = 0;
       std::size_t size = 0;
-      assert(task->streams[i].reduce_count > 0);
+      assert(parent_task->streams[i].reduce_count > 0);
+      if (rtask->is_first) {
+        if (0 == (parent_task->streams[i].reduce_count.fetch_sub(1, std::memory_order_acq_rel)-1)) {
+          /* we were the first and there is nothing to be done */
+          if (obj->tracing()) {
+            if constexpr (!ttg::meta::is_void_v<keyT>)
+              ttg::trace(obj->get_world().rank(), ":", obj->get_name(), " : ", parent_task->key, ": first reducer empty");
+            else
+              ttg::trace(obj->get_world().rank(), ":", obj->get_name(), " : first reducer empty");
+          }
+
+          return PARSEC_HOOK_RETURN_DONE;
+        }
+      }
+
+      assert(parsec_ttg_caller == NULL);
+      detail::parsec_ttg_caller = rtask->parent_task;
+
       do {
         if constexpr(!val_is_void) {
           /* the copies to reduce out of */
           detail::ttg_data_copy_t *source_copy;
           parsec_list_item_t *item;
-          item = parsec_lifo_pop(&task->streams[i].reduce_copies);
+          item = parsec_lifo_pop(&parent_task->streams[i].reduce_copies);
           if (nullptr == item) {
             // maybe someone is changing the goal right now
             break;
           }
           source_copy = ((detail::ttg_data_copy_self_t *)(item))->self;
-          reducer(*reinterpret_cast<std::decay_t<value_t> *>(target_copy->device_private),
-                  *reinterpret_cast<std::decay_t<value_t> *>(source_copy->device_private));
+          reducer(*reinterpret_cast<std::decay_t<value_t> *>(target_copy->get_ptr()),
+                  *reinterpret_cast<std::decay_t<value_t> *>(source_copy->get_ptr()));
           detail::release_data_copy(source_copy);
         } else if constexpr(val_is_void) {
           reducer(); // invoke control reducer
         }
         // there is only one task working on this stream, so no need to be atomic here
-        size = ++task->streams[i].size;
-        //std::cout << "static_reducer_op key " << task->key << " size " << size << " of " << task->streams[i].goal << std::endl;
-      } while ((c = (task->streams[i].reduce_count.fetch_sub(1, std::memory_order_acq_rel)-1)) > 0);
+        size = ++parent_task->streams[i].size;
+        //std::cout << "static_reducer_op size " << size << " of " << parent_task->streams[i].goal << std::endl;
+      } while ((c = (parent_task->streams[i].reduce_count.fetch_sub(1, std::memory_order_acq_rel)-1)) > 0);
       //} while ((c = (--task->streams[i].reduce_count)) > 0);
 
       /* finalize_argstream sets goal to 1, so size may be larger than goal */
-      bool complete = (size >= task->streams[i].goal);
-
+      bool complete = (size >= parent_task->streams[i].goal);
 
+      //std::cout << "static_reducer_op size " << size
+      //          << " of " << parent_task->streams[i].goal << " complete " << complete
+      //          << " c " << c << std::endl;
       if (complete && c == 0) {
         /* task is still in the hash table, have release_task remove it */
-        task->remove_from_hash = true;
-        task->release_task(task);
+        parent_task->remove_from_hash = true;
+        parent_task->release_task(parent_task);
       }
 
-      parsec_ttg_caller = NULL;
+      detail::parsec_ttg_caller = NULL;
 
       if (obj->tracing()) {
         if constexpr (!ttg::meta::is_void_v<keyT>)
-          ttg::trace(obj->get_world().rank(), ":", obj->get_name(), " : ", task->key, ": done executing");
+          ttg::trace(obj->get_world().rank(), ":", obj->get_name(), " : ", parent_task->key, ": done executing");
         else
           ttg::trace(obj->get_world().rank(), ":", obj->get_name(), " : done executing");
       }
@@ -2028,12 +2025,12 @@ namespace ttg_parsec {
 
 
     template <std::size_t i>
-    reducer_task_t *create_new_reducer_task(task_t *task) {
+    detail::reducer_task_t *create_new_reducer_task(task_t *task, bool is_first) {
       /* make sure we can reuse the existing memory pool and don't have to create a new one */
-      static_assert(sizeof(task_t) >= sizeof(reducer_task_t));
+      static_assert(sizeof(task_t) >= sizeof(detail::reducer_task_t));
       constexpr const bool keyT_is_Void = ttg::meta::is_void_v<keyT>;
       auto &world_impl = world.impl();
-      reducer_task_t *newtask;
+      detail::reducer_task_t *newtask;
       parsec_thread_mempool_t *mempool = get_task_mempool();
       char *taskobj = (char *)parsec_thread_mempool_allocate(mempool);
       // use the priority of the task we stream into
@@ -2046,8 +2043,8 @@ namespace ttg_parsec {
         ttg::trace(world.rank(), ":", get_name(), ": creating reducer task");
       }
       /* placement-new the task */
-      newtask = new (taskobj) reducer_task_t(task, mempool, inpute_reducers_taskclass[i],
-                                                    world_impl.taskpool(), priority);
+      newtask = new (taskobj) detail::reducer_task_t(task, mempool, inpute_reducers_taskclass[i],
+                                                     world_impl.taskpool(), priority, is_first);
 
       return newtask;
     }
@@ -2138,30 +2135,30 @@ namespace ttg_parsec {
 #endif
       }
 
-      /* transfer ownership changes: readers, coherency state, ownership
-       * version of CPU copy must be changed when used inside a CPU task
-       * PTG increases version: complete_execution (if W)
-       * PTG calls transfer_ownership at beginning of CPU task (hook) if RW access
-       * DSL never modifies GPU data copy
-       * Pushout happens at the end of GPU task if successor can be CPU?
-       */
-
-      //parsec_data_transfer_ownership_to_copy(data, 0, PARSEC_FLOW_ACCESS_RW);
+      auto get_copy_fn = [&](detail::parsec_ttg_task_base_t *task, auto&& value, bool is_const){
+        detail::ttg_data_copy_t *copy = copy_in;
+        if (nullptr != detail::parsec_ttg_caller) {
+          copy = detail::find_copy_in_task(detail::parsec_ttg_caller, &value);
+        }
+        if (nullptr != copy) {
+          /* retain the data copy */
+          copy = detail::register_data_copy<valueT>(copy, task, true);
+        } else {
+          /* create a new copy */
+          copy = detail::create_new_datacopy(std::forward<Value>(value));
+        }
+        return copy;
+      };
 
-      if (reducer) {  // is this a streaming input? reduce the received value
-        auto submit_reducer_task = [&](auto *task){
+      if (reducer && 1 != task->streams[i].goal) {  // is this a streaming input? reduce the received value
+        auto submit_reducer_task = [&](auto *parent_task){
           /* check if we need to create a task */
-          std::size_t c = task->streams[i].reduce_count.fetch_add(1, std::memory_order_release);
-          //std::size_t c = task->streams[i].reduce_count++;
+          std::size_t c = parent_task->streams[i].reduce_count.fetch_add(1, std::memory_order_release);
           if (0 == c) {
-            //std::cout << "submit reducer task for task " << task->key
-            //          << " size " << task->streams[i].size << " of " << task->streams[i].goal << std::endl;
             /* we are responsible for creating the reduction task */
-            reducer_task_t *reduce_task;
-            reduce_task = create_new_reducer_task<i>(task);
-            parsec_task_t *vp_task_rings[1] = { &reduce_task->parsec_task };
-            parsec_execution_stream_t *es = world_impl.execution_stream();
-            __parsec_schedule_vp(es, vp_task_rings, 0);
+            detail::reducer_task_t *reduce_task;
+            reduce_task = create_new_reducer_task<i>(parent_task, false);
+            reduce_task->release_task(reduce_task); // release immediately
           }
         };
 
@@ -2170,29 +2167,34 @@ namespace ttg_parsec {
           detail::ttg_data_copy_t *copy = nullptr;
           if (nullptr == (copy = task->copies[i])) {
             using decay_valueT = std::decay_t<valueT>;
-            /* For now, we always create a copy because we cannot rely on the task_release
-             * mechanism (it would release the task, not the reduction value). */
-            task->parsec_task.data[i].data_in = detail::create_new_datacopy(std::forward<Value>(value));
-            task->streams[i].size++;
-            if (task->streams[i].size == task->streams[i].goal) {
-              release = true;
+
+            /* first input value, create a task and bind it to the copy */
+            detail::reducer_task_t *reduce_task;
+            reduce_task = create_new_reducer_task<i>(task, true);
+
+            /* get the copy to use as input for this task */
+            detail::ttg_data_copy_t *copy = get_copy_fn(reduce_task, std::forward<Value>(value), false);
+
+            /* put the copy into the task */
+            task->parsec_task.data[i].data_in = copy;
+
+            /* protected by the bucket lock */
+            task->streams[i].size = 1;
+            task->streams[i].reduce_count.store(1, std::memory_order_relaxed);
+
+            if (copy->get_next_task() != &reduce_task->parsec_task) {
+              reduce_task->release_task(reduce_task);
             }
+
             /* now we can unlock the bucket */
             parsec_hash_table_unlock_bucket(&tasks_table, hk);
           } else {
-            detail::ttg_data_copy_t *copy = nullptr;
             /* unlock the bucket, the lock is not needed anymore */
             parsec_hash_table_unlock_bucket(&tasks_table, hk);
-            if (nullptr != parsec_ttg_caller) {
-              copy = detail::find_copy_in_task(parsec_ttg_caller, &value);
-            }
-            if (nullptr != copy) {
-              /* retain the data copy */
-              copy = detail::register_data_copy<valueT>(copy, task, true);
-            } else {
-              /* create a new copy */
-              copy = detail::create_new_datacopy(std::forward<Value>(value));
-            }
+
+            /* get the copy to use as input for this task */
+            detail::ttg_data_copy_t *copy = get_copy_fn(task, std::forward<Value>(value), true);
+
             /* enqueue the data copy to be reduced */
             parsec_lifo_push(&task->streams[i].reduce_copies, &copy->super);
             submit_reducer_task(task);
@@ -2216,17 +2218,8 @@ namespace ttg_parsec {
             throw std::logic_error("bad set arg");
           }
 
-          detail::ttg_data_copy_t *copy = copy_in;
-          if (nullptr == copy_in && nullptr != detail::parsec_ttg_caller) {
-            copy = detail::find_copy_in_task(detail::parsec_ttg_caller, &value);
-          }
-
-          if (nullptr != copy) {
-            /* register_data_copy might provide us with a different copy if !input_is_const */
-            copy = detail::register_data_copy<valueT>(copy, task, input_is_const);
-          } else {
-            copy = detail::create_new_datacopy(std::forward<Value>(value));
-          }
+          /* get the copy to use as input for this task */
+          detail::ttg_data_copy_t *copy = get_copy_fn(task, std::forward<Value>(value), input_is_const);
 
           /* if we registered as a writer and were the first to register with this copy
            * we need to defer the release of this task to give other tasks a chance to
diff --git a/ttg/ttg/parsec/ttg_data_copy.h b/ttg/ttg/parsec/ttg_data_copy.h
index 98d952777..ee97afd0b 100644
--- a/ttg/ttg/parsec/ttg_data_copy.h
+++ b/ttg/ttg/parsec/ttg_data_copy.h
@@ -34,11 +34,14 @@ namespace ttg_parsec {
      * because ttg_data_copy_t has virtual functions so we cannot cast from parsec_data_copy_t
      * to ttg_data_copy_t (offsetof is not supported for virtual classes).
      * The self pointer is a back-pointer to the ttg_data_copy_t. */
-    struct ttg_data_copy_self_t : public parsec_data_copy_t {
+    struct ttg_data_copy_self_t {
+      parsec_list_item_t super;
       ttg_data_copy_t *self;
       ttg_data_copy_self_t(ttg_data_copy_t* dc)
       : self(dc)
-      { }
+      {
+        PARSEC_OBJ_CONSTRUCT(&super, parsec_list_item_t);
+      }
     };
 
     /* Non-owning copy-tracking wrapper, accounting for N readers or 1 writer.
@@ -57,6 +60,7 @@ namespace ttg_parsec {
       { }
 
       ttg_data_copy_t(const ttg_data_copy_t& c)
+      : ttg_data_copy_self_t(this)
       {
         /* we allow copying but do not copy any data over from the original
          * device copies will have to be allocated again

From 2c6a603143f0b2836bf666f8c82a79db0af8a59e Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 12 Jul 2023 15:29:00 +0000
Subject: [PATCH 093/259] SPMM: make multi-process runs work

ttg_data_copy now tracks the buffers, not the parsec_data_t.
The ttg_data_copy also changes the owner of the buffer if moved.

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/spmm/spmm_cuda.cc     |  18 ++-
 ttg/CMakeLists.txt             |   2 +-
 ttg/ttg/parsec/buffer.h        | 125 ++++------------
 ttg/ttg/parsec/task.h          |   4 +-
 ttg/ttg/parsec/ttg.h           |  29 ++--
 ttg/ttg/parsec/ttg_data_copy.h | 251 +++++++++++++++++++++++++++------
 6 files changed, 277 insertions(+), 152 deletions(-)

diff --git a/examples/spmm/spmm_cuda.cc b/examples/spmm/spmm_cuda.cc
index fa2e95fda..bef438e33 100644
--- a/examples/spmm/spmm_cuda.cc
+++ b/examples/spmm/spmm_cuda.cc
@@ -184,8 +184,14 @@ struct DeviceTensor : public ttg::TTValue<DeviceTensor<_T, _Range, _Storage>>
     DeviceTensor(DeviceTensor&& x) noexcept
     : ttvalue_type(std::move(x))
     , tensor_type(std::move(x))
-    , b(std::move(x.b))
+    /* Grrrr, moving a Tensor does not guarantee to move the pointer */
+    , b((this->size() == 0 ||
+         this->data() == x.b.host_ptr()) ? std::move(x.b)
+                                         : ttg::buffer<_T>(this->size() ? this->data()
+                                                                        : nullptr,
+                                                           this->size()))
     {
+      assert(this->data() == b.host_ptr());
       //std::cout << "DeviceTensor move ctor" << std::endl;
     }
 
@@ -224,7 +230,12 @@ struct DeviceTensor : public ttg::TTValue<DeviceTensor<_T, _Range, _Storage>>
     DeviceTensor& operator=(DeviceTensor&& x) noexcept {
       ttvalue_type::operator=(std::move(x));
       tensor_type::operator=(std::move(x));
-      std::swap(x.b, b);
+      if (this->size() == 0 || this->data() == x.b.host_ptr()){
+        b = std::move(x.b);
+      } else  {
+        b = ttg::buffer<_T>(this->size() ? this->data() : nullptr, this->size());
+      }
+      //std::swap(x.b, b);
       //std::cout << "DeviceTensor move ctor" << std::endl;
       return *this;
     }
@@ -1510,6 +1521,9 @@ static void timed_measurement(SpMatrix<> &A, SpMatrix<> &B, const std::function<
   gettimeofday(&start, nullptr);
   // ready, go! need only 1 kick, so must be done by 1 thread only
   if (ttg::default_execution_context().rank() == 0) control.start(P, Q);
+  static volatile int debug_signal = 0;
+  std::cout << "Waiting on debug signal (int*)" << &debug_signal << std::endl;
+  //while (!debug_signal) {}
   fence();
   gettimeofday(&end, nullptr);
   timersub(&end, &start, &diff);
diff --git a/ttg/CMakeLists.txt b/ttg/CMakeLists.txt
index 12f1df749..715a7fd56 100644
--- a/ttg/CMakeLists.txt
+++ b/ttg/CMakeLists.txt
@@ -117,7 +117,7 @@ if (TARGET std::coroutine)
           )
 endif ()
 if (TARGET CUDA::cudart)
-  list(APPEND ttg-deps CUDA::cudart)
+  list(APPEND ttg-deps CUDA::cudart CUDA::cublas)
 endif (TARGET CUDA::cudart)
 
 set(ttg-public-headers ${ttg-headers};${ttg-impl-headers};${ttg-base-headers};${ttg-util-headers};${ttg_device_headers})
diff --git a/ttg/ttg/parsec/buffer.h b/ttg/ttg/parsec/buffer.h
index f65b34f65..700c9a22f 100644
--- a/ttg/ttg/parsec/buffer.h
+++ b/ttg/ttg/parsec/buffer.h
@@ -35,7 +35,7 @@ namespace detail {
  * tracking of the containing object.
  */
 template<typename T>
-struct buffer {
+struct buffer : public detail::ttg_parsec_data_wrapper_t {
 
   using element_type = std::decay_t<T>;
 
@@ -48,11 +48,8 @@ struct buffer {
   using delete_fn_t = std::add_pointer_t<void(element_type*)>;
 
   using host_data_ptr   = std::unique_ptr<element_type[], delete_fn_t>;
-  using parsec_data_ptr = std::unique_ptr<parsec_data_t, decltype(&parsec_data_destroy)>;
   host_data_ptr m_host_data;
-  parsec_data_ptr m_data;
   std::size_t m_count = 0;
-  detail::ttg_data_copy_t *m_ttg_copy = nullptr;
 
   static void delete_owned(element_type *ptr) {
     delete[] ptr;
@@ -62,47 +59,6 @@ struct buffer {
     // nothing to be done, we don't own the memory
   }
 
-  static void delete_parsec_data(parsec_data_t *data) {
-#if defined(PARSEC_HAVE_DEV_CUDA_SUPPORT)
-    if (data->device_copies[0]->flags & TTG_PARSEC_DATA_FLAG_REGISTERED) {
-      // register the memory for faster access
-      cudaError_t status;
-      status = cudaHostUnregister(data->device_copies[0]->device_private);
-      assert(cudaSuccess == status);
-      data->device_copies[0]->flags ^= TTG_PARSEC_DATA_FLAG_REGISTERED;
-    }
-#endif // PARSEC_HAVE_DEV_CUDA_SUPPORT
-    parsec_data_destroy(data);
-  }
-
-  static void delete_null_parsec_data(parsec_data_t *) {
-    // nothing to be done, only used for nullptr
-  }
-
-  static parsec_data_t* create_parsec_data(void *ptr, size_t count) {
-    parsec_data_t *data = parsec_data_create_with_type(nullptr, 0, ptr,
-                                                       sizeof(element_type)*count,
-                                                       parsec_datatype_int8_t);
-    data->device_copies[0]->flags |= PARSEC_DATA_FLAG_PARSEC_MANAGED;
-    data->device_copies[0]->coherency_state = PARSEC_DATA_COHERENCY_SHARED;
-    data->device_copies[0]->version = 1;
-    return data;
-  }
-
-  static void register_memory(void *ptr, size_t count) {
-
-  }
-
-  void reset() {
-    if (m_data) {
-      if (nullptr != m_ttg_copy) {
-        m_ttg_copy->remove_device_data(m_data.get());
-      }
-      m_data.reset();
-      m_count = 0;
-    }
-  }
-
   friend parsec_data_t* detail::get_parsec_data<T>(const ttg_parsec::buffer<T>&);
 
 public:
@@ -114,49 +70,40 @@ struct buffer {
   { }
 
   buffer(std::size_t count)
-  : m_host_data(new element_type[count](), &delete_owned)
-  , m_data(create_parsec_data(m_host_data.get(), count), &delete_parsec_data)
+  : ttg_parsec_data_wrapper_t()
+  , m_host_data(new element_type[count](), &delete_owned)
   , m_count(count)
-  , m_ttg_copy(detail::ttg_data_copy_container())
   {
-    //std::cout << "buffer " << this << " ctor ttg_copy " << m_ttg_copy << std::endl;
-    if (nullptr != m_ttg_copy) {
-      /* register with the new ttg_copy */
-      m_ttg_copy->add_device_data(m_data.get());
-    }
+    //std::cout << "buffer " << this << " ctor count "
+    //          << count << "(" << m_host_data.get() << ") ttg_copy "
+    //          << m_ttg_copy
+    //          << " parsec_data " << m_data.get() << std::endl;
+    this->reset_parsec_data(m_host_data.get(), count*sizeof(element_type));
   }
 
   /* Constructing a buffer using application-managed memory.
    * The memory pointed to by ptr must be accessible during
    * the life-time of the buffer. */
   buffer(element_type* ptr, std::size_t count = 1)
-  : m_host_data(ptr, &delete_non_owned)
-  , m_data(nullptr == ptr ? nullptr : create_parsec_data(ptr, count),
-           nullptr == ptr ? &delete_null_parsec_data : &delete_parsec_data)
+  : ttg_parsec_data_wrapper_t()
+  , m_host_data(ptr, &delete_non_owned)
   , m_count(count)
-  , m_ttg_copy(detail::ttg_data_copy_container())
   {
-    //std::cout << "buffer " << this << " ctor ttg_copy " << m_ttg_copy << std::endl;
-    if (nullptr != m_ttg_copy && m_data) {
-      /* register with the new ttg_copy */
-      m_ttg_copy->add_device_data(m_data.get());
-    }
+    this->reset_parsec_data(m_host_data.get(), count*sizeof(element_type));
+    //std::cout << "buffer " << this << " ctor ptr " << ptr << " count " << count
+    //          << " ttg_copy " << m_ttg_copy
+    //          << " parsec_data " << m_data.get() << std::endl;
   }
 
-  ~buffer() {
+  virtual ~buffer() {
     unpin(); // make sure the copies are not pinned
-    /* remove the tracked copy */
-    if (nullptr != m_ttg_copy && m_data) {
-      m_ttg_copy->remove_device_data(m_data.get());
-    }
   }
 
   /* allow moving device buffers */
   buffer(buffer&& db)
-  : m_host_data(std::move(db.m_host_data))
-  , m_data(std::move(db.m_data))
+  : ttg_parsec_data_wrapper_t(std::move(db))
+  , m_host_data(std::move(db.m_host_data))
   , m_count(db.m_count)
-  , m_ttg_copy(detail::ttg_data_copy_container())
   {
     //std::cout << "buffer " << this << " other " << &db << " mv ctor ttg_copy " << m_ttg_copy << std::endl;
     db.m_count = 0;
@@ -166,18 +113,7 @@ struct buffer {
     //          << " parsec-data " << m_data.get()
     //          << std::endl;
 
-#if 0 // This is erroneous: the ttg copy will move its content over by itself
-    if (nullptr != db.m_ttg_copy) {
-      /* remove from old ttg copy */
-      db.m_ttg_copy->remove_device_data(m_data.get());
-      db.m_ttg_copy = nullptr;
-    }
-
-    if (nullptr != m_ttg_copy) {
-      /* register with the new ttg_copy */
-      m_ttg_copy->add_device_data(m_data.get());
-    }
-#endif // 0
+//#endif // 0
   }
 
   /* explicitly disable copying of buffers
@@ -187,7 +123,7 @@ struct buffer {
 
   /* allow moving device buffers */
   buffer& operator=(buffer&& db) {
-    m_data = std::move(db.m_data);
+    ttg_parsec_data_wrapper_t::operator=(std::move(db));
     m_host_data = std::move(db.m_host_data);
     m_count = db.m_count;
     db.m_count = 0;
@@ -315,17 +251,15 @@ struct buffer {
     /* drop the current data and reallocate */
     reset();
     if (count == 0) {
-      m_data = parsec_data_ptr(nullptr, &delete_null_parsec_data);
       m_host_data = host_data_ptr(nullptr, &delete_non_owned);
     } else {
       m_host_data = host_data_ptr(new element_type[count], &delete_owned);
-      m_data = parsec_data_ptr(create_parsec_data(m_host_data.get(), count), &delete_parsec_data);
-      if (nullptr != m_ttg_copy) {
-        m_ttg_copy->add_device_data(m_data.get());
-      }
     }
+    reset_parsec_data(m_host_data.get(), count*sizeof(element_type));
+    //std::cout << "buffer::reset(" << count << ") ptr " << m_host_data.get()
+    //          << " ttg_copy " << m_ttg_copy
+    //          << " parsec_data " << m_data.get() << std::endl;
     m_count = count;
-    /* don't touch the ttg_copy, we still belong to the same container */
   }
 
   /* Reset the buffer to use the ptr to count elements */
@@ -334,21 +268,18 @@ struct buffer {
     if (count == m_count) {
       return;
     }
-    /* drop the current data and reallocate */
-    reset();
+
     if (nullptr == ptr) {
       m_host_data = host_data_ptr(nullptr, &delete_non_owned);
-      m_data = parsec_data_ptr(nullptr, &delete_null_parsec_data);
       m_count = 0;
     } else {
       m_host_data = host_data_ptr(ptr, &delete_non_owned);
-      m_data = parsec_data_ptr(create_parsec_data(m_host_data.get(), count), &delete_parsec_data);
-      if (nullptr != m_ttg_copy) {
-        m_ttg_copy->add_device_data(m_data.get());
-      }
       m_count = count;
     }
-    /* don't touch the ttg_copy, we still belong to the same container */
+    reset_parsec_data(m_host_data.get(), count*sizeof(element_type));
+    //std::cout << "buffer::reset(" << ptr << ", " << count << ") ptr " << m_host_data.get()
+    //          << " ttg_copy " << m_ttg_copy
+    //          << " parsec_data " << m_data.get() << std::endl;
   }
 
   /* serialization support */
diff --git a/ttg/ttg/parsec/task.h b/ttg/ttg/parsec/task.h
index c5edc148f..09d084c08 100644
--- a/ttg/ttg/parsec/task.h
+++ b/ttg/ttg/parsec/task.h
@@ -184,7 +184,7 @@ namespace ttg_parsec {
                                                                       : (num_streams+1);
       TT* tt;
       key_type key;
-      stream_info_t streams[num_streams] = {};
+      std::array<stream_info_t, num_streams> streams;
 #ifdef TTG_HAS_COROUTINE
       void* suspended_task_address = nullptr;  // if not null the function is suspended
 #endif
@@ -239,7 +239,7 @@ namespace ttg_parsec {
     struct parsec_ttg_task_t<TT, true> : public parsec_ttg_task_base_t {
       static constexpr size_t num_streams = TT::numins;
       TT* tt;
-      stream_info_t streams[num_streams] = {};
+      std::array<stream_info_t, num_streams> streams;
 #ifdef TTG_HAS_COROUTINE
       void* suspended_task_address = nullptr;  // if not null the function is suspended
 #endif
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 8b9c12f4c..2d5d9be20 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1571,7 +1571,7 @@ namespace ttg_parsec {
 
       /* the copy to reduce into */
       detail::ttg_data_copy_t *target_copy;
-      target_copy = static_cast<detail::ttg_data_copy_t *>(parent_task->parsec_task.data[i].data_in);
+      target_copy = parent_task->copies[i];
       assert(val_is_void || nullptr != target_copy);
       /* once we hit 0 we have to stop since another thread might enqueue a new reduction task */
       std::size_t c = 0;
@@ -1591,7 +1591,7 @@ namespace ttg_parsec {
         }
       }
 
-      assert(parsec_ttg_caller == NULL);
+      assert(detail::parsec_ttg_caller == NULL);
       detail::parsec_ttg_caller = rtask->parent_task;
 
       do {
@@ -1813,12 +1813,14 @@ namespace ttg_parsec {
             metadata_t metadata;
             pos = unpack(metadata, msg->bytes, pos);
 
+            //std::cout << "set_arg_from_msg splitmd num_iovecs " << num_iovecs << std::endl;
+
             copy = detail::create_new_datacopy(descr.create_from_metadata(metadata));
           } else if constexpr (!ttg::has_split_metadata<decvalueT>::value) {
             copy = detail::create_new_datacopy(decvalueT{});
             /* unpack the object, potentially discovering iovecs */
             pos = unpack(*static_cast<decvalueT *>(copy->get_ptr()), msg->bytes, pos);
-            //std::cout << "num_iovecs " << num_iovecs << " distance " << std::distance(copy->iovec_begin(), copy->iovec_end()) << std::endl;
+            //std::cout << "set_arg_from_msg iovec_begin num_iovecs " << num_iovecs << " distance " << std::distance(copy->iovec_begin(), copy->iovec_end()) << std::endl;
             assert(std::distance(copy->iovec_begin(), copy->iovec_end()) == num_iovecs);
           }
 
@@ -2016,7 +2018,7 @@ namespace ttg_parsec {
       }
 
       for (int i = 0; i < static_stream_goal.size(); ++i) {
-        newtask->stream[i].goal = static_stream_goal[i];
+        newtask->streams[i].goal = static_stream_goal[i];
       }
 
       ttg::trace(world.rank(), ":", get_name(), " : ", key, ": creating task");
@@ -2176,7 +2178,7 @@ namespace ttg_parsec {
             detail::ttg_data_copy_t *copy = get_copy_fn(reduce_task, std::forward<Value>(value), false);
 
             /* put the copy into the task */
-            task->parsec_task.data[i].data_in = copy;
+            task->copies[i] = copy;
 
             /* protected by the bucket lock */
             task->streams[i].size = 1;
@@ -2211,6 +2213,8 @@ namespace ttg_parsec {
         //}
         //parsec_hash_table_unlock_bucket(&tasks_table, hk);
       } else {
+        /* unlock the bucket, the lock is not needed anymore */
+        parsec_hash_table_unlock_bucket(&tasks_table, hk);
         /* whether the task needs to be deferred or not */
         if constexpr (!valueT_is_Void) {
           if (nullptr != task->copies[i]) {
@@ -2419,6 +2423,7 @@ namespace ttg_parsec {
           auto metadata = descr.get_metadata(*const_cast<decvalueT *>(value_ptr));
           size_t metadata_size = sizeof(metadata);
           pos = pack(metadata, msg->bytes, pos);
+          //std::cout << "set_arg_impl splitmd num_iovecs " << num_iovecs << std::endl;
           handle_iovec_fn(iovs);
         } else if constexpr (!ttg::has_split_metadata<std::decay_t<Value>>::value) {
           /* serialize the object */
@@ -2571,6 +2576,7 @@ namespace ttg_parsec {
           auto metadata = descr.get_metadata(value);
           size_t metadata_size = sizeof(metadata);
           pos = pack(metadata, msg->bytes, pos);
+          //std::cout << "broadcast_arg splitmd num_iovecs " << num_iovs << std::endl;
         } else if constexpr (!ttg::has_split_metadata<std::decay_t<Value>>::value) {
           /* serialize the object once */
           pos = pack(value, msg->bytes, pos, copy);
@@ -2946,7 +2952,7 @@ namespace ttg_parsec {
 
       assert(detail::parsec_ttg_caller->dev_ptr && detail::parsec_ttg_caller->dev_ptr->gpu_task);
       parsec_gpu_task_t *gpu_task = detail::parsec_ttg_caller->dev_ptr->gpu_task;
-      for (auto data : *copy) {
+      auto check_parsec_data = [&](parsec_data_t* data) {
         if (data->owner_device != 0) {
           /* find the flow */
           int flowidx = 0;
@@ -2970,7 +2976,8 @@ namespace ttg_parsec {
           ((parsec_flow_t *)gpu_task->flow[flowidx])->flow_flags |= PARSEC_FLOW_ACCESS_RW;
           gpu_task->pushout |= 1<<flowidx;
         }
-      }
+      };
+      copy->foreach_parsec_data(check_parsec_data);
     }
 
 
@@ -3400,12 +3407,14 @@ namespace ttg_parsec {
         ((__parsec_chore_t *)self.incarnations)[0].type = PARSEC_DEV_CUDA;
         ((__parsec_chore_t *)self.incarnations)[0].evaluate = NULL;
         ((__parsec_chore_t *)self.incarnations)[0].hook = &detail::hook_cuda<TT>;
+#if 0
         ((__parsec_chore_t *)self.incarnations)[1].type = PARSEC_DEV_CPU;
         ((__parsec_chore_t *)self.incarnations)[1].evaluate = NULL;
         ((__parsec_chore_t *)self.incarnations)[1].hook = &detail::hook<TT>;
-        ((__parsec_chore_t *)self.incarnations)[2].type = PARSEC_DEV_NONE;
-        ((__parsec_chore_t *)self.incarnations)[2].evaluate = NULL;
-        ((__parsec_chore_t *)self.incarnations)[2].hook = NULL;
+#endif // 0
+        ((__parsec_chore_t *)self.incarnations)[1].type = PARSEC_DEV_NONE;
+        ((__parsec_chore_t *)self.incarnations)[1].evaluate = NULL;
+        ((__parsec_chore_t *)self.incarnations)[1].hook = NULL;
       } else {
         self.incarnations = (__parsec_chore_t *)malloc(2 * sizeof(__parsec_chore_t));
         ((__parsec_chore_t *)self.incarnations)[0].type = PARSEC_DEV_CPU;
diff --git a/ttg/ttg/parsec/ttg_data_copy.h b/ttg/ttg/parsec/ttg_data_copy.h
index ee97afd0b..a0b764902 100644
--- a/ttg/ttg/parsec/ttg_data_copy.h
+++ b/ttg/ttg/parsec/ttg_data_copy.h
@@ -8,9 +8,14 @@
 #include <atomic>
 #include <type_traits>
 
+#if defined(PARSEC_HAVE_DEV_CUDA_SUPPORT)
+#include <cuda_runtime.h>
+#endif // PARSEC_HAVE_DEV_CUDA_SUPPORT
+
 #include <parsec.h>
 
 #include "ttg/parsec/thread_local.h"
+#include "ttg/parsec/parsec-ext.h"
 #include "ttg/util/span.h"
 
 
@@ -21,6 +26,78 @@ namespace ttg_parsec {
     // fwd-decl
     struct ttg_data_copy_t;
 
+    /* Wrapper managing the relationship between a ttg data copy and the parsec_data_t object */
+    struct ttg_parsec_data_wrapper_t {
+
+    protected:
+      using parsec_data_ptr = std::unique_ptr<parsec_data_t, decltype(&parsec_data_destroy)>;
+
+      ttg_data_copy_t *m_ttg_copy = nullptr;
+      parsec_data_ptr m_data;
+
+      friend ttg_data_copy_t;
+
+      static parsec_data_t* create_parsec_data(void *ptr, size_t size) {
+        parsec_data_t *data = parsec_data_create_with_type(nullptr, 0, ptr, size,
+                                                          parsec_datatype_int8_t);
+        data->device_copies[0]->flags |= PARSEC_DATA_FLAG_PARSEC_MANAGED;
+        data->device_copies[0]->coherency_state = PARSEC_DATA_COHERENCY_SHARED;
+        data->device_copies[0]->version = 1;
+        return data;
+      }
+
+      parsec_data_t* parsec_data() {
+        return m_data.get();
+      }
+
+      const parsec_data_t* parsec_data() const {
+        return m_data.get();
+      }
+
+      static void delete_parsec_data(parsec_data_t *data) {
+#if defined(PARSEC_HAVE_DEV_CUDA_SUPPORT)
+        if (data->device_copies[0]->flags & TTG_PARSEC_DATA_FLAG_REGISTERED) {
+          // register the memory for faster access
+          cudaError_t status;
+          status = cudaHostUnregister(data->device_copies[0]->device_private);
+          assert(cudaSuccess == status);
+          data->device_copies[0]->flags ^= TTG_PARSEC_DATA_FLAG_REGISTERED;
+      }
+#endif // PARSEC_HAVE_DEV_CUDA_SUPPORT
+        parsec_data_destroy(data);
+      }
+
+      static void delete_null_parsec_data(parsec_data_t *) {
+        // nothing to be done, only used for nullptr
+      }
+
+    protected:
+
+      /* remove the the data from the owning data copy */
+      void remove_from_owner();
+
+      /* add the data to the owning data copy */
+      void reset_parsec_data(void *ptr, size_t size);
+
+      ttg_parsec_data_wrapper_t();
+
+      ttg_parsec_data_wrapper_t(const ttg_parsec_data_wrapper_t& other) = delete;
+
+      ttg_parsec_data_wrapper_t(ttg_parsec_data_wrapper_t&& other);
+
+      ttg_parsec_data_wrapper_t& operator=(const ttg_parsec_data_wrapper_t& other) = delete;
+
+      ttg_parsec_data_wrapper_t& operator=(ttg_parsec_data_wrapper_t&& other);
+
+      virtual ~ttg_parsec_data_wrapper_t();
+
+      /* set a new owning data copy object */
+      void set_owner(ttg_data_copy_t& new_copy) {
+        m_ttg_copy = &new_copy;
+      }
+    };
+
+
     /* templated to break cyclic dependency with ttg_data_copy_container */
     template<typename T = ttg_data_copy_t>
     struct ttg_data_copy_container_setter {
@@ -79,6 +156,10 @@ namespace ttg_parsec {
         c.m_num_dev_data = 0;
         c.m_readers = 0;
         c.m_single_dev_data = nullptr;
+
+        foreach_wrapper([&](ttg_parsec_data_wrapper_t* data){
+          data->set_owner(*this);
+        });
       }
 
       ttg_data_copy_t& operator=(ttg_data_copy_t&& c)
@@ -94,6 +175,11 @@ namespace ttg_parsec {
         c.m_single_dev_data = nullptr;
         m_num_dev_data = c.m_num_dev_data;
         c.m_num_dev_data = 0;
+
+        /* move all data to the new owner */
+        foreach_wrapper([&](ttg_parsec_data_wrapper_t* data){
+          data->set_owner(*this);
+        });
         return *this;
       }
 
@@ -190,47 +276,28 @@ namespace ttg_parsec {
 
       /* increment the version of the current copy */
       void inc_current_version() {
-        if (m_num_dev_data == 1) {
-          //std::cout << "data-copy " << this << " inc version " << m_single_dev_data << std::endl;
-          assert(m_single_dev_data->device_copies[0] != nullptr);
-          m_single_dev_data->device_copies[0]->version++;
-          //std::cout << "Incrementing version of data " << m_single_dev_data << " copy "
-          //          << m_single_dev_data->device_copies[m_single_dev_data->owner_device]
-          //          << " on device " << (int)m_single_dev_data->owner_device << " to "
-          //          << m_single_dev_data->device_copies[m_single_dev_data->owner_device]->version << std::endl;
-        } else if (m_num_dev_data > 1) {
-          std::for_each(m_dev_data.begin(), m_dev_data.end(),
-                        [](parsec_data_t *data){
-                           assert(data->device_copies[0] != nullptr);
-                           data->device_copies[0]->version++;
-                           //std::cout << "Incrementing version of copy "
-                           //          << " on device " << (int)data->owner_device << data->device_copies[data->owner_device] << " to "
-                           //          << data->device_copies[data->owner_device]->version << std::endl;
-                        });
-        }
+        //std::cout << "data-copy " << this << " inc_current_version " << " count " << m_num_dev_data << std::endl;
+        foreach_parsec_data([](parsec_data_t* data){
+          assert(data->device_copies[0] != nullptr);
+          data->device_copies[0]->version++;
+        });
       }
 
       void transfer_ownership(int access, int device = 0) {
-
-        if (m_num_dev_data == 1) {
-          parsec_data_transfer_ownership_to_copy(m_single_dev_data, device, access);
-        } else if (m_num_dev_data > 1) {
-          std::for_each(m_dev_data.begin(), m_dev_data.end(),
-                        [&](parsec_data_t *data){
-                          parsec_data_transfer_ownership_to_copy(data, device, access);
-                        });
-        }
+        foreach_parsec_data([&](parsec_data_t* data){
+          parsec_data_transfer_ownership_to_copy(data, device, access);
+        });
       }
 
       /* manage device copies owned by this object
        * we only touch the vector if we have more than one copies to track
        * and otherwise use the single-element member.
        */
-      using iterator = parsec_data_t**;
+      using iterator = ttg_parsec_data_wrapper_t**;
 
-      void add_device_data(parsec_data_t* data) {
-        //std::cout << "data-copy " << this << " add data " << data << std::endl;
-        PARSEC_OBJ_RETAIN(data);
+      void add_device_data(ttg_parsec_data_wrapper_t* data) {
+        // TODO: REMOVE AFTER DEBUG
+        //assert(m_num_dev_data == 0);
         switch (m_num_dev_data) {
           case 0:
             m_single_dev_data = data;
@@ -238,18 +305,23 @@ namespace ttg_parsec {
           case 1:
             /* move single copy into vector and add new copy below */
             m_dev_data.push_back(m_single_dev_data);
+            m_single_dev_data = nullptr;
             /* fall-through */
           default:
             /* store in multi-copy vector */
             m_dev_data.push_back(data);
             break;
         }
+        //std::cout << "data-copy " << this << " add data " << data << " count " << m_num_dev_data << std::endl;
         m_num_dev_data++;
       }
 
-      void remove_device_data(parsec_data_t* data) {
-        //std::cout << "data-copy " << this << " remove data " << data << std::endl;
-        assert(m_num_dev_data >= 1);
+      void remove_device_data(ttg_parsec_data_wrapper_t* data) {
+        //std::cout << "data-copy " << this << " remove data " << data << " count " << m_num_dev_data << std::endl;
+        if (m_num_dev_data == 0) {
+          /* this may happen if we're integrated into the object and have been moved */
+          return;
+        }
         if (m_num_dev_data == 1) {
           assert(m_single_dev_data == data);
           m_single_dev_data = nullptr;
@@ -265,13 +337,42 @@ namespace ttg_parsec {
           m_single_dev_data = m_dev_data[0];
           m_dev_data.clear();
         }
-        PARSEC_OBJ_RELEASE(data);
+        // TODO: REMOVE AFTER DEBUG
+        //assert(m_num_dev_data == 0);
       }
 
       int num_dev_data() const {
         return m_num_dev_data;
       }
 
+      template<typename Fn>
+      void foreach_wrapper(Fn&& fn) {
+        if (m_num_dev_data == 1) {
+          fn(m_single_dev_data);
+        } else if (m_num_dev_data > 1) {
+          std::for_each(m_dev_data.begin(), m_dev_data.end(), fn);
+        }
+      }
+
+      template<typename Fn>
+      void foreach_parsec_data(Fn&& fn) {
+        if (m_num_dev_data == 1) {
+          if (m_single_dev_data->parsec_data()) {
+            fn(m_single_dev_data->parsec_data());
+          }
+        } else if (m_num_dev_data > 1) {
+          std::for_each(m_dev_data.begin(), m_dev_data.end(),
+            [&](ttg_parsec_data_wrapper_t* data){
+              if (data->parsec_data()) {
+                fn(data->parsec_data());
+              }
+            }
+          );
+        }
+      }
+
+
+#if 0
       iterator begin() {
         switch(m_num_dev_data) {
           // no device copies
@@ -290,6 +391,7 @@ namespace ttg_parsec {
             return m_dev_data.data() + m_dev_data.size();
         }
       }
+#endif // 0
 
       using iovec_iterator = typename std::vector<ttg::iovec>::iterator;
 
@@ -324,14 +426,14 @@ namespace ttg_parsec {
     protected:
       parsec_task_t *m_next_task = nullptr;
       int32_t        m_readers  = 1;
-      std::atomic<int32_t>  m_refs = 1; // number of entities referencing this copy (TTGs, external)
+      std::atomic<int32_t>  m_refs = 1;                     //< number of entities referencing this copy (TTGs, external)
 
       std::vector<ttg::iovec> m_iovecs;
 
-      std::vector<parsec_data_t*> m_dev_data;   //< used if there are multiple device copies
-                                                //  that belong to this object
-      parsec_data_t *m_single_dev_data;         //< used if there is a single device copy
-      int m_num_dev_data = 0;                   //< number of device copies
+      std::vector<ttg_parsec_data_wrapper_t*> m_dev_data;   //< used if there are multiple device copies
+                                                            //  that belong to this object
+      ttg_parsec_data_wrapper_t *m_single_dev_data;         //< used if there is a single device copy
+      int m_num_dev_data = 0;                               //< number of device copies
     };
 
 
@@ -410,6 +512,75 @@ namespace ttg_parsec {
       }
     };
 
+    /**
+     * definition of ttg_parsec_data_wrapper_t members that depend on ttg_data_copy_t
+     */
+
+    void ttg_parsec_data_wrapper_t::remove_from_owner() {
+      if (nullptr != m_ttg_copy) {
+        m_ttg_copy->remove_device_data(this);
+        m_ttg_copy = nullptr;
+      }
+    }
+
+    void ttg_parsec_data_wrapper_t::reset_parsec_data(void *ptr, size_t size) {
+      if (ptr == m_data.get()) return;
+
+      if (nullptr == ptr) {
+        m_data = parsec_data_ptr(nullptr, &delete_null_parsec_data);
+      } else {
+        m_data = parsec_data_ptr(create_parsec_data(ptr, size), &delete_parsec_data);
+      }
+    }
+
+    ttg_parsec_data_wrapper_t::ttg_parsec_data_wrapper_t()
+    : m_data(nullptr, delete_null_parsec_data)
+    , m_ttg_copy(detail::ttg_data_copy_container())
+    {
+      if (m_ttg_copy) {
+        m_ttg_copy->add_device_data(this);
+      }
+    }
+    ttg_parsec_data_wrapper_t::ttg_parsec_data_wrapper_t(ttg_parsec_data_wrapper_t&& other)
+    : m_data(std::move(other.m_data))
+    , m_ttg_copy(detail::ttg_data_copy_container())
+    {
+      /* the ttg_data_copy may have moved us already */
+      if (other.m_ttg_copy != m_ttg_copy) {
+        // try to remove the old buffer from the *old* ttg_copy
+        other.remove_from_owner();
+
+        // register with the new ttg_copy
+        if (nullptr != m_ttg_copy) {
+          m_ttg_copy->add_device_data(this);
+        }
+      }
+    }
+
+    ttg_parsec_data_wrapper_t& ttg_parsec_data_wrapper_t::operator=(ttg_parsec_data_wrapper_t&& other) {
+      m_data = std::move(other.m_data);
+      /* check whether the owning ttg_data_copy has already moved us */
+      if (other.m_ttg_copy != m_ttg_copy) {
+        /* remove from old ttg copy */
+        other.remove_from_owner();
+
+        if (nullptr != m_ttg_copy) {
+          /* register with the new ttg_copy */
+          m_ttg_copy->add_device_data(this);
+        }
+      }
+      return *this;
+    }
+
+
+    ttg_parsec_data_wrapper_t::~ttg_parsec_data_wrapper_t() {
+      if (nullptr != m_ttg_copy) {
+        m_ttg_copy->remove_device_data(this);
+        m_ttg_copy = nullptr;
+      }
+    }
+
+
   } // namespace detail
 
 } // namespace ttg_parsec

From 0ef697a4c63c29f6be1f20ccc649135db5bf8bfc Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Wed, 12 Jul 2023 17:12:40 -0400
Subject: [PATCH 094/259] fixes to make compile with clang (rever the use of
 atomic_ref, obscure compile error) + partial support for building without
 CUDA

---
 tests/unit/CMakeLists.txt      |  2 +-
 ttg/CMakeLists.txt             |  4 +++-
 ttg/ttg/parsec/ttg.h           |  2 ++
 ttg/ttg/parsec/ttg_data_copy.h | 12 ++++++------
 ttg/ttg/parsec/ttvalue.h       |  4 ++--
 5 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt
index ac48e70f5..154f03183 100644
--- a/tests/unit/CMakeLists.txt
+++ b/tests/unit/CMakeLists.txt
@@ -17,7 +17,7 @@ set(ut_libs Catch2::Catch2)
     endif(TTG_HAVE_CUDA)
     list(APPEND ut_libs std::coroutine)
 #endif()
-add_ttg_executable(core-unittests-ttg "${ut_src}" LINK_LIBRARIES "${ut_libs}" COMPILE_DEFINITIONS "-DCATCH_CONFIG_NO_POSIX_SIGNALS=1" )
+add_ttg_executable(core-unittests-ttg "${ut_src}" LINK_LIBRARIES "${ut_libs}" COMPILE_DEFINITIONS "CATCH_CONFIG_NO_POSIX_SIGNALS=1" )
 
 # serialization test: probes serialization via all supported serialization methods (MADNESS, Boost::serialization, cereal) that are available
 add_executable(serialization "serialization.cc;unit_main.cpp")
diff --git a/ttg/CMakeLists.txt b/ttg/CMakeLists.txt
index 715a7fd56..8accd57ac 100644
--- a/ttg/CMakeLists.txt
+++ b/ttg/CMakeLists.txt
@@ -81,7 +81,6 @@ set(ttg-sources
         ${ttg-util-headers}
         ${ttg_device_headers}
         ${ttg-external-headers}
-        ${CMAKE_CURRENT_SOURCE_DIR}/ttg/device/cublas_helper.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/backtrace.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/bug.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/env.cpp
@@ -117,6 +116,9 @@ if (TARGET std::coroutine)
           )
 endif ()
 if (TARGET CUDA::cudart)
+  list(APPEND ttg-sources
+          ${CMAKE_CURRENT_SOURCE_DIR}/ttg/device/cublas_helper.cpp
+          )
   list(APPEND ttg-deps CUDA::cudart CUDA::cublas)
 endif (TARGET CUDA::cudart)
 
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 2d5d9be20..2d40b7064 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -100,7 +100,9 @@
 #include "ttg/parsec/thread_local.h"
 #include "ttg/parsec/ptr.h"
 #include "ttg/parsec/task.h"
+#ifdef TTG_HAVE_CUDART
 #include "ttg/device/cublas_helper.h"
+#endif  // TTG_HAVE_CUDART
 #include "ttg/parsec/parsec-ext.h"
 
 #undef TTG_PARSEC_DEBUG_TRACK_DATA_COPIES
diff --git a/ttg/ttg/parsec/ttg_data_copy.h b/ttg/ttg/parsec/ttg_data_copy.h
index a0b764902..27814ecf5 100644
--- a/ttg/ttg/parsec/ttg_data_copy.h
+++ b/ttg/ttg/parsec/ttg_data_copy.h
@@ -210,9 +210,9 @@ namespace ttg_parsec {
       template<bool Atomic = true>
       int increment_readers() {
         if constexpr(Atomic) {
-          //return parsec_atomic_fetch_inc_int32(&m_readers);
-          std::atomic_ref<int32_t> a{m_readers};
-          return a.fetch_add(1, std::memory_order_relaxed);
+          return parsec_atomic_fetch_inc_int32(&m_readers);
+//          std::atomic_ref<int32_t> a{m_readers};
+//          return a.fetch_add(1, std::memory_order_relaxed);
         } else {
           return m_readers++;
         }
@@ -233,9 +233,9 @@ namespace ttg_parsec {
       template<bool Atomic = true>
       int decrement_readers() {
         if constexpr(Atomic) {
-          //return parsec_atomic_fetch_dec_int32(&m_readers);
-          std::atomic_ref<int32_t> a{m_readers};
-          return a.fetch_sub(1, std::memory_order_relaxed);
+          return parsec_atomic_fetch_dec_int32(&m_readers);
+//          std::atomic_ref<int32_t> a{m_readers};
+//          return a.fetch_sub(1, std::memory_order_relaxed);
         } else {
           return m_readers--;
         }
diff --git a/ttg/ttg/parsec/ttvalue.h b/ttg/ttg/parsec/ttvalue.h
index 6ed31c9fc..6c3942854 100644
--- a/ttg/ttg/parsec/ttvalue.h
+++ b/ttg/ttg/parsec/ttvalue.h
@@ -69,11 +69,11 @@ namespace ttg {
 
   namespace detail {
 
-    template<typename T, bool Enabler = false>
+    template<typename T, typename Enabler = void>
     struct is_ttvalue_base : std::false_type {};
 
     template<typename T>
-    struct is_ttvalue_base<T, std::is_base_of_v<TTValue<std::decay_t<T>>, std::decay_t<T>>>
+    struct is_ttvalue_base<T, std::is_base_of<TTValue<std::decay_t<T>>, std::decay_t<T>>>
     : std::true_type
     { };
 

From 20690b9942e8d13a9d125f729d6ac1395fde8d56 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Wed, 12 Jul 2023 17:14:00 -0400
Subject: [PATCH 095/259] introduced FindOrFetchTiledArray, use tiledarray
 instead of BTAS to provide needed functionality to spmm

---
 CMakeLists.txt                            |  4 +-
 cmake/modules/FindOrFetchTiledArray.cmake | 65 +++++++++++++++++++++++
 examples/CMakeLists.txt                   |  7 ++-
 3 files changed, 72 insertions(+), 4 deletions(-)
 create mode 100644 cmake/modules/FindOrFetchTiledArray.cmake

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4f0048c2c..7a619a0af 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -132,8 +132,8 @@ if (TTG_EXAMPLES)
         set_property(TARGET eigen3 PROPERTY
                 INTERFACE_INCLUDE_DIRECTORIES ${EIGEN3_INCLUDE_DIR})
     endif (EIGEN3_FOUND)
-    # BTAS brings in linear algebra (BLAS++/LAPACK++)
-    include(FindOrFetchBTAS)
+    # TiledArray brings in BTAS AND linear algebra (BLAS++/LAPACK++)
+    include(FindOrFetchTiledArray)
     # OpenMP may also be used by some examples
     find_package(OpenMP COMPONENTS CXX)
     # std::execution may also be used by some examples
diff --git a/cmake/modules/FindOrFetchTiledArray.cmake b/cmake/modules/FindOrFetchTiledArray.cmake
new file mode 100644
index 000000000..b0f3d0bcc
--- /dev/null
+++ b/cmake/modules/FindOrFetchTiledArray.cmake
@@ -0,0 +1,65 @@
+if (NOT TARGET tiledarray AND NOT MPQC_BUILD_DEPENDENCIES_FROM_SOURCE)
+  if(TiledArray_INSTALL_DIR)
+    set(TiledArray_DIR ${TiledArray_INSTALL_DIR}/lib/cmake/tiledarray)
+  endif()
+  find_package(TiledArray CONFIG QUIET COMPONENTS tiledarray)
+endif ()
+
+set(TA_PYTHON OFF)
+
+if (TARGET tiledarray)
+  message(STATUS "Found TiledArray CONFIG at ${TiledArray_CONFIG}")
+
+  if ((NOT TA_ASSUMES_ASLR_DISABLED AND MPQC_ASSUMES_ASLR_DISABLED) OR (TA_ASSUMES_ASLR_DISABLED AND NOT MPQC_ASSUMES_ASLR_DISABLED))
+    message(FATAL_ERROR "Found TiledArray configured with TA_ASSUMES_ASLR_DISABLED=${TA_ASSUMES_ASLR_DISABLED} but MPQC is configured with MPQC_ASSUMES_ASLR_DISABLED=${MPQC_ASSUMES_ASLR_DISABLED}; MPQC_ASSUMES_ASLR_DISABLED and TA_ASSUMES_ASLR_DISABLED should be the same")
+  endif()
+
+else (TARGET tiledarray)
+
+  # enable CUDA if TTG has it
+  if (TTG_HAVE_CUDA)
+    set(ENABLE_CUDA ON CACHE BOOL "Enable CUDA")
+  endif()
+
+  # update CMake cache for TA
+  if (DEFINED MADNESS_CMAKE_EXTRA_ARGS)
+    set(MADNESS_CMAKE_EXTRA_ARGS "${MADNESS_CMAKE_EXTRA_ARGS};-DENABLE_DQ_PREBUF=OFF" CACHE STRING "Extra CMake arguments to MADNESS" FORCE)
+  else(DEFINED MADNESS_CMAKE_EXTRA_ARGS)
+    set(MADNESS_CMAKE_EXTRA_ARGS "-DENABLE_DQ_PREBUF=OFF" CACHE STRING "Extra CMake arguments to MADNESS")
+  endif(DEFINED MADNESS_CMAKE_EXTRA_ARGS)
+  if (NOT DEFINED TA_ASSUMES_ASLR_DISABLED)
+    set(TA_ASSUMES_ASLR_DISABLED ${MPQC_ASSUMES_ASLR_DISABLED} CACHE BOOL "TA assumes the Address Space Layout Randomization (ASLR) to be disabled")
+  endif(NOT DEFINED TA_ASSUMES_ASLR_DISABLED)
+  if (NOT DEFINED TA_ASSERT_POLICY)
+    string(REPLACE "MPQC_" "TA_" TA_ASSERT_POLICY "${MPQC_ASSERT_POLICY}")
+    set(TA_ASSERT_POLICY ${TA_ASSERT_POLICY} CACHE STRING "Controls the behavior of TA_ASSERT")
+  endif (NOT DEFINED TA_ASSERT_POLICY)
+  if (NOT DEFINED TA_BUILD_UNITTEST)
+    set(TA_BUILD_UNITTEST FALSE CACHE BOOL "Whether to build TA unit tests")
+  endif (NOT DEFINED TA_BUILD_UNITTEST)
+
+  include(FetchContent)
+  FetchContent_Declare(
+      TILEDARRAY
+      GIT_REPOSITORY      https://github.com/ValeevGroup/tiledarray.git
+      GIT_TAG             ${MPQC_TRACKED_TILEDARRAY_TAG}
+  )
+  FetchContent_MakeAvailable(TILEDARRAY)
+  FetchContent_GetProperties(TILEDARRAY
+      SOURCE_DIR TILEDARRAY_SOURCE_DIR
+      BINARY_DIR TILEDARRAY_BINARY_DIR
+      )
+  # TA includes dependencies that are built manually, not using FetchContent, hence make sure we build them before building any MPQC code
+  # add_dependencies(deps-mpqc External-tiledarray)
+
+  set(TTG_DOWNLOADED_TILEDARRAY ON CACHE BOOL "Whether TTG downloaded TiledArray")
+
+  include("${TILEDARRAY_BINARY_DIR}/cmake/modules/ReimportTargets.cmake")
+  if (NOT TARGET MADworld)
+    message(FATAL_ERROR "did not find re-imported target MADworld")
+  endif(NOT TARGET MADworld)
+
+  # this is where tiledarray-config.cmake will end up
+  # must be in sync with the "install(FILES ...tiledarray-config.cmake" statement in https://github.com/ValeevGroup/tiledarray/blob/${MPQC_TRACKED_TILEDARRAY_TAG}/CMakeLists.txt
+  set(TiledArray_CONFIG "${CMAKE_INSTALL_PREFIX}/${TILEDARRAY_INSTALL_CMAKEDIR}" CACHE INTERNAL "The location of installed tiledarray-config.cmake file")
+endif(TARGET tiledarray)
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 56895a057..d72712b81 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -11,10 +11,13 @@ if (TARGET eigen3)
     # block-sparse needs BTAS
     if (TARGET BTAS::BTAS)
         # since only need to use matrices, limit BTAS_TARGET_MAX_INDEX_RANK to 2
-        add_ttg_executable(bspmm spmm/spmm.cc LINK_LIBRARIES eigen3 BTAS Boost::boost COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2)
+        add_ttg_executable(bspmm spmm/spmm.cc LINK_LIBRARIES tiledarray eigen3 BTAS Boost::boost COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2)
         if (TARGET CUDA::cublas)
+          if (NOT TARGET tiledarray)
+              message(FATAL_ERROR "TiledArray is not found; it is required for CUDA-based block-sparse SPMM")
+          endif()
           add_ttg_executable(bspmm-cuda spmm/spmm_cuda.cc spmm/cuda_gemm.cc
-                             LINK_LIBRARIES eigen3 BTAS Boost::boost CUDA::cublas
+                             LINK_LIBRARIES tiledarray eigen3 BTAS Boost::boost CUDA::cublas
                              COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2
                              RUNTIMES "parsec")
         endif (TARGET CUDA::cublas)

From f6d0afa9579565a5ac47857d244b0f32c733fc84 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Thu, 13 Jul 2023 23:16:42 -0400
Subject: [PATCH 096/259] Pin to specific TA revision

---
 cmake/modules/ExternalDependenciesVersions.cmake | 1 +
 cmake/modules/FindOrFetchTiledArray.cmake        | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/cmake/modules/ExternalDependenciesVersions.cmake b/cmake/modules/ExternalDependenciesVersions.cmake
index b0c10abf0..123a6f7bd 100644
--- a/cmake/modules/ExternalDependenciesVersions.cmake
+++ b/cmake/modules/ExternalDependenciesVersions.cmake
@@ -8,3 +8,4 @@ set(TTG_TRACKED_CEREAL_VERSION 1.3.0)
 set(TTG_TRACKED_MADNESS_TAG 31b2470ca722a6a2d84d4de08d32fb72ae8fdeda)
 set(TTG_TRACKED_PARSEC_TAG 9fc74b6f165605a133125d8a5b62cf55642c1907) 
 set(TTG_TRACKED_BTAS_TAG d73153ad9bc41a177e441ef04eceff7fab0c766d)
+set(TTG_TRACKED_TILEDARRAY_TAG a9947ce6e64c871bc6f3f3610bf2cb2d1c02cac1)
diff --git a/cmake/modules/FindOrFetchTiledArray.cmake b/cmake/modules/FindOrFetchTiledArray.cmake
index b0f3d0bcc..a8318425d 100644
--- a/cmake/modules/FindOrFetchTiledArray.cmake
+++ b/cmake/modules/FindOrFetchTiledArray.cmake
@@ -42,7 +42,7 @@ else (TARGET tiledarray)
   FetchContent_Declare(
       TILEDARRAY
       GIT_REPOSITORY      https://github.com/ValeevGroup/tiledarray.git
-      GIT_TAG             ${MPQC_TRACKED_TILEDARRAY_TAG}
+      GIT_TAG             ${TTG_TRACKED_TILEDARRAY_TAG}
   )
   FetchContent_MakeAvailable(TILEDARRAY)
   FetchContent_GetProperties(TILEDARRAY

From e07140e648f8f7becee6cc8b077ed5bdbdba0a83 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Fri, 14 Jul 2023 00:01:25 -0400
Subject: [PATCH 097/259] removed cublas v1 + SPMM tiles use pinned allocator
 ... does not compile yet

---
 examples/CMakeLists.txt          |  2 +-
 examples/spmm/cuda_gemm.cc       | 29 ----------------------------
 examples/spmm/cuda_gemm.h        | 24 -----------------------
 examples/spmm/spmm.cc            |  5 +++--
 ttg/ttg/device/cublas_helper.cpp | 33 ++++++++++++++++++++------------
 ttg/ttg/device/cublas_helper.h   |  3 ++-
 ttg/ttg/parsec/ttg.h             |  4 ++--
 7 files changed, 29 insertions(+), 71 deletions(-)
 delete mode 100644 examples/spmm/cuda_gemm.cc
 delete mode 100644 examples/spmm/cuda_gemm.h

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index d72712b81..8750204a9 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -16,7 +16,7 @@ if (TARGET eigen3)
           if (NOT TARGET tiledarray)
               message(FATAL_ERROR "TiledArray is not found; it is required for CUDA-based block-sparse SPMM")
           endif()
-          add_ttg_executable(bspmm-cuda spmm/spmm_cuda.cc spmm/cuda_gemm.cc
+          add_ttg_executable(bspmm-cuda spmm/spmm_cuda.cc
                              LINK_LIBRARIES tiledarray eigen3 BTAS Boost::boost CUDA::cublas
                              COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2
                              RUNTIMES "parsec")
diff --git a/examples/spmm/cuda_gemm.cc b/examples/spmm/cuda_gemm.cc
deleted file mode 100644
index 68987f64f..000000000
--- a/examples/spmm/cuda_gemm.cc
+++ /dev/null
@@ -1,29 +0,0 @@
-
-#include <cublas.h>
-
-
-void my_cublas_init_because_cublas_is_stupid() {
-    cublasInit();
-}
-
-void my_cublas_shutdown_because_cublas_is_stupid() {
-    cublasShutdown();
-}
-
-void my_cublas_dgemm_because_cublas_is_stupid(
-    char transa,
-    char transb,
-    int m,
-    int n,
-    int k,
-    double alpha,
-    const double* A,
-    int lda,
-    const double* B,
-    int ldb,
-    double beta,
-    double* C,
-    int ldc)
-{
-  cublasDgemm(transa, transb, m, n, k, alpha, A, lda, B, ldb, beta, C, ldc);
-}
diff --git a/examples/spmm/cuda_gemm.h b/examples/spmm/cuda_gemm.h
deleted file mode 100644
index cc6aba6c8..000000000
--- a/examples/spmm/cuda_gemm.h
+++ /dev/null
@@ -1,24 +0,0 @@
-
-
-/* We need wrap cublas v1 functions because some header (blaspp)
- * includes cublas_v2 and cublas is braindead and overwrites the v1
- * signatures. What a clusterfuck. */
-
-void my_cublas_init_because_cublas_is_stupid();
-
-void my_cublas_shutdown_because_cublas_is_stupid();
-
-void my_cublas_dgemm_because_cublas_is_stupid(
-    char transa,
-    char transb,
-    int m,
-    int n,
-    int k,
-    double alpha,
-    const double* A,
-    int lda,
-    const double* B,
-    int ldb,
-    double beta,
-    double* C,
-    int ldc);
\ No newline at end of file
diff --git a/examples/spmm/spmm.cc b/examples/spmm/spmm.cc
index 4403c1012..1610477b7 100644
--- a/examples/spmm/spmm.cc
+++ b/examples/spmm/spmm.cc
@@ -14,8 +14,9 @@
 #include <btas/features.h>
 #ifdef BTAS_IS_USABLE
 #include <btas/btas.h>
-#include <btas/optimize/contract.h>
 #include <btas/util/mohndle.h>
+#include <btas/optimize/contract.h>
+#include <TiledArray/cuda/allocators.h>
 #else
 #warning "found btas/features.h but Boost.Iterators is missing, hence BTAS is unusable ... add -I/path/to/boost"
 #endif
@@ -40,7 +41,7 @@ using namespace ttg;
 #include "ttg/util/bug.h"
 
 #if defined(BLOCK_SPARSE_GEMM) && defined(BTAS_IS_USABLE)
-using blk_t = btas::Tensor<double, btas::DEFAULT::range, btas::mohndle<btas::varray<double>, btas::Handle::shared_ptr>>;
+using blk_t = btas::Tensor<double, btas::DEFAULT::range, btas::mohndle<btas::varray<double, TiledArray::cuda_pinned_allocator<double>>, btas::Handle::shared_ptr>>;
 
 #if defined(TTG_USE_PARSEC)
 namespace ttg {
diff --git a/ttg/ttg/device/cublas_helper.cpp b/ttg/ttg/device/cublas_helper.cpp
index 0f77c2a5f..f86eda05d 100644
--- a/ttg/ttg/device/cublas_helper.cpp
+++ b/ttg/ttg/device/cublas_helper.cpp
@@ -1,23 +1,32 @@
-#ifdef TTG_HAVE_CUDART
-#include <cublas.h>
-#endif // TTG_HAVE_CUDART
-
-#include <exception>
-#include <stdexcept>
-
 #include "ttg/config.h"
 #include "ttg/device/cublas_helper.h"
 
+#include <memory>
+#include <stdexcept>
+#include <optional>
+
 namespace ttg::detail {
 
-/* shim wrapper to work around the fact that cublas
- * deliberately breaks its API depending on the order
- * in which header are included */
+#ifdef TTG_HAVE_CUDART
+/// \brief Returns the cuBLAS handle to be used for launching cuBLAS kernels from the current thread
+/// \return the cuBLAS handle for the current thread
+inline const cublasHandle_t& cublas_get_handle() {
+  static thread_local std::optional<cublasHandle_t> handle;
+  if (!handle.has_value()) {
+    auto status = cublasCreate_v2(&handle.emplace());
+    if (CUBLAS_STATUS_SUCCESS != status) {
+      throw std::runtime_error("cublasCreate_v2 failed");
+    }
+  }
+  return *handle;
+}
+#endif // TTG_HAVE_CUDART
+
 void cublas_set_kernel_stream(cudaStream_t stream) {
 #ifdef TTG_HAVE_CUDART
-    cublasStatus_t status = cublasSetKernelStream(stream);
+    cublasStatus_t status = cublasSetStream_v2(cublas_get_handle(), stream);
     if (CUBLAS_STATUS_SUCCESS != status) {
-        throw std::runtime_error("cublasSetKernelStream failed");
+        throw std::runtime_error("cublasSetStream_v2 failed");
     }
 #else
     throw std::runtime_error("Support for cublas missing during installation!");
diff --git a/ttg/ttg/device/cublas_helper.h b/ttg/ttg/device/cublas_helper.h
index 51c13601c..5aae3df9b 100644
--- a/ttg/ttg/device/cublas_helper.h
+++ b/ttg/ttg/device/cublas_helper.h
@@ -4,10 +4,11 @@
 #include "ttg/config.h"
 
 #ifdef TTG_HAVE_CUDART
-#include <cublas.h>
+#include <cublas_v2.h>
 
 namespace ttg::detail {
 
+/// \brief Returns the current CUDA stream used by cuBLAS
 void cublas_set_kernel_stream(cudaStream_t stream);
 
 } // namespace ttg::detail
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 2d40b7064..59575f13b 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1362,8 +1362,8 @@ namespace ttg_parsec {
       /* TODO: is this the right place to set the mask? */
       task->parsec_task.chore_mask = PARSEC_DEV_ALL;
       /* get a device and come back if we need another one */
-      int64_t task_load = 1;
-      dev_index = parsec_get_best_device(parsec_task, &task_load);
+      double task_load = 1.;
+      dev_index = parsec_get_best_device(parsec_task, task_load);
       assert(dev_index >= 0);
       if (dev_index < 2) {
           return PARSEC_HOOK_RETURN_NEXT; /* Fall back */

From bcf5998786a4da0efb004ec739fa3c367367671e Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Fri, 14 Jul 2023 09:32:04 -0400
Subject: [PATCH 098/259] introduce ability to override detection of usability
 of std::memcpy for situations (like std::pair<int,int>) where
 std::is_trivially_copyable is overly strong ... user needs to specialize
 inline constexpr variable ttg::detail::is_memcpyable_override_v<T>

---
 tests/unit/serialization.cc             | 16 ++++++++++++++++
 ttg/ttg/serialization/data_descriptor.h |  8 ++++----
 ttg/ttg/serialization/traits.h          | 22 ++++++++++++++++++++++
 3 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/tests/unit/serialization.cc b/tests/unit/serialization.cc
index 434c828b7..647f7d1ec 100644
--- a/tests/unit/serialization.cc
+++ b/tests/unit/serialization.cc
@@ -40,6 +40,7 @@ class POD {
   bool operator==(const POD& other) const { return value == other.value; }
 };
 static_assert(std::is_trivially_copyable_v<POD>);
+static_assert(ttg::detail::is_memcpyable_v<POD>);
 
 #ifdef TTG_SERIALIZATION_SUPPORTS_CEREAL
 // WTF?! std::array of non-Serializable is Serializable
@@ -91,7 +92,17 @@ class NonPOD {
 
   int get() const { return value; }
 };
+// non-default ctor breaks trivial copyability
 static_assert(!std::is_trivially_copyable_v<NonPOD>);
+static_assert(!ttg::detail::is_memcpyable_v<NonPOD>);
+
+// but can allow use of std::memcpy on type
+class MemcpyableNonPOD : public NonPOD {};
+namespace ttg::detail {
+  template<> inline constexpr bool is_memcpyable_override_v<MemcpyableNonPOD> = true;
+}  // namespace ttg::detail
+static_assert(!std::is_trivially_copyable_v<MemcpyableNonPOD>);
+static_assert(ttg::detail::is_memcpyable_v<MemcpyableNonPOD>);
 
 namespace intrusive::symmetric::mc {
 
@@ -471,6 +482,11 @@ static_assert(!ttg::detail::is_cereal_user_buffer_serializable_v<int[4]>);
 static_assert(!ttg::detail::is_user_buffer_serializable_v<int[4]>);
 static_assert(!ttg::detail::is_user_buffer_serializable_v<std::array<int, 4>>);
 
+// default_data_descriptor<std::pair<int,int>> should be defined but std::is_trivially_copyable_v<std::pair<int,int>> is false
+// is_memcpyable_v<std::pair<int,int>> is true
+static_assert(!std::is_trivially_copyable_v<std::pair<int,int>>);
+static_assert(ttg::detail::is_memcpyable_v<std::pair<int,int>>);
+
 #ifdef TTG_SERIALIZATION_SUPPORTS_MADNESS
 
 static_assert(ttg::detail::is_madness_serializable_v<madness::archive::BufferOutputArchive, int>);
diff --git a/ttg/ttg/serialization/data_descriptor.h b/ttg/ttg/serialization/data_descriptor.h
index 265897e94..fb26b1d55 100644
--- a/ttg/ttg/serialization/data_descriptor.h
+++ b/ttg/ttg/serialization/data_descriptor.h
@@ -44,7 +44,7 @@ namespace ttg {
   /// @tparam T a trivially-copyable type
   template <typename T>
   struct default_data_descriptor<
-      T, std::enable_if_t<std::is_trivially_copyable_v<T> && !detail::is_user_buffer_serializable_v<T> &&
+      T, std::enable_if_t<detail::is_memcpyable_v<T> && !detail::is_user_buffer_serializable_v<T> &&
                           !ttg::has_split_metadata<T>::value>> {
     static constexpr const bool serialize_size_is_const = true;
 
@@ -154,7 +154,7 @@ namespace ttg {
   /// and support MADNESS serialization
   template <typename T>
   struct default_data_descriptor<
-      T, std::enable_if_t<((!std::is_trivially_copyable_v<T> && detail::is_madness_buffer_serializable_v<T>) ||
+      T, std::enable_if_t<((!detail::is_memcpyable_v<T> && detail::is_madness_buffer_serializable_v<T>) ||
                            detail::is_madness_user_buffer_serializable_v<T>)&&!ttg::has_split_metadata<T>::value>> {
     static constexpr const bool serialize_size_is_const = false;
 
@@ -200,7 +200,7 @@ namespace ttg {
   /// do not support MADNESS serialization, and support Boost serialization
   template <typename T>
   struct default_data_descriptor<
-      T, std::enable_if_t<(!std::is_trivially_copyable_v<T> && !detail::is_madness_buffer_serializable_v<T> &&
+      T, std::enable_if_t<(!detail::is_memcpyable_v<T> && !detail::is_madness_buffer_serializable_v<T> &&
                            detail::is_boost_buffer_serializable_v<T>) ||
                           (!detail::is_madness_user_buffer_serializable_v<T> &&
                            detail::is_boost_user_buffer_serializable_v<T>)>> {
@@ -244,7 +244,7 @@ namespace ttg {
   /// do not support MADNESS or Boost serialization, and support Cereal serialization
   template <typename T>
   struct default_data_descriptor<
-      T, std::enable_if_t<(!std::is_trivially_copyable_v<T> && !detail::is_madness_buffer_serializable_v<T> &&
+      T, std::enable_if_t<(!detail::is_memcpyable_v<T> && !detail::is_madness_buffer_serializable_v<T> &&
                            !detail::is_boost_buffer_serializable_v<T> && detail::is_cereal_buffer_serializable_v<T>) ||
                           (!detail::is_madness_user_buffer_serializable_v<T> &&
                            !detail::is_boost_user_buffer_serializable_v<T> &&
diff --git a/ttg/ttg/serialization/traits.h b/ttg/ttg/serialization/traits.h
index 431b86098..bd6d8f825 100644
--- a/ttg/ttg/serialization/traits.h
+++ b/ttg/ttg/serialization/traits.h
@@ -185,6 +185,28 @@ namespace ttg::detail {
   template <typename T>
   inline constexpr bool is_user_buffer_serializable_v = is_user_buffer_serializable<T>::value;
 
+  /// \brief can be used to override the default value of is_memcpyable<T>::value
+
+  /// std::is_trivially_copyable_v<T> is sufficient to guarantee that T std::memcpy is safe to use on object of type T
+  /// however, sometimes is_trivially_copyable_v<T> reports false for objects that can be copied with memcpy ,
+  /// e.g., std::pair<int, int> (see https://danlark.org/2020/04/13/why-is-stdpair-broken/).
+  /// In this case specialize this trait
+  template <typename T>
+  inline constexpr bool is_memcpyable_override_v = std::is_trivially_copyable_v<T>;
+
+  // std::pair of trivially-copyable types is trivially copyable
+  template <typename T1, typename T2>
+  inline constexpr bool is_memcpyable_override_v<std::pair<T1,T2>> = std::is_trivially_copyable_v<T1> && std::is_trivially_copyable_v<T2>;
+
+  /// \brief reports whether objects of type T are safe to std::memcpy
+
+  /// True if either std::is_trivially_copyable_v<T> or is_memcpyable_override_v<T> are true
+  template <typename T>
+  struct is_memcpyable : std::bool_constant<std::is_trivially_copyable_v<T> || is_memcpyable_override_v<T>> {};
+
+  template <typename T>
+  inline constexpr bool is_memcpyable_v = is_memcpyable<T>::value;
+
 }  // namespace ttg::detail
 
 #endif  // TTG_SERIALIZATION_TRAITS_H

From a7ed933f1f538f7e1c641c4caa453586dd4e0e86 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Fri, 14 Jul 2023 09:34:51 -0400
Subject: [PATCH 099/259] moved the uses of pinned allocator from spmm to
 spmm_cuda

---
 examples/spmm/spmm.cc      | 5 ++---
 examples/spmm/spmm_cuda.cc | 3 ++-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/examples/spmm/spmm.cc b/examples/spmm/spmm.cc
index 1610477b7..ed051d32b 100644
--- a/examples/spmm/spmm.cc
+++ b/examples/spmm/spmm.cc
@@ -16,7 +16,6 @@
 #include <btas/btas.h>
 #include <btas/util/mohndle.h>
 #include <btas/optimize/contract.h>
-#include <TiledArray/cuda/allocators.h>
 #else
 #warning "found btas/features.h but Boost.Iterators is missing, hence BTAS is unusable ... add -I/path/to/boost"
 #endif
@@ -41,7 +40,7 @@ using namespace ttg;
 #include "ttg/util/bug.h"
 
 #if defined(BLOCK_SPARSE_GEMM) && defined(BTAS_IS_USABLE)
-using blk_t = btas::Tensor<double, btas::DEFAULT::range, btas::mohndle<btas::varray<double, TiledArray::cuda_pinned_allocator<double>>, btas::Handle::shared_ptr>>;
+using blk_t = btas::Tensor<double, btas::DEFAULT::range, btas::mohndle<btas::varray<double>, btas::Handle::shared_ptr>>;
 
 #if defined(TTG_USE_PARSEC)
 namespace ttg {
@@ -461,7 +460,7 @@ class SpMM25D {
                 {"c_ij", "c_ijk"}, ijk_keymap)
         , a_rowidx_to_colidx_(a_rowidx_to_colidx)
         , b_colidx_to_rowidx_(b_colidx_to_rowidx) {
-      this->set_priomap([=](const Key<3> &ijk) { return this->prio(ijk); });  // map a key to an integral priority value
+      this->set_priomap([=,this](const Key<3> &ijk) { return this->prio(ijk); });  // map a key to an integral priority value
 
       // for each {i,j} determine first k that contributes AND belongs to this node,
       // initialize input {i,j,first_k} flow to 0
diff --git a/examples/spmm/spmm_cuda.cc b/examples/spmm/spmm_cuda.cc
index bef438e33..e4c3710a5 100644
--- a/examples/spmm/spmm_cuda.cc
+++ b/examples/spmm/spmm_cuda.cc
@@ -17,6 +17,7 @@
 #include <btas/btas.h>
 #include <btas/optimize/contract.h>
 #include <btas/util/mohndle.h>
+#include <TiledArray/cuda/allocators.h>
 #else
 #warning "found btas/features.h but Boost.Iterators is missing, hence BTAS is unusable ... add -I/path/to/boost"
 #endif
@@ -248,7 +249,7 @@ struct DeviceTensor : public ttg::TTValue<DeviceTensor<_T, _Range, _Storage>>
 };
 
 using blk_t = DeviceTensor<double, btas::DEFAULT::range,
-                           btas::mohndle<btas::varray<double>,
+                           btas::mohndle<btas::varray<double, TiledArray::cuda_pinned_allocator<double>>,
                                          btas::Handle::shared_ptr>>;
 
 

From e6d94c9137bd46baa68c3eb36e8bc782310a390a Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Sun, 16 Jul 2023 17:40:31 -0400
Subject: [PATCH 100/259] bspm-cuda ready to try but need to teach MADNESS how
 to use already-initialized PaRSEC context

---
 CMakeLists.txt                                |  9 ------
 .../ExternalDependenciesVersions.cmake        |  2 +-
 examples/CMakeLists.txt                       | 31 ++++++++-----------
 examples/spmm/spmm_cuda.cc                    | 22 +++++--------
 ttg/ttg/device/cublas_helper.cpp              |  2 +-
 ttg/ttg/device/cublas_helper.h                |  5 ++-
 6 files changed, 27 insertions(+), 44 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7a619a0af..4665313d4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -103,7 +103,6 @@ if (TARGET CUDA::cudart)
   set(TTG_HAVE_CUDART True CACHE BOOL "True support for CUDART was found.")
 endif(TARGET CUDA::cudart)
 
-
 ##########################
 #### prerequisite runtimes
 ##########################
@@ -124,14 +123,6 @@ endif(TARGET MADworld)
 ##########################
 # N.B. discover last so that we do not affect core dependencies
 if (TTG_EXAMPLES)
-    # sparse tensor algorithms need Eigen
-    find_package(Eigen3)
-    message (STATUS "EIGEN3_FOUND=${EIGEN3_FOUND}")
-    if (EIGEN3_FOUND)
-        add_library(eigen3 INTERFACE IMPORTED)
-        set_property(TARGET eigen3 PROPERTY
-                INTERFACE_INCLUDE_DIRECTORIES ${EIGEN3_INCLUDE_DIR})
-    endif (EIGEN3_FOUND)
     # TiledArray brings in BTAS AND linear algebra (BLAS++/LAPACK++)
     include(FindOrFetchTiledArray)
     # OpenMP may also be used by some examples
diff --git a/cmake/modules/ExternalDependenciesVersions.cmake b/cmake/modules/ExternalDependenciesVersions.cmake
index 123a6f7bd..451dbc7b5 100644
--- a/cmake/modules/ExternalDependenciesVersions.cmake
+++ b/cmake/modules/ExternalDependenciesVersions.cmake
@@ -8,4 +8,4 @@ set(TTG_TRACKED_CEREAL_VERSION 1.3.0)
 set(TTG_TRACKED_MADNESS_TAG 31b2470ca722a6a2d84d4de08d32fb72ae8fdeda)
 set(TTG_TRACKED_PARSEC_TAG 9fc74b6f165605a133125d8a5b62cf55642c1907) 
 set(TTG_TRACKED_BTAS_TAG d73153ad9bc41a177e441ef04eceff7fab0c766d)
-set(TTG_TRACKED_TILEDARRAY_TAG a9947ce6e64c871bc6f3f3610bf2cb2d1c02cac1)
+set(TTG_TRACKED_TILEDARRAY_TAG b3a4e00e5dc9b1f857558339548e09e936d8a32a)
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 8750204a9..7d95198c5 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -4,25 +4,20 @@ add_ttg_executable(test test/test.cc)
 add_ttg_executable(t9 t9/t9.cc)
 add_ttg_executable(t9-streaming t9/t9_streaming.cc)
 
-# sparse matmul
-if (TARGET eigen3)
+# sparse matmul need Eigen ... it's always provided by TA
+if (TARGET tiledarray)
     # MADworld used for MADNESS serialization
-    add_ttg_executable(spmm spmm/spmm.cc LINK_LIBRARIES eigen3)
-    # block-sparse needs BTAS
-    if (TARGET BTAS::BTAS)
-        # since only need to use matrices, limit BTAS_TARGET_MAX_INDEX_RANK to 2
-        add_ttg_executable(bspmm spmm/spmm.cc LINK_LIBRARIES tiledarray eigen3 BTAS Boost::boost COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2)
-        if (TARGET CUDA::cublas)
-          if (NOT TARGET tiledarray)
-              message(FATAL_ERROR "TiledArray is not found; it is required for CUDA-based block-sparse SPMM")
-          endif()
-          add_ttg_executable(bspmm-cuda spmm/spmm_cuda.cc
-                             LINK_LIBRARIES tiledarray eigen3 BTAS Boost::boost CUDA::cublas
-                             COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2
-                             RUNTIMES "parsec")
-        endif (TARGET CUDA::cublas)
-    endif (TARGET BTAS::BTAS)
-endif(TARGET eigen3)
+    add_ttg_executable(spmm spmm/spmm.cc LINK_LIBRARIES TiledArray_Eigen)
+    # block-sparse needs BTAS ... it's always provided by TA
+    # since only need to use matrices, limit BTAS_TARGET_MAX_INDEX_RANK to 2
+    add_ttg_executable(bspmm spmm/spmm.cc LINK_LIBRARIES tiledarray TiledArray_Eigen BTAS Boost::boost COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2)
+    if (TARGET CUDA::cublas)
+      add_ttg_executable(bspmm-cuda spmm/spmm_cuda.cc
+                         LINK_LIBRARIES tiledarray TiledArray_Eigen BTAS Boost::boost CUDA::cublas
+                         COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2
+                         RUNTIMES "parsec")
+    endif (TARGET CUDA::cublas)
+endif()
 
 if (TARGET MADworld)
   add_ttg_executable(madness-1d madness/madness-1d/madness-1d.cc RUNTIMES "mad")
diff --git a/examples/spmm/spmm_cuda.cc b/examples/spmm/spmm_cuda.cc
index e4c3710a5..f9458ae66 100644
--- a/examples/spmm/spmm_cuda.cc
+++ b/examples/spmm/spmm_cuda.cc
@@ -18,6 +18,7 @@
 #include <btas/optimize/contract.h>
 #include <btas/util/mohndle.h>
 #include <TiledArray/cuda/allocators.h>
+#include <ttg/device/cublas_helper.h>
 #else
 #warning "found btas/features.h but Boost.Iterators is missing, hence BTAS is unusable ... add -I/path/to/boost"
 #endif
@@ -43,12 +44,6 @@ using namespace ttg;
 
 #include "ttg/serialization/std/pair.h"
 
-#include "cuda_gemm.h"
-
-#ifdef CUBLAS_V2_H_
-#define CUBLAS_V2_H_ 1
-#endif
-
 #if defined(BLOCK_SPARSE_GEMM) && defined(BTAS_IS_USABLE)
 
 template <typename _T, class _Range, class _Storage>
@@ -270,10 +265,10 @@ static void cuda_gemm(blk_t &C, const blk_t &A, const blk_t &B) {
   //assert(B.b.get_current_device() != 0);
   int device = C.b.get_current_device();
   assert(device != 0);
-  my_cublas_dgemm_because_cublas_is_stupid(
-              'N', 'N', C.extent(0), C.extent(1), A.extent(1), alpha,
+  cublasDgemm(ttg::detail::cublas_get_handle(),
+              CUBLAS_OP_N, CUBLAS_OP_N, C.extent(0), C.extent(1), A.extent(1), &alpha,
               A.b.device_ptr_on(device), A.extent(0),
-              B.b.device_ptr_on(device), B.extent(0), beta,
+              B.b.device_ptr_on(device), B.extent(0), &beta,
               C.b.current_device_ptr(), C.extent(0));
 }
 
@@ -1598,8 +1593,6 @@ int main(int argc, char **argv) {
   bool timing;
   double gflops;
 
-  my_cublas_init_because_cublas_is_stupid();
-
   // warm up silicon by calling gemm a few times
 #ifdef BTAS_IS_USABLE
   for (int i = 0; i < 20; i++) {
@@ -1624,6 +1617,9 @@ int main(int argc, char **argv) {
     initialize(1, argv, cores);
   }
 
+  // initialize MADNESS so that TA allocators can be created
+  madness::initialize(argc, argv, /* nthread = */ 0, /* quiet = */ true);
+
   std::string debugStr(getCmdOption(argv, argv + argc, "-d"));
   auto debug = (unsigned int)parseOption(debugStr, 0);
 
@@ -1864,10 +1860,8 @@ int main(int argc, char **argv) {
       }
     }
 
+    madness::finalize();
     ttg_finalize();
-
-    my_cublas_shutdown_because_cublas_is_stupid();
-
     return 0;
   }
 }
diff --git a/ttg/ttg/device/cublas_helper.cpp b/ttg/ttg/device/cublas_helper.cpp
index f86eda05d..1a1fe36fa 100644
--- a/ttg/ttg/device/cublas_helper.cpp
+++ b/ttg/ttg/device/cublas_helper.cpp
@@ -10,7 +10,7 @@ namespace ttg::detail {
 #ifdef TTG_HAVE_CUDART
 /// \brief Returns the cuBLAS handle to be used for launching cuBLAS kernels from the current thread
 /// \return the cuBLAS handle for the current thread
-inline const cublasHandle_t& cublas_get_handle() {
+const cublasHandle_t& cublas_get_handle() {
   static thread_local std::optional<cublasHandle_t> handle;
   if (!handle.has_value()) {
     auto status = cublasCreate_v2(&handle.emplace());
diff --git a/ttg/ttg/device/cublas_helper.h b/ttg/ttg/device/cublas_helper.h
index 5aae3df9b..6056cdb30 100644
--- a/ttg/ttg/device/cublas_helper.h
+++ b/ttg/ttg/device/cublas_helper.h
@@ -8,7 +8,10 @@
 
 namespace ttg::detail {
 
-/// \brief Returns the current CUDA stream used by cuBLAS
+/// \brief Returns the cuBLAS context used by this thread
+const cublasHandle_t& cublas_get_handle();
+
+/// associates the given CUDA stream with the cuBLAS context used by this thread
 void cublas_set_kernel_stream(cudaStream_t stream);
 
 } // namespace ttg::detail

From 5e219a2bab8e4f03ea5dced3ff1018df30f66d04 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Mon, 17 Jul 2023 08:13:56 -0400
Subject: [PATCH 101/259] Initialize TA/madness runtime with TTG's parsec
 context

---
 cmake/modules/ExternalDependenciesVersions.cmake | 2 +-
 examples/spmm/spmm_cuda.cc                       | 8 +++++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/cmake/modules/ExternalDependenciesVersions.cmake b/cmake/modules/ExternalDependenciesVersions.cmake
index 451dbc7b5..e97e5b8db 100644
--- a/cmake/modules/ExternalDependenciesVersions.cmake
+++ b/cmake/modules/ExternalDependenciesVersions.cmake
@@ -5,7 +5,7 @@
 set(TTG_TRACKED_BOOST_VERSION 1.66)
 set(TTG_TRACKED_CATCH2_VERSION 2.13.1)
 set(TTG_TRACKED_CEREAL_VERSION 1.3.0)
-set(TTG_TRACKED_MADNESS_TAG 31b2470ca722a6a2d84d4de08d32fb72ae8fdeda)
+set(TTG_TRACKED_MADNESS_TAG bbc9f5ad8cbd0726d9efde9f0b886bfac526848e)
 set(TTG_TRACKED_PARSEC_TAG 9fc74b6f165605a133125d8a5b62cf55642c1907) 
 set(TTG_TRACKED_BTAS_TAG d73153ad9bc41a177e441ef04eceff7fab0c766d)
 set(TTG_TRACKED_TILEDARRAY_TAG b3a4e00e5dc9b1f857558339548e09e936d8a32a)
diff --git a/examples/spmm/spmm_cuda.cc b/examples/spmm/spmm_cuda.cc
index f9458ae66..d505853ab 100644
--- a/examples/spmm/spmm_cuda.cc
+++ b/examples/spmm/spmm_cuda.cc
@@ -19,6 +19,7 @@
 #include <btas/util/mohndle.h>
 #include <TiledArray/cuda/allocators.h>
 #include <ttg/device/cublas_helper.h>
+#include <madness/world/parsec.h>  // need to initialize MADNESS purely for the purposes of TA allocators
 #else
 #warning "found btas/features.h but Boost.Iterators is missing, hence BTAS is unusable ... add -I/path/to/boost"
 #endif
@@ -1617,8 +1618,11 @@ int main(int argc, char **argv) {
     initialize(1, argv, cores);
   }
 
+#ifdef BTAS_IS_USABLE
   // initialize MADNESS so that TA allocators can be created
-  madness::initialize(argc, argv, /* nthread = */ 0, /* quiet = */ true);
+  madness::ParsecRuntime::initialize_with_existing_context(ttg::default_execution_context().impl().context());
+  madness::initialize(argc, argv, /* nthread = */ 1, /* quiet = */ true);
+#endif  // BTAS_IS_USABLE
 
   std::string debugStr(getCmdOption(argv, argv + argc, "-d"));
   auto debug = (unsigned int)parseOption(debugStr, 0);
@@ -1860,7 +1864,9 @@ int main(int argc, char **argv) {
       }
     }
 
+#ifdef BTAS_IS_USABLE
     madness::finalize();
+#endif  // BTAS_IS_USABLE
     ttg_finalize();
     return 0;
   }

From c24accaad1e402d3706a4b42d856c2486c07f91a Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Mon, 17 Jul 2023 08:34:48 -0400
Subject: [PATCH 102/259] switch therault/madness -> m-a-d-n-e-s-s/madness

---
 cmake/modules/FindOrFetchMADNESS.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/modules/FindOrFetchMADNESS.cmake b/cmake/modules/FindOrFetchMADNESS.cmake
index f112e4ff7..24f0bc798 100644
--- a/cmake/modules/FindOrFetchMADNESS.cmake
+++ b/cmake/modules/FindOrFetchMADNESS.cmake
@@ -10,7 +10,7 @@ if (NOT TARGET MADworld)
   set(MADNESS_TASK_BACKEND PaRSEC CACHE STRING "The task backend to use for MADNESS tasks")
   FetchContent_Declare(
           MADNESS
-          GIT_REPOSITORY https://github.com/therault/madness.git
+          GIT_REPOSITORY https://github.com/m-a-d-n-e-s-s/madness.git
           GIT_TAG ${TTG_TRACKED_MADNESS_TAG}
   )
   FetchContent_MakeAvailable(MADNESS)

From e527dfeb350c8fccbb441a30e7176f001ca7f352 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Mon, 17 Jul 2023 09:02:22 -0400
Subject: [PATCH 103/259] bump TA tag to actually use pinned allocator

---
 cmake/modules/ExternalDependenciesVersions.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/modules/ExternalDependenciesVersions.cmake b/cmake/modules/ExternalDependenciesVersions.cmake
index e97e5b8db..6e8fafd05 100644
--- a/cmake/modules/ExternalDependenciesVersions.cmake
+++ b/cmake/modules/ExternalDependenciesVersions.cmake
@@ -8,4 +8,4 @@ set(TTG_TRACKED_CEREAL_VERSION 1.3.0)
 set(TTG_TRACKED_MADNESS_TAG bbc9f5ad8cbd0726d9efde9f0b886bfac526848e)
 set(TTG_TRACKED_PARSEC_TAG 9fc74b6f165605a133125d8a5b62cf55642c1907) 
 set(TTG_TRACKED_BTAS_TAG d73153ad9bc41a177e441ef04eceff7fab0c766d)
-set(TTG_TRACKED_TILEDARRAY_TAG b3a4e00e5dc9b1f857558339548e09e936d8a32a)
+set(TTG_TRACKED_TILEDARRAY_TAG c445f4c04c93448803371b64f902d3ef8aa9e55a)

From de1332804e0fe3925fc6c63885b19e4c711bf9d7 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Mon, 17 Jul 2023 13:38:09 -0400
Subject: [PATCH 104/259] bump TA tag to be able to use gcc 13

---
 cmake/modules/ExternalDependenciesVersions.cmake | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmake/modules/ExternalDependenciesVersions.cmake b/cmake/modules/ExternalDependenciesVersions.cmake
index 6e8fafd05..0beddf284 100644
--- a/cmake/modules/ExternalDependenciesVersions.cmake
+++ b/cmake/modules/ExternalDependenciesVersions.cmake
@@ -5,7 +5,7 @@
 set(TTG_TRACKED_BOOST_VERSION 1.66)
 set(TTG_TRACKED_CATCH2_VERSION 2.13.1)
 set(TTG_TRACKED_CEREAL_VERSION 1.3.0)
-set(TTG_TRACKED_MADNESS_TAG bbc9f5ad8cbd0726d9efde9f0b886bfac526848e)
+set(TTG_TRACKED_MADNESS_TAG 31d803325623de75371774feffb0270c796bea24)
 set(TTG_TRACKED_PARSEC_TAG 9fc74b6f165605a133125d8a5b62cf55642c1907) 
 set(TTG_TRACKED_BTAS_TAG d73153ad9bc41a177e441ef04eceff7fab0c766d)
-set(TTG_TRACKED_TILEDARRAY_TAG c445f4c04c93448803371b64f902d3ef8aa9e55a)
+set(TTG_TRACKED_TILEDARRAY_TAG f6f5039b54ee5a1180106351cac05e0d33e14c62)

From c2cae26a4cb2515e4ad9099a929069b4f270ed32 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Mon, 17 Jul 2023 13:39:17 -0400
Subject: [PATCH 105/259] [cmake] TTG_ENABLE_CUDA to control whether PaRSEC
 etc. try to discover CUDA

---
 CMakeLists.txt                            | 21 ++++++++++++---------
 cmake/modules/FindOrFetchPARSEC.cmake     |  1 +
 cmake/modules/FindOrFetchTiledArray.cmake |  4 +---
 examples/CMakeLists.txt                   |  4 ++--
 ttg/CMakeLists.txt                        |  4 ++--
 5 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4665313d4..cd8e0c192 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -51,6 +51,7 @@ set(CMAKE_INSTALL_CMAKEDIR "lib/cmake/ttg"
 #### user-defined configuration options
 ########################################
 option(TTG_PARSEC_USE_BOOST_SERIALIZATION "Whether to select Boost serialization methods in PaRSEC backend" ON)
+option(TTG_ENABLE_CUDA "Whether to TTG will look for CUDA" OFF)
 option(TTG_EXAMPLES "Whether to build examples" OFF)
 
 option(TTG_FETCH_BOOST "Whether to fetch+build Boost, if missing" OFF)
@@ -91,17 +92,19 @@ find_package(CXXStdCoroutine MODULE REQUIRED COMPONENTS Final Experimental)
 ##########################
 #### CUDA: must come before PaRSEC
 ##########################
-include(CheckLanguage)
-check_language(CUDA)
-if(CMAKE_CUDA_COMPILER)
+if (TTG_ENABLE_CUDA)
+  include(CheckLanguage)
+  check_language(CUDA)
+  if(CMAKE_CUDA_COMPILER)
     enable_language(CUDA)
-endif(CMAKE_CUDA_COMPILER)
-set(TTG_HAVE_CUDA ${CMAKE_CUDA_COMPILER} CACHE BOOL "True if TTG supports compiling .cu files")
+  endif(CMAKE_CUDA_COMPILER)
+  set(TTG_HAVE_CUDA ${CMAKE_CUDA_COMPILER} CACHE BOOL "True if TTG supports compiling .cu files")
 
-find_package(CUDAToolkit)
-if (TARGET CUDA::cudart)
-  set(TTG_HAVE_CUDART True CACHE BOOL "True support for CUDART was found.")
-endif(TARGET CUDA::cudart)
+  find_package(CUDAToolkit)
+  if (TARGET CUDA::cudart)
+    set(TTG_HAVE_CUDART True CACHE BOOL "TTG supports execution on CUDA devices")
+  endif()
+endif()
 
 ##########################
 #### prerequisite runtimes
diff --git a/cmake/modules/FindOrFetchPARSEC.cmake b/cmake/modules/FindOrFetchPARSEC.cmake
index 82d6ac0d9..b3fd5faa3 100644
--- a/cmake/modules/FindOrFetchPARSEC.cmake
+++ b/cmake/modules/FindOrFetchPARSEC.cmake
@@ -13,6 +13,7 @@ if (NOT TARGET PaRSEC::parsec)
   set(CMAKE_SYSTEM_PROCESSOR ${CMAKE_HOST_SYSTEM_PROCESSOR})
   set(PARSEC_WITH_DEVEL_HEADERS ON CACHE BOOL "Install PaRSEC headers")
   set(BUILD_TOOLS ON CACHE BOOL "Do not build PaRSEC tools")
+  set(PARSEC_GPU_WITH_CUDA ${TTG_ENABLE_CUDA} CACHE BOOL "Enable CUDA support in PaRSEC runtime?")
 
   FetchContent_Declare(
           PARSEC
diff --git a/cmake/modules/FindOrFetchTiledArray.cmake b/cmake/modules/FindOrFetchTiledArray.cmake
index a8318425d..314ddff9f 100644
--- a/cmake/modules/FindOrFetchTiledArray.cmake
+++ b/cmake/modules/FindOrFetchTiledArray.cmake
@@ -17,9 +17,7 @@ if (TARGET tiledarray)
 else (TARGET tiledarray)
 
   # enable CUDA if TTG has it
-  if (TTG_HAVE_CUDA)
-    set(ENABLE_CUDA ON CACHE BOOL "Enable CUDA")
-  endif()
+  set(ENABLE_CUDA ${TTG_HAVE_CUDA} CACHE BOOL "Enable CUDA")
 
   # update CMake cache for TA
   if (DEFINED MADNESS_CMAKE_EXTRA_ARGS)
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 7d95198c5..6d2f1cbb6 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -11,12 +11,12 @@ if (TARGET tiledarray)
     # block-sparse needs BTAS ... it's always provided by TA
     # since only need to use matrices, limit BTAS_TARGET_MAX_INDEX_RANK to 2
     add_ttg_executable(bspmm spmm/spmm.cc LINK_LIBRARIES tiledarray TiledArray_Eigen BTAS Boost::boost COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2)
-    if (TARGET CUDA::cublas)
+    if (TTG_HAVE_CUDA)
       add_ttg_executable(bspmm-cuda spmm/spmm_cuda.cc
                          LINK_LIBRARIES tiledarray TiledArray_Eigen BTAS Boost::boost CUDA::cublas
                          COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2
                          RUNTIMES "parsec")
-    endif (TARGET CUDA::cublas)
+    endif()
 endif()
 
 if (TARGET MADworld)
diff --git a/ttg/CMakeLists.txt b/ttg/CMakeLists.txt
index 8accd57ac..7bdc0a4b6 100644
--- a/ttg/CMakeLists.txt
+++ b/ttg/CMakeLists.txt
@@ -115,12 +115,12 @@ if (TARGET std::coroutine)
           ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/coroutine.h
           )
 endif ()
-if (TARGET CUDA::cudart)
+if (TTG_HAVE_CUDA)
   list(APPEND ttg-sources
           ${CMAKE_CURRENT_SOURCE_DIR}/ttg/device/cublas_helper.cpp
           )
   list(APPEND ttg-deps CUDA::cudart CUDA::cublas)
-endif (TARGET CUDA::cudart)
+endif ()
 
 set(ttg-public-headers ${ttg-headers};${ttg-impl-headers};${ttg-base-headers};${ttg-util-headers};${ttg_device_headers})
 if (NOT TTG_IGNORE_BUNDLED_EXTERNALS)

From 31af8204bce16ccac90cb22067967397163c2327 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 28 Aug 2023 22:16:13 +0000
Subject: [PATCH 106/259] Add support for HIP

Most functionality is an extension/copy of the CUDA support. Also ported
the bspmm example to HIP.

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 CMakeLists.txt                   |  19 +++++-
 examples/CMakeLists.txt          |   9 ++-
 examples/spmm/spmm_cuda.cc       |  36 +++++++++--
 ttg/CMakeLists.txt               |  12 +++-
 ttg/ttg/config.in.h              |   2 +
 ttg/ttg/device/cublas_helper.cpp |  28 +++++++--
 ttg/ttg/device/cublas_helper.h   |  14 +++++
 ttg/ttg/execution.h              |   1 +
 ttg/ttg/parsec/task.h            |   6 +-
 ttg/ttg/parsec/ttg.h             | 100 ++++++++++++++++++++++++++-----
 10 files changed, 194 insertions(+), 33 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index cd8e0c192..aae70683d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -52,6 +52,7 @@ set(CMAKE_INSTALL_CMAKEDIR "lib/cmake/ttg"
 ########################################
 option(TTG_PARSEC_USE_BOOST_SERIALIZATION "Whether to select Boost serialization methods in PaRSEC backend" ON)
 option(TTG_ENABLE_CUDA "Whether to TTG will look for CUDA" OFF)
+option(TTG_ENABLE_HIP "Whether to TTG will look for HIP" OFF)
 option(TTG_EXAMPLES "Whether to build examples" OFF)
 
 option(TTG_FETCH_BOOST "Whether to fetch+build Boost, if missing" OFF)
@@ -67,6 +68,7 @@ if (FORCE_COLORED_OUTPUT)
     endif ()
 endif (FORCE_COLORED_OUTPUT)
 
+set(TTG_HIP_PLATFORM "__HIP_PLATFORM_AMD__" CACHE STRING "Which platform to use when compiling HIP-related code (default: __HIP_PLATFORM_AMD__)")
 ##########################
 #### prerequisites
 ##########################
@@ -104,7 +106,22 @@ if (TTG_ENABLE_CUDA)
   if (TARGET CUDA::cudart)
     set(TTG_HAVE_CUDART True CACHE BOOL "TTG supports execution on CUDA devices")
   endif()
-endif()
+endif(TTG_ENABLE_CUDA)
+
+if (TTG_ENABLE_HIP)
+  include(CheckLanguage)
+  check_language(HIP)
+  if(CMAKE_HIP_COMPILER)
+    enable_language(HIP)
+  endif(CMAKE_HIP_COMPILER)
+  set(TTG_HAVE_HIP ${CMAKE_HIP_COMPILER} CACHE BOOL "True if TTG supports compiling .hip files")
+
+  find_package(hipblas)
+  if (TARGET roc::hipblas)
+    set(TTG_HAVE_HIPBLAS True CACHE BOOL "TTG detected support for hipBLAS")
+  endif()
+  add_compile_definitions(${TTG_HIP_PLATFORM})
+endif(TTG_ENABLE_HIP)
 
 ##########################
 #### prerequisite runtimes
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 6d2f1cbb6..b2e398ddf 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -11,11 +11,16 @@ if (TARGET tiledarray)
     # block-sparse needs BTAS ... it's always provided by TA
     # since only need to use matrices, limit BTAS_TARGET_MAX_INDEX_RANK to 2
     add_ttg_executable(bspmm spmm/spmm.cc LINK_LIBRARIES tiledarray TiledArray_Eigen BTAS Boost::boost COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2)
-    if (TTG_HAVE_CUDA)
+    if (TARGET CUDA::cublas)
       add_ttg_executable(bspmm-cuda spmm/spmm_cuda.cc
                          LINK_LIBRARIES tiledarray TiledArray_Eigen BTAS Boost::boost CUDA::cublas
-                         COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2
+                         COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2;TTG_HAVE_CUDA=1
                          RUNTIMES "parsec")
+    elseif (TARGET roc::hipblas)
+      add_ttg_executable(bspmm-hip spmm/spmm_cuda.cc
+                        LINK_LIBRARIES tiledarray TiledArray_Eigen BTAS Boost::boost roc::hipblas
+                        COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2
+                        RUNTIMES "parsec")
     endif()
 endif()
 
diff --git a/examples/spmm/spmm_cuda.cc b/examples/spmm/spmm_cuda.cc
index d505853ab..d121ca258 100644
--- a/examples/spmm/spmm_cuda.cc
+++ b/examples/spmm/spmm_cuda.cc
@@ -10,6 +10,8 @@
 #include <utility>
 #include <vector>
 
+#include <madness/world/world.h>
+
 #if __has_include(<btas/features.h>)
 #pragma message("C Preprocessor got here!")
 #include <btas/features.h>
@@ -244,10 +246,15 @@ struct DeviceTensor : public ttg::TTValue<DeviceTensor<_T, _Range, _Storage>>
 
 };
 
+#if defined(TTG_HAVE_CUDA)
 using blk_t = DeviceTensor<double, btas::DEFAULT::range,
                            btas::mohndle<btas::varray<double, TiledArray::cuda_pinned_allocator<double>>,
                                          btas::Handle::shared_ptr>>;
-
+#else
+// TODO: no hip pinned allocator in TA?
+using blk_t = DeviceTensor<double, btas::DEFAULT::range,
+                           btas::mohndle<btas::varray<double>, btas::Handle::shared_ptr>>;
+#endif
 
 
 //inline blk_t operator*(const blk_t &A, const blk_t &B) {
@@ -257,7 +264,7 @@ using blk_t = DeviceTensor<double, btas::DEFAULT::range,
 //}
 
 /* TODO: call CUDA gemm here */
-static void cuda_gemm(blk_t &C, const blk_t &A, const blk_t &B) {
+static void device_gemm(blk_t &C, const blk_t &A, const blk_t &B) {
   double alpha = 1.0;
   double beta  = 1.0;
   // make sure all memory is on the device
@@ -266,11 +273,20 @@ static void cuda_gemm(blk_t &C, const blk_t &A, const blk_t &B) {
   //assert(B.b.get_current_device() != 0);
   int device = C.b.get_current_device();
   assert(device != 0);
+#if defined(TTG_HAVE_CUDA)
   cublasDgemm(ttg::detail::cublas_get_handle(),
               CUBLAS_OP_N, CUBLAS_OP_N, C.extent(0), C.extent(1), A.extent(1), &alpha,
               A.b.device_ptr_on(device), A.extent(0),
               B.b.device_ptr_on(device), B.extent(0), &beta,
               C.b.current_device_ptr(), C.extent(0));
+#elif defined(TTG_HAVE_HIPBLAS)
+  hipblasDgemm(ttg::detail::hipblas_get_handle(),
+                HIPBLAS_OP_N, HIPBLAS_OP_N,
+                C.extent(0), C.extent(1), A.extent(1), &alpha,
+                A.b.device_ptr_on(device), A.extent(0),
+                B.b.device_ptr_on(device), B.extent(0), &beta,
+                C.b.current_device_ptr(), C.extent(0));
+#endif
 }
 
 //using blk_t = btas::Tensor<double, btas::DEFAULT::range, btas::mohndle<btas::varray<double>, btas::Handle::shared_ptr>>;
@@ -687,7 +703,13 @@ class SpMM25D {
    public:
     using baseT = typename MultiplyAdd::ttT;
 
+#if defined(TTG_HAVE_CUDA)
     static constexpr bool have_cuda_op = true;
+#elif defined(TTG_HAVE_HIPBLAS)
+    static constexpr bool have_hip_op  = true;
+#endif
+
+    static_assert(have_hip_op);
 
     MultiplyAdd(Edge<Key<3>, Blk> &a_ijk, Edge<Key<3>, Blk> &b_ijk, Edge<Key<3>, Blk> &c_ijk, Edge<Key<2>, Blk> &c,
                 const std::vector<std::vector<long>> &a_rowidx_to_colidx,
@@ -747,7 +769,7 @@ class SpMM25D {
       co_await ttg::to_device(A.b, B.b, C.b);
 
       /* everything is on the device, call the gemm */
-      cuda_gemm(C, A, B);
+      device_gemm(C, A, B);
 
       /* compute next k while the kernel is running */
       std::tie(next_k, have_next_k) = compute_next_k(i, j, k, p);
@@ -1518,9 +1540,6 @@ static void timed_measurement(SpMatrix<> &A, SpMatrix<> &B, const std::function<
   gettimeofday(&start, nullptr);
   // ready, go! need only 1 kick, so must be done by 1 thread only
   if (ttg::default_execution_context().rank() == 0) control.start(P, Q);
-  static volatile int debug_signal = 0;
-  std::cout << "Waiting on debug signal (int*)" << &debug_signal << std::endl;
-  //while (!debug_signal) {}
   fence();
   gettimeofday(&end, nullptr);
   timersub(&end, &start, &diff);
@@ -1608,6 +1627,11 @@ int main(int argc, char **argv) {
   }
 #endif  // BTAS_IS_USABLE
 
+//  static volatile int debug_signal = 0;
+//  std::cout << "Waiting on debug signal (int*)" << &debug_signal << std::endl;
+//  while (!debug_signal) {}
+
+
   int cores = -1;
   std::string nbCoreStr(getCmdOption(argv, argv + argc, "-c"));
   cores = parseOption(nbCoreStr, cores);
diff --git a/ttg/CMakeLists.txt b/ttg/CMakeLists.txt
index 7bdc0a4b6..08614f7e1 100644
--- a/ttg/CMakeLists.txt
+++ b/ttg/CMakeLists.txt
@@ -120,7 +120,17 @@ if (TTG_HAVE_CUDA)
           ${CMAKE_CURRENT_SOURCE_DIR}/ttg/device/cublas_helper.cpp
           )
   list(APPEND ttg-deps CUDA::cudart CUDA::cublas)
-endif ()
+endif (TTG_HAVE_CUDA)
+
+if (TTG_HAVE_HIP)
+  # add the helper file if not added already
+  if (NOT TTG_HAVE_CUDA)
+    list(APPEND ttg-sources
+      ${CMAKE_CURRENT_SOURCE_DIR}/ttg/device/cublas_helper.cpp
+    )
+  endif (NOT TTG_HAVE_CUDA)
+  list(APPEND ttg-deps roc::hipblas)
+endif (TTG_HAVE_HIP)
 
 set(ttg-public-headers ${ttg-headers};${ttg-impl-headers};${ttg-base-headers};${ttg-util-headers};${ttg_device_headers})
 if (NOT TTG_IGNORE_BUNDLED_EXTERNALS)
diff --git a/ttg/ttg/config.in.h b/ttg/ttg/config.in.h
index 36b3adced..44628784d 100644
--- a/ttg/ttg/config.in.h
+++ b/ttg/ttg/config.in.h
@@ -15,4 +15,6 @@
 
 #cmakedefine TTG_HAVE_CUDART
 
+#cmakedefine TTG_HAVE_HIPBLAS
+
 #endif  // TTG_CONFIG_IN_H
diff --git a/ttg/ttg/device/cublas_helper.cpp b/ttg/ttg/device/cublas_helper.cpp
index 1a1fe36fa..06be13e95 100644
--- a/ttg/ttg/device/cublas_helper.cpp
+++ b/ttg/ttg/device/cublas_helper.cpp
@@ -20,17 +20,37 @@ const cublasHandle_t& cublas_get_handle() {
   }
   return *handle;
 }
-#endif // TTG_HAVE_CUDART
 
 void cublas_set_kernel_stream(cudaStream_t stream) {
-#ifdef TTG_HAVE_CUDART
     cublasStatus_t status = cublasSetStream_v2(cublas_get_handle(), stream);
     if (CUBLAS_STATUS_SUCCESS != status) {
         throw std::runtime_error("cublasSetStream_v2 failed");
     }
-#else
-    throw std::runtime_error("Support for cublas missing during installation!");
+}
 #endif // TTG_HAVE_CUDART
+
+#ifdef TTG_HAVE_HIPBLAS
+/// \brief Returns the rocBLAS handle to be used for launching rocBLAS kernels from the current thread
+/// \return the rocBLAS handle for the current thread
+const hipblasHandle_t& hipblas_get_handle() {
+  static thread_local std::optional<hipblasHandle_t> handle;
+  if (!handle.has_value()) {
+    hipblasStatus_t status = hipblasCreate(&handle.emplace());
+    if (HIPBLAS_STATUS_SUCCESS != status) {
+      throw std::runtime_error("hipblasCreate failed");
+    }
+  }
+  return *handle;
+}
+
+void hipblas_set_kernel_stream(hipStream_t stream) {
+    hipblasStatus_t status = hipblasSetStream(hipblas_get_handle(), stream);
+    if (HIPBLAS_STATUS_SUCCESS != status) {
+        throw std::runtime_error("hipblasSetStream failed");
+    }
 }
 
+#endif // TTG_HAVE_HIPBLAS
+
+
 } // namespace
\ No newline at end of file
diff --git a/ttg/ttg/device/cublas_helper.h b/ttg/ttg/device/cublas_helper.h
index 6056cdb30..6ab106bfb 100644
--- a/ttg/ttg/device/cublas_helper.h
+++ b/ttg/ttg/device/cublas_helper.h
@@ -18,4 +18,18 @@ void cublas_set_kernel_stream(cudaStream_t stream);
 #endif // TTG_HAVE_CUDART
 
 
+#ifdef TTG_HAVE_HIPBLAS
+#include <hipblas/hipblas.h>
+
+namespace ttg::detail {
+
+const hipblasHandle_t& hipblas_get_handle();
+
+/// associates the given HIP stream with the hipblas context used by this thread
+void hipblas_set_kernel_stream(hipStream_t stream);
+
+} // namespace ttg::detail
+#endif // TTG_HAVE_HIPBLAS
+
+
 #endif // TTG_DEVICE_CUBLAS_HELPER_H
\ No newline at end of file
diff --git a/ttg/ttg/execution.h b/ttg/ttg/execution.h
index 29775a24b..cbdd65ff1 100644
--- a/ttg/ttg/execution.h
+++ b/ttg/ttg/execution.h
@@ -17,6 +17,7 @@ enum class Execution {
 enum class ExecutionSpace {
   Host,   // a CPU
   CUDA,   // an NVIDIA CUDA device
+  HIP,    // an AMD HIP device
   Invalid
 };
 
diff --git a/ttg/ttg/parsec/task.h b/ttg/ttg/parsec/task.h
index 09d084c08..104642f08 100644
--- a/ttg/ttg/parsec/task.h
+++ b/ttg/ttg/parsec/task.h
@@ -180,7 +180,7 @@ namespace ttg_parsec {
       using key_type = typename TT::key_type;
       static constexpr size_t num_streams = TT::numins;
       /* device tasks may have to store more copies than it's inputs as their sends are aggregated */
-      static constexpr size_t num_copies  = TT::derived_has_cuda_op() ? static_cast<size_t>(MAX_PARAM_COUNT)
+      static constexpr size_t num_copies  = TT::derived_has_device_op() ? static_cast<size_t>(MAX_PARAM_COUNT)
                                                                       : (num_streams+1);
       TT* tt;
       key_type key;
@@ -189,7 +189,7 @@ namespace ttg_parsec {
       void* suspended_task_address = nullptr;  // if not null the function is suspended
 #endif
       ttg_data_copy_t *copies[num_copies] = { nullptr };  // the data copies tracked by this task
-      device_state_t<TT::derived_has_cuda_op()> dev_state;
+      device_state_t<TT::derived_has_device_op()> dev_state;
 
       parsec_ttg_task_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class)
           : parsec_ttg_task_base_t(mempool, task_class, num_streams, copies, dev_state.dev_ptr()) {
@@ -245,7 +245,7 @@ namespace ttg_parsec {
 #endif
       ttg_data_copy_t *copies[num_streams+1] = { nullptr };  // the data copies tracked by this task
                                                              // +1 for the copy needed during send/bcast
-      device_state_t<TT::derived_has_cuda_op()> dev_state;
+      device_state_t<TT::derived_has_device_op()> dev_state;
 
       parsec_ttg_task_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class)
           : parsec_ttg_task_base_t(mempool, task_class, num_streams, copies, dev_state.dev_ptr()) {
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 59575f13b..4c70ee713 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -79,8 +79,14 @@
 #include <parsec/parsec_internal.h>
 #include <parsec/scheduling.h>
 #include <parsec/remote_dep.h>
-/* TODO: once we use parsec master we need to switch this include */
+
+#ifdef PARSEC_HAVE_DEV_CUDA_SUPPORT
 #include <parsec/mca/device/cuda/device_cuda.h>
+#endif // PARSEC_HAVE_DEV_CUDA_SUPPORT
+#ifdef PARSEC_HAVE_DEV_HIP_SUPPORT
+#include <parsec/mca/device/hip/device_hip.h>
+#endif // PARSEC_HAVE_DEV_HIP_SUPPORT
+
 #include <parsec/mca/device/device_gpu.h>
 #if defined(PARSEC_PROF_TRACE)
 #include <parsec/profiling.h>
@@ -706,7 +712,17 @@ namespace ttg_parsec {
         parsec_ttg_task_t<TT> *me = (parsec_ttg_task_t<TT> *)parsec_task;
         return me->template invoke_op<ttg::ExecutionSpace::CUDA>();
       } else {
-        assert(TT::derived_has_cuda_op());
+        throw std::runtime_error("PaRSEC CUDA hook invoked on a TT that does not support CUDA operations!");
+      }
+    }
+
+    template<typename TT>
+    inline parsec_hook_return_t hook_hip(struct parsec_execution_stream_s *es, parsec_task_t *parsec_task) {
+      if constexpr(TT::derived_has_hip_op()) {
+        parsec_ttg_task_t<TT> *me = (parsec_ttg_task_t<TT> *)parsec_task;
+        return me->template invoke_op<ttg::ExecutionSpace::HIP>();
+      } else {
+        throw std::runtime_error("PaRSEC HIP hook invoked on a TT that does not support HIP operations!");
       }
     }
 
@@ -1042,6 +1058,9 @@ namespace ttg_parsec {
     template <typename T>
     using have_cuda_op_non_type_t = decltype(T::have_cuda_op);
 
+    template <typename T>
+    using have_hip_op_non_type_t = decltype(T::have_hip_op);
+
     bool alive = true;
 
     static constexpr int numinedges = std::tuple_size_v<input_tuple_type>;     // number of input edges
@@ -1059,6 +1078,20 @@ namespace ttg_parsec {
       }
     }
 
+    /// @return true if derivedT::have_hip_op exists and is defined to true
+    static constexpr bool derived_has_hip_op() {
+      if constexpr (ttg::meta::is_detected_v<have_hip_op_non_type_t, derivedT>) {
+        return derivedT::have_hip_op;
+      } else {
+        return false;
+      }
+    }
+
+    /// @return true if the TT supports device execution
+    static constexpr bool derived_has_device_op() {
+      return (derived_has_cuda_op() || derived_has_hip_op());
+    }
+
     using ttT = TT;
     using key_type = keyT;
     using input_terminals_type = ttg::detail::input_terminals_tuple_t<keyT, input_tuple_type>;
@@ -1266,6 +1299,11 @@ namespace ttg_parsec {
       ttg::detail::cublas_set_kernel_stream(cuda_stream->cuda_stream);
 #endif // PARSEC_HAVE_DEV_CUDA_SUPPORT
 
+#if defined(PARSEC_HAVE_DEV_HIP_SUPPORT)
+      parsec_hip_exec_stream_t *hip_stream = (parsec_hip_exec_stream_t *)gpu_stream;
+      ttg::detail::hipblas_set_kernel_stream(hip_stream->hip_stream);
+#endif // PARSEC_HAVE_DEV_HIP_SUPPORT
+
       /* Here we call back into the coroutine again after the transfers have completed */
       static_op<Space>(&task->parsec_task);
 
@@ -1295,8 +1333,8 @@ namespace ttg_parsec {
       return rc;
     }
 
-    static int
-    static_cuda_stage_in(parsec_gpu_task_t        *gtask,
+    static void
+    static_device_stage_in(parsec_gpu_task_t        *gtask,
                          uint32_t                  flow_mask,
                          parsec_gpu_exec_stream_t *gpu_stream) {
       /* register any memory that hasn't been registered yet */
@@ -1308,19 +1346,36 @@ namespace ttg_parsec {
 #if defined(PARSEC_HAVE_DEV_CUDA_SUPPORT)
             // register host memory for faster device access
             cudaError_t status;
-            status = cudaHostRegister(copy->device_private, gtask->flow_nb_elts[i], cudaHostRegisterPortable);
-            assert(cudaSuccess == status);
+            //status = cudaHostRegister(copy->device_private, gtask->flow_nb_elts[i], cudaHostRegisterPortable);
+            //assert(cudaSuccess == status);
 #endif // PARSEC_HAVE_DEV_CUDA_SUPPORT
-            copy->flags |= TTG_PARSEC_DATA_FLAG_REGISTERED;
+            //copy->flags |= TTG_PARSEC_DATA_FLAG_REGISTERED;
           }
         }
       }
+    }
+#if defined(PARSEC_HAVE_DEV_CUDA_SUPPORT)
+    static int
+    static_cuda_stage_in(parsec_gpu_task_t        *gtask,
+                         uint32_t                  flow_mask,
+                         parsec_gpu_exec_stream_t *gpu_stream) {
+      static_device_stage_in(gtask, flow_mask, gpu_stream);
       return parsec_default_cuda_stage_in(gtask, flow_mask, gpu_stream);
     }
+#endif // PARSEC_HAVE_DEV_CUDA_SUPPORT
+#if defined(PARSEC_HAVE_DEV_HIP_SUPPORT)
+    static int
+    static_hip_stage_in(parsec_gpu_task_t        *gtask,
+                         uint32_t                  flow_mask,
+                         parsec_gpu_exec_stream_t *gpu_stream) {
+      static_device_stage_in(gtask, flow_mask, gpu_stream);
+      return parsec_default_hip_stage_in(gtask, flow_mask, gpu_stream);
+    }
+#endif
 
     template <ttg::ExecutionSpace Space>
     static parsec_hook_return_t device_static_op(parsec_task_t* parsec_task) {
-      static_assert(derived_has_cuda_op());
+      static_assert(derived_has_device_op());
 
       int dev_index;
       double ratio = 1.0;
@@ -1362,8 +1417,8 @@ namespace ttg_parsec {
       /* TODO: is this the right place to set the mask? */
       task->parsec_task.chore_mask = PARSEC_DEV_ALL;
       /* get a device and come back if we need another one */
-      double task_load = 1.;
-      dev_index = parsec_get_best_device(parsec_task, task_load);
+      int64_t task_load = 1;
+      dev_index = parsec_get_best_device(parsec_task, &task_load);
       assert(dev_index >= 0);
       if (dev_index < 2) {
           return PARSEC_HOOK_RETURN_NEXT; /* Fall back */
@@ -1384,10 +1439,19 @@ namespace ttg_parsec {
           }
           break;
 #endif
+#if defined(PARSEC_HAVE_DEV_HIP_SUPPORT)
+        case PARSEC_DEV_HIP:
+          if constexpr (Space == ttg::ExecutionSpace::HIP) {
+            gpu_task->stage_in  = static_hip_stage_in;
+            gpu_task->stage_out = parsec_default_hip_stage_out;
+            return parsec_hip_kernel_scheduler(es, gpu_task, dev_index);
+          }
+          break;
+#endif // PARSEC_HAVE_DEV_HIP_SUPPORT
         default:
           break;
       }
-      ttg::print_error(task->tt->get_name(), " : received mismatching device type ", device->type, " from PaRSEC");
+      ttg::print_error(task->tt->get_name(), " : received mismatching device type ", (int)device->type, " from PaRSEC");
       ttg::abort();
       return PARSEC_HOOK_RETURN_DONE; // will not be reacehed
     }
@@ -3042,7 +3106,7 @@ namespace ttg_parsec {
         }
       }
 
-      if constexpr (!derived_has_cuda_op()) {
+      if constexpr (!derived_has_device_op()) {
         need_pushout = true;
       }
 
@@ -3409,11 +3473,15 @@ namespace ttg_parsec {
         ((__parsec_chore_t *)self.incarnations)[0].type = PARSEC_DEV_CUDA;
         ((__parsec_chore_t *)self.incarnations)[0].evaluate = NULL;
         ((__parsec_chore_t *)self.incarnations)[0].hook = &detail::hook_cuda<TT>;
-#if 0
-        ((__parsec_chore_t *)self.incarnations)[1].type = PARSEC_DEV_CPU;
+        ((__parsec_chore_t *)self.incarnations)[1].type = PARSEC_DEV_NONE;
         ((__parsec_chore_t *)self.incarnations)[1].evaluate = NULL;
-        ((__parsec_chore_t *)self.incarnations)[1].hook = &detail::hook<TT>;
-#endif // 0
+        ((__parsec_chore_t *)self.incarnations)[1].hook = NULL;
+      } else if (derived_has_hip_op()) {
+        self.incarnations = (__parsec_chore_t *)malloc(3 * sizeof(__parsec_chore_t));
+        ((__parsec_chore_t *)self.incarnations)[0].type = PARSEC_DEV_HIP;
+        ((__parsec_chore_t *)self.incarnations)[0].evaluate = NULL;
+        ((__parsec_chore_t *)self.incarnations)[0].hook = &detail::hook_hip<TT>;
+
         ((__parsec_chore_t *)self.incarnations)[1].type = PARSEC_DEV_NONE;
         ((__parsec_chore_t *)self.incarnations)[1].evaluate = NULL;
         ((__parsec_chore_t *)self.incarnations)[1].hook = NULL;

From 9b817f35933a177f42d0f8265f6a04de66ba5af3 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 28 Aug 2023 22:11:41 +0000
Subject: [PATCH 107/259] Bump PaRSEC version to current master

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 cmake/modules/ExternalDependenciesVersions.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/modules/ExternalDependenciesVersions.cmake b/cmake/modules/ExternalDependenciesVersions.cmake
index 0beddf284..4afbf4e0e 100644
--- a/cmake/modules/ExternalDependenciesVersions.cmake
+++ b/cmake/modules/ExternalDependenciesVersions.cmake
@@ -6,6 +6,6 @@ set(TTG_TRACKED_BOOST_VERSION 1.66)
 set(TTG_TRACKED_CATCH2_VERSION 2.13.1)
 set(TTG_TRACKED_CEREAL_VERSION 1.3.0)
 set(TTG_TRACKED_MADNESS_TAG 31d803325623de75371774feffb0270c796bea24)
-set(TTG_TRACKED_PARSEC_TAG 9fc74b6f165605a133125d8a5b62cf55642c1907) 
+set(TTG_TRACKED_PARSEC_TAG e59bed9d4934775792a58980cf5500bd41381bc4)
 set(TTG_TRACKED_BTAS_TAG d73153ad9bc41a177e441ef04eceff7fab0c766d)
 set(TTG_TRACKED_TILEDARRAY_TAG f6f5039b54ee5a1180106351cac05e0d33e14c62)

From 30d7ee00d3a1c485026f5baf3d2611b94e016b81 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 28 Aug 2023 22:24:27 +0000
Subject: [PATCH 108/259] SPMM: move tiles in the non-cuda version

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/spmm/spmm.cc | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/examples/spmm/spmm.cc b/examples/spmm/spmm.cc
index ed051d32b..c0e6dd881 100644
--- a/examples/spmm/spmm.cc
+++ b/examples/spmm/spmm.cc
@@ -36,6 +36,7 @@ using namespace ttg;
 #include "ttg/util/future.h"
 
 #include "ttg/util/multiindex.h"
+#include "ttg/serialization/std/pair.h"
 
 #include "ttg/util/bug.h"
 
@@ -327,7 +328,7 @@ class SpMM25D {
           ijk_keys.emplace_back(Key<3>({i, j, k}));
         }
       }
-      ::broadcast<0>(ijk_keys, baseT::template get<0>(a_ik), a_ijk);
+      ::broadcast<0>(ijk_keys, std::move(baseT::template get<0>(a_ik)), a_ijk);
     }
 
    private:
@@ -364,7 +365,7 @@ class SpMM25D {
           procmap[p] = true;
         }
       }
-      ::broadcast<0>(ikp_keys, baseT::template get<0>(a_ik), a_ikp);
+      ::broadcast<0>(ikp_keys, std::move(baseT::template get<0>(a_ik)), a_ikp);
     }
 
    private:
@@ -401,7 +402,7 @@ class SpMM25D {
           ijk_keys.emplace_back(Key<3>({i, j, k}));
         }
       }
-      ::broadcast<0>(ijk_keys, baseT::template get<0>(b_kj), b_ijk);
+      ::broadcast<0>(ijk_keys, std::move(baseT::template get<0>(b_kj)), b_ijk);
     }
 
    private:
@@ -437,7 +438,7 @@ class SpMM25D {
           procmap[p] = true;
         }
       }
-      ::broadcast<0>(kjp_keys, baseT::template get<0>(b_kj), b_kjp);
+      ::broadcast<0>(kjp_keys, std::move(baseT::template get<0>(b_kj)), b_kjp);
     }
 
    private:
@@ -645,7 +646,7 @@ class SpMM25D {
 
     void op(const Key<2> &ij, typename baseT::input_values_tuple_type &&c_ij_p, std::tuple<Out<Key<2>, Blk>> &c_ij) {
       ttg::trace("ReduceC(", ij[0], ", ", ij[1], ")");
-      ::send<0>(ij, baseT::template get<0>(c_ij_p), c_ij);
+      ::send<0>(ij, std::move(baseT::template get<0>(c_ij_p)), c_ij);
     }
   };  // class ReduceC
 

From 81cca941c3dfe6d29bb77030b5b2efb3e3768d72 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 30 Aug 2023 22:35:02 +0000
Subject: [PATCH 109/259] Fixes to make Clang happy

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/spmm/spmm_cuda.cc |  8 ++++----
 ttg/CMakeLists.txt         |  4 ++--
 ttg/ttg/device/task.h      |  4 ++--
 ttg/ttg/parsec/ptr.h       |  6 ++++--
 ttg/ttg/parsec/task.h      | 26 +++++++++++++-------------
 ttg/ttg/parsec/ttg.h       | 20 ++++++++++++++------
 6 files changed, 39 insertions(+), 29 deletions(-)

diff --git a/examples/spmm/spmm_cuda.cc b/examples/spmm/spmm_cuda.cc
index d121ca258..f5728231e 100644
--- a/examples/spmm/spmm_cuda.cc
+++ b/examples/spmm/spmm_cuda.cc
@@ -56,10 +56,10 @@ struct DeviceTensor : public ttg::TTValue<DeviceTensor<_T, _Range, _Storage>>
   using ttvalue_type = ttg::TTValue<DeviceTensor<_T, _Range, _Storage>>;
   ttg::buffer<_T> b; // does not own the host buffer
 
-  using value_type = tensor_type::value_type;
-  using size_type = tensor_type::size_type;
-  using storage_type = tensor_type::storage_type;
-  using range_type = tensor_type::range_type;
+  using value_type = typename tensor_type::value_type;
+  using size_type = typename tensor_type::size_type;
+  using storage_type = typename tensor_type::storage_type;
+  using range_type = typename tensor_type::range_type;
 
 
    public:
diff --git a/ttg/CMakeLists.txt b/ttg/CMakeLists.txt
index 08614f7e1..bf1f71b74 100644
--- a/ttg/CMakeLists.txt
+++ b/ttg/CMakeLists.txt
@@ -122,7 +122,7 @@ if (TTG_HAVE_CUDA)
   list(APPEND ttg-deps CUDA::cudart CUDA::cublas)
 endif (TTG_HAVE_CUDA)
 
-if (TTG_HAVE_HIP)
+if (TTG_HAVE_HIPBLAS)
   # add the helper file if not added already
   if (NOT TTG_HAVE_CUDA)
     list(APPEND ttg-sources
@@ -130,7 +130,7 @@ if (TTG_HAVE_HIP)
     )
   endif (NOT TTG_HAVE_CUDA)
   list(APPEND ttg-deps roc::hipblas)
-endif (TTG_HAVE_HIP)
+endif (TTG_HAVE_HIPBLAS)
 
 set(ttg-public-headers ${ttg-headers};${ttg-impl-headers};${ttg-base-headers};${ttg-util-headers};${ttg_device_headers})
 if (NOT TTG_IGNORE_BUNDLED_EXTERNALS)
diff --git a/ttg/ttg/device/task.h b/ttg/ttg/device/task.h
index b811c4484..da7067194 100644
--- a/ttg/ttg/device/task.h
+++ b/ttg/ttg/device/task.h
@@ -29,14 +29,14 @@ namespace ttg {
   namespace detail {
     template<typename... Ts>
     struct to_device_t {
-      std::tuple<Ts&...> ties;
+      std::tuple<std::add_lvalue_reference_t<Ts>...> ties;
     };
   } // namespace detail
 
   template<typename... Args>
   [[nodiscard]]
   inline auto to_device(Args&&... args) {
-    return detail::to_device_t{std::tie(std::forward<Args>(args)...)};
+    return detail::to_device_t<std::remove_reference_t<Args>...>{std::tie(std::forward<Args>(args)...)};
   }
 
   namespace detail {
diff --git a/ttg/ttg/parsec/ptr.h b/ttg/ttg/parsec/ptr.h
index 6499e050b..6ab48345d 100644
--- a/ttg/ttg/parsec/ptr.h
+++ b/ttg/ttg/parsec/ptr.h
@@ -126,11 +126,13 @@ namespace ttg_parsec {
     ttg_parsec::detail::ttg_data_copy_t* get_copy(ttg_parsec::ptr<T>& p);
   } // namespace detail
 
+  // fwd decl
   template<typename T, typename... Args>
-  ptr<T> ttg_parsec::make_ptr(Args&&... args);
+  ptr<T> make_ptr(Args&&... args);
 
+  // fwd decl
   template<typename T>
-  ptr<std::decay_t<T>> ttg_parsec::get_ptr(T&& obj);
+  ptr<std::decay_t<T>> get_ptr(T&& obj);
 
   template<typename T>
   struct ptr {
diff --git a/ttg/ttg/parsec/task.h b/ttg/ttg/parsec/task.h
index 104642f08..30ce6edc8 100644
--- a/ttg/ttg/parsec/task.h
+++ b/ttg/ttg/parsec/task.h
@@ -139,11 +139,10 @@ namespace ttg_parsec {
        */
 
       parsec_ttg_task_base_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class,
-                             int data_count, ttg_data_copy_t **copies, device_ptr_t *dev_ptr,
+                             int data_count, ttg_data_copy_t **copies,
                              bool defer_writer = TTG_PARSEC_DEFER_WRITER)
           : data_count(data_count)
           , copies(copies)
-          , dev_ptr(dev_ptr)
           , defer_writer(defer_writer) {
         PARSEC_LIST_ITEM_SINGLETON(&parsec_task.super);
         parsec_task.mempool_owner = mempool;
@@ -153,13 +152,12 @@ namespace ttg_parsec {
 
       parsec_ttg_task_base_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class,
                              parsec_taskpool_t *taskpool, int32_t priority,
-                             int data_count, ttg_data_copy_t **copies, device_ptr_t *dev_ptr,
+                             int data_count, ttg_data_copy_t **copies,
                              release_task_fn *release_fn,
                              bool defer_writer = TTG_PARSEC_DEFER_WRITER)
           : data_count(data_count)
           , copies(copies)
           , release_task_cb(release_fn)
-          , dev_ptr(dev_ptr)
           , defer_writer(defer_writer) {
         PARSEC_LIST_ITEM_SINGLETON(&parsec_task.super);
         parsec_task.mempool_owner = mempool;
@@ -182,19 +180,19 @@ namespace ttg_parsec {
       /* device tasks may have to store more copies than it's inputs as their sends are aggregated */
       static constexpr size_t num_copies  = TT::derived_has_device_op() ? static_cast<size_t>(MAX_PARAM_COUNT)
                                                                       : (num_streams+1);
-      TT* tt;
+      TT* tt = nullptr;
       key_type key;
       std::array<stream_info_t, num_streams> streams;
 #ifdef TTG_HAS_COROUTINE
       void* suspended_task_address = nullptr;  // if not null the function is suspended
 #endif
-      ttg_data_copy_t *copies[num_copies] = { nullptr };  // the data copies tracked by this task
       device_state_t<TT::derived_has_device_op()> dev_state;
+      ttg_data_copy_t *copies[num_copies] = { nullptr };  // the data copies tracked by this task
 
       parsec_ttg_task_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class)
-          : parsec_ttg_task_base_t(mempool, task_class, num_streams, copies, dev_state.dev_ptr()) {
+          : parsec_ttg_task_base_t(mempool, task_class, num_streams, copies) {
         tt_ht_item.key = pkey();
-
+        this->dev_ptr = this->dev_state.dev_ptr();
         // We store the hash of the key and the address where it can be found in locals considered as a scratchpad
         *(uintptr_t*)&(parsec_task.locals[0]) = 0; //there is no key
         *(uintptr_t*)&(parsec_task.locals[2]) = 0; //there is no key
@@ -204,10 +202,11 @@ namespace ttg_parsec {
                         parsec_task_class_t *task_class, parsec_taskpool_t *taskpool,
                         TT *tt_ptr, int32_t priority)
           : parsec_ttg_task_base_t(mempool, task_class, taskpool, priority,
-                                   num_streams, copies, dev_state.dev_ptr(),
+                                   num_streams, copies,
                                    &release_task, tt_ptr->m_defer_writer)
           , tt(tt_ptr), key(key) {
         tt_ht_item.key = pkey();
+        this->dev_ptr = this->dev_state.dev_ptr();
 
         // We store the hash of the key and the address where it can be found in locals considered as a scratchpad
         uint64_t hv = ttg::hash<std::decay_t<decltype(key)>>{}(key);
@@ -243,23 +242,24 @@ namespace ttg_parsec {
 #ifdef TTG_HAS_COROUTINE
       void* suspended_task_address = nullptr;  // if not null the function is suspended
 #endif
+      device_state_t<TT::derived_has_device_op()> dev_state;
       ttg_data_copy_t *copies[num_streams+1] = { nullptr };  // the data copies tracked by this task
                                                              // +1 for the copy needed during send/bcast
-      device_state_t<TT::derived_has_device_op()> dev_state;
 
       parsec_ttg_task_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class)
-          : parsec_ttg_task_base_t(mempool, task_class, num_streams, copies, dev_state.dev_ptr()) {
+          : parsec_ttg_task_base_t(mempool, task_class, num_streams, copies) {
         tt_ht_item.key = pkey();
+        this->dev_ptr = this->dev_state.dev_ptr();
       }
 
       parsec_ttg_task_t(parsec_thread_mempool_t *mempool, parsec_task_class_t *task_class,
                         parsec_taskpool_t *taskpool, TT *tt_ptr, int32_t priority)
           : parsec_ttg_task_base_t(mempool, task_class, taskpool, priority,
-                                   num_streams, copies, dev_state.dev_ptr(),
+                                   num_streams, copies,
                                    &release_task, tt_ptr->m_defer_writer)
           , tt(tt_ptr) {
         tt_ht_item.key = pkey();
-
+        this->dev_ptr = this->dev_state.dev_ptr();
         init_stream_info(tt, streams);
       }
 
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 4c70ee713..407f63764 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -129,18 +129,21 @@ namespace ttg_parsec {
 
   struct msg_header_t {
     typedef enum fn_id : std::int8_t {
+      MSG_INVALID = -1,
       MSG_SET_ARG = 0,
       MSG_SET_ARGSTREAM_SIZE = 1,
       MSG_FINALIZE_ARGSTREAM_SIZE = 2,
       MSG_GET_FROM_PULL = 3 } fn_id_t;
-    uint32_t taskpool_id;
-    uint64_t op_id;
+    uint32_t taskpool_id = -1;
+    uint64_t op_id = -1;
     std::size_t key_offset = 0;
-    fn_id_t fn_id;
+    fn_id_t fn_id = MSG_INVALID;
     std::int8_t num_iovecs = 0;
-    int32_t param_id;
+    int32_t param_id = -1;
     int num_keys = 0;
-    int sender;
+    int sender = -1;
+
+    msg_header_t() = default;
 
     msg_header_t(fn_id_t fid, uint32_t tid, uint64_t oid, int32_t pid, int sender, int nk)
     : fn_id(fid)
@@ -2147,8 +2150,10 @@ namespace ttg_parsec {
       bool remove_from_hash = true;
       bool discover_task = true;
       bool get_pull_data = false;
+      bool has_lock = false;
       /* If we have only one input and no reducer on that input we can skip the hash table */
       if (numins > 1 || reducer) {
+        has_lock = true;
         parsec_hash_table_lock_bucket(&tasks_table, hk);
         if (nullptr == (task = (task_t *)parsec_hash_table_nolock_find(&tasks_table, hk))) {
           task = create_new_task(key);
@@ -2169,6 +2174,7 @@ namespace ttg_parsec {
         /* if we have a reducer, we need to hold on to the lock for just a little longer */
         if (!reducer) {
           parsec_hash_table_unlock_bucket(&tasks_table, hk);
+          has_lock = false;
         }
       } else {
         task = create_new_task(key);
@@ -2280,7 +2286,9 @@ namespace ttg_parsec {
         //parsec_hash_table_unlock_bucket(&tasks_table, hk);
       } else {
         /* unlock the bucket, the lock is not needed anymore */
-        parsec_hash_table_unlock_bucket(&tasks_table, hk);
+        if (has_lock) {
+          parsec_hash_table_unlock_bucket(&tasks_table, hk);
+        }
         /* whether the task needs to be deferred or not */
         if constexpr (!valueT_is_Void) {
           if (nullptr != task->copies[i]) {

From 975e878487a846994f213b4753976fbce341811f Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 6 Sep 2023 16:55:43 +0000
Subject: [PATCH 110/259] Try to send from accelerator device directly

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 CMakeLists.txt             | 14 ++++++++
 examples/spmm/spmm_cuda.cc |  6 ++--
 ttg/ttg/config.in.h        |  3 ++
 ttg/ttg/parsec/buffer.h    | 11 +++---
 ttg/ttg/parsec/task.h      |  2 +-
 ttg/ttg/parsec/ttg.h       | 69 ++++++++++++++++++++++++++++++++++----
 6 files changed, 92 insertions(+), 13 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index aae70683d..06169f4e8 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -137,6 +137,20 @@ if (TARGET MADworld)
   message(STATUS "MADNESS_FOUND=1")
 endif(TARGET MADworld)
 
+####################################################
+#### Check for MPIX_Query_[cuda|rocm]_support
+#### Open MPI provides mpi-ext.h for such extensions
+#### so check for that first.
+####################################################
+find_package(MPI)
+set(TTG_HAVE_MPI MPI_FOUND)
+if (MPI_FOUND)
+  include(CheckIncludeFiles)
+  set(CMAKE_REQUIRED_INCLUDES ${MPI_C_INCLUDE_DIRS})
+  check_include_file("mpi-ext.h" TTG_HAVE_MPIEXT)
+else(MPI_FOUND)
+  set(TTG_HAVE_MPIEXT $<BOOL:false>)
+endif(MPI_FOUND)
 
 ##########################
 #### Examples
diff --git a/examples/spmm/spmm_cuda.cc b/examples/spmm/spmm_cuda.cc
index f5728231e..9e479dc61 100644
--- a/examples/spmm/spmm_cuda.cc
+++ b/examples/spmm/spmm_cuda.cc
@@ -705,12 +705,14 @@ class SpMM25D {
 
 #if defined(TTG_HAVE_CUDA)
     static constexpr bool have_cuda_op = true;
+#warning SPMM using CUDA implementation
 #elif defined(TTG_HAVE_HIPBLAS)
     static constexpr bool have_hip_op  = true;
+#warning SPMM using HIP implementation
+#else
+#error No valid device implementation found!
 #endif
 
-    static_assert(have_hip_op);
-
     MultiplyAdd(Edge<Key<3>, Blk> &a_ijk, Edge<Key<3>, Blk> &b_ijk, Edge<Key<3>, Blk> &c_ijk, Edge<Key<2>, Blk> &c,
                 const std::vector<std::vector<long>> &a_rowidx_to_colidx,
                 const std::vector<std::vector<long>> &b_colidx_to_rowidx, const std::vector<int> &mTiles,
diff --git a/ttg/ttg/config.in.h b/ttg/ttg/config.in.h
index 44628784d..bfb27daa6 100644
--- a/ttg/ttg/config.in.h
+++ b/ttg/ttg/config.in.h
@@ -17,4 +17,7 @@
 
 #cmakedefine TTG_HAVE_HIPBLAS
 
+#cmakedefine TTG_HAVE_MPI
+#cmakedefine TTG_HAVE_MPIEXT
+
 #endif  // TTG_CONFIG_IN_H
diff --git a/ttg/ttg/parsec/buffer.h b/ttg/ttg/parsec/buffer.h
index 700c9a22f..0e9c58cf9 100644
--- a/ttg/ttg/parsec/buffer.h
+++ b/ttg/ttg/parsec/buffer.h
@@ -333,8 +333,10 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t {
       std::size_t s = size();
       ar& s;
       assert(m_ttg_copy != nullptr); // only tracked objects allowed
-      /* TODO: move to device pointer once we are sure we can send from the device */
-      m_ttg_copy->iovec_add(ttg::iovec{s*sizeof(T), host_ptr()});
+      /* transfer from the current device
+       * note: if the transport layer (MPI) does not support device transfers
+       *       the data will have been pushed out */
+      m_ttg_copy->iovec_add(ttg::iovec{s*sizeof(T), current_device_ptr()});
     } else {
       std::size_t s;
       ar & s;
@@ -342,8 +344,9 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t {
       /* initialize internal pointers and then reset */
       reset(s);
       assert(m_ttg_copy != nullptr); // only tracked objects allowed
-      /* TODO: move to device pointer once we are sure we can send from the device */
-      m_ttg_copy->iovec_add(ttg::iovec{s*sizeof(T), host_ptr()});
+      /* transfer to the current device
+       * TODO: how can we make sure the device copy is not evicted? */
+      m_ttg_copy->iovec_add(ttg::iovec{s*sizeof(T), current_device_ptr()});
     }
   }
 #endif // TTG_SERIALIZATION_SUPPORTS_MADNESS
diff --git a/ttg/ttg/parsec/task.h b/ttg/ttg/parsec/task.h
index 30ce6edc8..b8bfc2aa5 100644
--- a/ttg/ttg/parsec/task.h
+++ b/ttg/ttg/parsec/task.h
@@ -297,7 +297,7 @@ namespace ttg_parsec {
                      parsec_task_class_t *task_class, parsec_taskpool_t *taskpool,
                      int32_t priority, bool is_first)
       : parsec_ttg_task_base_t(mempool, task_class, taskpool, priority,
-                               0, nullptr, nullptr,
+                               0, nullptr,
                                &release_task,
                                true /* deferred until other readers have completed */)
       , parent_task(task)
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 407f63764..b9291beb8 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -65,8 +65,12 @@
 #include <vector>
 
 // needed for MPIX_CUDA_AWARE_SUPPORT
+#if defined(TTG_HAVE_MPI)
 #include <mpi.h>
+#if defined(TTG_HAVE_MPIEXT)
 #include <mpi-ext.h>
+#endif // TTG_HAVE_MPIEXT
+#endif // TTG_HAVE_MPI
 
 
 #include <parsec.h>
@@ -202,6 +206,8 @@ namespace ttg_parsec {
     ttg::Edge<> m_ctl_edge;
     bool _dag_profiling;
     bool _task_profiling;
+    std::array<bool, static_cast<std::size_t>(ttg::ExecutionSpace::Invalid)>
+               mpi_space_support = {true, false, false};
 
     int query_comm_size() {
       int comm_size;
@@ -249,6 +255,19 @@ namespace ttg_parsec {
       ttg::detail::register_world(*this);
       if (own_ctx) ctx = parsec_init(ncores, argc, argv);
 
+      /* query MPI device support */
+#if defined(MPIX_CUDA_AWARE_SUPPORT) && MPIX_CUDA_AWARE_SUPPORT
+      if (MPIX_Query_cuda_support()) {
+        mpi_space_support[static_cast<ttg::ExecutionSpace::CUDA>] = true;
+      }
+#endif // MPIX_CUDA_AWARE_SUPPORT
+
+#if defined(MPIX_HIP_AWARE_SUPPORT) && MPIX_HIP_AWARE_SUPPORT
+      if (MPIX_Query_hip_support()) {
+        mpi_space_support[static_cast<ttg::ExecutionSpace::HIP>] = true;
+      }
+#endif // MPIX_HIP_AWARE_SUPPORT
+
 #if defined(PARSEC_PROF_TRACE)
       if(parsec_profile_enabled) {
         profile_on();
@@ -447,6 +466,10 @@ namespace ttg_parsec {
 
     virtual bool profiling() override { return _task_profiling; }
 
+    bool mpi_support(ttg::ExecutionSpace space) {
+      return mpi_space_support[static_cast<std::size_t>(space)];
+    }
+
     virtual void final_task() override {
 #ifdef TTG_USE_USER_TERMDET
       if(parsec_taskpool_started) {
@@ -1420,8 +1443,9 @@ namespace ttg_parsec {
       /* TODO: is this the right place to set the mask? */
       task->parsec_task.chore_mask = PARSEC_DEV_ALL;
       /* get a device and come back if we need another one */
-      int64_t task_load = 1;
-      dev_index = parsec_get_best_device(parsec_task, &task_load);
+      //int64_t task_load = 1;
+      gpu_task->load = 1.0;
+      dev_index = parsec_get_best_device(parsec_task, &gpu_task->load);
       assert(dev_index >= 0);
       if (dev_index < 2) {
           return PARSEC_HOOK_RETURN_NEXT; /* Fall back */
@@ -1887,6 +1911,30 @@ namespace ttg_parsec {
             copy = detail::create_new_datacopy(descr.create_from_metadata(metadata));
           } else if constexpr (!ttg::has_split_metadata<decvalueT>::value) {
             copy = detail::create_new_datacopy(decvalueT{});
+#if 0
+            // TODO: first attempt at sending directly to the device
+            parsec_gpu_data_copy_t* gpu_elem;
+            gpu_elem = PARSEC_DATA_GET_COPY(master, gpu_device->super.device_index);
+            int i = 2; // 0: cpu, 1: recursive -> start with 2 (first accelerator)
+            int devid = 2;
+            while (i < parsec_nb_devices) {
+              if (nullptr == gpu_elem) {
+                gpu_elem = PARSEC_OBJ_NEW(parsec_data_copy_t);
+                gpu_elem->flags = PARSEC_DATA_FLAG_PARSEC_OWNED | PARSEC_DATA_FLAG_PARSEC_MANAGED;
+                gpu_elem->coherency_state = PARSEC_DATA_COHERENCY_INVALID;
+                gpu_elem->version = 0;
+                gpu_elem->coherency_state = PARSEC_DATA_COHERENCY_OWNED;
+              }
+              if (nullptr == gpu_elem->device_private) {
+                gpu_elem->device_private = zone_malloc(gpu_device->memory, gpu_task->flow_nb_elts[i]);
+                if (nullptr == gpu_elem->device_private) {
+                  devid++;
+                  continue;
+                }
+                break;
+              }
+            }
+#endif // 0
             /* unpack the object, potentially discovering iovecs */
             pos = unpack(*static_cast<decvalueT *>(copy->get_ptr()), msg->bytes, pos);
             //std::cout << "set_arg_from_msg iovec_begin num_iovecs " << num_iovecs << " distance " << std::distance(copy->iovec_begin(), copy->iovec_end()) << std::endl;
@@ -3083,7 +3131,7 @@ namespace ttg_parsec {
       /* TODO: remove this once we support reductions on the GPU */
       auto &reducer = std::get<i>(input_reducers);
       if (reducer) {
-        /* non-gpu task, check if we need to push back to the host */
+        /* reductions are currently done only on the host so push out */
         copy_mark_pushout(copy);
         caller->data_flags |= detail::ttg_parsec_data_flags::MARKED_PUSHOUT;
         return;
@@ -3118,13 +3166,22 @@ namespace ttg_parsec {
         need_pushout = true;
       }
 
-      /* check if there are non-local successors if it's a cuda task */
+      /* check if there are non-local successors if it's a device task */
       if (!need_pushout) {
-        need_pushout = remote_check();
+        bool device_supported = false;
+        if constexpr (derived_has_cuda_op()) {
+          device_supported = !world.impl().mpi_support(ttg::ExecutionSpace::CUDA);
+        } else if constexpr (derived_has_hip_op()) {
+          device_supported = !world.impl().mpi_support(ttg::ExecutionSpace::HIP);
+        }
+        /* if MPI supports the device we don't care whether we have remote peers
+         * because we can send from the device directly */
+        if (!device_supported) {
+          need_pushout = remote_check();
+        }
       }
 
       if (need_pushout) {
-        /* non-gpu task, check if we need to push back to the host */
         copy_mark_pushout(copy);
         caller->data_flags |= detail::ttg_parsec_data_flags::MARKED_PUSHOUT;
       }

From 2b7e67c93fd92df24296c128a209cd8a9c19e545 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 6 Sep 2023 17:18:55 +0000
Subject: [PATCH 111/259] Fix cpu-only spmm compilation

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/ttg.h | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index b9291beb8..d3d294860 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -110,9 +110,7 @@
 #include "ttg/parsec/thread_local.h"
 #include "ttg/parsec/ptr.h"
 #include "ttg/parsec/task.h"
-#ifdef TTG_HAVE_CUDART
 #include "ttg/device/cublas_helper.h"
-#endif  // TTG_HAVE_CUDART
 #include "ttg/parsec/parsec-ext.h"
 
 #undef TTG_PARSEC_DEBUG_TRACK_DATA_COPIES
@@ -1320,12 +1318,12 @@ namespace ttg_parsec {
       assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_TRANSFER ||
              dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_KERNEL);
 
-#if defined(PARSEC_HAVE_DEV_CUDA_SUPPORT)
+#if defined(PARSEC_HAVE_DEV_CUDA_SUPPORT) && defined(TTG_HAVE_CUDART)
       parsec_cuda_exec_stream_t *cuda_stream = (parsec_cuda_exec_stream_t *)gpu_stream;
       ttg::detail::cublas_set_kernel_stream(cuda_stream->cuda_stream);
 #endif // PARSEC_HAVE_DEV_CUDA_SUPPORT
 
-#if defined(PARSEC_HAVE_DEV_HIP_SUPPORT)
+#if defined(PARSEC_HAVE_DEV_HIP_SUPPORT) && defined(TTG_HAVE_HIPBLAS)
       parsec_hip_exec_stream_t *hip_stream = (parsec_hip_exec_stream_t *)gpu_stream;
       ttg::detail::hipblas_set_kernel_stream(hip_stream->hip_stream);
 #endif // PARSEC_HAVE_DEV_HIP_SUPPORT

From 7faf783e36749aa93a8abdd7345346610da15a74 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 22 Sep 2023 19:25:15 +0000
Subject: [PATCH 112/259] Add have_hip_op to TT produced by make_tt

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/make_tt.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ttg/ttg/make_tt.h b/ttg/ttg/make_tt.h
index 1c01a2554..ef52228b8 100644
--- a/ttg/ttg/make_tt.h
+++ b/ttg/ttg/make_tt.h
@@ -161,6 +161,7 @@ class CallableWrapTTArgs
 
 public:
   static constexpr bool have_cuda_op = (space == ttg::ExecutionSpace::CUDA);
+  static constexpr bool have_hip_op  = (space == ttg::ExecutionSpace::HIP);
 
 protected:
 

From a423e87aeb392a261f2f3a241808978aede3e774 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 25 Sep 2023 20:32:44 +0000
Subject: [PATCH 113/259] Add TTG_FORCE_DEVICE_COMM to force use of device-side
 comm buffers

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/task.h |  4 ++--
 ttg/ttg/parsec/ttg.h  | 20 ++++++++++++--------
 ttg/ttg/util/env.cpp  | 11 +++++++++++
 ttg/ttg/util/env.h    |  8 ++++++++
 4 files changed, 33 insertions(+), 10 deletions(-)

diff --git a/ttg/ttg/parsec/task.h b/ttg/ttg/parsec/task.h
index b8bfc2aa5..0d4c890b2 100644
--- a/ttg/ttg/parsec/task.h
+++ b/ttg/ttg/parsec/task.h
@@ -118,7 +118,7 @@ namespace ttg_parsec {
        * need offsetof for the mempool and scheduling.
        */
       release_task_fn* release_task_cb = nullptr;
-      device_ptr_t* dev_ptr;
+      device_ptr_t* dev_ptr = nullptr;
       bool remove_from_hash = true;
       bool is_dummy = false;
       bool defer_writer = TTG_PARSEC_DEFER_WRITER; // whether to defer writer instead of creating a new copy
@@ -237,7 +237,7 @@ namespace ttg_parsec {
     template <typename TT>
     struct parsec_ttg_task_t<TT, true> : public parsec_ttg_task_base_t {
       static constexpr size_t num_streams = TT::numins;
-      TT* tt;
+      TT* tt = nullptr;
       std::array<stream_info_t, num_streams> streams;
 #ifdef TTG_HAS_COROUTINE
       void* suspended_task_address = nullptr;  // if not null the function is suspended
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index d3d294860..3bfad35a7 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -254,17 +254,21 @@ namespace ttg_parsec {
       if (own_ctx) ctx = parsec_init(ncores, argc, argv);
 
       /* query MPI device support */
+      if (ttg::detail::force_device_comm()
 #if defined(MPIX_CUDA_AWARE_SUPPORT) && MPIX_CUDA_AWARE_SUPPORT
-      if (MPIX_Query_cuda_support()) {
+          || MPIX_Query_cuda_support()
+#endif // MPIX_CUDA_AWARE_SUPPORT
+         ) {
         mpi_space_support[static_cast<ttg::ExecutionSpace::CUDA>] = true;
       }
-#endif // MPIX_CUDA_AWARE_SUPPORT
 
+      if (ttg::detail::force_device_comm()
 #if defined(MPIX_HIP_AWARE_SUPPORT) && MPIX_HIP_AWARE_SUPPORT
-      if (MPIX_Query_hip_support()) {
+          || MPIX_Query_hip_support()
+#endif // MPIX_HIP_AWARE_SUPPORT
+         ) {
         mpi_space_support[static_cast<ttg::ExecutionSpace::HIP>] = true;
       }
-#endif // MPIX_HIP_AWARE_SUPPORT
 
 #if defined(PARSEC_PROF_TRACE)
       if(parsec_profile_enabled) {
@@ -1416,7 +1420,7 @@ namespace ttg_parsec {
       PARSEC_OBJ_CONSTRUCT(gpu_task, parsec_list_item_t);
       gpu_task->ec = parsec_task;
       gpu_task->task_type = 0; // user task
-      gpu_task->load = 1.0;    // TODO: can we do better?
+      gpu_task->load = 1;    // TODO: can we do better?
       gpu_task->last_data_check_epoch = -1; // used internally
       gpu_task->pushout = 0;
       gpu_task->submit = &TT::device_static_submit<Space>;
@@ -1441,9 +1445,9 @@ namespace ttg_parsec {
       /* TODO: is this the right place to set the mask? */
       task->parsec_task.chore_mask = PARSEC_DEV_ALL;
       /* get a device and come back if we need another one */
-      //int64_t task_load = 1;
-      gpu_task->load = 1.0;
-      dev_index = parsec_get_best_device(parsec_task, &gpu_task->load);
+      int64_t task_load = 1;
+      dev_index = parsec_get_best_device(parsec_task, &task_load);
+      gpu_task->load = task_load;
       assert(dev_index >= 0);
       if (dev_index < 2) {
           return PARSEC_HOOK_RETURN_NEXT; /* Fall back */
diff --git a/ttg/ttg/util/env.cpp b/ttg/ttg/util/env.cpp
index a4de47b09..12d1d6051 100644
--- a/ttg/ttg/util/env.cpp
+++ b/ttg/ttg/util/env.cpp
@@ -30,5 +30,16 @@ namespace ttg {
       return static_cast<int>(result);
     }
 
+    bool force_device_comm() {
+      bool result = false;
+      const char* ttg_force_device_comm_cstr = std::getenv("TTG_FORCE_DEVICE_COMM");
+      if (ttg_force_device_comm_cstr) {
+        const auto result_int = std::atoi(ttg_force_device_comm_cstr);
+        if (result_int) {
+          result = true;
+        }
+      }
+      return result;
+    }
   }  // namespace detail
 }  // namespace ttg
diff --git a/ttg/ttg/util/env.h b/ttg/ttg/util/env.h
index 7681480ef..2cbb20fea 100644
--- a/ttg/ttg/util/env.h
+++ b/ttg/ttg/util/env.h
@@ -16,6 +16,14 @@ namespace ttg {
     /// @post `num_threads()>0`
     int num_threads();
 
+    /// Override whether TTG should attempt to communicate to and from device buffers.
+    /// TTG will attempt to query device support from the underlying MPI implementation (e.g.,
+    /// using the unofficial extension MPIX_Query_cuda_support). However, since not all MPI implementations
+    /// support this extension, users can force the use of device buffers in communication by setting
+    /// `TTG_FORCE_DEVICE_COMM` to a non-negative number.
+    /// @return true if the user wants to force the use of device-side buffers in communicaton.
+    bool enable_device_comm();
+
   }  // namespace detail
 }  // namespace ttg
 

From a7fe69a205eeb853234a7e1a869e5760f087c647 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 25 Sep 2023 20:39:53 +0000
Subject: [PATCH 114/259] Bump TA to support ROCM allocators

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 cmake/modules/ExternalDependenciesVersions.cmake |  2 +-
 examples/spmm/spmm_cuda.cc                       | 11 +++++------
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/cmake/modules/ExternalDependenciesVersions.cmake b/cmake/modules/ExternalDependenciesVersions.cmake
index 4afbf4e0e..74337e482 100644
--- a/cmake/modules/ExternalDependenciesVersions.cmake
+++ b/cmake/modules/ExternalDependenciesVersions.cmake
@@ -8,4 +8,4 @@ set(TTG_TRACKED_CEREAL_VERSION 1.3.0)
 set(TTG_TRACKED_MADNESS_TAG 31d803325623de75371774feffb0270c796bea24)
 set(TTG_TRACKED_PARSEC_TAG e59bed9d4934775792a58980cf5500bd41381bc4)
 set(TTG_TRACKED_BTAS_TAG d73153ad9bc41a177e441ef04eceff7fab0c766d)
-set(TTG_TRACKED_TILEDARRAY_TAG f6f5039b54ee5a1180106351cac05e0d33e14c62)
+set(TTG_TRACKED_TILEDARRAY_TAG 055cf56681e5e55f4c7d415597540f6d65a31097)
diff --git a/examples/spmm/spmm_cuda.cc b/examples/spmm/spmm_cuda.cc
index 9e479dc61..2feb690f8 100644
--- a/examples/spmm/spmm_cuda.cc
+++ b/examples/spmm/spmm_cuda.cc
@@ -19,7 +19,7 @@
 #include <btas/btas.h>
 #include <btas/optimize/contract.h>
 #include <btas/util/mohndle.h>
-#include <TiledArray/cuda/allocators.h>
+#include <TiledArray/device/allocators.h>
 #include <ttg/device/cublas_helper.h>
 #include <madness/world/parsec.h>  // need to initialize MADNESS purely for the purposes of TA allocators
 #else
@@ -52,8 +52,8 @@ using namespace ttg;
 template <typename _T, class _Range, class _Storage>
 struct DeviceTensor : public ttg::TTValue<DeviceTensor<_T, _Range, _Storage>>
                     , public btas::Tensor<_T, _Range, _Storage> {
-  using tensor_type = btas::Tensor<_T, _Range, _Storage>;
-  using ttvalue_type = ttg::TTValue<DeviceTensor<_T, _Range, _Storage>>;
+  using tensor_type = typename btas::Tensor<_T, _Range, _Storage>;
+  using ttvalue_type = typename ttg::TTValue<DeviceTensor<_T, _Range, _Storage>>;
   ttg::buffer<_T> b; // does not own the host buffer
 
   using value_type = typename tensor_type::value_type;
@@ -246,12 +246,11 @@ struct DeviceTensor : public ttg::TTValue<DeviceTensor<_T, _Range, _Storage>>
 
 };
 
-#if defined(TTG_HAVE_CUDA)
+#if defined(TTG_HAVE_CUDA) || defined(TTG_HAVE_HIPBLAS)
 using blk_t = DeviceTensor<double, btas::DEFAULT::range,
-                           btas::mohndle<btas::varray<double, TiledArray::cuda_pinned_allocator<double>>,
+                           btas::mohndle<btas::varray<double, TiledArray::device_pinned_allocator<double>>,
                                          btas::Handle::shared_ptr>>;
 #else
-// TODO: no hip pinned allocator in TA?
 using blk_t = DeviceTensor<double, btas::DEFAULT::range,
                            btas::mohndle<btas::varray<double>, btas::Handle::shared_ptr>>;
 #endif

From 6d736213b7c284e247e06bff7455c56a43fd60cf Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 25 Sep 2023 20:52:44 +0000
Subject: [PATCH 115/259] Fix device comm code

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/ttg.h | 4 ++--
 ttg/ttg/util/env.h   | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 3bfad35a7..288545dbb 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -259,7 +259,7 @@ namespace ttg_parsec {
           || MPIX_Query_cuda_support()
 #endif // MPIX_CUDA_AWARE_SUPPORT
          ) {
-        mpi_space_support[static_cast<ttg::ExecutionSpace::CUDA>] = true;
+        mpi_space_support[static_cast<std::size_t>(ttg::ExecutionSpace::CUDA)] = true;
       }
 
       if (ttg::detail::force_device_comm()
@@ -267,7 +267,7 @@ namespace ttg_parsec {
           || MPIX_Query_hip_support()
 #endif // MPIX_HIP_AWARE_SUPPORT
          ) {
-        mpi_space_support[static_cast<ttg::ExecutionSpace::HIP>] = true;
+        mpi_space_support[static_cast<std::size_t>(ttg::ExecutionSpace::HIP)] = true;
       }
 
 #if defined(PARSEC_PROF_TRACE)
diff --git a/ttg/ttg/util/env.h b/ttg/ttg/util/env.h
index 2cbb20fea..4e07bff7b 100644
--- a/ttg/ttg/util/env.h
+++ b/ttg/ttg/util/env.h
@@ -22,7 +22,7 @@ namespace ttg {
     /// support this extension, users can force the use of device buffers in communication by setting
     /// `TTG_FORCE_DEVICE_COMM` to a non-negative number.
     /// @return true if the user wants to force the use of device-side buffers in communicaton.
-    bool enable_device_comm();
+    bool force_device_comm();
 
   }  // namespace detail
 }  // namespace ttg

From 88f8da9c785960cfe9cf5bd530a8b869a8e2d77d Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 25 Sep 2023 21:37:18 +0000
Subject: [PATCH 116/259] SPMM: restore write-back to Eigen matrix for
 verification

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/spmm/spmm_cuda.cc | 31 ++++++++++++++++++-------------
 1 file changed, 18 insertions(+), 13 deletions(-)

diff --git a/examples/spmm/spmm_cuda.cc b/examples/spmm/spmm_cuda.cc
index 2feb690f8..b0b879717 100644
--- a/examples/spmm/spmm_cuda.cc
+++ b/examples/spmm/spmm_cuda.cc
@@ -264,8 +264,8 @@ using blk_t = DeviceTensor<double, btas::DEFAULT::range,
 
 /* TODO: call CUDA gemm here */
 static void device_gemm(blk_t &C, const blk_t &A, const blk_t &B) {
-  double alpha = 1.0;
-  double beta  = 1.0;
+  static const double alpha = 1.0;
+  static const double beta  = 1.0;
   // make sure all memory is on the device
   // TODO: A and B are read-only so the owner device will be 0. How to fix?
   //assert(A.b.get_current_device() != 0);
@@ -438,18 +438,22 @@ class Write_SpMatrix : public TT<Key<2>, std::tuple<>, Write_SpMatrix<Blk>, ttg:
   using baseT = typename Write_SpMatrix::ttT;
 
   template <typename Keymap2>
-  Write_SpMatrix(SpMatrix<Blk> &matrix, Edge<Key<2>, Blk> &in, Keymap2 &&ij_keymap)
-      : baseT(edges(in), edges(), "write_spmatrix", {"Cij"}, {}, ij_keymap), matrix_(matrix) {}
+  Write_SpMatrix(SpMatrix<Blk> &matrix, Edge<Key<2>, Blk> &in, Keymap2 &&ij_keymap, bool write_back = false)
+      : baseT(edges(in), edges(), "write_spmatrix", {"Cij"}, {}, ij_keymap)
+      , matrix_(matrix)
+      , write_back(write_back)
+  { }
 
   void op(const Key<2> &key, typename baseT::input_refs_tuple_type &&elem, std::tuple<> &) {
-#if 0
-    std::lock_guard<std::mutex> lock(mtx_);
-    ttg::trace("rank =", default_execution_context().rank(),
-               "/ thread_id =", reinterpret_cast<std::uintptr_t>(pthread_self()), "spmm.cc Write_SpMatrix wrote {",
-               key[0], ",", key[1], "} = ", baseT::template get<0>(elem), " in ", static_cast<void *>(&matrix_),
-               " with mutex @", static_cast<void *>(&mtx_), " for object @", static_cast<void *>(this));
-    values_.emplace_back(key[0], key[1], std::move(baseT::template get<0>(elem)));
-#endif // 0
+
+    if (write_back) {
+      std::lock_guard<std::mutex> lock(mtx_);
+      ttg::trace("rank =", default_execution_context().rank(),
+                 "/ thread_id =", reinterpret_cast<std::uintptr_t>(pthread_self()), "spmm.cc Write_SpMatrix wrote {",
+                 key[0], ",", key[1], "} = ", baseT::template get<0>(elem), " in ", static_cast<void *>(&matrix_),
+                 " with mutex @", static_cast<void *>(&mtx_), " for object @", static_cast<void *>(this));
+      values_.emplace_back(key[0], key[1], std::move(baseT::template get<0>(elem)));
+    }
   }
 
   /// grab completion status as a future<void>
@@ -473,6 +477,7 @@ class Write_SpMatrix : public TT<Key<2>, std::tuple<>, Write_SpMatrix<Blk>, ttg:
   SpMatrix<Blk> &matrix_;
   std::vector<SpMatrixTriplet<Blk>> values_;
   mutable std::shared_ptr<std::shared_future<void>> completion_status_;
+  bool write_back = false;
 };
 
 /// sparse mm via 2.5D SUMMA
@@ -1834,7 +1839,7 @@ int main(int argc, char **argv) {
         Edge<Key<2>, blk_t> eA, eB, eC;
         Read_SpMatrix a("A", A, ctl, eA, ij_keymap);
         Read_SpMatrix b("B", B, ctl, eB, ij_keymap);
-        Write_SpMatrix<> c(C, eC, keymap_write);
+        Write_SpMatrix<> c(C, eC, keymap_write, true);
         auto &c_status = c.status();
         assert(!has_value(c_status));
         //  SpMM25D a_times_b(world, eA, eB, eC, A, B);

From 3ba4a4f536170d79b1e71158079c7c8105a97fa8 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 28 Sep 2023 16:09:03 +0000
Subject: [PATCH 117/259] We need one cublas/hipblas context per device

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/device/cublas_helper.cpp | 41 +++++++++++++++++++++++++-------
 1 file changed, 33 insertions(+), 8 deletions(-)

diff --git a/ttg/ttg/device/cublas_helper.cpp b/ttg/ttg/device/cublas_helper.cpp
index 06be13e95..6bb9d6a3e 100644
--- a/ttg/ttg/device/cublas_helper.cpp
+++ b/ttg/ttg/device/cublas_helper.cpp
@@ -4,6 +4,14 @@
 #include <memory>
 #include <stdexcept>
 #include <optional>
+#include <map>
+#ifdef TTG_HAVE_CUDART
+#include <cuda_runtime.h>
+#endif // TTG_HAVE_CUDART
+
+#ifdef TTG_HAVE_HIPBLAS
+#include <hip.h>
+#endif // TTG_HAVE_HIPBLAS
 
 namespace ttg::detail {
 
@@ -11,14 +19,23 @@ namespace ttg::detail {
 /// \brief Returns the cuBLAS handle to be used for launching cuBLAS kernels from the current thread
 /// \return the cuBLAS handle for the current thread
 const cublasHandle_t& cublas_get_handle() {
-  static thread_local std::optional<cublasHandle_t> handle;
-  if (!handle.has_value()) {
-    auto status = cublasCreate_v2(&handle.emplace());
+  static thread_local std::map<int, cublasHandle_t> handles;
+  int device;
+  if (cudaSuccess != cudaGetDevice(&device)){
+      throw std::runtime_error("cudaGetDevice failed");
+  }
+  std::map<int, cublasHandle_t>::iterator it;
+  if ((it = handles.find(device)) == handles.end()){
+    cublasHandle_t handle;
+    auto status = cublasCreate_v2(&handle);
     if (CUBLAS_STATUS_SUCCESS != status) {
       throw std::runtime_error("cublasCreate_v2 failed");
     }
+    auto [iterator, success] = handles.insert({device, handle});
+    it = iterator;
   }
-  return *handle;
+
+  return it->second;
 }
 
 void cublas_set_kernel_stream(cudaStream_t stream) {
@@ -33,14 +50,22 @@ void cublas_set_kernel_stream(cudaStream_t stream) {
 /// \brief Returns the rocBLAS handle to be used for launching rocBLAS kernels from the current thread
 /// \return the rocBLAS handle for the current thread
 const hipblasHandle_t& hipblas_get_handle() {
-  static thread_local std::optional<hipblasHandle_t> handle;
-  if (!handle.has_value()) {
-    hipblasStatus_t status = hipblasCreate(&handle.emplace());
+  static thread_local std::map<int, hipblasHandle_t> handles;
+  int device;
+  if (hipSuccess != hipGetDevice(&device)){
+      throw std::runtime_error("hipGetDevice failed");
+  }
+  std::map<int, hipblasHandle_t>::iterator it;
+  if ((it = handles.find(device)) == handles.end()){
+    hipblasHandle_t handle;
+    auto status = hipblasCreate(&handle);
     if (HIPBLAS_STATUS_SUCCESS != status) {
       throw std::runtime_error("hipblasCreate failed");
     }
+    auto [iterator, success] = handles.insert({device, handle});
+    it = iterator;
   }
-  return *handle;
+  return it->second;
 }
 
 void hipblas_set_kernel_stream(hipStream_t stream) {

From b570108df144f08a5aa2b2ceaa8a9a9a69a3a3b1 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 28 Sep 2023 16:09:40 +0000
Subject: [PATCH 118/259] Add missing header to install

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ttg/CMakeLists.txt b/ttg/CMakeLists.txt
index bf1f71b74..7bf16f0da 100644
--- a/ttg/CMakeLists.txt
+++ b/ttg/CMakeLists.txt
@@ -230,6 +230,7 @@ endif(TARGET MADworld)
 if (TARGET PaRSEC::parsec)
   set(ttg-parsec-headers
           ${CMAKE_CURRENT_SOURCE_DIR}/ttg/parsec/buffer.h
+          ${CMAKE_CURRENT_SOURCE_DIR}/ttg/parsec/devicefunc.h
           ${CMAKE_CURRENT_SOURCE_DIR}/ttg/parsec/devicescratch.h
           ${CMAKE_CURRENT_SOURCE_DIR}/ttg/parsec/fwd.h
           ${CMAKE_CURRENT_SOURCE_DIR}/ttg/parsec/import.h

From 5afb32b08db1ff881e36feaa170eb17184e39e3f Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 29 Sep 2023 02:07:01 +0000
Subject: [PATCH 119/259] Bump TA sha to pull in newer Umpire

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 cmake/modules/ExternalDependenciesVersions.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/modules/ExternalDependenciesVersions.cmake b/cmake/modules/ExternalDependenciesVersions.cmake
index 74337e482..1f4d47771 100644
--- a/cmake/modules/ExternalDependenciesVersions.cmake
+++ b/cmake/modules/ExternalDependenciesVersions.cmake
@@ -8,4 +8,4 @@ set(TTG_TRACKED_CEREAL_VERSION 1.3.0)
 set(TTG_TRACKED_MADNESS_TAG 31d803325623de75371774feffb0270c796bea24)
 set(TTG_TRACKED_PARSEC_TAG e59bed9d4934775792a58980cf5500bd41381bc4)
 set(TTG_TRACKED_BTAS_TAG d73153ad9bc41a177e441ef04eceff7fab0c766d)
-set(TTG_TRACKED_TILEDARRAY_TAG 055cf56681e5e55f4c7d415597540f6d65a31097)
+set(TTG_TRACKED_TILEDARRAY_TAG dfa3f76ebd58c05c64acf6e57cbdc85e90c880a8)

From 3aff11c563e38c13436566a2ffc626743569538b Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 29 Sep 2023 02:07:51 +0000
Subject: [PATCH 120/259] SPMM: Flush out PaRSEC data between measurements

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/spmm/spmm_cuda.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/examples/spmm/spmm_cuda.cc b/examples/spmm/spmm_cuda.cc
index b0b879717..05a6599b9 100644
--- a/examples/spmm/spmm_cuda.cc
+++ b/examples/spmm/spmm_cuda.cc
@@ -1826,9 +1826,11 @@ int main(int argc, char **argv) {
         // Start up engine
         execute();
         for (int nrun = 0; nrun < nb_runs; nrun++) {
+          parsec_devices_release_memory();
           timed_measurement(A, B, ij_keymap, ijk_keymap, tiling_type, gflops, avg_nb, Adensity, Bdensity,
                             a_rowidx_to_colidx, a_colidx_to_rowidx, b_rowidx_to_colidx, b_colidx_to_rowidx, mTiles,
                             nTiles, kTiles, M, N, K, minTs, maxTs, P, Q, R);
+          parsec_devices_reset_load(default_execution_context().impl().context());
         }
       } else {
         // flow graph needs to exist on every node

From 3fa800187453ad8890ee62cf320b1a77db392d10 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 29 Sep 2023 02:09:34 +0000
Subject: [PATCH 121/259] Minor fixes to device cide

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/CMakeLists.txt               | 3 +++
 ttg/ttg/config.in.h              | 2 ++
 ttg/ttg/device/cublas_helper.cpp | 2 +-
 ttg/ttg/device/task.h            | 2 +-
 4 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/ttg/CMakeLists.txt b/ttg/CMakeLists.txt
index 7bf16f0da..a60f4f833 100644
--- a/ttg/CMakeLists.txt
+++ b/ttg/CMakeLists.txt
@@ -130,6 +130,9 @@ if (TTG_HAVE_HIPBLAS)
     )
   endif (NOT TTG_HAVE_CUDA)
   list(APPEND ttg-deps roc::hipblas)
+  find_package(hip REQUIRED)
+  list(APPEND ttg-deps hip::host)
+
 endif (TTG_HAVE_HIPBLAS)
 
 set(ttg-public-headers ${ttg-headers};${ttg-impl-headers};${ttg-base-headers};${ttg-util-headers};${ttg_device_headers})
diff --git a/ttg/ttg/config.in.h b/ttg/ttg/config.in.h
index bfb27daa6..22b92608d 100644
--- a/ttg/ttg/config.in.h
+++ b/ttg/ttg/config.in.h
@@ -15,6 +15,8 @@
 
 #cmakedefine TTG_HAVE_CUDART
 
+#cmakedefine TTG_HAVE_HIP
+
 #cmakedefine TTG_HAVE_HIPBLAS
 
 #cmakedefine TTG_HAVE_MPI
diff --git a/ttg/ttg/device/cublas_helper.cpp b/ttg/ttg/device/cublas_helper.cpp
index 6bb9d6a3e..24e34f52c 100644
--- a/ttg/ttg/device/cublas_helper.cpp
+++ b/ttg/ttg/device/cublas_helper.cpp
@@ -10,7 +10,7 @@
 #endif // TTG_HAVE_CUDART
 
 #ifdef TTG_HAVE_HIPBLAS
-#include <hip.h>
+#include <hip_runtime.h>
 #endif // TTG_HAVE_HIPBLAS
 
 namespace ttg::detail {
diff --git a/ttg/ttg/device/task.h b/ttg/ttg/device/task.h
index da7067194..e5998d0ae 100644
--- a/ttg/ttg/device/task.h
+++ b/ttg/ttg/device/task.h
@@ -206,7 +206,7 @@ namespace ttg {
     // clang-format on
     template <size_t i, typename keyT, typename valueT>
     inline auto send(const keyT &key, valueT &&value) {
-      return send(i, key, std::forward<valueT>(value));
+      return ttg::device::send(i, key, std::forward<valueT>(value));
     }
 
 

From 4d60c3fac1f97092166493e1f81cb35544ca2f43 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Fri, 29 Sep 2023 15:29:05 -0400
Subject: [PATCH 122/259] [cmake] bump MADNESS tag to pull in
 https://github.com/m-a-d-n-e-s-s/madness/pull/501

avoids CMake errors related to lapack including OpenMP_C dependence that MADNESS does not know anything about
---
 cmake/modules/ExternalDependenciesVersions.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/modules/ExternalDependenciesVersions.cmake b/cmake/modules/ExternalDependenciesVersions.cmake
index 1f4d47771..b4aa2e8e1 100644
--- a/cmake/modules/ExternalDependenciesVersions.cmake
+++ b/cmake/modules/ExternalDependenciesVersions.cmake
@@ -5,7 +5,7 @@
 set(TTG_TRACKED_BOOST_VERSION 1.66)
 set(TTG_TRACKED_CATCH2_VERSION 2.13.1)
 set(TTG_TRACKED_CEREAL_VERSION 1.3.0)
-set(TTG_TRACKED_MADNESS_TAG 31d803325623de75371774feffb0270c796bea24)
+set(TTG_TRACKED_MADNESS_TAG e4ee892717e7f91f824a9652d53d4da2acf6920e)
 set(TTG_TRACKED_PARSEC_TAG e59bed9d4934775792a58980cf5500bd41381bc4)
 set(TTG_TRACKED_BTAS_TAG d73153ad9bc41a177e441ef04eceff7fab0c766d)
 set(TTG_TRACKED_TILEDARRAY_TAG dfa3f76ebd58c05c64acf6e57cbdc85e90c880a8)

From 4f21ccd9322df779280b477fbaf754130a0529d1 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 29 Sep 2023 18:10:08 +0000
Subject: [PATCH 123/259] Add missing device::sendk and sendv routines

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/device/task.h | 64 ++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 60 insertions(+), 4 deletions(-)

diff --git a/ttg/ttg/device/task.h b/ttg/ttg/device/task.h
index e5998d0ae..b275da195 100644
--- a/ttg/ttg/device/task.h
+++ b/ttg/ttg/device/task.h
@@ -157,6 +157,21 @@ namespace ttg {
         co_await ttg::Void{}; // we'll come back once the task is done
         t.sendv(std::forward<Value>(value));
       };
+
+      template<typename Key, ttg::Runtime Runtime = ttg::ttg_runtime>
+      inline send_coro_state sendk_coro (const Key& key, ttg::Out<Key, void> &t) {
+        // no need to prepare the send but we have to suspend once
+        Key k = key;
+        co_await ttg::Void{}; // we'll come back once the task is done
+        t.sendk(k);
+      };
+
+      template<ttg::Runtime Runtime = ttg::ttg_runtime>
+      inline send_coro_state send_coro (ttg::Out<void, void> &t) {
+        // no need to prepare the send but we have to suspend once
+        co_await ttg::Void{}; // we'll come back once the task is done
+        t.send();
+      };
     } // namespace detail
 
 
@@ -183,6 +198,12 @@ namespace ttg {
       return detail::send_t{detail::sendv_coro(copy_handler(std::forward<valueT>(value)), std::get<i>(t), copy_handler)};
     }
 
+    template <size_t i, typename Key, typename... out_keysT, typename... out_valuesT,
+              ttg::Runtime Runtime = ttg::ttg_runtime>
+    inline detail::send_t sendk(
+        const Key& key, std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
+      return detail::send_t{detail::sendk_coro(key, std::get<i>(t))};
+    }
 
     // clang-format off
     /// \brief Sends a task id and a value to the template tasks attached to the output terminal of this template task
@@ -210,6 +231,41 @@ namespace ttg {
     }
 
 
+    template <typename valueT, ttg::Runtime Runtime = ttg::ttg_runtime>
+    inline detail::send_t sendv(std::size_t i, valueT &&value) {
+      auto *terminal_ptr = ttg::detail::get_out_terminal<void, valueT>(i, "ttg::device::send(i, key, value)");
+      ttg::detail::value_copy_handler<Runtime> copy_handler;
+      return detail::send_t{detail::sendv_coro(copy_handler(std::forward<valueT>(value)), *terminal_ptr, copy_handler)};
+    }
+
+    template <typename Key, ttg::Runtime Runtime = ttg::ttg_runtime>
+    inline detail::send_t sendk(std::size_t i, const Key& key) {
+      auto *terminal_ptr = ttg::detail::get_out_terminal<Key, void>(i, "ttg::device::send(i, key, value)");
+      return detail::send_t{detail::sendk_coro(key, *terminal_ptr)};
+    }
+
+    template <ttg::Runtime Runtime = ttg::ttg_runtime>
+    inline detail::send_t send(std::size_t i) {
+      auto *terminal_ptr = ttg::detail::get_out_terminal<void, void>(i, "ttg::device::send(i, key, value)");
+      return detail::send_t{detail::send_coro(*terminal_ptr)};
+    }
+
+
+    template <std::size_t i, typename valueT, typename... out_keysT, typename... out_valuesT,
+              ttg::Runtime Runtime = ttg::ttg_runtime>
+    inline detail::send_t sendv(valueT &&value) {
+      return sendv(i, std::forward<valueT>(value));
+    }
+
+    template <size_t i, typename Key, ttg::Runtime Runtime = ttg::ttg_runtime>
+    inline detail::send_t sendk(const Key& key) {
+      return sendk(i, key);
+    }
+
+    template <size_t i, ttg::Runtime Runtime = ttg::ttg_runtime>
+    inline detail::send_t sendk() {
+      return send(i);
+    }
 
     namespace detail {
 
@@ -504,12 +560,12 @@ namespace ttg {
       return {};
     }
 
-#if 0
-    void return_value(std::vector<device::detail::send_t>&& sends) {
-      m_sends = std::forward<std::vector<device::detail::send_t>>(sends);
+    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(device::detail::send_t&& v) {
+      m_sends.clear();
+      m_sends.push_back(std::forward<device::detail::send_t>(v));
       m_state = TTG_DEVICE_CORO_SENDOUT;
+      return {};
     }
-#endif // 0
 
     void return_void() {
       m_state = TTG_DEVICE_CORO_COMPLETE;

From f91fa921334233dad54239dc124a4000a1036762 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Sun, 1 Oct 2023 18:12:12 +0000
Subject: [PATCH 124/259] Add device-based chain task benchmark

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/CMakeLists.txt                   |   3 +
 examples/task-benchmarks/chain-ttg-dev.cc | 251 ++++++++++++++++++++++
 examples/task-benchmarks/chrono.h         |  22 ++
 ttg/ttg/make_tt.h                         | 152 ++++++++-----
 ttg/ttg/parsec/devicefunc.h               |   5 +-
 5 files changed, 373 insertions(+), 60 deletions(-)
 create mode 100644 examples/task-benchmarks/chain-ttg-dev.cc
 create mode 100644 examples/task-benchmarks/chrono.h

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index b2e398ddf..24b6f5897 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -22,6 +22,8 @@ if (TARGET tiledarray)
                         COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2
                         RUNTIMES "parsec")
     endif()
+
+    add_ttg_executable(chain-ttg-dev task-benchmarks/chain-ttg-dev.cc LINK_LIBRARIES tiledarray RUNTIMES "parsec")
 endif()
 
 if (TARGET MADworld)
@@ -61,3 +63,4 @@ add_ttg_executable(sw sw/sw.cc)
 if (TARGET MADworld)
   add_ttg_executable(randomaccess randomaccess/randomaccess.cc RUNTIMES "mad")
 endif (TARGET MADworld)
+
diff --git a/examples/task-benchmarks/chain-ttg-dev.cc b/examples/task-benchmarks/chain-ttg-dev.cc
new file mode 100644
index 000000000..5ccf1297d
--- /dev/null
+++ b/examples/task-benchmarks/chain-ttg-dev.cc
@@ -0,0 +1,251 @@
+//#define TTG_USE_USER_TERMDET 1
+#include "ttg.h"
+
+#include "chrono.h"
+
+#if defined(TTG_HAVE_CUDA)
+#define ES ttg::ExecutionSpace::CUDA
+#elif defined(TTG_HAVE_HIP)
+#define ES ttg::ExecutionSpace::HIP
+#else
+#error "Either CUDA OR HIP is required to build this test!"
+#endif // 0
+
+#define NUM_TASKS 10000
+
+using namespace ttg;
+
+std::atomic<int> task_counter = 0;
+
+struct A : public ttg::TTValue<A> {
+  // TODO: allocate pinned memory
+  int v = 0;
+  ttg::buffer<int> b;
+  A() : b(&v, 1) { }
+
+  A(A&& a) = default;
+  A(const A& a) : v(a.v), b(&v, 1) { }
+
+  template <typename Archive>
+  void serialize(Archive& ar) {
+    ttg_abort();
+  }
+  template <typename Archive>
+  void serialize(Archive& ar, const unsigned int) {
+    ttg_abort();
+  }
+
+};
+
+template <int num_flows>
+auto make_ttg(bool do_move);
+
+// flows task ids via values
+template <>
+auto make_ttg<1>(bool do_move) {
+  Edge<int, A> I2N, N2N;
+  Edge<void, A> N2S;
+
+  auto init = make_tt<void>(
+    []() {
+      ++task_counter;
+      std::cout << "init 1 " << std::endl;
+      send<0>(0, A{});
+    }, edges(), edges(I2N));
+
+  auto next = make_tt<ES, int>([=](const int &key, auto&& value) -> ttg::device_task {
+    //++task_counter;
+    co_await ttg::to_device(value.b);
+    co_await ttg::wait_kernel(); // empty kernel
+    if (key < NUM_TASKS) {
+      if (do_move) {
+        co_await ttg::device::forward(ttg::device::send<0>(key+1, std::move(value)));
+      } else {
+        co_await ttg::device::forward(ttg::device::send<0>(key+1, value));
+      }
+    } else {
+    }
+  } , edges(fuse(I2N, N2N)), edges(N2N));
+
+  return std::make_tuple(std::move(init), std::move(next));
+}
+
+template <>
+auto make_ttg<2>(bool do_move) {
+  Edge<int, A> I2N1, I2N2;
+  Edge<int, A> N2N1, N2N2;
+  Edge<void, A> N2S1, N2S2;
+
+  auto init = make_tt<void>([]() {
+    send<0>(0, A{});
+    send<1>(0, A{});
+  }, edges(), edges(I2N1, I2N2));
+
+  auto next = make_tt<ES, int>([=](const int &key, A&& v1, A&& v2) -> ttg::device_task {
+    co_await ttg::to_device(v1.b, v2.b);
+    co_await ttg::wait_kernel(); // empty kernel
+    if (key < NUM_TASKS) {
+      if (do_move) {
+        co_await ttg::device::forward(ttg::device::send<0>(key+1, std::move(v1)),
+                                      ttg::device::send<1>(key+1, std::move(v2)));
+      } else {
+        co_await ttg::device::forward(ttg::device::send<0>(key+1, v1),
+                                      ttg::device::send<1>(key+1, v2));
+      }
+    }
+  } , edges(fuse(I2N1, N2N1), fuse(I2N2, N2N2)), edges(N2N1, N2N2));
+
+  return std::make_tuple(std::move(init), std::move(next));
+}
+
+template <>
+auto make_ttg<4>(bool do_move) {
+  Edge<int, A> I2N1, I2N2, I2N3, I2N4;
+  Edge<int, A> N2N1, N2N2, N2N3, N2N4;
+  Edge<void, A> N2S1, N2S2, N2S3, N2S4;
+
+  auto init = make_tt<void>(
+    []() {
+      send<0>(0, A{});
+      send<1>(0, A{});
+      send<2>(0, A{});
+      send<3>(0, A{});
+    }, edges(), edges(I2N1, I2N2, I2N3, I2N4));
+
+  auto next = make_tt<ES, int>([=](const int &key, A&& v1, A&& v2, A&& v3, A&& v4) -> ttg::device_task {
+    co_await ttg::to_device(v1.b, v2.b, v3.b, v4.b);
+    co_await ttg::wait_kernel(); // empty kernel
+    if (key < NUM_TASKS) {
+      if (do_move) {
+        co_await ttg::device::forward(ttg::device::send<0>(key+1, std::move(v1)),
+                                      ttg::device::send<1>(key+1, std::move(v2)),
+                                      ttg::device::send<2>(key+1, std::move(v3)),
+                                      ttg::device::send<3>(key+1, std::move(v4)));
+      } else {
+        co_await ttg::device::forward(ttg::device::send<0>(key+1, v1),
+                                      ttg::device::send<1>(key+1, v2),
+                                      ttg::device::send<2>(key+1, v3),
+                                      ttg::device::send<3>(key+1, v4));
+      }
+    }
+  }, edges(fuse(I2N1, N2N1), fuse(I2N2, N2N2),
+           fuse(I2N3, N2N3), fuse(I2N4, N2N4)),
+     edges(N2N1, N2N2, N2N3, N2N4));
+
+  return std::make_tuple(std::move(init), std::move(next));
+}
+
+template <>
+auto make_ttg<8>(bool do_move) {
+  Edge<int, A> I2N1, I2N2, I2N3, I2N4, I2N5, I2N6, I2N7, I2N8;
+  Edge<int, A> N2N1, N2N2, N2N3, N2N4, N2N5, N2N6, N2N7, N2N8;
+  Edge<void, A> N2S1, N2S2, N2S3, N2S4, N2S5, N2S6, N2S7, N2S8;
+
+  auto init = make_tt<void>(
+    []() {
+      send<0>(0, A{});
+      send<1>(0, A{});
+      send<2>(0, A{});
+      send<3>(0, A{});
+      send<4>(0, A{});
+      send<5>(0, A{});
+      send<6>(0, A{});
+      send<7>(0, A{});
+    }, edges(), edges(I2N1, I2N2, I2N3, I2N4, I2N5, I2N6, I2N7, I2N8));
+
+  auto next = make_tt<ES, int>([=](const int &key, auto&& v1, auto&& v2, auto&& v3, auto&& v4, auto&& v5, auto&& v6, auto&& v7, auto&& v8) -> ttg::device_task {
+    co_await ttg::to_device(v1.b, v2.b, v3.b, v4.b, v5.b, v6.b, v7.b, v8.b);
+    co_await ttg::wait_kernel(); // empty kernel
+    if (key < NUM_TASKS) {
+      if (do_move) {
+        co_await ttg::device::forward(ttg::device::send<0>(key+1, std::move(v1)),
+                                      ttg::device::send<1>(key+1, std::move(v2)),
+                                      ttg::device::send<2>(key+1, std::move(v3)),
+                                      ttg::device::send<3>(key+1, std::move(v4)),
+                                      ttg::device::send<4>(key+1, std::move(v5)),
+                                      ttg::device::send<5>(key+1, std::move(v6)),
+                                      ttg::device::send<6>(key+1, std::move(v7)),
+                                      ttg::device::send<7>(key+1, std::move(v8)));
+      } else {
+        co_await ttg::device::forward(ttg::device::send<0>(key+1, v1),
+                                      ttg::device::send<1>(key+1, v2),
+                                      ttg::device::send<2>(key+1, v3),
+                                      ttg::device::send<3>(key+1, v4),
+                                      ttg::device::send<4>(key+1, v5),
+                                      ttg::device::send<5>(key+1, v6),
+                                      ttg::device::send<6>(key+1, v7),
+                                      ttg::device::send<7>(key+1, v8));
+      }
+    }
+  }, edges(fuse(I2N1, N2N1), fuse(I2N2, N2N2), fuse(I2N3, N2N3), fuse(I2N4, N2N4), fuse(I2N5, N2N5), fuse(I2N6, N2N6), fuse(I2N7, N2N7), fuse(I2N8, N2N8)),
+     edges(N2N1, N2N2, N2N3, N2N4, N2N5, N2N6, N2N7, N2N8));
+
+  return std::make_tuple(std::move(init), std::move(next));
+}
+
+// flows task ids via keys
+template <>
+auto make_ttg<0>(bool do_move) {
+  Edge<int, void> I2N, N2N;
+  Edge<void, int> N2S;
+
+  auto init = make_tt<void>([](std::tuple<Out<int, void>> &outs) { sendk<0>(0, outs); }, edges(), edges(I2N));
+
+  auto next = make_tt<ES>([](const int& key) -> ttg::device_task {
+    co_await ttg::to_device();
+    co_await ttg::wait_kernel();
+    if (key < NUM_TASKS) {
+      co_await ttg::device::forward(ttg::device::sendk<0>(key+1));
+    }
+  }, edges(fuse(I2N, N2N)), edges(N2N));
+
+  return std::make_tuple(std::move(init), std::move(next));
+}
+
+template<int num_flows>
+void run_bench(bool do_move)
+{
+  auto [init, next] = make_ttg<num_flows>(do_move);
+
+  auto connected = make_graph_executable(init.get());
+  assert(connected);
+  std::cout << "Graph " << num_flows << " is connected.\n";
+
+  auto t0 = now();
+  if (ttg::default_execution_context().rank() == 0) init->invoke();
+
+  ttg_execute(ttg_default_execution_context());
+  ttg_fence(ttg_default_execution_context());
+  auto t1 = now();
+
+  std::cout << "# of tasks = " << task_counter.load() << std::endl;
+  std::cout << "time elapsed (microseconds) = " << duration_in_mus(t0, t1) << std::endl;
+}
+
+int main(int argc, char* argv[]) {
+
+  int num_flows = 0;
+  int do_move = 1;
+  ttg_initialize(argc, argv, -1);
+
+  if (argc > 1) {
+    num_flows = std::atoi(argv[1]);
+  }
+
+  if (argc > 2) {
+    do_move = std::atoi(argv[2]);
+  }
+
+  switch(num_flows) {
+  case 0: run_bench<0>(do_move); break;
+  case 1: run_bench<1>(do_move); break;
+  case 2: run_bench<2>(do_move); break;
+  case 4: run_bench<4>(do_move); break;
+  case 8: run_bench<8>(do_move); break;
+  default: std::cout << "Unsupported number of flows: " << num_flows << std::endl;
+  }
+
+  ttg_finalize();
+  return 0;
+}
+
diff --git a/examples/task-benchmarks/chrono.h b/examples/task-benchmarks/chrono.h
new file mode 100644
index 000000000..358d6dcc4
--- /dev/null
+++ b/examples/task-benchmarks/chrono.h
@@ -0,0 +1,22 @@
+//
+// Created by Eduard Valeyev on 10/24/21.
+//
+
+#ifndef TEST_BENCHMARKS_CHRONO_H
+#define TEST_BENCHMARKS_CHRONO_H
+
+#include <chrono>
+
+using time_point = std::chrono::high_resolution_clock::time_point;
+
+inline time_point now() { return std::chrono::high_resolution_clock::now(); }
+
+inline std::chrono::system_clock::time_point system_now() {
+  return std::chrono::system_clock::now();
+}
+
+inline int64_t duration_in_mus(time_point const &t0, time_point const &t1) {
+  return std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
+}
+
+#endif // TEST_BENCHMARKS_CHRONO_H
diff --git a/ttg/ttg/make_tt.h b/ttg/ttg/make_tt.h
index ef52228b8..5dd18402d 100644
--- a/ttg/ttg/make_tt.h
+++ b/ttg/ttg/make_tt.h
@@ -165,49 +165,50 @@ class CallableWrapTTArgs
 
 protected:
 
-  /// @return coroutine handle<> (if funcT is a coroutine), else void
-  template <typename Key, typename Tuple, std::size_t... S>
-  auto call_func(Key &&key, Tuple &&args_tuple, output_terminalsT &out, std::index_sequence<S...>) {
-    using func_args_t = ttg::meta::tuple_concat_t<std::tuple<const Key &>, input_refs_tuple_type, output_edges_type>;
-
-    auto process_return = [&out](auto &&ret) {
-      static_assert(std::is_same_v<std::remove_reference_t<decltype(ret)>, returnT>,
-                    "CallableWrapTTArgs<funcT,returnT,...>: returnT does not match the actual return type of funcT");
-      if constexpr (!std::is_void_v<returnT>) {  // protect from compiling for void returnT
+  template<typename ReturnT>
+  auto process_return(ReturnT&& ret, output_terminalsT &out) {
+    static_assert(std::is_same_v<std::remove_reference_t<decltype(ret)>, returnT>,
+                  "CallableWrapTTArgs<funcT,returnT,...>: returnT does not match the actual return type of funcT");
+    if constexpr (!std::is_void_v<returnT>) {  // protect from compiling for void returnT
 #ifdef TTG_HAS_COROUTINE
-        if constexpr (std::is_same_v<returnT, ttg::resumable_task>) {
-          ttg::coroutine_handle<> coro_handle;
-          // if task completed destroy it
-          if (ret.completed()) {
-            ret.destroy();
-          } else {  // if task is suspended return the coroutine promise ptr
-            coro_handle = ret;
-          }
-          return coro_handle;
-        } else if constexpr (std::is_same_v<returnT, ttg::device_task>) {
-          ttg::device_task::base_type coro_handle = ret;
-          return coro_handle;
+      if constexpr (std::is_same_v<returnT, ttg::resumable_task>) {
+        ttg::coroutine_handle<> coro_handle;
+        // if task completed destroy it
+        if (ret.completed()) {
+          ret.destroy();
+        } else {  // if task is suspended return the coroutine promise ptr
+          coro_handle = ret;
         }
-        if constexpr (!(std::is_same_v<returnT, ttg::resumable_task> || std::is_same_v<returnT, ttg::device_task>))
+        return coro_handle;
+      } else if constexpr (std::is_same_v<returnT, ttg::device_task>) {
+        ttg::device_task::base_type coro_handle = ret;
+        return coro_handle;
+      }
+      if constexpr (!(std::is_same_v<returnT, ttg::resumable_task> || std::is_same_v<returnT, ttg::device_task>))
 #endif
-        {
-          static_assert(std::tuple_size_v<std::remove_reference_t<decltype(out)>> == 1,
-                        "CallableWrapTTArgs<funcT,returnT,funcT_receives_outterm_tuple=true,...): funcT can return a "
-                        "value only if there is only 1 out terminal");
-          static_assert(std::tuple_size_v<returnT> <= 2,
-                        "CallableWrapTTArgs<funcT,returnT,funcT_receives_outterm_tuple=true,...): funcT can return a "
-                        "value only if it is a plain value (then sent with null key), a tuple-like containing a single "
-                        "key (hence value is void), or a tuple-like containing a key and a value");
-          if constexpr (std::tuple_size_v<returnT> == 0)
-            std::get<0>(out).sendv(std::move(ret));
-          else if constexpr (std::tuple_size_v<returnT> == 1)
-            std::get<0>(out).sendk(std::move(std::get<0>(ret)));
-          else if constexpr (std::tuple_size_v<returnT> == 2)
-            std::get<0>(out).send(std::move(std::get<0>(ret)), std::move(std::get<1>(ret)));
-          return;
-        }
+      {
+        static_assert(std::tuple_size_v<std::remove_reference_t<decltype(out)>> == 1,
+                      "CallableWrapTTArgs<funcT,returnT,funcT_receives_outterm_tuple=true,...): funcT can return a "
+                      "value only if there is only 1 out terminal");
+        static_assert(std::tuple_size_v<returnT> <= 2,
+                      "CallableWrapTTArgs<funcT,returnT,funcT_receives_outterm_tuple=true,...): funcT can return a "
+                      "value only if it is a plain value (then sent with null key), a tuple-like containing a single "
+                      "key (hence value is void), or a tuple-like containing a key and a value");
+        if constexpr (std::tuple_size_v<returnT> == 0)
+          std::get<0>(out).sendv(std::move(ret));
+        else if constexpr (std::tuple_size_v<returnT> == 1)
+          std::get<0>(out).sendk(std::move(std::get<0>(ret)));
+        else if constexpr (std::tuple_size_v<returnT> == 2)
+          std::get<0>(out).send(std::move(std::get<0>(ret)), std::move(std::get<1>(ret)));
+        return;
       }
-    };
+    }
+  }
+
+  /// @return coroutine handle<> (if funcT is a coroutine), else void
+  template <typename Key, typename Tuple, std::size_t... S>
+  auto call_func(Key &&key, Tuple &&args_tuple, output_terminalsT &out, std::index_sequence<S...>) {
+    using func_args_t = ttg::meta::tuple_concat_t<std::tuple<const Key &>, input_refs_tuple_type, output_edges_type>;
 
     if constexpr (funcT_receives_outterm_tuple) {
       if constexpr (std::is_void_v<returnT>) {
@@ -219,7 +220,7 @@ class CallableWrapTTArgs
             std::forward<Key>(key),
             baseT::template get<S, std::tuple_element_t<S + 1, func_args_t>>(std::forward<Tuple>(args_tuple))..., out);
 
-        return process_return(std::move(ret));
+        return process_return(std::move(ret), out);
       }
     } else {
       auto old_output_tls_ptr = this->outputs_tls_ptr_accessor();
@@ -234,45 +235,78 @@ class CallableWrapTTArgs
             func(std::forward<Key>(key),
                  baseT::template get<S, std::tuple_element_t<S + 1, func_args_t>>(std::forward<Tuple>(args_tuple))...);
         this->set_outputs_tls_ptr(old_output_tls_ptr);
-        return process_return(std::move(ret));
+        return process_return(std::move(ret), out);
       }
     }
   }
 
   template <typename Tuple, std::size_t... S>
-  void call_func(Tuple &&args_tuple, output_terminalsT &out, std::index_sequence<S...>) {
+  auto call_func(Tuple &&args_tuple, output_terminalsT &out, std::index_sequence<S...>) {
     using func_args_t = ttg::meta::tuple_concat_t<input_refs_tuple_type, output_edges_type>;
-    if constexpr (funcT_receives_outterm_tuple)
-      func(baseT::template get<S, std::tuple_element_t<S, func_args_t>>(std::forward<Tuple>(args_tuple))..., out);
-    else {
+    if constexpr (funcT_receives_outterm_tuple) {
+      if constexpr (std::is_void_v<returnT>) {
+        func(baseT::template get<S, std::tuple_element_t<S, func_args_t>>(std::forward<Tuple>(args_tuple))..., out);
+      } else {
+        auto ret = func(baseT::template get<S, std::tuple_element_t<S, func_args_t>>(std::forward<Tuple>(args_tuple))..., out);
+        return process_return(std::move(ret), out);
+      }
+    } else {
       auto old_output_tls_ptr = this->outputs_tls_ptr_accessor();
       this->set_outputs_tls_ptr();
-      func(baseT::template get<S, std::tuple_element_t<S, func_args_t>>(std::forward<Tuple>(args_tuple))...);
-      this->set_outputs_tls_ptr(old_output_tls_ptr);
+      if constexpr (std::is_void_v<returnT>) {
+        func(baseT::template get<S, std::tuple_element_t<S, func_args_t>>(std::forward<Tuple>(args_tuple))...);
+        this->set_outputs_tls_ptr(old_output_tls_ptr);
+      } else {
+        auto ret = func(baseT::template get<S, std::tuple_element_t<S, func_args_t>>(std::forward<Tuple>(args_tuple))...);
+        this->set_outputs_tls_ptr(old_output_tls_ptr);
+        return process_return(std::move(ret), out);
+      }
     }
   }
 
   template <typename Key>
-  void call_func(Key &&key, output_terminalsT &out) {
-    if constexpr (funcT_receives_outterm_tuple)
-      func(std::forward<Key>(key), out);
-    else {
+  auto call_func(Key &&key, output_terminalsT &out) {
+    if constexpr (funcT_receives_outterm_tuple) {
+      if constexpr (std::is_void_v<returnT>) {
+        func(std::forward<Key>(key), out);
+      } else {
+        auto ret = func(std::forward<Key>(key), out);
+        return process_return(std::move(ret), out);
+      }
+    } else {
       auto old_output_tls_ptr = this->outputs_tls_ptr_accessor();
       this->set_outputs_tls_ptr();
-      func(std::forward<Key>(key));
-      this->set_outputs_tls_ptr(old_output_tls_ptr);
+      if constexpr (std::is_void_v<returnT>) {
+        func(std::forward<Key>(key));
+        this->set_outputs_tls_ptr(old_output_tls_ptr);
+      } else {
+        auto ret = func(std::forward<Key>(key));
+        this->set_outputs_tls_ptr(old_output_tls_ptr);
+        return process_return(std::move(ret), out);
+      }
     }
   }
 
   template <typename OutputTerminals>
-  void call_func(OutputTerminals &out) {
-    if constexpr (funcT_receives_outterm_tuple)
-      func(out);
-    else {
+  auto call_func(OutputTerminals &out) {
+    if constexpr (funcT_receives_outterm_tuple) {
+      if constexpr (std::is_void_v<returnT>) {
+        func(out);
+      } else {
+        auto ret = func(out);
+        return process_return(std::move(ret), out);
+      }
+    } else {
       auto old_output_tls_ptr = this->outputs_tls_ptr_accessor();
       this->set_outputs_tls_ptr();
-      func();
-      this->set_outputs_tls_ptr(old_output_tls_ptr);
+      if constexpr (std::is_void_v<returnT>) {
+        func();
+        this->set_outputs_tls_ptr(old_output_tls_ptr);
+      } else {
+        auto ret = func(out);
+        this->set_outputs_tls_ptr(old_output_tls_ptr);
+        return process_return(std::move(ret), out);
+      }
     }
   }
 
diff --git a/ttg/ttg/parsec/devicefunc.h b/ttg/ttg/parsec/devicefunc.h
index 773218e4d..d250e060e 100644
--- a/ttg/ttg/parsec/devicefunc.h
+++ b/ttg/ttg/parsec/devicefunc.h
@@ -78,6 +78,7 @@ namespace ttg_parsec {
    * is current on the target device, false if transfers are required. */
   template<typename... Views>
   inline bool register_device_memory(std::tuple<Views&...> &views) {
+    bool is_current = true;
     if (nullptr == detail::parsec_ttg_caller) {
       throw std::runtime_error("register_device_memory may only be invoked from inside a task!");
     }
@@ -86,7 +87,9 @@ namespace ttg_parsec {
       throw std::runtime_error("register_device_memory called inside a non-gpu task!");
     }
 
-    bool is_current = detail::register_device_memory(views, std::index_sequence_for<Views...>{});
+    if constexpr (sizeof...(Views) > 0) {
+      is_current = detail::register_device_memory(views, std::index_sequence_for<Views...>{});
+    }
 
     /* reset all entries in the current task */
     for (int i = sizeof...(Views); i < MAX_PARAM_COUNT; ++i) {

From d1a6ef3ef935634b61011ad132a6a6f2924cf7a3 Mon Sep 17 00:00:00 2001
From: Thomas Herault <herault@icl.utk.edu>
Date: Wed, 6 Sep 2023 13:57:26 -0400
Subject: [PATCH 125/259] User needs to include any std:: serialization they
 use

---
 examples/potrf/testing_dlauum.cc | 1 +
 examples/potrf/testing_dpoinv.cc | 1 +
 examples/potrf/testing_dpotrf.cc | 1 +
 examples/potrf/testing_dtrtri.cc | 1 +
 4 files changed, 4 insertions(+)

diff --git a/examples/potrf/testing_dlauum.cc b/examples/potrf/testing_dlauum.cc
index 51dc7f81a..717600094 100644
--- a/examples/potrf/testing_dlauum.cc
+++ b/examples/potrf/testing_dlauum.cc
@@ -6,6 +6,7 @@
 #endif  // TTG_USE_PARSEC
 
 #include <ttg.h>
+#include <ttg/serialization/std/tuple.h>
 
 #include "lauum.h"
 #include "plgsy.h"
diff --git a/examples/potrf/testing_dpoinv.cc b/examples/potrf/testing_dpoinv.cc
index 550e6435d..a1e419cc8 100644
--- a/examples/potrf/testing_dpoinv.cc
+++ b/examples/potrf/testing_dpoinv.cc
@@ -6,6 +6,7 @@
 #endif  // TTG_USE_PARSEC
 
 #include <ttg.h>
+#include <ttg/serialization/std/tuple.h>
 
 #include "plgsy.h"
 #include "pmw.h"
diff --git a/examples/potrf/testing_dpotrf.cc b/examples/potrf/testing_dpotrf.cc
index 5cef0bd20..d5686042f 100644
--- a/examples/potrf/testing_dpotrf.cc
+++ b/examples/potrf/testing_dpotrf.cc
@@ -1,4 +1,5 @@
 #include <ttg.h>
+#include <ttg/serialization/std/tuple.h>
 
 #include "plgsy.h"
 #include "pmw.h"
diff --git a/examples/potrf/testing_dtrtri.cc b/examples/potrf/testing_dtrtri.cc
index 4a24dec4a..107fbf8c2 100644
--- a/examples/potrf/testing_dtrtri.cc
+++ b/examples/potrf/testing_dtrtri.cc
@@ -6,6 +6,7 @@
 #endif  // TTG_USE_PARSEC
 
 #include <ttg.h>
+#include <ttg/serialization/std/tuple.h>
 
 #include "plgsy.h"
 #include "pmw.h"

From 7576321f426a18f60d4c46a1a7f197202fea8935 Mon Sep 17 00:00:00 2001
From: Thomas Herault <herault@icl.utk.edu>
Date: Fri, 29 Sep 2023 12:12:05 -0400
Subject: [PATCH 126/259] Work in progress: adding CUDA version of POTRF task
 in POTRF example -- not compiling

---
 examples/matrixtile.h  | 46 ++++++++++------------
 examples/potrf/potrf.h | 86 +++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 104 insertions(+), 28 deletions(-)

diff --git a/examples/matrixtile.h b/examples/matrixtile.h
index e23007803..b3e5a3b8e 100644
--- a/examples/matrixtile.h
+++ b/examples/matrixtile.h
@@ -6,12 +6,13 @@
 
 #include <ttg/serialization/splitmd_data_descriptor.h>
 
-template <typename T>
-class MatrixTile {
+template <typename T, class _Storage>
+class MatrixTile : public ttg::TTValue<MatrixTile<T, _Storage>> {
  public:
   using metadata_t = typename std::tuple<int, int, int>;
 
-  using pointer_t = typename std::shared_ptr<T>;
+  using pointer_t = typename ttg::buffer<T>;
+  using ttvalue_type = ttg::TTValue<MatrixTile<T, _Storage>>;
 
  private:
   pointer_t _data;
@@ -20,18 +21,18 @@ class MatrixTile {
   // (Re)allocate the tile memory
   void realloc() {
     // std::cout << "Reallocating new tile" << std::endl;
-    _data = std::shared_ptr<T>(new T[_lda * _cols], [](T* p) { delete[] p; });
+    _data = ttg::buffer<T>(_data, _lda * _cols);
   }
 
  public:
   MatrixTile() {}
 
-  MatrixTile(int rows, int cols, int lda) : _rows(rows), _cols(cols), _lda(lda) { realloc(); }
+  MatrixTile(int rows, int cols, int lda) : ttvalue_type(), _rows(rows), _cols(cols), _lda(lda) { realloc(); }
 
   MatrixTile(const metadata_t& metadata)
       : MatrixTile(std::get<0>(metadata), std::get<1>(metadata), std::get<2>(metadata)) {}
 
-  MatrixTile(int rows, int cols, pointer_t data, int lda) : _data(data), _rows(rows), _cols(cols), _lda(lda) {}
+  MatrixTile(int rows, int cols, pointer_t data, int lda) : ttvalue_type(), _data(data), _rows(rows), _cols(cols), _lda(lda) {}
 
   MatrixTile(const metadata_t& metadata, pointer_t data)
       : MatrixTile(std::get<0>(metadata), std::get<1>(metadata), std::forward(data), std::get<2>(metadata)) {}
@@ -40,38 +41,20 @@ class MatrixTile {
    * Constructor with outside memory. The tile will *not* delete this memory
    * upon destruction.
    */
-  MatrixTile(int rows, int cols, T* data, int lda) : _data(data, [](T*) {}), _rows(rows), _cols(cols), _lda(lda) {}
+  MatrixTile(int rows, int cols, T* data, int lda) : ttvalue_type(), _data(data), _rows(rows), _cols(cols), _lda(lda) {}
 
   MatrixTile(const metadata_t& metadata, T* data)
       : MatrixTile(std::get<0>(metadata), std::get<1>(metadata), data, std::get<2>(metadata)) {}
 
-#if 0
-  /* Copy dtor and operator with a static_assert to catch unexpected copying */
-  MatrixTile(const MatrixTile& other) {
-    static_assert("Oops, copy ctor called?!");
-  }
-
-  MatrixTile& operator=(const MatrixTile& other) {
-    static_assert("Oops, copy ctor called?!");
-  }
-#endif
-
   MatrixTile(MatrixTile<T>&& other) = default;
 
   MatrixTile& operator=(MatrixTile<T>&& other) = default;
 
-#if 0
-  /* Defaulted copy ctor and op for shallow copies, see comment below */
-  MatrixTile(const MatrixTile<T>& other)  = default;
-
-  MatrixTile& operator=(const MatrixTile<T>& other)  = default;
-#endif  // 0
   /* Deep copy ctor und op are not needed for PO since tiles will never be read
    * and written concurrently. Hence shallow copies are enough, will all
    * receiving tasks sharing tile data. Re-enable this once the PaRSEC backend
    * can handle data sharing without excessive copying */
-#if 1
-  MatrixTile(const MatrixTile<T>& other) : _rows(other._rows), _cols(other._cols), _lda(other._lda) {
+  MatrixTile(const MatrixTile<T>& other) : ttvalue_type(), _rows(other._rows), _cols(other._cols), _lda(other._lda) {
     this->realloc();
     std::copy_n(other.data(), _lda * _cols, this->data());
   }
@@ -84,7 +67,6 @@ class MatrixTile {
     std::copy_n(other.data(), _lda * _cols, this->data());
     return *this;
   }
-#endif  // 1
 
   void set_metadata(metadata_t meta) {
     _rows = std::get<0>(meta);
@@ -144,6 +126,16 @@ class MatrixTile {
   }
 };
 
+#if defined(TTG_HAVE_CUDA)
+using blk_t = DeviceTensor<double, btas::DEFAULT::range,
+                           btas::mohndle<btas::varray<double, TiledArray::cuda_pinned_allocator<double>>,
+                                         btas::Handle::shared_ptr>>;
+#else
+// TODO: no hip pinned allocator in TA?
+using blk_t = DeviceTensor<double, btas::DEFAULT::range,
+                           btas::mohndle<btas::varray<double>, btas::Handle::shared_ptr>>;
+#endif
+
 namespace ttg {
 
   template <typename T>
diff --git a/examples/potrf/potrf.h b/examples/potrf/potrf.h
index 6079bc6bc..e536c655a 100644
--- a/examples/potrf/potrf.h
+++ b/examples/potrf/potrf.h
@@ -19,7 +19,90 @@ namespace potrf {
                   ttg::Edge<Key1, MatrixTile<typename MatrixT::element_type>>& input,
                   ttg::Edge<Key2, MatrixTile<typename MatrixT::element_type>>& output_trsm,
                   ttg::Edge<Key2, MatrixTile<typename MatrixT::element_type>>& output_result) {
-    using T = typename MatrixT::element_type;
+      using T = typename MatrixT::element_type;
+#if defined(TTG_HAS_CUDART) || defined(TTG_HAS_HIP)
+    static int device_potrf_workspace_size(blk_t &A) {
+      int Lwork;
+      #if defined(TTG_HAVE_CUDA)
+        cusolverDnDpotrf_bufferSize(ttg::detail::cublas_get_handle(),
+                                    CUBLAS_FILL_MODE_LOWER, A.extent(1),
+                                    nullptr, A.extent(0),
+                                    &Lwork);
+        return Lwork;
+      #elif defined(TTG_HAVE_HIPBLAS)
+        #error TBCoded
+      #else
+        return 0;
+      #endif
+    }
+
+    static void device_potrf(blk_t &A, double *workspace, int Lwork, int *devInfo) {
+      int device = A.b.get_current_device();
+      assert(device != 0);
+    #if defined(TTG_HAVE_CUDA)
+      cusolverDnDpotrf(ttg::detail::cublas_get_handle(),
+                       CUBLAS_FILL_MODE_LOWER, A.extent(1),
+                       A.b.device_ptr_on(device), A.extent(0),
+                       workspace, Lwork,
+                       devInfo);
+    #elif defined(TTG_HAVE_HIPBLAS)
+      hipsolverDpotrf(ttg::detail::hipblas_get_handle(),
+                       HIPSOLVER_FILL_MODE_LOWER, A.extent(1),
+                       A.b.device_ptr_on(device), A.extent(0),
+                       workspace, Lwork,
+                       devInfo);
+    #endif
+    }
+
+    auto f_dev = [=](const Key1& key, MatrixTile<T>&& A,
+                     std::tuple<ttg::Out<Key2, MatrixTile<T>>, ttg::Out<Key2, MatrixTile<T>>>& out) -> ttg::device_task {
+      const auto K = key[0]; 
+
+      /* pull the matrix onto the device, as computing the workspace size might in theory depend on the data */
+      //TODO: extend MatrixTile<T> to be heterogeneous-aware. Look at spmm-cuda.cc 50-253
+      //       Need to include a ttg::buffer<T> _data instead of a shared_ptr;
+      //       Check pmw.h: when we generate the MatrixTile
+      //       Also check pinned allocator at the end of DeviceTensor (250-253)
+
+      int Lwork = device_potrf_workspace_size(A);
+
+      // Instead of using scratch here, we should have hostWS and hostInfo globals and use to_device
+      // this would reduce the number of I/O operations to devices
+      double hostWS[Lwork];
+      ttg::devicescratch<double> devWS = ttg::make_scratch(hostWS, ttg::scope::Allocate);
+      int hostInfo = -1;
+      ttg::devicescratch<int> devInfo = ttg::make_scratch(&hostInfo, ttg::scope::Allocate);
+
+      /* the workspace and the devInfo must be device-level pointers */
+      co_await ttg::to_device(A._data, devWS, devInfo);
+
+      /* everything is on the device, call the POTRF */
+      device_potrf(A, devWS, Lwork, devInfo);
+
+      /* compute successors while the kernel is running */
+      std::vector<Key2> keylist;
+      keylist.reserve(A.rows() - K);
+      /* TODO: reverse order of arrays */
+      for (int m = K + 1; m < A.rows(); ++m) {
+        /* send tile to trsm */
+        keylist.push_back(Key2(m, K));
+      }
+
+      /* wait for the kernel to complete */
+      co_await ttg::wait_kernel(devInfo);
+
+      if( hostInfo == 0 ) {
+        co_await ttg::device::forward(ttg::device::broadcast<0, 1>(std::make_tuple(Key2(K, K), keylist), std::move(A), out));
+        // Anything after this co_await is never executed
+        // co_return would look better, but co_return would destroy keylist before the runtime can handle it
+      } else {
+        // Well... Here we should interrupt the DAG of tasks, there is an error. Raise?
+        std::cerr << "Factorization is SUSPICIOUS (the matrix might not be diagonal dominant)" << std::endl;
+      }
+    }
+    return ttg::make_tt<ttg::ExecutionSpace::CUDA>(f_dev, ttg::edges(ttg::fuse(input, input_disp)), ttg::edges(output_result, output_trsm), "POTRF",
+                        {"tile_kk/dispatcher"}, {"output_result", "output_trsm"});
+#else /* defined(TTG_HAS_CUDART) || defined(TTG_HAS_HIP) */
     auto f = [=](const Key1& key, MatrixTile<T>&& tile_kk,
                  std::tuple<ttg::Out<Key2, MatrixTile<T>>, ttg::Out<Key2, MatrixTile<T>>>& out) {
       const int K = key[0];
@@ -49,6 +132,7 @@ namespace potrf {
     };
     return ttg::make_tt(f, ttg::edges(ttg::fuse(input, input_disp)), ttg::edges(output_result, output_trsm), "POTRF",
                         {"tile_kk/dispatcher"}, {"output_result", "output_trsm"});
+#endif
   }
 
   template <typename MatrixT>

From 4a23f42ba4465c0ffe7697f667f1e76a67a10174 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Sun, 1 Oct 2023 19:27:27 +0000
Subject: [PATCH 127/259] Further steps towards device-enabled POTRF

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/matrixtile.h  |  64 ++++++------
 examples/potrf/potrf.h | 216 ++++++++++++++++++++++++++++++++++++++---
 2 files changed, 230 insertions(+), 50 deletions(-)

diff --git a/examples/matrixtile.h b/examples/matrixtile.h
index b3e5a3b8e..929b6a07e 100644
--- a/examples/matrixtile.h
+++ b/examples/matrixtile.h
@@ -6,34 +6,38 @@
 
 #include <ttg/serialization/splitmd_data_descriptor.h>
 
-template <typename T, class _Storage>
+template <typename T, class _Storage = TiledArray::cuda_pinned_allocator<double>>
 class MatrixTile : public ttg::TTValue<MatrixTile<T, _Storage>> {
  public:
   using metadata_t = typename std::tuple<int, int, int>;
 
-  using pointer_t = typename ttg::buffer<T>;
+  using buffer_t  = typename ttg::buffer<T>;
   using ttvalue_type = ttg::TTValue<MatrixTile<T, _Storage>>;
 
  private:
-  pointer_t _data;
+  buffer_t b;
   int _rows = 0, _cols = 0, _lda = 0;
 
   // (Re)allocate the tile memory
   void realloc() {
     // std::cout << "Reallocating new tile" << std::endl;
-    _data = ttg::buffer<T>(_data, _lda * _cols);
+    b.reset(_lda * _cols);
   }
 
  public:
   MatrixTile() {}
 
-  MatrixTile(int rows, int cols, int lda) : ttvalue_type(), _rows(rows), _cols(cols), _lda(lda) { realloc(); }
+  MatrixTile(int rows, int cols, int lda)
+  : ttvalue_type()
+  , b(lda*cols)
+  , _rows(rows)
+  , _cols(cols)
+  , _lda(lda)
+  { }
 
   MatrixTile(const metadata_t& metadata)
       : MatrixTile(std::get<0>(metadata), std::get<1>(metadata), std::get<2>(metadata)) {}
 
-  MatrixTile(int rows, int cols, pointer_t data, int lda) : ttvalue_type(), _data(data), _rows(rows), _cols(cols), _lda(lda) {}
-
   MatrixTile(const metadata_t& metadata, pointer_t data)
       : MatrixTile(std::get<0>(metadata), std::get<1>(metadata), std::forward(data), std::get<2>(metadata)) {}
 
@@ -41,7 +45,13 @@ class MatrixTile : public ttg::TTValue<MatrixTile<T, _Storage>> {
    * Constructor with outside memory. The tile will *not* delete this memory
    * upon destruction.
    */
-  MatrixTile(int rows, int cols, T* data, int lda) : ttvalue_type(), _data(data), _rows(rows), _cols(cols), _lda(lda) {}
+  MatrixTile(int rows, int cols, T* data, int lda)
+  : ttvalue_type()
+  , _data(data, lda*cols)
+  , _rows(rows)
+  , _cols(cols)
+  , _lda(lda)
+  { }
 
   MatrixTile(const metadata_t& metadata, T* data)
       : MatrixTile(std::get<0>(metadata), std::get<1>(metadata), data, std::get<2>(metadata)) {}
@@ -54,8 +64,12 @@ class MatrixTile : public ttg::TTValue<MatrixTile<T, _Storage>> {
    * and written concurrently. Hence shallow copies are enough, will all
    * receiving tasks sharing tile data. Re-enable this once the PaRSEC backend
    * can handle data sharing without excessive copying */
-  MatrixTile(const MatrixTile<T>& other) : ttvalue_type(), _rows(other._rows), _cols(other._cols), _lda(other._lda) {
-    this->realloc();
+  MatrixTile(const MatrixTile<T>& other)
+  : ttvalue_type()
+  , b(other._lda*other._cols)
+  , _rows(other._rows)
+  , _cols(other._cols)
+  , _lda(other._lda) {
     std::copy_n(other.data(), _lda * _cols, this->data());
   }
 
@@ -77,22 +91,9 @@ class MatrixTile : public ttg::TTValue<MatrixTile<T, _Storage>> {
   metadata_t get_metadata(void) const { return metadata_t{_rows, _cols, _lda}; }
 
   // Accessing the raw data
-  T* data() { return _data.get(); }
-
-  const T* data() const { return _data.get(); }
-
-  /// @return shared_ptr to data
-  pointer_t data_shared() & { return _data; }
+  T* data() { return b.host_ptr(); }
 
-  /// @return shared_ptr to data
-  pointer_t data_shared() const& { return _data; }
-
-  /// yields data and resets this object to a default-constucted state
-  pointer_t yield_data() && {
-    pointer_t result = _data;
-    *this = MatrixTile();
-    return std::move(result);
-  }
+  const T* data() const { return b.host_ptr(); }
 
   size_t size() const { return _cols * _lda; }
 
@@ -103,7 +104,8 @@ class MatrixTile : public ttg::TTValue<MatrixTile<T, _Storage>> {
   int lda() const { return _lda; }
 
   auto& fill(T value) {
-    std::fill(_data.get(), _data.get() + size(), value);
+    std::fill(data().get(), data().get() + size(), value);
+    b.set_current_device(0);
     return *this;
   }
 
@@ -126,16 +128,6 @@ class MatrixTile : public ttg::TTValue<MatrixTile<T, _Storage>> {
   }
 };
 
-#if defined(TTG_HAVE_CUDA)
-using blk_t = DeviceTensor<double, btas::DEFAULT::range,
-                           btas::mohndle<btas::varray<double, TiledArray::cuda_pinned_allocator<double>>,
-                                         btas::Handle::shared_ptr>>;
-#else
-// TODO: no hip pinned allocator in TA?
-using blk_t = DeviceTensor<double, btas::DEFAULT::range,
-                           btas::mohndle<btas::varray<double>, btas::Handle::shared_ptr>>;
-#endif
-
 namespace ttg {
 
   template <typename T>
diff --git a/examples/potrf/potrf.h b/examples/potrf/potrf.h
index e536c655a..e6966bb2b 100644
--- a/examples/potrf/potrf.h
+++ b/examples/potrf/potrf.h
@@ -6,6 +6,19 @@
 
 #undef DEBUG_TILES_VALUES
 
+#if defined(TTG_HAS_CUDART)
+#define ES ttg::ExecutionSpace::CUDA
+#define TASKRET -> ttg::device_task
+#elif defined(TTG_HAS_HIP)
+#define ES ttg::ExecutionSpace::HIP
+#define TASKRET -> ttg::device_task
+#else
+#define ES ttg::ExecutionSpace::Host
+#define TASKRET -> void
+#endif
+
+
+
 namespace potrf {
 
   /* FLOP macros taken from DPLASMA */
@@ -55,8 +68,19 @@ namespace potrf {
     }
 
     auto f_dev = [=](const Key1& key, MatrixTile<T>&& A,
-                     std::tuple<ttg::Out<Key2, MatrixTile<T>>, ttg::Out<Key2, MatrixTile<T>>>& out) -> ttg::device_task {
-      const auto K = key[0]; 
+                     std::tuple<ttg::Out<Key2, MatrixTile<T>>, ttg::Out<Key2, MatrixTile<T>>>& out) TASKRET {
+      const auto K = key[0];
+
+      /* compute successors before submitting the kernel running
+       * TODO: this is parsec specific since this code is still executing on the worker threads
+       */
+      std::vector<Key2> keylist;
+      keylist.reserve(A.rows() - K);
+      /* TODO: reverse order of arrays */
+      for (int m = K + 1; m < A.rows(); ++m) {
+        /* send tile to trsm */
+        keylist.push_back(Key2(m, K));
+      }
 
       /* pull the matrix onto the device, as computing the workspace size might in theory depend on the data */
       //TODO: extend MatrixTile<T> to be heterogeneous-aware. Look at spmm-cuda.cc 50-253
@@ -79,28 +103,20 @@ namespace potrf {
       /* everything is on the device, call the POTRF */
       device_potrf(A, devWS, Lwork, devInfo);
 
-      /* compute successors while the kernel is running */
-      std::vector<Key2> keylist;
-      keylist.reserve(A.rows() - K);
-      /* TODO: reverse order of arrays */
-      for (int m = K + 1; m < A.rows(); ++m) {
-        /* send tile to trsm */
-        keylist.push_back(Key2(m, K));
-      }
-
       /* wait for the kernel to complete */
       co_await ttg::wait_kernel(devInfo);
 
       if( hostInfo == 0 ) {
-        co_await ttg::device::forward(ttg::device::broadcast<0, 1>(std::make_tuple(Key2(K, K), keylist), std::move(A), out));
+        co_await ttg::device::forward(ttg::device::broadcast<0, 1>(std::make_tuple(Key2(K, K), std::move(keylist)), std::move(A), out));
         // Anything after this co_await is never executed
         // co_return would look better, but co_return would destroy keylist before the runtime can handle it
       } else {
         // Well... Here we should interrupt the DAG of tasks, there is an error. Raise?
-        std::cerr << "Factorization is SUSPICIOUS (the matrix might not be diagonal dominant)" << std::endl;
+        std::cerr << "Factorization is SUSPICIOUS (the matrix might not be diagonally dominant)" << std::endl;
+        ttg::abort();
       }
     }
-    return ttg::make_tt<ttg::ExecutionSpace::CUDA>(f_dev, ttg::edges(ttg::fuse(input, input_disp)), ttg::edges(output_result, output_trsm), "POTRF",
+    return ttg::make_tt<ES>(f_dev, ttg::edges(ttg::fuse(input, input_disp)), ttg::edges(output_result, output_trsm), "POTRF",
                         {"tile_kk/dispatcher"}, {"output_result", "output_trsm"});
 #else /* defined(TTG_HAS_CUDART) || defined(TTG_HAS_HIP) */
     auto f = [=](const Key1& key, MatrixTile<T>&& tile_kk,
@@ -145,6 +161,69 @@ namespace potrf {
                  ttg::Edge<Key3, MatrixTile<typename MatrixT::element_type>>& output_col,   // to GEMM
                  ttg::Edge<Key2, MatrixTile<typename MatrixT::element_type>>& output_result) {
     using T = typename MatrixT::element_type;
+#if defined(TTG_HAS_CUDART) || defined(TTG_HAS_HIP)
+    auto f = [=](const Key2& key, const MatrixTile<T>& tile_kk, MatrixTile<T>&& tile_mk,
+                 std::tuple<ttg::Out<Key2, MatrixTile<T>>, ttg::Out<Key2, MatrixTile<T>>, ttg::Out<Key3, MatrixTile<T>>,
+                            ttg::Out<Key3, MatrixTile<T>>>& out) TASKRET {
+      const int M = key[0];
+      const int K = key[1];  // the column equals the outer most look K (same as PO)
+
+      auto mb = tile_mk.rows();
+      auto nb = tile_mk.cols();
+
+      /* in trsm, tile_mk is mb x nb, and tile_kk needs to be lda x nb because side = Right */
+      assert(nb == tile_kk.rows());
+
+      if (ttg::tracing()) ttg::print("TRSM(", key, ")");
+
+      /* populate successor keys while we're on the worker thread */
+      std::vector<Key3> keylist_row;
+      keylist_row.reserve(M - K);
+      std::vector<Key3> keylist_col;
+      keylist_col.reserve(A.rows() - M - 1);
+
+      /* send tile to syrk on diagonal */
+      if (ttg::tracing()) ttg::print("TRSM(", key, "): sending output to syrk(", Key2{K, M}, ")");
+
+      /* send the tile to all gemms across in row i */
+      for (int n = K + 1; n < M; ++n) {
+        if (ttg::tracing()) ttg::print("TRSM(", key, "): sending output to gemm( ", Key3{M, n, K}, ")");
+        keylist_row.push_back(Key3(M, n, K));
+      }
+
+      /* send the tile to all gemms down in column i */
+      for (int m = M + 1; m < A.rows(); ++m) {
+        if (ttg::tracing()) ttg::print("TRSM(", key, "): sending output to gemm( ", Key3{m, M, K}, ")");
+        keylist_col.push_back(Key3(m, M, K));
+      }
+
+
+      co_await ttg::to_device(tile_kk.b, tile_mk.b);
+      int device = tile_kk.b.get_current_device();
+      double alpha = 1.0;
+#if defined(TTG_HAVE_CUDA)
+      cublasDtrsm(ttg::detail::cublas_get_handle(),
+                  CUBLAS_SIDE_RIGHT, CUBLAS_FILL_MODE_LOWER,
+                  CUBLAS_OP_T, CUBLAS_DIAG_NON_UNIT,
+                  mb, nb, &alpha,
+                  tile_kk.b.device_ptr_on(device), tile_kk.lda(),
+                  tile_mk.b.device_ptr_on(device), tile_mk.lda());
+#elif defined(TTG_HAVE_HIPBLAS)
+      hipblasDtrsm(ttg::detail:hipblas_get_handle(),
+                   HIPBLAS_SIDE_RIGHT, HIPBLAS_FILL_MODE_LOWER,
+                   HIPBLAS_OP_T, HIPBLAS_DIAG_NON_UNIT,
+                   mb, nb, &alpha,
+                   tile_kk.b.device_ptr_on(device), tile_kk.lda(),
+                   tile_mk.b.device_ptr_on(device), tile_mk.lda());
+#endif
+
+      co_await ttg::device::forward(ttg::device::broadcast<0, 1, 2, 3>(std::make_tuple(key, Key2(K, M), keylist_row, keylist_col),
+                                                                       std::move(tile_mk), out));
+    };
+    return ttg::make_tt<ES>(f, ttg::edges(input_kk, ttg::fuse(input_mk, input_disp)),
+                            ttg::edges(output_result, output_diag, output_row, output_col), "TRSM",
+                            {"tile_kk", "tile_mk/dispatcher"}, {"output_result", "tile_mk", "output_row", "output_col"});
+#else
     auto f = [=](const Key2& key, const MatrixTile<T>& tile_kk, MatrixTile<T>&& tile_mk,
                  std::tuple<ttg::Out<Key2, MatrixTile<T>>, ttg::Out<Key2, MatrixTile<T>>, ttg::Out<Key3, MatrixTile<T>>,
                             ttg::Out<Key3, MatrixTile<T>>>& out) {
@@ -195,6 +274,7 @@ namespace potrf {
     return ttg::make_tt(f, ttg::edges(input_kk, ttg::fuse(input_mk, input_disp)),
                         ttg::edges(output_result, output_diag, output_row, output_col), "TRSM",
                         {"tile_kk", "tile_mk/dispatcher"}, {"output_result", "tile_mk", "output_row", "output_col"});
+#endif
   }
 
   template <typename MatrixT>
@@ -205,6 +285,54 @@ namespace potrf {
                  ttg::Edge<Key1, MatrixTile<typename MatrixT::element_type>>& output_potrf,  // to POTRF
                  ttg::Edge<Key2, MatrixTile<typename MatrixT::element_type>>& output_syrk) {
     using T = typename MatrixT::element_type;
+#if defined(TTG_HAS_CUDART) || defined(TTG_HAS_HIP)
+    auto f = [=](const Key2& key, const MatrixTile<T>& tile_mk, MatrixTile<T>&& tile_kk,
+                 std::tuple<ttg::Out<Key1, MatrixTile<T>>, ttg::Out<Key2, MatrixTile<T>>>& out) TASKRET {
+      const int K = key[0];
+      const int M = key[1];
+
+      /* tile_kk is mb x mb and tile_mk is mb x nb */
+      assert(tile_kk.rows() == tile_kk.cols());
+      assert(tile_mk.rows() == tile_kk.rows());
+
+      auto mb = tile_mk.rows();
+      auto nb = tile_mk.cols();
+
+      if (ttg::tracing()) ttg::print("SYRK(", key, ")");
+
+      co_await ttg::to_device(tile_kk.b, tile_mk.b);
+
+      double alpha = -1.0;
+      double beta  =  1.0;
+#if defined(TTG_HAVE_CUDA)
+      cublasDsyrk(ttg::detail::cublas_get_handle(),
+                  CUBLAS_FILL_MODE_LOWER,
+                  CUBLAS_OP_N,
+                  mb, nb, &alpha,
+                  tile_nk.b.device_ptr_on(device), tile_mk.lda(), &beta,
+                  tile_kk.b.device_ptr_on(device), tile_kk.lda());
+#elif defined(TTG_HAVE_HIPBLAS)
+      hipblasDsyrk(ttg::detail:hipblas_get_handle(),
+                   HIPBLAS_FILL_MODE_LOWER,
+                   HIPBLAS_OP_N,
+                   mb, nb, &alpha,
+                   tile_kk.b.device_ptr_on(device), tile_kk.lda(), &beta,
+                   tile_mk.b.device_ptr_on(device), tile_mk.lda());
+#endif
+
+      if (M == K + 1) {
+        /* send the tile to potrf */
+        if (ttg::tracing()) ttg::print("SYRK(", key, "): sending output to POTRF(", Key1{K + 1}, ")");
+        co_await ttg::device::send<0>(Key1(K + 1), std::move(tile_kk), out);
+      } else {
+        /* send output to next syrk */
+        if (ttg::tracing()) ttg::print("SYRK(", key, "): sending output to SYRK(", Key2{K + 1, M}, ")");
+        co_await ttg::device::send<1>(Key2(K + 1, M), std::move(tile_kk), out);
+      }
+    };
+    return ttg::make_tt<ES>(f, ttg::edges(input_mk, ttg::fuse(input_kk, input_disp)), ttg::edges(output_potrf, output_syrk),
+                            "SYRK", {"tile_mk", "tile_kk/dispatcher"}, {"output_potrf", "output_syrk"});
+#else
     auto f = [=](const Key2& key, const MatrixTile<T>& tile_mk, MatrixTile<T>&& tile_kk,
                  std::tuple<ttg::Out<Key1, MatrixTile<T>>, ttg::Out<Key2, MatrixTile<T>>>& out) {
       const int K = key[0];
@@ -242,6 +370,7 @@ namespace potrf {
     };
     return ttg::make_tt(f, ttg::edges(input_mk, ttg::fuse(input_kk, input_disp)), ttg::edges(output_potrf, output_syrk),
                         "SYRK", {"tile_mk", "tile_kk/dispatcher"}, {"output_potrf", "output_syrk"});
+#endif
   }
 
   template <typename MatrixT>
@@ -253,6 +382,64 @@ namespace potrf {
                  ttg::Edge<Key2, MatrixTile<typename MatrixT::element_type>>& output_trsm,  // to TRSM
                  ttg::Edge<Key3, MatrixTile<typename MatrixT::element_type>>& output_gemm) {
     using T = typename MatrixT::element_type;
+#if defined(TTG_HAS_CUDART) || defined(TTG_HAS_HIP)
+    auto f = [=](const Key3& key, const MatrixTile<T>& tile_mk, const MatrixTile<T>& tile_nk, MatrixTile<T>&& tile_mn,
+                 std::tuple<ttg::Out<Key2, MatrixTile<T>>, ttg::Out<Key3, MatrixTile<T>>>& out) TASKRET {
+      const int M = key[0];
+      const int N = key[1];
+      const int K = key[2];
+      assert(M != N && M > K && N > K);
+
+      assert(tile_mk.cols() == tile_nk.cols());
+      assert(tile_mk.rows() == tile_mn.rows());
+      assert(tile_nk.rows() == tile_mn.cols());
+
+      if (ttg::tracing()) ttg::print("GEMM(", key, ")");
+#if defined(DEBUG_TILES_VALUES)
+      std::cout << "Before GEMM(" << key << "), A(" << M << ", " << K << ") is " << tile_mk << " and A(" << K << ", "
+                << N << ") is " << tile_nk << " and A(" << M << ", " << N << ") is " << tile_mn;
+#endif
+
+      co_await ttg::to_device(tile_mk.b, tile_nk.b, tile_mn.b);
+
+      double alpha = -1.0;
+      double beta  =  1.0;
+#if defined(TTG_HAVE_CUDA)
+      cublasDgemm(ttg::detail:cublas_get_handle(),
+                  CUBLAS_OP_N, CUBLAS_OP_T,
+                  tile_mk.rows(), tile_nk.rows(),
+                  tile_nk.cols(), &alpha,
+                  tile_mk.data(), tile_mk.lda(),
+                  tile_nk.data(), tile_nk.lda(), &beta,
+                  tile_mn.data(), tile_mn.lda());
+#elif defined(TTG_HAVE_HIPBLAS)
+      hipblasDgemm(ttg::detail:hipblas_get_handle(),
+                   HIPBLAS_OP_N, HIPBLAS_OP_T,
+                   tile_mk.rows(), tile_nk.rows(),
+                   tile_nk.cols(), &alpha,
+                   tile_mk.data(), tile_mk.lda(),
+                   tile_nk.data(), tile_nk.lda(), &beta,
+                   tile_mn.data(), tile_mn.lda());
+#endif
+
+#if defined(DEBUG_TILES_VALUES)
+      std::cout << "After GEMM(" << key << "), A(" << M << ", " << N << ") is " << tile_mn << std::endl;
+#endif
+
+      if (N == K + 1) {
+        /* send the tile to trsm */
+        if (ttg::tracing()) ttg::print("GEMM(", key, "): sending output to TRSM(", Key2{M, N}, ")");
+        co_await ttg::device::send<0>(Key2(M, N), std::move(tile_mn), out);
+      } else {
+        /* send the tile to the next gemm */
+        if (ttg::tracing()) ttg::print("GEMM(", key, "): sending output to GEMM(", Key3{M, N, K + 1}, ")");
+        co_await ttg::device::send<1>(Key3(M, N, K + 1), std::move(tile_mn), out);
+      }
+    };
+    return ttg::make_tt<ES>(f, ttg::edges(input_mk, input_nk, ttg::fuse(input_disp, input_mn)),
+                            ttg::edges(output_trsm, output_gemm), "GEMM", {"input_mk", "input_kn", "input_mn/dispatcher"},
+                            {"output_trsm", "outout_gemm"});
+#else
     auto f = [=](const Key3& key, const MatrixTile<T>& tile_mk, const MatrixTile<T>& tile_nk, MatrixTile<T>&& tile_mn,
                  std::tuple<ttg::Out<Key2, MatrixTile<T>>, ttg::Out<Key3, MatrixTile<T>>>& out) {
       const int M = key[0];
@@ -291,6 +478,7 @@ namespace potrf {
     return ttg::make_tt(f, ttg::edges(input_mk, input_nk, ttg::fuse(input_disp, input_mn)),
                         ttg::edges(output_trsm, output_gemm), "GEMM", {"input_mk", "input_kn", "input_mn/dispatcher"},
                         {"output_trsm", "outout_gemm"});
+#endif
   }
 
   template <typename T>

From 57faac424de2f510d828196d131ec03a82a5f04d Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 2 Oct 2023 02:16:45 +0000
Subject: [PATCH 128/259] Add allocator to ttg::buffer

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/matrixtile.h   | 25 +++++++-------
 ttg/ttg/buffer.h        |  8 ++---
 ttg/ttg/parsec/buffer.h | 72 ++++++++++++++++++++++-------------------
 ttg/ttg/parsec/fwd.h    |  2 +-
 4 files changed, 55 insertions(+), 52 deletions(-)

diff --git a/examples/matrixtile.h b/examples/matrixtile.h
index 929b6a07e..693ae3d1e 100644
--- a/examples/matrixtile.h
+++ b/examples/matrixtile.h
@@ -6,13 +6,15 @@
 
 #include <ttg/serialization/splitmd_data_descriptor.h>
 
-template <typename T, class _Storage = TiledArray::cuda_pinned_allocator<double>>
-class MatrixTile : public ttg::TTValue<MatrixTile<T, _Storage>> {
+#include <TiledArray/device/allocators.h>
+
+template <typename T, class Allocator = TiledArray::device_pinned_allocator<double>>
+class MatrixTile : public ttg::TTValue<MatrixTile<T, Allocator>> {
  public:
   using metadata_t = typename std::tuple<int, int, int>;
 
-  using buffer_t  = typename ttg::buffer<T>;
-  using ttvalue_type = ttg::TTValue<MatrixTile<T, _Storage>>;
+  using buffer_t  = typename ttg::buffer<T, Allocator>;
+  using ttvalue_type = ttg::TTValue<MatrixTile<T, Allocator>>;
 
  private:
   buffer_t b;
@@ -38,7 +40,7 @@ class MatrixTile : public ttg::TTValue<MatrixTile<T, _Storage>> {
   MatrixTile(const metadata_t& metadata)
       : MatrixTile(std::get<0>(metadata), std::get<1>(metadata), std::get<2>(metadata)) {}
 
-  MatrixTile(const metadata_t& metadata, pointer_t data)
+  MatrixTile(const metadata_t& metadata, T* data)
       : MatrixTile(std::get<0>(metadata), std::get<1>(metadata), std::forward(data), std::get<2>(metadata)) {}
 
   /**
@@ -47,24 +49,21 @@ class MatrixTile : public ttg::TTValue<MatrixTile<T, _Storage>> {
    */
   MatrixTile(int rows, int cols, T* data, int lda)
   : ttvalue_type()
-  , _data(data, lda*cols)
+  , b(data, lda*cols)
   , _rows(rows)
   , _cols(cols)
   , _lda(lda)
   { }
 
-  MatrixTile(const metadata_t& metadata, T* data)
-      : MatrixTile(std::get<0>(metadata), std::get<1>(metadata), data, std::get<2>(metadata)) {}
-
-  MatrixTile(MatrixTile<T>&& other) = default;
+  MatrixTile(MatrixTile<T, Allocator>&& other) = default;
 
-  MatrixTile& operator=(MatrixTile<T>&& other) = default;
+  MatrixTile& operator=(MatrixTile<T, Allocator>&& other) = default;
 
   /* Deep copy ctor und op are not needed for PO since tiles will never be read
    * and written concurrently. Hence shallow copies are enough, will all
    * receiving tasks sharing tile data. Re-enable this once the PaRSEC backend
    * can handle data sharing without excessive copying */
-  MatrixTile(const MatrixTile<T>& other)
+  MatrixTile(const MatrixTile<T, Allocator>& other)
   : ttvalue_type()
   , b(other._lda*other._cols)
   , _rows(other._rows)
@@ -73,7 +72,7 @@ class MatrixTile : public ttg::TTValue<MatrixTile<T, _Storage>> {
     std::copy_n(other.data(), _lda * _cols, this->data());
   }
 
-  MatrixTile& operator=(const MatrixTile<T>& other) {
+  MatrixTile& operator=(const MatrixTile<T, Allocator>& other) {
     this->_rows = other._rows;
     this->_cols = other._cols;
     this->_lda = other._lda;
diff --git a/ttg/ttg/buffer.h b/ttg/ttg/buffer.h
index 1868b7e0c..6d252cb09 100644
--- a/ttg/ttg/buffer.h
+++ b/ttg/ttg/buffer.h
@@ -7,16 +7,16 @@
 
 namespace ttg {
 
-template<typename T>
-using buffer = TTG_IMPL_NS::buffer<T>;
+template<typename T, typename Allocator = std::allocator<T>>
+using buffer = TTG_IMPL_NS::buffer<T, Allocator>;
 
 namespace detail {
   template<typename T>
   struct is_buffer : std::false_type
   { };
 
-  template<typename T>
-  struct is_buffer<ttg::buffer<T>> : std::true_type
+  template<typename T, typename A>
+  struct is_buffer<ttg::buffer<T, A>> : std::true_type
   { };
 
   template<typename T>
diff --git a/ttg/ttg/parsec/buffer.h b/ttg/ttg/parsec/buffer.h
index 0e9c58cf9..00c7c996e 100644
--- a/ttg/ttg/parsec/buffer.h
+++ b/ttg/ttg/parsec/buffer.h
@@ -34,33 +34,43 @@ namespace detail {
  * in order for TTG to properly facilitate ownership
  * tracking of the containing object.
  */
-template<typename T>
-struct buffer : public detail::ttg_parsec_data_wrapper_t {
+template<typename T, typename Allocator>
+struct buffer : public detail::ttg_parsec_data_wrapper_t
+              , private Allocator {
 
   using element_type = std::decay_t<T>;
 
+  using allocator_traits = std::allocator_traits<Allocator>;
+  using allocator_type = typename  allocator_traits::allocator_type;
+
   static_assert(std::is_trivially_copyable_v<element_type>,
                 "Only trivially copyable types are supported for devices.");
   static_assert(std::is_default_constructible_v<element_type>,
                 "Only default constructible types are supported for devices.");
 
 private:
-  using delete_fn_t = std::add_pointer_t<void(element_type*)>;
+  using delete_fn_t = std::function<void(element_type*)>;
 
   using host_data_ptr   = std::unique_ptr<element_type[], delete_fn_t>;
   host_data_ptr m_host_data;
   std::size_t m_count = 0;
 
-  static void delete_owned(element_type *ptr) {
-    delete[] ptr;
-  }
-
   static void delete_non_owned(element_type *ptr) {
     // nothing to be done, we don't own the memory
   }
 
   friend parsec_data_t* detail::get_parsec_data<T>(const ttg_parsec::buffer<T>&);
 
+  allocator_type& get_allocator_reference() { return static_cast<allocator_type&>(*this); }
+
+  element_type* allocate(std::size_t n) {
+    return allocator_traits::allocate(get_allocator_reference(), n);
+  }
+
+  void deallocate(element_type *ptr, std::size_t n) {
+    allocator_traits::deallocate(get_allocator_reference(), ptr, n);
+  }
+
 public:
 
   /* The device ID of the CPU. */
@@ -69,30 +79,29 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t {
   buffer() : buffer(nullptr, 0)
   { }
 
-  buffer(std::size_t count)
+  buffer(std::size_t n)
   : ttg_parsec_data_wrapper_t()
-  , m_host_data(new element_type[count](), &delete_owned)
-  , m_count(count)
+  , allocator_type()
+  , m_host_data(allocate(n), [this, n](element_type* ptr){ deallocate(ptr, n); })
+  , m_count(n)
   {
     //std::cout << "buffer " << this << " ctor count "
     //          << count << "(" << m_host_data.get() << ") ttg_copy "
     //          << m_ttg_copy
     //          << " parsec_data " << m_data.get() << std::endl;
-    this->reset_parsec_data(m_host_data.get(), count*sizeof(element_type));
+    this->reset_parsec_data(m_host_data.get(), n*sizeof(element_type));
   }
 
   /* Constructing a buffer using application-managed memory.
    * The memory pointed to by ptr must be accessible during
    * the life-time of the buffer. */
-  buffer(element_type* ptr, std::size_t count = 1)
+  buffer(element_type* ptr, std::size_t n = 1)
   : ttg_parsec_data_wrapper_t()
+  , allocator_type()
   , m_host_data(ptr, &delete_non_owned)
-  , m_count(count)
+  , m_count(n)
   {
-    this->reset_parsec_data(m_host_data.get(), count*sizeof(element_type));
-    //std::cout << "buffer " << this << " ctor ptr " << ptr << " count " << count
-    //          << " ttg_copy " << m_ttg_copy
-    //          << " parsec_data " << m_data.get() << std::endl;
+    this->reset_parsec_data(m_host_data.get(), n*sizeof(element_type));
   }
 
   virtual ~buffer() {
@@ -102,18 +111,11 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t {
   /* allow moving device buffers */
   buffer(buffer&& db)
   : ttg_parsec_data_wrapper_t(std::move(db))
+  , allocator_type(std::move(db))
   , m_host_data(std::move(db.m_host_data))
   , m_count(db.m_count)
   {
-    //std::cout << "buffer " << this << " other " << &db << " mv ctor ttg_copy " << m_ttg_copy << std::endl;
     db.m_count = 0;
-
-    //std::cout << "buffer::move-ctor from " << &db << " ttg-copy " << db.m_ttg_copy
-    //          << " to " << this << " ttg-copy " << m_ttg_copy
-    //          << " parsec-data " << m_data.get()
-    //          << std::endl;
-
-//#endif // 0
   }
 
   /* explicitly disable copying of buffers
@@ -124,6 +126,7 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t {
   /* allow moving device buffers */
   buffer& operator=(buffer&& db) {
     ttg_parsec_data_wrapper_t::operator=(std::move(db));
+    allocator_type::operator=(std::move(db));
     m_host_data = std::move(db.m_host_data);
     m_count = db.m_count;
     db.m_count = 0;
@@ -202,6 +205,7 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t {
 
   void allocate_on(int device_id) {
     /* TODO: need exposed PaRSEC memory allocator */
+    throw std::runtime_error("not implemented yet");
   }
 
   /* TODO: can we do this automatically?
@@ -246,26 +250,26 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t {
   }
 
   /* Reallocate the buffer with count elements */
-  void reset(std::size_t count) {
+  void reset(std::size_t n) {
     /* TODO: can we resize if count is smaller than m_count? */
     /* drop the current data and reallocate */
     reset();
-    if (count == 0) {
+    if (n == 0) {
       m_host_data = host_data_ptr(nullptr, &delete_non_owned);
     } else {
-      m_host_data = host_data_ptr(new element_type[count], &delete_owned);
+      m_host_data = host_data_ptr(allocate(n), [this, n](element_type* ptr){ deallocate(ptr, n); });
     }
-    reset_parsec_data(m_host_data.get(), count*sizeof(element_type));
+    reset_parsec_data(m_host_data.get(), n*sizeof(element_type));
     //std::cout << "buffer::reset(" << count << ") ptr " << m_host_data.get()
     //          << " ttg_copy " << m_ttg_copy
     //          << " parsec_data " << m_data.get() << std::endl;
-    m_count = count;
+    m_count = n;
   }
 
   /* Reset the buffer to use the ptr to count elements */
-  void reset(T* ptr, std::size_t count = 1) {
+  void reset(T* ptr, std::size_t n = 1) {
     /* TODO: can we resize if count is smaller than m_count? */
-    if (count == m_count) {
+    if (n == m_count) {
       return;
     }
 
@@ -274,9 +278,9 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t {
       m_count = 0;
     } else {
       m_host_data = host_data_ptr(ptr, &delete_non_owned);
-      m_count = count;
+      m_count = n;
     }
-    reset_parsec_data(m_host_data.get(), count*sizeof(element_type));
+    reset_parsec_data(m_host_data.get(), n*sizeof(element_type));
     //std::cout << "buffer::reset(" << ptr << ", " << count << ") ptr " << m_host_data.get()
     //          << " ttg_copy " << m_ttg_copy
     //          << " parsec_data " << m_data.get() << std::endl;
diff --git a/ttg/ttg/parsec/fwd.h b/ttg/ttg/parsec/fwd.h
index f184548d4..098f51220 100644
--- a/ttg/ttg/parsec/fwd.h
+++ b/ttg/ttg/parsec/fwd.h
@@ -16,7 +16,7 @@ namespace ttg_parsec {
   template<typename T>
   struct ptr;
 
-  template<typename T>
+  template<typename T, typename Allocator = std::allocator<T>>
   struct buffer;
   template<typename T>
   struct devicescratch;

From 194e93da4ab653476c133603a7602a2798cf6aef Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 2 Oct 2023 02:19:35 +0000
Subject: [PATCH 129/259] Add first full draft of device-enabled POTRF

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/CMakeLists.txt          | 10 +++++++++-
 examples/potrf/potrf.h           |  6 ++----
 examples/potrf/testing_dpotrf.cc |  8 +++++++-
 ttg/ttg.h                        |  1 +
 4 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 24b6f5897..b599d97f6 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -14,13 +14,21 @@ if (TARGET tiledarray)
     if (TARGET CUDA::cublas)
       add_ttg_executable(bspmm-cuda spmm/spmm_cuda.cc
                          LINK_LIBRARIES tiledarray TiledArray_Eigen BTAS Boost::boost CUDA::cublas
-                         COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2;TTG_HAVE_CUDA=1
+                         COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2
+                         RUNTIMES "parsec")
+
+      add_ttg_executable(testing_dpotrf_cuda potrf/testing_dpotrf.cc
+                         LINK_LIBRARIES lapackpp tiledarray CUDA::cublas
+                         COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2
                          RUNTIMES "parsec")
     elseif (TARGET roc::hipblas)
       add_ttg_executable(bspmm-hip spmm/spmm_cuda.cc
                         LINK_LIBRARIES tiledarray TiledArray_Eigen BTAS Boost::boost roc::hipblas
                         COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2
                         RUNTIMES "parsec")
+      add_ttg_executable(testing_dpotrf_hip potrf/testing_dpotrf.cc
+                         LINK_LIBRARIES lapackpp tiledarray roc::hipblas
+                         RUNTIMES "parsec")
     endif()
 
     add_ttg_executable(chain-ttg-dev task-benchmarks/chain-ttg-dev.cc LINK_LIBRARIES tiledarray RUNTIMES "parsec")
diff --git a/examples/potrf/potrf.h b/examples/potrf/potrf.h
index e6966bb2b..039a0065b 100644
--- a/examples/potrf/potrf.h
+++ b/examples/potrf/potrf.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <ttg.h>
+#include <ttg/config.h>
 #include "lapack.hh"
 #include "pmw.h"
 
@@ -34,6 +35,7 @@ namespace potrf {
                   ttg::Edge<Key2, MatrixTile<typename MatrixT::element_type>>& output_result) {
       using T = typename MatrixT::element_type;
 #if defined(TTG_HAS_CUDART) || defined(TTG_HAS_HIP)
+    std::cout << "Creating CUDA POTRF task " << std::endl;
     static int device_potrf_workspace_size(blk_t &A) {
       int Lwork;
       #if defined(TTG_HAVE_CUDA)
@@ -422,10 +424,6 @@ namespace potrf {
                    tile_mn.data(), tile_mn.lda());
 #endif
 
-#if defined(DEBUG_TILES_VALUES)
-      std::cout << "After GEMM(" << key << "), A(" << M << ", " << N << ") is " << tile_mn << std::endl;
-#endif
-
       if (N == K + 1) {
         /* send the tile to trsm */
         if (ttg::tracing()) ttg::print("GEMM(", key, "): sending output to TRSM(", Key2{M, N}, ")");
diff --git a/examples/potrf/testing_dpotrf.cc b/examples/potrf/testing_dpotrf.cc
index d5686042f..76055b4d4 100644
--- a/examples/potrf/testing_dpotrf.cc
+++ b/examples/potrf/testing_dpotrf.cc
@@ -56,7 +56,12 @@ int main(int argc, char **argv)
   bool check = !cmdOptionExists(argv+1, argv+argc, "-x");
   bool cow_hint = !cmdOptionExists(argv+1, argv+argc, "-w");
 
-  ttg::initialize(argc, argv, nthreads);
+  // TODO: need to filter out our arguments to make parsec happy
+  ttg::initialize(1, argv, nthreads);
+
+  // initialize MADNESS so that TA allocators can be created
+  madness::ParsecRuntime::initialize_with_existing_context(ttg::default_execution_context().impl().context());
+  madness::initialize(argc, argv, /* nthread = */ 1, /* quiet = */ true);
 
   auto world = ttg::default_execution_context();
   if(nullptr != prof_filename) {
@@ -207,6 +212,7 @@ int main(int argc, char **argv)
 
   world.profile_off();
 
+  madness::finalize();
   ttg::finalize();
   return ret;
 }
diff --git a/ttg/ttg.h b/ttg/ttg.h
index 09ccfaf90..c84af7854 100644
--- a/ttg/ttg.h
+++ b/ttg/ttg.h
@@ -1,6 +1,7 @@
 #ifndef TTG_H_INCLUDED
 #define TTG_H_INCLUDED
 
+#include "ttg/config.h"
 #include "ttg/fwd.h"
 
 #include "ttg/runtimes.h"

From e41a4b1354c92cd63ff544cdbdd0c2e6036ed742 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 2 Oct 2023 02:44:45 +0000
Subject: [PATCH 130/259] POTRF: Use the right precompiler guard for CUDA and
 HIP

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/potrf/potrf.h | 82 +++++++++++++++++++++---------------------
 1 file changed, 42 insertions(+), 40 deletions(-)

diff --git a/examples/potrf/potrf.h b/examples/potrf/potrf.h
index 039a0065b..e6230371a 100644
--- a/examples/potrf/potrf.h
+++ b/examples/potrf/potrf.h
@@ -7,10 +7,10 @@
 
 #undef DEBUG_TILES_VALUES
 
-#if defined(TTG_HAS_CUDART)
+#if defined(TTG_HAVE_CUDART)
 #define ES ttg::ExecutionSpace::CUDA
 #define TASKRET -> ttg::device_task
-#elif defined(TTG_HAS_HIP)
+#elif defined(TTG_HAVE_HIP)
 #define ES ttg::ExecutionSpace::HIP
 #define TASKRET -> ttg::device_task
 #else
@@ -27,6 +27,41 @@ namespace potrf {
   inline double FADDS_POTRF(double __n) { return (__n * (((1. / 6.) * __n) * __n - (1. / 6.))); }
   inline double FLOPS_DPOTRF(double __n) { return FMULS_POTRF(__n) + FADDS_POTRF(__n); }
 
+#if defined(TTG_HAVE_CUDART) || defined(TTG_HAVE_HIP)
+  static int device_potrf_workspace_size(MatrixTile<double> &A) {
+    int Lwork;
+    #if defined(TTG_HAVE_CUDA)
+      cusolverDnDpotrf_bufferSize(ttg::detail::cublas_get_handle(),
+                                  CUBLAS_FILL_MODE_LOWER, A.extent(1),
+                                  nullptr, A.extent(0),
+                                  &Lwork);
+      return Lwork;
+    #elif defined(TTG_HAVE_HIPBLAS)
+      #error TBCoded
+    #else
+      return 0;
+    #endif
+  }
+
+  static void device_potrf(MatrixTile<double> &A, double *workspace, int Lwork, int *devInfo) {
+    int device = A.b.get_current_device();
+    assert(device != 0);
+  #if defined(TTG_HAVE_CUDA)
+    cusolverDnDpotrf(ttg::detail::cublas_get_handle(),
+                      CUBLAS_FILL_MODE_LOWER, A.extent(1),
+                      A.b.device_ptr_on(device), A.extent(0),
+                      workspace, Lwork,
+                      devInfo);
+  #elif defined(TTG_HAVE_HIPBLAS)
+    hipsolverDpotrf(ttg::detail::hipblas_get_handle(),
+                      HIPSOLVER_FILL_MODE_LOWER, A.extent(1),
+                      A.b.device_ptr_on(device), A.extent(0),
+                      workspace, Lwork,
+                      devInfo);
+  #endif
+  }
+#endif // defined(TTG_HAVE_CUDART) || defined(TTG_HAVE_HIP)
+
   template <typename MatrixT>
   auto make_potrf(MatrixT& A,
                   ttg::Edge<Key1, MatrixTile<typename MatrixT::element_type>>& input_disp,  // from the dispatcher
@@ -34,41 +69,8 @@ namespace potrf {
                   ttg::Edge<Key2, MatrixTile<typename MatrixT::element_type>>& output_trsm,
                   ttg::Edge<Key2, MatrixTile<typename MatrixT::element_type>>& output_result) {
       using T = typename MatrixT::element_type;
-#if defined(TTG_HAS_CUDART) || defined(TTG_HAS_HIP)
+#if defined(TTG_HAVE_CUDART) || defined(TTG_HAVE_HIP)
     std::cout << "Creating CUDA POTRF task " << std::endl;
-    static int device_potrf_workspace_size(blk_t &A) {
-      int Lwork;
-      #if defined(TTG_HAVE_CUDA)
-        cusolverDnDpotrf_bufferSize(ttg::detail::cublas_get_handle(),
-                                    CUBLAS_FILL_MODE_LOWER, A.extent(1),
-                                    nullptr, A.extent(0),
-                                    &Lwork);
-        return Lwork;
-      #elif defined(TTG_HAVE_HIPBLAS)
-        #error TBCoded
-      #else
-        return 0;
-      #endif
-    }
-
-    static void device_potrf(blk_t &A, double *workspace, int Lwork, int *devInfo) {
-      int device = A.b.get_current_device();
-      assert(device != 0);
-    #if defined(TTG_HAVE_CUDA)
-      cusolverDnDpotrf(ttg::detail::cublas_get_handle(),
-                       CUBLAS_FILL_MODE_LOWER, A.extent(1),
-                       A.b.device_ptr_on(device), A.extent(0),
-                       workspace, Lwork,
-                       devInfo);
-    #elif defined(TTG_HAVE_HIPBLAS)
-      hipsolverDpotrf(ttg::detail::hipblas_get_handle(),
-                       HIPSOLVER_FILL_MODE_LOWER, A.extent(1),
-                       A.b.device_ptr_on(device), A.extent(0),
-                       workspace, Lwork,
-                       devInfo);
-    #endif
-    }
-
     auto f_dev = [=](const Key1& key, MatrixTile<T>&& A,
                      std::tuple<ttg::Out<Key2, MatrixTile<T>>, ttg::Out<Key2, MatrixTile<T>>>& out) TASKRET {
       const auto K = key[0];
@@ -120,7 +122,7 @@ namespace potrf {
     }
     return ttg::make_tt<ES>(f_dev, ttg::edges(ttg::fuse(input, input_disp)), ttg::edges(output_result, output_trsm), "POTRF",
                         {"tile_kk/dispatcher"}, {"output_result", "output_trsm"});
-#else /* defined(TTG_HAS_CUDART) || defined(TTG_HAS_HIP) */
+#else /* defined(TTG_HAVE_CUDART) || defined(TTG_HAVE_HIP) */
     auto f = [=](const Key1& key, MatrixTile<T>&& tile_kk,
                  std::tuple<ttg::Out<Key2, MatrixTile<T>>, ttg::Out<Key2, MatrixTile<T>>>& out) {
       const int K = key[0];
@@ -163,7 +165,7 @@ namespace potrf {
                  ttg::Edge<Key3, MatrixTile<typename MatrixT::element_type>>& output_col,   // to GEMM
                  ttg::Edge<Key2, MatrixTile<typename MatrixT::element_type>>& output_result) {
     using T = typename MatrixT::element_type;
-#if defined(TTG_HAS_CUDART) || defined(TTG_HAS_HIP)
+#if defined(TTG_HAVE_CUDART) || defined(TTG_HAVE_HIP)
     auto f = [=](const Key2& key, const MatrixTile<T>& tile_kk, MatrixTile<T>&& tile_mk,
                  std::tuple<ttg::Out<Key2, MatrixTile<T>>, ttg::Out<Key2, MatrixTile<T>>, ttg::Out<Key3, MatrixTile<T>>,
                             ttg::Out<Key3, MatrixTile<T>>>& out) TASKRET {
@@ -287,7 +289,7 @@ namespace potrf {
                  ttg::Edge<Key1, MatrixTile<typename MatrixT::element_type>>& output_potrf,  // to POTRF
                  ttg::Edge<Key2, MatrixTile<typename MatrixT::element_type>>& output_syrk) {
     using T = typename MatrixT::element_type;
-#if defined(TTG_HAS_CUDART) || defined(TTG_HAS_HIP)
+#if defined(TTG_HAVE_CUDART) || defined(TTG_HAVE_HIP)
     auto f = [=](const Key2& key, const MatrixTile<T>& tile_mk, MatrixTile<T>&& tile_kk,
                  std::tuple<ttg::Out<Key1, MatrixTile<T>>, ttg::Out<Key2, MatrixTile<T>>>& out) TASKRET {
       const int K = key[0];
@@ -384,7 +386,7 @@ namespace potrf {
                  ttg::Edge<Key2, MatrixTile<typename MatrixT::element_type>>& output_trsm,  // to TRSM
                  ttg::Edge<Key3, MatrixTile<typename MatrixT::element_type>>& output_gemm) {
     using T = typename MatrixT::element_type;
-#if defined(TTG_HAS_CUDART) || defined(TTG_HAS_HIP)
+#if defined(TTG_HAVE_CUDART) || defined(TTG_HAVE_HIP)
     auto f = [=](const Key3& key, const MatrixTile<T>& tile_mk, const MatrixTile<T>& tile_nk, MatrixTile<T>&& tile_mn,
                  std::tuple<ttg::Out<Key2, MatrixTile<T>>, ttg::Out<Key3, MatrixTile<T>>>& out) TASKRET {
       const int M = key[0];

From 18577f96d8a75472fb8e5e23753577cf53392449 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 2 Oct 2023 22:59:48 +0000
Subject: [PATCH 131/259] POTRF: make it run, still fails factorization

Notable changes:
- add ttg::device::current_device and ttg::device::current_stream to
  query current streams
- Consistently start counting devices at 0
- The host has ID -2
- Fixes in the broadcast

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/CMakeLists.txt   |  18 +++--
 examples/devblas_helper.h | 158 ++++++++++++++++++++++++++++++++++++++
 examples/matrixtile.h     |  25 ++++--
 examples/potrf/potrf.h    | 121 +++++++++++++++++------------
 ttg/CMakeLists.txt        |   1 +
 ttg/ttg/device/device.h   |  61 +++++++++++++++
 ttg/ttg/device/task.h     |  38 +++++----
 ttg/ttg/devicescratch.h   |   2 +-
 ttg/ttg/parsec/buffer.h   |  41 +++++-----
 ttg/ttg/parsec/ttg.h      |  30 +++++++-
 10 files changed, 388 insertions(+), 107 deletions(-)
 create mode 100644 examples/devblas_helper.h
 create mode 100644 ttg/ttg/device/device.h

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index b599d97f6..4d8896444 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -17,18 +17,22 @@ if (TARGET tiledarray)
                          COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2
                          RUNTIMES "parsec")
 
-      add_ttg_executable(testing_dpotrf_cuda potrf/testing_dpotrf.cc
-                         LINK_LIBRARIES lapackpp tiledarray CUDA::cublas
-                         COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2
-                         RUNTIMES "parsec")
+      if (TARGET CUDA::cusolver)
+        add_ttg_executable(testing_dpotrf_cuda potrf/testing_dpotrf.cc
+                          LINK_LIBRARIES lapackpp tiledarray CUDA::cublas CUDA::cusolver
+                          COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2;TTG_ENABLE_CUDA=1
+                          RUNTIMES "parsec")
+      endif(TARGET CUDA::cusolver)
     elseif (TARGET roc::hipblas)
       add_ttg_executable(bspmm-hip spmm/spmm_cuda.cc
                         LINK_LIBRARIES tiledarray TiledArray_Eigen BTAS Boost::boost roc::hipblas
                         COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2
                         RUNTIMES "parsec")
-      add_ttg_executable(testing_dpotrf_hip potrf/testing_dpotrf.cc
-                         LINK_LIBRARIES lapackpp tiledarray roc::hipblas
-                         RUNTIMES "parsec")
+      if (TARGET roc::hipsolver)
+        add_ttg_executable(testing_dpotrf_hip potrf/testing_dpotrf.cc
+                           LINK_LIBRARIES lapackpp tiledarray roc::hipblas roc::hipsolver
+                           RUNTIMES "parsec")
+      endif(TARGET roc::hipsolver)
     endif()
 
     add_ttg_executable(chain-ttg-dev task-benchmarks/chain-ttg-dev.cc LINK_LIBRARIES tiledarray RUNTIMES "parsec")
diff --git a/examples/devblas_helper.h b/examples/devblas_helper.h
new file mode 100644
index 000000000..704a2bc85
--- /dev/null
+++ b/examples/devblas_helper.h
@@ -0,0 +1,158 @@
+#include "ttg/config.h"
+
+#include <memory>
+#include <stdexcept>
+#include <optional>
+#include <map>
+
+#ifdef TTG_HAVE_CUDART
+
+#include <cuda_runtime.h>
+#include <cublas_v2.h>
+#include <cusolverDn.h>
+
+namespace detail {
+  template<typename Stream>
+  struct device_stream_t {
+    int device;
+    Stream stream;
+    device_stream_t(int device, Stream stream)
+    : device(device)
+    , stream(stream)
+    { }
+
+    bool operator<(const device_stream_t& ds) const {
+      bool result = ((device < ds.device) && (reinterpret_cast<uintptr_t>(stream) < reinterpret_cast<uintptr_t>(ds.stream)));
+      std::cout << *this << " < " << ds << ": " << result << std::endl;
+      return result;
+    }
+
+    bool operator==(const device_stream_t& ds) const {
+      bool result = ((device == ds.device) && (stream == ds.stream));
+      std::cout << *this << " == " << ds << ": " << result << std::endl;
+      return result;
+    }
+  };
+} // namespace detail
+
+namespace std {
+template<typename Stream>
+  std::ostream& operator<<(std::ostream& os, const ::detail::device_stream_t<Stream>& ds) {
+    os << "[" << ds.device << ", " << ds.stream << "]";
+    return os;
+  }
+
+} //namespace std
+
+/// \brief Returns the cuBLAS handle to be used for launching cuBLAS kernels from the current thread
+/// \return the cuBLAS handle for the current thread
+inline const cublasHandle_t& cublas_handle() {
+  using map_type = std::map<std::pair<int, cudaStream_t>, cublasHandle_t>;
+  static thread_local map_type handles;
+
+  int device = ttg::device::current_device();
+  cudaStream_t stream = ttg::device::current_stream();
+
+  map_type::iterator it;
+  if ((it = handles.find({device, stream})) == handles.end()){
+    cublasHandle_t handle;
+    auto status = cublasCreate_v2(&handle);
+    if (CUBLAS_STATUS_SUCCESS != status) {
+      std::cerr << "cublasCreate_v2 failed: " << status << std::endl;
+      throw std::runtime_error("cublasCreate_v2 failed");
+    }
+    status = cublasSetStream_v2(handle, ttg::device::current_stream());
+    if (CUBLAS_STATUS_SUCCESS != status) {
+      std::cerr << "cublasSetStream_v2 failed: " << status << std::endl;
+      throw std::runtime_error("cublasSetStream_v2 failed");
+    }
+    auto [iterator, success] = handles.insert({{device, stream}, handle});
+    it = iterator;
+  }
+  return it->second;
+}
+
+inline const cusolverDnHandle_t& cusolver_handle() {
+
+  //using map_type = std::map<detail::device_stream_t<cudaStream_t>, cusolverDnHandle_t>;
+  using map_type = std::map<std::pair<int, cudaStream_t>, cusolverDnHandle_t>;
+  static thread_local map_type handles;
+
+  int device = ttg::device::current_device();
+  cudaStream_t stream = ttg::device::current_stream();
+
+  map_type::iterator it;
+  if ((it = handles.find({device, stream})) == handles.end()){
+    cusolverDnHandle_t handle;
+    auto status = cusolverDnCreate(&handle);
+    if (CUSOLVER_STATUS_SUCCESS != status) {
+      std::cerr << "cusolverDnCreate failed: " << status << std::endl;
+      throw std::runtime_error("cusolverDnCreate failed");
+    }
+    status = cusolverDnSetStream(handle, stream);
+    if (CUSOLVER_STATUS_SUCCESS != status) {
+      std::cerr << "cusolverDnSetStream failed: " << status << std::endl;
+      throw std::runtime_error("cusolverDnSetStream failed");
+    }
+
+    std::cout << "Creating cusolver handle " << handle << " for device " << device << " stream " << stream << std::endl;
+    auto [iterator, success] = handles.insert({{device, stream}, handle});
+    it = iterator;
+  } else {
+    std::cout << "Found cusolver handle " << it->second << " for device " << device << " stream " << stream << std::endl;
+  }
+
+  return it->second;
+}
+#endif // TTG_HAVE_CUDART
+
+#ifdef TTG_HAVE_HIPBLAS
+
+#include <hip_runtime.h>
+#include <hipblas.h>
+#include <hipsolverDn.h>
+
+/// \brief Returns the rocBLAS handle to be used for launching rocBLAS kernels from the current thread
+/// \return the rocBLAS handle for the current thread
+const hipblasHandle_t& hipblas_handle() {
+  static thread_local std::map<int, hipblasHandle_t> handles;
+  int device = ttg::device::current_device();
+  std::map<int, hipblasHandle_t>::iterator it;
+  if ((it = handles.find(device)) == handles.end()){
+    hipblasHandle_t handle;
+    auto status = hipblasCreate(&handle);
+    if (HIPBLAS_STATUS_SUCCESS != status) {
+      throw std::runtime_error("hipblasCreate failed");
+    }
+    auto [iterator, success] = handles.insert({device, handle});
+    it = iterator;
+  }
+  hipblasStatus_t status = hipblasSetStream(it->second, ttg::device::current_stream());
+  if (HIPBLAS_STATUS_SUCCESS != status) {
+    throw std::runtime_error("hipblasSetStream failed");
+  }
+  return it->second;
+}
+
+/// \brief Returns the hipsolver handle to be used for launching rocBLAS kernels from the current thread
+/// \return the hipsolver handle for the current thread
+const hipsolverDnHandle_t& hipsolver_handle() {
+  static thread_local std::map<int, hipsolverDnHandle_t> handles;
+  int device = ttg::device::current_device();
+  std::map<int, hipsolverDnHandle_t>::iterator it;
+  if ((it = handles.find(device)) == handles.end()){
+    hipsolverDnHandle_t handle;
+    auto status = hipsolverDnCreate(&handle);
+    if (HIPSOLVER_STATUS_SUCCESS != status) {
+      throw std::runtime_error("hipsolverCreate failed");
+    }
+    auto [iterator, success] = handles.insert({device, handle});
+    it = iterator;
+  }
+  hipsolverStatus_t status = hipsolverDnSetStream(it->second, ttg::device::current_stream());
+  if (HIPSOLVER_STATUS_SUCCESS != status) {
+    throw std::runtime_error("hipsolverSetStream failed");
+  }
+  return it->second;
+}
+#endif // TTG_HAVE_HIPBLAS
diff --git a/examples/matrixtile.h b/examples/matrixtile.h
index 693ae3d1e..ba2fcf9cd 100644
--- a/examples/matrixtile.h
+++ b/examples/matrixtile.h
@@ -17,13 +17,13 @@ class MatrixTile : public ttg::TTValue<MatrixTile<T, Allocator>> {
   using ttvalue_type = ttg::TTValue<MatrixTile<T, Allocator>>;
 
  private:
-  buffer_t b;
+  buffer_t _buffer;
   int _rows = 0, _cols = 0, _lda = 0;
 
   // (Re)allocate the tile memory
   void realloc() {
     // std::cout << "Reallocating new tile" << std::endl;
-    b.reset(_lda * _cols);
+    _buffer.reset(_lda * _cols);
   }
 
  public:
@@ -31,7 +31,7 @@ class MatrixTile : public ttg::TTValue<MatrixTile<T, Allocator>> {
 
   MatrixTile(int rows, int cols, int lda)
   : ttvalue_type()
-  , b(lda*cols)
+  , _buffer(lda*cols)
   , _rows(rows)
   , _cols(cols)
   , _lda(lda)
@@ -49,7 +49,7 @@ class MatrixTile : public ttg::TTValue<MatrixTile<T, Allocator>> {
    */
   MatrixTile(int rows, int cols, T* data, int lda)
   : ttvalue_type()
-  , b(data, lda*cols)
+  , _buffer(data, lda*cols)
   , _rows(rows)
   , _cols(cols)
   , _lda(lda)
@@ -65,7 +65,7 @@ class MatrixTile : public ttg::TTValue<MatrixTile<T, Allocator>> {
    * can handle data sharing without excessive copying */
   MatrixTile(const MatrixTile<T, Allocator>& other)
   : ttvalue_type()
-  , b(other._lda*other._cols)
+  , _buffer(other._lda*other._cols)
   , _rows(other._rows)
   , _cols(other._cols)
   , _lda(other._lda) {
@@ -85,14 +85,15 @@ class MatrixTile : public ttg::TTValue<MatrixTile<T, Allocator>> {
     _rows = std::get<0>(meta);
     _cols = std::get<1>(meta);
     _lda = std::get<2>(meta);
+    this->realloc();
   }
 
   metadata_t get_metadata(void) const { return metadata_t{_rows, _cols, _lda}; }
 
   // Accessing the raw data
-  T* data() { return b.host_ptr(); }
+  T* data() { return _buffer.host_ptr(); }
 
-  const T* data() const { return b.host_ptr(); }
+  const T* data() const { return _buffer.host_ptr(); }
 
   size_t size() const { return _cols * _lda; }
 
@@ -102,9 +103,17 @@ class MatrixTile : public ttg::TTValue<MatrixTile<T, Allocator>> {
 
   int lda() const { return _lda; }
 
+  buffer_t& buffer() {
+    return _buffer;
+  }
+
+  const buffer_t& buffer() const {
+    return _buffer;
+  }
+
   auto& fill(T value) {
     std::fill(data().get(), data().get() + size(), value);
-    b.set_current_device(0);
+    _buffer.set_current_device(0);
     return *this;
   }
 
diff --git a/examples/potrf/potrf.h b/examples/potrf/potrf.h
index e6230371a..50a57a766 100644
--- a/examples/potrf/potrf.h
+++ b/examples/potrf/potrf.h
@@ -4,22 +4,24 @@
 #include <ttg/config.h>
 #include "lapack.hh"
 #include "pmw.h"
+#include "../devblas_helper.h"
 
 #undef DEBUG_TILES_VALUES
 
 #if defined(TTG_HAVE_CUDART)
 #define ES ttg::ExecutionSpace::CUDA
 #define TASKRET -> ttg::device_task
+#include <cusolverDn.h>
 #elif defined(TTG_HAVE_HIP)
 #define ES ttg::ExecutionSpace::HIP
 #define TASKRET -> ttg::device_task
+#include <hip/hipsolver.h>
+#include <hip/hipblas.h>
 #else
 #define ES ttg::ExecutionSpace::Host
 #define TASKRET -> void
 #endif
 
-
-
 namespace potrf {
 
   /* FLOP macros taken from DPLASMA */
@@ -31,9 +33,9 @@ namespace potrf {
   static int device_potrf_workspace_size(MatrixTile<double> &A) {
     int Lwork;
     #if defined(TTG_HAVE_CUDA)
-      cusolverDnDpotrf_bufferSize(ttg::detail::cublas_get_handle(),
-                                  CUBLAS_FILL_MODE_LOWER, A.extent(1),
-                                  nullptr, A.extent(0),
+      cusolverDnDpotrf_bufferSize(cusolver_handle(),
+                                  CUBLAS_FILL_MODE_LOWER, A.cols(),
+                                  nullptr, A.lda(),
                                   &Lwork);
       return Lwork;
     #elif defined(TTG_HAVE_HIPBLAS)
@@ -44,18 +46,21 @@ namespace potrf {
   }
 
   static void device_potrf(MatrixTile<double> &A, double *workspace, int Lwork, int *devInfo) {
-    int device = A.b.get_current_device();
-    assert(device != 0);
+    int device = ttg::device::current_device();
+    assert(device >= 0);
   #if defined(TTG_HAVE_CUDA)
-    cusolverDnDpotrf(ttg::detail::cublas_get_handle(),
-                      CUBLAS_FILL_MODE_LOWER, A.extent(1),
-                      A.b.device_ptr_on(device), A.extent(0),
+    //std::cout << "POTRF A " << A.buffer().device_ptr_on(device) << " device " << device << " cols " << A.cols() << " lda " << A.lda() << " Lwork " << Lwork << " WS " << workspace << " devInfo " << devInfo << std::endl;
+    auto handle = cusolver_handle();
+    std::cout << "POTRF handle  " << handle << " device " << device << " stream " << ttg::device::current_stream() << std::endl;
+    cusolverDnDpotrf(handle,
+                      CUBLAS_FILL_MODE_LOWER, A.cols(),
+                      A.buffer().device_ptr_on(device), A.lda(),
                       workspace, Lwork,
                       devInfo);
   #elif defined(TTG_HAVE_HIPBLAS)
-    hipsolverDpotrf(ttg::detail::hipblas_get_handle(),
-                      HIPSOLVER_FILL_MODE_LOWER, A.extent(1),
-                      A.b.device_ptr_on(device), A.extent(0),
+    hipsolverDpotrf(hipsolver_handle(),
+                      HIPSOLVER_FILL_MODE_LOWER, A.cols(),
+                      A.buffer().device_ptr_on(device), A.lda(),
                       workspace, Lwork,
                       devInfo);
   #endif
@@ -69,9 +74,11 @@ namespace potrf {
                   ttg::Edge<Key2, MatrixTile<typename MatrixT::element_type>>& output_trsm,
                   ttg::Edge<Key2, MatrixTile<typename MatrixT::element_type>>& output_result) {
       using T = typename MatrixT::element_type;
+      auto iallocator = std::make_shared<TiledArray::device_pinned_allocator<int>>();
 #if defined(TTG_HAVE_CUDART) || defined(TTG_HAVE_HIP)
-    std::cout << "Creating CUDA POTRF task " << std::endl;
-    auto f_dev = [=](const Key1& key, MatrixTile<T>&& A,
+    //std::cout << "Creating CUDA POTRF task " << std::endl;
+    auto f_dev = [=, iallocator = std::move(iallocator)]
+                    (const Key1& key, MatrixTile<T>&& tile_kk,
                      std::tuple<ttg::Out<Key2, MatrixTile<T>>, ttg::Out<Key2, MatrixTile<T>>>& out) TASKRET {
       const auto K = key[0];
 
@@ -81,8 +88,10 @@ namespace potrf {
       std::vector<Key2> keylist;
       keylist.reserve(A.rows() - K);
       /* TODO: reverse order of arrays */
+      //std::cout << "POTRF K " << K << " A.rows " << A.rows() << std::endl;
       for (int m = K + 1; m < A.rows(); ++m) {
         /* send tile to trsm */
+        //std::cout << "POTRF successor " << Key2(m, K) << std::endl;
         keylist.push_back(Key2(m, K));
       }
 
@@ -92,34 +101,43 @@ namespace potrf {
       //       Check pmw.h: when we generate the MatrixTile
       //       Also check pinned allocator at the end of DeviceTensor (250-253)
 
-      int Lwork = device_potrf_workspace_size(A);
+      int Lwork = device_potrf_workspace_size(tile_kk);
 
       // Instead of using scratch here, we should have hostWS and hostInfo globals and use to_device
       // this would reduce the number of I/O operations to devices
-      double hostWS[Lwork];
-      ttg::devicescratch<double> devWS = ttg::make_scratch(hostWS, ttg::scope::Allocate);
-      int hostInfo = -1;
-      ttg::devicescratch<int> devInfo = ttg::make_scratch(&hostInfo, ttg::scope::Allocate);
+      double *hostWS = new double[Lwork];
+      ttg::devicescratch<double> devWS = ttg::make_scratch(hostWS, ttg::scope::Allocate, Lwork);
+      int *hostInfo = iallocator->allocate(1);
+      ttg::devicescratch<int> devInfo = ttg::make_scratch(hostInfo, ttg::scope::Allocate);
+
+      *hostInfo = -32;
 
       /* the workspace and the devInfo must be device-level pointers */
-      co_await ttg::to_device(A._data, devWS, devInfo);
+      co_await ttg::to_device(tile_kk.buffer(), devWS, devInfo);
+
+      //std::cout << "devWS host ptr " << hostWS << " device ptr " << devWS.device_ptr() << " size " <<  devWS.size()
+      //          << " devInfo host ptr " << hostInfo << " device ptr " << devInfo.device_ptr() << "size "  << devInfo.size() << std::endl;
 
       /* everything is on the device, call the POTRF */
-      device_potrf(A, devWS, Lwork, devInfo);
+      device_potrf(tile_kk, devWS.device_ptr(), Lwork, devInfo.device_ptr());
 
       /* wait for the kernel to complete */
       co_await ttg::wait_kernel(devInfo);
 
-      if( hostInfo == 0 ) {
-        co_await ttg::device::forward(ttg::device::broadcast<0, 1>(std::make_tuple(Key2(K, K), std::move(keylist)), std::move(A), out));
+      delete[] hostWS;
+      int info = *hostInfo;
+      assert(info == 0);
+      iallocator->deallocate(hostInfo, 1);
+      if( info == 0 ) {
+        co_await ttg::device::forward(ttg::device::broadcast<0, 1>(std::make_tuple(Key2(K, K), std::move(keylist)), std::move(tile_kk), out));
         // Anything after this co_await is never executed
         // co_return would look better, but co_return would destroy keylist before the runtime can handle it
       } else {
         // Well... Here we should interrupt the DAG of tasks, there is an error. Raise?
-        std::cerr << "Factorization is SUSPICIOUS (the matrix might not be diagonally dominant)" << std::endl;
+        std::cerr << "Factorization of tile " << K << " failed: " << info << std::endl;
         ttg::abort();
       }
-    }
+    };
     return ttg::make_tt<ES>(f_dev, ttg::edges(ttg::fuse(input, input_disp)), ttg::edges(output_result, output_trsm), "POTRF",
                         {"tile_kk/dispatcher"}, {"output_result", "output_trsm"});
 #else /* defined(TTG_HAVE_CUDART) || defined(TTG_HAVE_HIP) */
@@ -202,23 +220,23 @@ namespace potrf {
       }
 
 
-      co_await ttg::to_device(tile_kk.b, tile_mk.b);
-      int device = tile_kk.b.get_current_device();
+      co_await ttg::to_device(tile_kk.buffer(), tile_mk.buffer());
+      int device = ttg::device::current_device();
       double alpha = 1.0;
 #if defined(TTG_HAVE_CUDA)
-      cublasDtrsm(ttg::detail::cublas_get_handle(),
+      cublasDtrsm(cublas_handle(),
                   CUBLAS_SIDE_RIGHT, CUBLAS_FILL_MODE_LOWER,
                   CUBLAS_OP_T, CUBLAS_DIAG_NON_UNIT,
                   mb, nb, &alpha,
-                  tile_kk.b.device_ptr_on(device), tile_kk.lda(),
-                  tile_mk.b.device_ptr_on(device), tile_mk.lda());
+                  tile_kk.buffer().device_ptr_on(device), tile_kk.lda(),
+                  tile_mk.buffer().device_ptr_on(device), tile_mk.lda());
 #elif defined(TTG_HAVE_HIPBLAS)
-      hipblasDtrsm(ttg::detail:hipblas_get_handle(),
+      hipblasDtrsm(hipblas_handle(),
                    HIPBLAS_SIDE_RIGHT, HIPBLAS_FILL_MODE_LOWER,
                    HIPBLAS_OP_T, HIPBLAS_DIAG_NON_UNIT,
                    mb, nb, &alpha,
-                   tile_kk.b.device_ptr_on(device), tile_kk.lda(),
-                   tile_mk.b.device_ptr_on(device), tile_mk.lda());
+                   tile_kk.buffer().device_ptr_on(device), tile_kk.lda(),
+                   tile_mk.buffer().device_ptr_on(device), tile_mk.lda());
 #endif
 
       co_await ttg::device::forward(ttg::device::broadcast<0, 1, 2, 3>(std::make_tuple(key, Key2(K, M), keylist_row, keylist_col),
@@ -304,24 +322,26 @@ namespace potrf {
 
       if (ttg::tracing()) ttg::print("SYRK(", key, ")");
 
-      co_await ttg::to_device(tile_kk.b, tile_mk.b);
+      co_await ttg::to_device(tile_kk.buffer(), tile_mk.buffer());
+
+      int device = ttg::device::current_device();
 
       double alpha = -1.0;
       double beta  =  1.0;
 #if defined(TTG_HAVE_CUDA)
-      cublasDsyrk(ttg::detail::cublas_get_handle(),
+      cublasDsyrk(cublas_handle(),
                   CUBLAS_FILL_MODE_LOWER,
                   CUBLAS_OP_N,
                   mb, nb, &alpha,
-                  tile_nk.b.device_ptr_on(device), tile_mk.lda(), &beta,
-                  tile_kk.b.device_ptr_on(device), tile_kk.lda());
+                  tile_mk.buffer().device_ptr_on(device), tile_mk.lda(), &beta,
+                  tile_kk.buffer().device_ptr_on(device), tile_kk.lda());
 #elif defined(TTG_HAVE_HIPBLAS)
-      hipblasDsyrk(ttg::detail:hipblas_get_handle(),
+      hipblasDsyrk(hipblas_handle(),
                    HIPBLAS_FILL_MODE_LOWER,
                    HIPBLAS_OP_N,
                    mb, nb, &alpha,
-                   tile_kk.b.device_ptr_on(device), tile_kk.lda(), &beta,
-                   tile_mk.b.device_ptr_on(device), tile_mk.lda());
+                   tile_mk.buffer().device_ptr_on(device), tile_mk.lda(), &beta,
+                   tile_kk.buffer().device_ptr_on(device), tile_kk.lda());
 #endif
 
       if (M == K + 1) {
@@ -404,26 +424,27 @@ namespace potrf {
                 << N << ") is " << tile_nk << " and A(" << M << ", " << N << ") is " << tile_mn;
 #endif
 
-      co_await ttg::to_device(tile_mk.b, tile_nk.b, tile_mn.b);
+      co_await ttg::to_device(tile_mk.buffer(), tile_nk.buffer(), tile_mn.buffer());
 
+      int device = ttg::device::current_device();
       double alpha = -1.0;
       double beta  =  1.0;
 #if defined(TTG_HAVE_CUDA)
-      cublasDgemm(ttg::detail:cublas_get_handle(),
+      cublasDgemm(cublas_handle(),
                   CUBLAS_OP_N, CUBLAS_OP_T,
                   tile_mk.rows(), tile_nk.rows(),
                   tile_nk.cols(), &alpha,
-                  tile_mk.data(), tile_mk.lda(),
-                  tile_nk.data(), tile_nk.lda(), &beta,
-                  tile_mn.data(), tile_mn.lda());
+                  tile_mk.buffer().device_ptr_on(device), tile_mk.lda(),
+                  tile_nk.buffer().device_ptr_on(device), tile_nk.lda(), &beta,
+                  tile_mn.buffer().device_ptr_on(device), tile_mn.lda());
 #elif defined(TTG_HAVE_HIPBLAS)
-      hipblasDgemm(ttg::detail:hipblas_get_handle(),
+      hipblasDgemm(hipblas_handle(),
                    HIPBLAS_OP_N, HIPBLAS_OP_T,
                    tile_mk.rows(), tile_nk.rows(),
                    tile_nk.cols(), &alpha,
-                   tile_mk.data(), tile_mk.lda(),
-                   tile_nk.data(), tile_nk.lda(), &beta,
-                   tile_mn.data(), tile_mn.lda());
+                   tile_mk.buffer().device_ptr_on(device), tile_mk.lda(),
+                   tile_nk.buffer().device_ptr_on(device), tile_nk.lda(), &beta,
+                   tile_mn.buffer().device_ptr_on(device), tile_mn.lda());
 #endif
 
       if (N == K + 1) {
diff --git a/ttg/CMakeLists.txt b/ttg/CMakeLists.txt
index a60f4f833..476118524 100644
--- a/ttg/CMakeLists.txt
+++ b/ttg/CMakeLists.txt
@@ -69,6 +69,7 @@ set(ttg-impl-headers
     )
 set(ttg_device_headers
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/device/cublas_helper.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/ttg/device/device.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/device/task.h
     )
 set(ttg-headers
diff --git a/ttg/ttg/device/device.h b/ttg/ttg/device/device.h
new file mode 100644
index 000000000..17f8364f1
--- /dev/null
+++ b/ttg/ttg/device/device.h
@@ -0,0 +1,61 @@
+
+#include <ttg/config.h>
+
+#if defined(TTG_HAVE_CUDA)
+#include <cuda_runtime.h>
+
+namespace ttg::device {
+  namespace detail {
+    inline thread_local int current_device_ts = 0;
+    inline thread_local cudaStream_t current_stream_ts = 0; // default stream
+
+    void reset_current() {
+      current_device_ts = 0;
+      current_stream_ts = 0;
+    }
+
+    void set_current(int device, cudaStream_t stream) {
+      current_device_ts = device;
+      current_stream_ts = stream;
+    }
+  } // namespace detail
+
+  int current_device() {
+    return detail::current_device_ts;
+  }
+
+  cudaStream_t current_stream() {
+    return detail::current_stream_ts;
+  }
+} // namespace ttg
+
+#elif defined(TTG_HAVE_HIP)
+
+#include <hip/hip_runtime.h>
+
+namespace ttg::device {
+  namespace detail {
+    inline thread_local int current_device_ts = 0;
+    inline thread_local hipStream_t current_stream_ts = 0; // default stream
+
+    void reset_current() {
+      current_device_ts = 0;
+      current_stream_ts = 0;
+    }
+
+    void set_current(int device, hipStream_t stream) {
+      current_device_ts = device;
+      current_stream_ts = stream;
+    }
+  } // namespace detail
+
+  int current_device() {
+    return detail::current_device_ts;
+  }
+
+  hipStream_t current_stream() {
+    return detail::current_stream_ts;
+  }
+} // namespace ttg
+
+#endif // defined(TTG_HAVE_HIP)
diff --git a/ttg/ttg/device/task.h b/ttg/ttg/device/task.h
index b275da195..c51990d99 100644
--- a/ttg/ttg/device/task.h
+++ b/ttg/ttg/device/task.h
@@ -134,7 +134,8 @@ namespace ttg {
         }
 
         void unhandled_exception() {
-
+          std::cerr << "Send coroutine caught an unhandled exception!" << std::endl;
+          throw; // fwd
         }
 
       };
@@ -284,9 +285,9 @@ namespace ttg {
                 typename... out_keysT, typename... out_valuesT>
       inline void prepare_broadcast(const std::tuple<RangesT...> &keylists, valueT &&value,
                                     std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
-        std::get<I>(t)->prepare_send(std::get<KeyId>(keylists), std::forward<valueT>(value));
+        std::get<I>(t).prepare_send(std::get<KeyId>(keylists), std::forward<valueT>(value));
         if constexpr (sizeof...(Is) > 0) {
-          detail::prepare_broadcast<KeyId+1, Is...>(keylists, std::forward<valueT>(value), t);
+          prepare_broadcast<KeyId+1, Is...>(keylists, std::forward<valueT>(value), t);
         }
       }
 
@@ -299,7 +300,7 @@ namespace ttg {
         auto *terminal_ptr = ttg::detail::get_out_terminal<key_t, valueT>(I, "ttg::device::broadcast(keylists, value)");
         terminal_ptr->prepare_send(std::get<KeyId>(keylists), value);
         if constexpr (sizeof...(Is) > 0) {
-          detail::prepare_broadcast<KeyId+1, Is...>(keylists, std::forward<valueT>(value));
+          prepare_broadcast<KeyId+1, Is...>(keylists, std::forward<valueT>(value));
         }
       }
 
@@ -307,7 +308,7 @@ namespace ttg {
                 typename... out_keysT, typename... out_valuesT>
       inline void broadcast(const std::tuple<RangesT...> &keylists, valueT &&value,
                                     std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
-        std::get<I>(t)->broadcast(std::get<KeyId>(keylists), std::forward<valueT>(value));
+        std::get<I>(t).broadcast(std::get<KeyId>(keylists), std::forward<valueT>(value));
         if constexpr (sizeof...(Is) > 0) {
           detail::broadcast<KeyId+1, Is...>(keylists, std::forward<valueT>(value), t);
         }
@@ -322,7 +323,7 @@ namespace ttg {
         auto *terminal_ptr = ttg::detail::get_out_terminal<key_t, valueT>(I, "ttg::device::broadcast(keylists, value)");
         terminal_ptr->broadcast(std::get<KeyId>(keylists), value);
         if constexpr (sizeof...(Is) > 0) {
-          detail::broadcast<KeyId+1, Is...>(keylists, std::forward<valueT>(value));
+          ttg::device::detail::broadcast<KeyId+1, Is...>(keylists, std::forward<valueT>(value));
         }
       }
 
@@ -340,12 +341,12 @@ namespace ttg {
           // treat as tuple
           prepare_broadcast<0, I, Is...>(kl, std::forward<std::decay_t<decltype(value)>>(value), t);
           co_await ttg::Void{}; // we'll come back once the task is done
-          broadcast<0, I, Is...>(kl, std::forward<std::decay_t<decltype(value)>>(value), t);
+          ttg::device::detail::broadcast<0, I, Is...>(kl, std::forward<std::decay_t<decltype(value)>>(value), t);
         } else if constexpr (!ttg::meta::is_tuple_v<RangesT>) {
           // create a tie to the captured keylist
           prepare_broadcast<0, I, Is...>(std::tie(kl), std::forward<std::decay_t<decltype(value)>>(value), t);
           co_await ttg::Void{}; // we'll come back once the task is done
-          broadcast<0, I, Is...>(std::tie(kl), std::forward<std::decay_t<decltype(value)>>(value), t);
+          ttg::device::detail::broadcast<0, I, Is...>(std::tie(kl), std::forward<std::decay_t<decltype(value)>>(value), t);
         }
       }
 
@@ -363,12 +364,12 @@ namespace ttg {
                         "Size of keylist tuple must match the number of output terminals");
           prepare_broadcast<0, I, Is...>(kl, std::forward<std::decay_t<decltype(value)>>(value));
           co_await ttg::Void{}; // we'll come back once the task is done
-          broadcast<0, I, Is...>(kl, std::forward<std::decay_t<decltype(value)>>(value));
+          ttg::device::detail::broadcast<0, I, Is...>(kl, std::forward<std::decay_t<decltype(value)>>(value));
         } else if constexpr (!ttg::meta::is_tuple_v<RangesT>) {
           // create a tie to the captured keylist
           prepare_broadcast<0, I, Is...>(std::tie(kl), std::forward<std::decay_t<decltype(value)>>(value));
           co_await ttg::Void{}; // we'll come back once the task is done
-          broadcast<0, I, Is...>(std::tie(kl), std::forward<std::decay_t<decltype(value)>>(value));
+          ttg::device::detail::broadcast<0, I, Is...>(std::tie(kl), std::forward<std::decay_t<decltype(value)>>(value));
         }
       }
     }  // namespace detail
@@ -379,12 +380,12 @@ namespace ttg {
     [[nodiscard]]
     inline detail::send_t broadcast(rangeT &&keylist,
                                     valueT &&value,
-                                    const std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
+                                    std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
       ttg::detail::value_copy_handler<Runtime> copy_handler;
       return detail::send_t{
-              broadcast_coro<0, I, Is...>(std::forward<rangeT>(keylist),
-                                          copy_handler(std::forward<valueT>(value)),
-                                          t, std::move(copy_handler))};
+             detail::broadcast_coro<I, Is...>(std::forward<rangeT>(keylist),
+                                              copy_handler(std::forward<valueT>(value)),
+                                              t, std::move(copy_handler))};
     }
 
     /* overload with implicit terminals and keylist passed by const reference */
@@ -392,8 +393,8 @@ namespace ttg {
               ttg::Runtime Runtime = ttg::ttg_runtime>
     inline detail::send_t broadcast(rangeT &&keylist, valueT &&value) {
       ttg::detail::value_copy_handler<Runtime> copy_handler;
-      return detail::send_t{broadcast_coro<0, i>(std::tie(keylist), copy_handler(std::forward<valueT>(value)),
-                                                 std::move(copy_handler))};
+      return detail::send_t{broadcast_coro<i>(std::tie(keylist), copy_handler(std::forward<valueT>(value)),
+                                              std::move(copy_handler))};
     }
 
     template<typename... Args, ttg::Runtime Runtime = ttg::ttg_runtime>
@@ -577,7 +578,10 @@ namespace ttg {
 
     device_task get_return_object() { return device_task{device_task_handle_type::from_promise(*this)}; }
 
-    void unhandled_exception() { }
+    void unhandled_exception() {
+      std::cerr << "Task coroutine caught an unhandled exception!" << std::endl;
+      throw; // fwd
+    }
 
     //using iterator = std::vector<device_obj_view>::iterator;
 
diff --git a/ttg/ttg/devicescratch.h b/ttg/ttg/devicescratch.h
index 37510436f..729361fa2 100644
--- a/ttg/ttg/devicescratch.h
+++ b/ttg/ttg/devicescratch.h
@@ -11,7 +11,7 @@ using devicescratch = TTG_IMPL_NS::devicescratch<T>;
 
 template<typename T>
 auto make_scratch(T* val, ttg::scope scope, std::size_t count = 1) {
-  return devicescratch<T>(val, scope, 1);
+  return devicescratch<T>(val, scope, count);
 }
 
 namespace detail {
diff --git a/ttg/ttg/parsec/buffer.h b/ttg/ttg/parsec/buffer.h
index 00c7c996e..aac5de428 100644
--- a/ttg/ttg/parsec/buffer.h
+++ b/ttg/ttg/parsec/buffer.h
@@ -19,8 +19,8 @@ namespace ttg_parsec {
 
 namespace detail {
   // fwd decl
-  template<typename T>
-  parsec_data_t* get_parsec_data(const ttg_parsec::buffer<T>& db);
+  template<typename T, typename A>
+  parsec_data_t* get_parsec_data(const ttg_parsec::buffer<T, A>& db);
 } // namespace detail
 
 /**
@@ -59,7 +59,7 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
     // nothing to be done, we don't own the memory
   }
 
-  friend parsec_data_t* detail::get_parsec_data<T>(const ttg_parsec::buffer<T>&);
+  friend parsec_data_t* detail::get_parsec_data<T, Allocator>(const ttg_parsec::buffer<T, Allocator>&);
 
   allocator_type& get_allocator_reference() { return static_cast<allocator_type&>(*this); }
 
@@ -74,7 +74,7 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
 public:
 
   /* The device ID of the CPU. */
-  static constexpr int cpu_device = 0;
+  static constexpr int cpu_device = -2;
 
   buffer() : buffer(nullptr, 0)
   { }
@@ -151,15 +151,16 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
     /* make sure it's a valid device */
     assert(parsec_nb_devices > device_id);
     /* make sure it's a valid copy */
-    assert(m_data->device_copies[device_id] != nullptr);
-    m_data->owner_device = device_id;
+    assert(m_data->device_copies[device_id+2] != nullptr);
+    m_data->owner_device = device_id+2;
   }
 
   /* get the current device ID, i.e., the last updated
-   * device buffer.  */
+   * device buffer. A value of -2 designates the host
+   * as the current device. */
   int get_current_device() const {
     assert(is_valid());
-    return m_data->owner_device;
+    return m_data->owner_device - 2; // 0: host, 1: recursive, 2: first device
   }
 
   /* get the current device pointer */
@@ -179,7 +180,7 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
    */
   element_type* device_ptr_on(int device_id) {
     assert(is_valid());
-    return static_cast<element_type*>(parsec_data_get_ptr(m_data.get(), device_id));
+    return static_cast<element_type*>(parsec_data_get_ptr(m_data.get(), device_id + 2));
   }
 
   /* get the device pointer at the given device
@@ -187,20 +188,20 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
    */
   const element_type* device_ptr_on(int device_id) const {
     assert(is_valid());
-    return static_cast<element_type*>(parsec_data_get_ptr(m_data.get(), device_id));
+    return static_cast<element_type*>(parsec_data_get_ptr(m_data.get(), device_id + 2)); // GPUs start at 2
   }
 
   element_type* host_ptr() {
-    return device_ptr_on(cpu_device);
+    return static_cast<element_type*>(parsec_data_get_ptr(m_data.get(), 0));
   }
 
   const element_type* host_ptr() const {
-    return device_ptr_on(cpu_device);
+    return static_cast<element_type*>(parsec_data_get_ptr(m_data.get(), 0));
   }
 
   bool is_valid_on(int device_id) const {
     assert(is_valid());
-    return (parsec_data_get_ptr(m_data.get(), device_id) != nullptr);
+    return (parsec_data_get_ptr(m_data.get(), device_id+2) != nullptr);
   }
 
   void allocate_on(int device_id) {
@@ -222,7 +223,7 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
   /* Unpin the memory on all devices we currently track. */
   void unpin() {
     if (!is_valid()) return;
-    for (int i = 1; i < parsec_nb_devices; ++i) {
+    for (int i = 0; i < parsec_nb_devices-2; ++i) {
       unpin_on(i);
     }
   }
@@ -362,20 +363,20 @@ template<typename T>
 struct is_buffer : std::false_type
 { };
 
-template<typename T>
-struct is_buffer<buffer<T>> : std::true_type
+template<typename T, typename A>
+struct is_buffer<buffer<T, A>> : std::true_type
 { };
 
-template<typename T>
-struct is_buffer<const buffer<T>> : std::true_type
+template<typename T, typename A>
+struct is_buffer<const buffer<T, A>> : std::true_type
 { };
 
 template<typename T>
 constexpr static const bool is_buffer_v = is_buffer<T>::value;
 
 namespace detail {
-  template<typename T>
-  parsec_data_t* get_parsec_data(const ttg_parsec::buffer<T>& db) {
+  template<typename T, typename A>
+  parsec_data_t* get_parsec_data(const ttg_parsec::buffer<T, A>& db) {
     return const_cast<parsec_data_t*>(db.m_data.get());
   }
 } // namespace detail
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 288545dbb..2bca31533 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -113,6 +113,8 @@
 #include "ttg/device/cublas_helper.h"
 #include "ttg/parsec/parsec-ext.h"
 
+#include "ttg/device/device.h"
+
 #undef TTG_PARSEC_DEBUG_TRACK_DATA_COPIES
 
 /* PaRSEC function declarations */
@@ -1323,15 +1325,35 @@ namespace ttg_parsec {
              dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_KERNEL);
 
 #if defined(PARSEC_HAVE_DEV_CUDA_SUPPORT) && defined(TTG_HAVE_CUDART)
-      parsec_cuda_exec_stream_t *cuda_stream = (parsec_cuda_exec_stream_t *)gpu_stream;
-      ttg::detail::cublas_set_kernel_stream(cuda_stream->cuda_stream);
+      {
+        parsec_cuda_exec_stream_t *cuda_stream = (parsec_cuda_exec_stream_t *)gpu_stream;
+        ttg::detail::cublas_set_kernel_stream(cuda_stream->cuda_stream);
+      }
 #endif // PARSEC_HAVE_DEV_CUDA_SUPPORT
 
+#if defined(PARSEC_HAVE_DEV_CUDA_SUPPORT) && defined(TTG_HAVE_CUDA)
+      {
+        parsec_cuda_exec_stream_t *cuda_stream = (parsec_cuda_exec_stream_t *)gpu_stream;
+        int device = gpu_device->super.device_index - 2; // 0: host, 1: recursive, 2: first GPU
+        ttg::device::detail::set_current(device, cuda_stream->cuda_stream);
+      }
+#endif // defined(PARSEC_HAVE_DEV_CUDA_SUPPORT) && defined(TTG_HAVE_CUDA)
+
 #if defined(PARSEC_HAVE_DEV_HIP_SUPPORT) && defined(TTG_HAVE_HIPBLAS)
-      parsec_hip_exec_stream_t *hip_stream = (parsec_hip_exec_stream_t *)gpu_stream;
-      ttg::detail::hipblas_set_kernel_stream(hip_stream->hip_stream);
+      {
+        parsec_hip_exec_stream_t *hip_stream = (parsec_hip_exec_stream_t *)gpu_stream;
+        ttg::detail::hipblas_set_kernel_stream(hip_stream->hip_stream);
+      }
 #endif // PARSEC_HAVE_DEV_HIP_SUPPORT
 
+#if defined(PARSEC_HAVE_DEV_HIP_SUPPORT) && defined(TTG_HAVE_HIP)
+      {
+        parsec_cuda_exec_stream_t *hip_stream = (parsec_hip_exec_stream_t *)gpu_stream;
+        int device = gpu_device->super.device_index - 2; // 0: host, 1: recursive, 2: first GPU
+        ttg::device::detail::set_current(device, hip_stream->hip_stream);
+      }
+#endif // defined(PARSEC_HAVE_DEV_CUDA_SUPPORT) && defined(TTG_HAVE_CUDA)
+
       /* Here we call back into the coroutine again after the transfers have completed */
       static_op<Space>(&task->parsec_task);
 

From b60577c62430b2b41e44b1898f4f3ab733430d57 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Tue, 3 Oct 2023 17:51:01 +0000
Subject: [PATCH 132/259] Fix release of data copies in buffer

This fixes a segfault in multi-gpu operations. We should not destroy the
full data_t but only remove the host-side copy and release our reference
on the data. The data will be destroyed once all gpu copies have been
removed.

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/devicefunc.h    |  4 ++--
 ttg/ttg/parsec/ttg_data_copy.h | 11 +++++++++--
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/ttg/ttg/parsec/devicefunc.h b/ttg/ttg/parsec/devicefunc.h
index d250e060e..703b42fb4 100644
--- a/ttg/ttg/parsec/devicefunc.h
+++ b/ttg/ttg/parsec/devicefunc.h
@@ -117,7 +117,7 @@ namespace ttg_parsec {
       /* enqueue the transfer into the compute stream to come back once the compute and transfer are complete */
 
 #if defined(TTG_HAVE_CUDART) && defined(PARSEC_HAVE_DEV_CUDA_SUPPORT)
-      std::cout << "cudaMemcpyAsync of " << data->nb_elts << "B" << std::endl;
+      //std::cout << "cudaMemcpyAsync of " << data->nb_elts << "B from " << data->device_copies[data->owner_device]->device_private << " device " << (int)data->owner_device << " to " << data->device_copies[0]->device_private << std::endl;
       parsec_cuda_exec_stream_t *cuda_stream = (parsec_cuda_exec_stream_t *)stream;
       cudaMemcpyAsync(data->device_copies[0]->device_private,
                       data->device_copies[data->owner_device]->device_private,
@@ -176,4 +176,4 @@ namespace ttg_parsec {
 
 } // namespace ttg_parsec
 
-#endif // TTG_PARSEC_DEVICEFUNC_H
\ No newline at end of file
+#endif // TTG_PARSEC_DEVICEFUNC_H
diff --git a/ttg/ttg/parsec/ttg_data_copy.h b/ttg/ttg/parsec/ttg_data_copy.h
index 27814ecf5..9d21cc2d1 100644
--- a/ttg/ttg/parsec/ttg_data_copy.h
+++ b/ttg/ttg/parsec/ttg_data_copy.h
@@ -62,9 +62,16 @@ namespace ttg_parsec {
           status = cudaHostUnregister(data->device_copies[0]->device_private);
           assert(cudaSuccess == status);
           data->device_copies[0]->flags ^= TTG_PARSEC_DATA_FLAG_REGISTERED;
-      }
+        }
 #endif // PARSEC_HAVE_DEV_CUDA_SUPPORT
-        parsec_data_destroy(data);
+        //std::fprintf(stderr, "parsec_data_destroy %p device_copy[0] %p\n", data, data->device_copies[0]);
+        //parsec_data_destroy(data);
+        assert(data->device_copies[0] != nullptr);
+        auto copy = data->device_copies[0];
+        parsec_data_copy_detach(data, data->device_copies[0], 0);
+        PARSEC_OBJ_RELEASE(copy);
+        PARSEC_OBJ_RELEASE(data);
+
       }
 
       static void delete_null_parsec_data(parsec_data_t *) {

From a2f4bbeed5f17cc433354ed88e7385e8fad3dd3b Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 4 Oct 2023 01:08:13 +0000
Subject: [PATCH 133/259] Enable POTRF on hip

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 CMakeLists.txt                   |  5 +++++
 examples/devblas_helper.h        |  6 +++---
 examples/potrf/potrf.h           | 12 ++++++++----
 examples/spmm/spmm_cuda.cc       |  6 ++++--
 ttg/ttg/device/cublas_helper.cpp |  2 +-
 ttg/ttg/parsec/devicefunc.h      |  5 +++++
 ttg/ttg/parsec/ttg.h             |  4 ++++
 7 files changed, 30 insertions(+), 10 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 06169f4e8..b6b9c39fe 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -120,6 +120,11 @@ if (TTG_ENABLE_HIP)
   if (TARGET roc::hipblas)
     set(TTG_HAVE_HIPBLAS True CACHE BOOL "TTG detected support for hipBLAS")
   endif()
+
+  find_package(hipsolver)
+  if (TARGET roc::hipsolver)
+    set(TTG_HAVE_HIPSOLVER True CACHE BOOL "TTG detected support for hipSolver")
+  endif()
   add_compile_definitions(${TTG_HIP_PLATFORM})
 endif(TTG_ENABLE_HIP)
 
diff --git a/examples/devblas_helper.h b/examples/devblas_helper.h
index 704a2bc85..10c420111 100644
--- a/examples/devblas_helper.h
+++ b/examples/devblas_helper.h
@@ -108,9 +108,9 @@ inline const cusolverDnHandle_t& cusolver_handle() {
 
 #ifdef TTG_HAVE_HIPBLAS
 
-#include <hip_runtime.h>
-#include <hipblas.h>
-#include <hipsolverDn.h>
+#include <hip/hip_runtime.h>
+#include <hipblas/hipblas.h>
+#include <hipsolver/hipsolver.h>
 
 /// \brief Returns the rocBLAS handle to be used for launching rocBLAS kernels from the current thread
 /// \return the rocBLAS handle for the current thread
diff --git a/examples/potrf/potrf.h b/examples/potrf/potrf.h
index 50a57a766..24bdd44d2 100644
--- a/examples/potrf/potrf.h
+++ b/examples/potrf/potrf.h
@@ -15,8 +15,8 @@
 #elif defined(TTG_HAVE_HIP)
 #define ES ttg::ExecutionSpace::HIP
 #define TASKRET -> ttg::device_task
-#include <hip/hipsolver.h>
-#include <hip/hipblas.h>
+#include <hipsolver/hipsolver.h>
+#include <hipblas/hipblas.h>
 #else
 #define ES ttg::ExecutionSpace::Host
 #define TASKRET -> void
@@ -39,7 +39,11 @@ namespace potrf {
                                   &Lwork);
       return Lwork;
     #elif defined(TTG_HAVE_HIPBLAS)
-      #error TBCoded
+      hipsolverDnDpotrf_bufferSize(hipsolver_handle(),
+                                   HIPSOLVER_FILL_MODE_LOWER, A.cols(),
+                                   nullptr, A.lda(),
+                                   &Lwork);
+      return Lwork;
     #else
       return 0;
     #endif
@@ -48,7 +52,7 @@ namespace potrf {
   static void device_potrf(MatrixTile<double> &A, double *workspace, int Lwork, int *devInfo) {
     int device = ttg::device::current_device();
     assert(device >= 0);
-  #if defined(TTG_HAVE_CUDA)
+#if defined(TTG_HAVE_CUDA)
     //std::cout << "POTRF A " << A.buffer().device_ptr_on(device) << " device " << device << " cols " << A.cols() << " lda " << A.lda() << " Lwork " << Lwork << " WS " << workspace << " devInfo " << devInfo << std::endl;
     auto handle = cusolver_handle();
     std::cout << "POTRF handle  " << handle << " device " << device << " stream " << ttg::device::current_stream() << std::endl;
diff --git a/examples/spmm/spmm_cuda.cc b/examples/spmm/spmm_cuda.cc
index 05a6599b9..3f6ef0dec 100644
--- a/examples/spmm/spmm_cuda.cc
+++ b/examples/spmm/spmm_cuda.cc
@@ -37,6 +37,8 @@
 
 #include "ttg.h"
 
+#include "../devblas_helper.h"
+
 using namespace ttg;
 
 #include "ttg/util/future.h"
@@ -273,13 +275,13 @@ static void device_gemm(blk_t &C, const blk_t &A, const blk_t &B) {
   int device = C.b.get_current_device();
   assert(device != 0);
 #if defined(TTG_HAVE_CUDA)
-  cublasDgemm(ttg::detail::cublas_get_handle(),
+  cublasDgemm(cublas_handle(),
               CUBLAS_OP_N, CUBLAS_OP_N, C.extent(0), C.extent(1), A.extent(1), &alpha,
               A.b.device_ptr_on(device), A.extent(0),
               B.b.device_ptr_on(device), B.extent(0), &beta,
               C.b.current_device_ptr(), C.extent(0));
 #elif defined(TTG_HAVE_HIPBLAS)
-  hipblasDgemm(ttg::detail::hipblas_get_handle(),
+  hipblasDgemm(hipblas_handle(),
                 HIPBLAS_OP_N, HIPBLAS_OP_N,
                 C.extent(0), C.extent(1), A.extent(1), &alpha,
                 A.b.device_ptr_on(device), A.extent(0),
diff --git a/ttg/ttg/device/cublas_helper.cpp b/ttg/ttg/device/cublas_helper.cpp
index 24e34f52c..768411282 100644
--- a/ttg/ttg/device/cublas_helper.cpp
+++ b/ttg/ttg/device/cublas_helper.cpp
@@ -10,7 +10,7 @@
 #endif // TTG_HAVE_CUDART
 
 #ifdef TTG_HAVE_HIPBLAS
-#include <hip_runtime.h>
+#include <hip/hip_runtime.h>
 #endif // TTG_HAVE_HIPBLAS
 
 namespace ttg::detail {
diff --git a/ttg/ttg/parsec/devicefunc.h b/ttg/ttg/parsec/devicefunc.h
index 703b42fb4..8686588f4 100644
--- a/ttg/ttg/parsec/devicefunc.h
+++ b/ttg/ttg/parsec/devicefunc.h
@@ -122,6 +122,11 @@ namespace ttg_parsec {
       cudaMemcpyAsync(data->device_copies[0]->device_private,
                       data->device_copies[data->owner_device]->device_private,
                       data->nb_elts, cudaMemcpyDeviceToHost, cuda_stream->cuda_stream);
+#elif defined(TTG_HAVE_HIP) && defined(PARSEC_HAVE_DEV_HIP_SUPPORT)
+      parsec_hip_exec_stream_t *hip_stream = (parsec_hip_exec_stream_t *)stream;
+      hipMemcpyAsync(data->device_copies[0]->device_private,
+                      data->device_copies[data->owner_device]->device_private,
+                      data->nb_elts, hipMemcpyDeviceToHost, hip_stream->hip_stream);
 #else
       static_assert(DeviceAvail, "No device implementation detected!");
 #endif // defined(PARSEC_HAVE_DEV_CUDA_SUPPORT)
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 2bca31533..422e9f1ef 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1324,12 +1324,14 @@ namespace ttg_parsec {
       assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_TRANSFER ||
              dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_KERNEL);
 
+#if 0
 #if defined(PARSEC_HAVE_DEV_CUDA_SUPPORT) && defined(TTG_HAVE_CUDART)
       {
         parsec_cuda_exec_stream_t *cuda_stream = (parsec_cuda_exec_stream_t *)gpu_stream;
         ttg::detail::cublas_set_kernel_stream(cuda_stream->cuda_stream);
       }
 #endif // PARSEC_HAVE_DEV_CUDA_SUPPORT
+#endif //0
 
 #if defined(PARSEC_HAVE_DEV_CUDA_SUPPORT) && defined(TTG_HAVE_CUDA)
       {
@@ -1339,12 +1341,14 @@ namespace ttg_parsec {
       }
 #endif // defined(PARSEC_HAVE_DEV_CUDA_SUPPORT) && defined(TTG_HAVE_CUDA)
 
+#if 0
 #if defined(PARSEC_HAVE_DEV_HIP_SUPPORT) && defined(TTG_HAVE_HIPBLAS)
       {
         parsec_hip_exec_stream_t *hip_stream = (parsec_hip_exec_stream_t *)gpu_stream;
         ttg::detail::hipblas_set_kernel_stream(hip_stream->hip_stream);
       }
 #endif // PARSEC_HAVE_DEV_HIP_SUPPORT
+#endif // 0
 
 #if defined(PARSEC_HAVE_DEV_HIP_SUPPORT) && defined(TTG_HAVE_HIP)
       {

From 965433580334b87d46f7e197fe3f9c47a5929181 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 4 Oct 2023 02:41:55 +0000
Subject: [PATCH 134/259] POTRF: add -n parameters to control number of
 iterations

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/potrf/testing_dpotrf.cc | 30 +++++++++++++++++++++---------
 1 file changed, 21 insertions(+), 9 deletions(-)

diff --git a/examples/potrf/testing_dpotrf.cc b/examples/potrf/testing_dpotrf.cc
index 76055b4d4..402f50dd9 100644
--- a/examples/potrf/testing_dpotrf.cc
+++ b/examples/potrf/testing_dpotrf.cc
@@ -36,6 +36,7 @@ int main(int argc, char **argv)
   const char* prof_filename = nullptr;
   char *opt = nullptr;
   int ret = EXIT_SUCCESS;
+  int niter = 3;
 
   if( (opt = getCmdOption(argv+1, argv+argc, "-N")) != nullptr ) {
     N = M = atoi(opt);
@@ -53,6 +54,10 @@ int main(int argc, char **argv)
     prof_filename = opt;
   }
 
+  if( (opt = getCmdOption(argv+1, argv+argc, "-n")) != nullptr) {
+    niter = atoi(opt);
+  }
+
   bool check = !cmdOptionExists(argv+1, argv+argc, "-x");
   bool cow_hint = !cmdOptionExists(argv+1, argv+argc, "-w");
 
@@ -128,16 +133,23 @@ int main(int argc, char **argv)
       std::cout << "==== end dot ====\n";
       beg = std::chrono::high_resolution_clock::now();
     }
-    init_tt->invoke();
 
-    ttg::execute(world);
-    ttg::fence(world);
-    if (world.rank() == 0) {
-      end = std::chrono::high_resolution_clock::now();
-      auto elapsed = (std::chrono::duration_cast<std::chrono::microseconds>(end - beg).count());
-      end = std::chrono::high_resolution_clock::now();
-      std::cout << "TTG Execution Time (milliseconds) : "
-                << elapsed / 1E3 << " : Flops " << (potrf::FLOPS_DPOTRF(N)) << " " << (potrf::FLOPS_DPOTRF(N)/1e9)/(elapsed/1e6) << " GF/s" << std::endl;
+    for (int i = 0; i < niter; ++i) {
+      if (world.rank() == 0) {
+        beg = std::chrono::high_resolution_clock::now();
+      }
+
+      init_tt->invoke();
+      ttg::execute(world);
+      ttg::fence(world);
+
+      if (world.rank() == 0) {
+        end = std::chrono::high_resolution_clock::now();
+        auto elapsed = (std::chrono::duration_cast<std::chrono::microseconds>(end - beg).count());
+        end = std::chrono::high_resolution_clock::now();
+        std::cout << "TTG Execution Time (milliseconds) : "
+                  << elapsed / 1E3 << " : Flops " << (potrf::FLOPS_DPOTRF(N)) << " " << (potrf::FLOPS_DPOTRF(N)/1e9)/(elapsed/1e6) << " GF/s" << std::endl;
+      }
     }
 
     world.dag_off();

From 410dc23120fdf7dbdb9147a0957f39e04464c78c Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 4 Oct 2023 02:43:11 +0000
Subject: [PATCH 135/259] Minor fixes to compile on Frontier

---
 examples/devblas_helper.h   | 46 +++++++------------------------------
 ttg/ttg/parsec/devicefunc.h |  2 ++
 ttg/ttg/parsec/ttg.h        |  2 +-
 3 files changed, 11 insertions(+), 39 deletions(-)

diff --git a/examples/devblas_helper.h b/examples/devblas_helper.h
index 10c420111..349025209 100644
--- a/examples/devblas_helper.h
+++ b/examples/devblas_helper.h
@@ -11,42 +11,10 @@
 #include <cublas_v2.h>
 #include <cusolverDn.h>
 
-namespace detail {
-  template<typename Stream>
-  struct device_stream_t {
-    int device;
-    Stream stream;
-    device_stream_t(int device, Stream stream)
-    : device(device)
-    , stream(stream)
-    { }
-
-    bool operator<(const device_stream_t& ds) const {
-      bool result = ((device < ds.device) && (reinterpret_cast<uintptr_t>(stream) < reinterpret_cast<uintptr_t>(ds.stream)));
-      std::cout << *this << " < " << ds << ": " << result << std::endl;
-      return result;
-    }
-
-    bool operator==(const device_stream_t& ds) const {
-      bool result = ((device == ds.device) && (stream == ds.stream));
-      std::cout << *this << " == " << ds << ": " << result << std::endl;
-      return result;
-    }
-  };
-} // namespace detail
-
-namespace std {
-template<typename Stream>
-  std::ostream& operator<<(std::ostream& os, const ::detail::device_stream_t<Stream>& ds) {
-    os << "[" << ds.device << ", " << ds.stream << "]";
-    return os;
-  }
-
-} //namespace std
-
 /// \brief Returns the cuBLAS handle to be used for launching cuBLAS kernels from the current thread
 /// \return the cuBLAS handle for the current thread
-inline const cublasHandle_t& cublas_handle() {
+template<typename T = int>
+inline const cublasHandle_t& cublas_handle(T _ = 0) {
   using map_type = std::map<std::pair<int, cudaStream_t>, cublasHandle_t>;
   static thread_local map_type handles;
 
@@ -72,9 +40,9 @@ inline const cublasHandle_t& cublas_handle() {
   return it->second;
 }
 
-inline const cusolverDnHandle_t& cusolver_handle() {
+template<typename T = int>
+inline const cusolverDnHandle_t& cusolver_handle(T _ = 0) {
 
-  //using map_type = std::map<detail::device_stream_t<cudaStream_t>, cusolverDnHandle_t>;
   using map_type = std::map<std::pair<int, cudaStream_t>, cusolverDnHandle_t>;
   static thread_local map_type handles;
 
@@ -114,7 +82,8 @@ inline const cusolverDnHandle_t& cusolver_handle() {
 
 /// \brief Returns the rocBLAS handle to be used for launching rocBLAS kernels from the current thread
 /// \return the rocBLAS handle for the current thread
-const hipblasHandle_t& hipblas_handle() {
+template<typename T = int>
+const hipblasHandle_t& hipblas_handle(T _ = 0) {
   static thread_local std::map<int, hipblasHandle_t> handles;
   int device = ttg::device::current_device();
   std::map<int, hipblasHandle_t>::iterator it;
@@ -136,7 +105,8 @@ const hipblasHandle_t& hipblas_handle() {
 
 /// \brief Returns the hipsolver handle to be used for launching rocBLAS kernels from the current thread
 /// \return the hipsolver handle for the current thread
-const hipsolverDnHandle_t& hipsolver_handle() {
+template<typename T = int>
+const hipsolverDnHandle_t& hipsolver_handle(T _ = 0) {
   static thread_local std::map<int, hipsolverDnHandle_t> handles;
   int device = ttg::device::current_device();
   std::map<int, hipsolverDnHandle_t>::iterator it;
diff --git a/ttg/ttg/parsec/devicefunc.h b/ttg/ttg/parsec/devicefunc.h
index 8686588f4..325750e04 100644
--- a/ttg/ttg/parsec/devicefunc.h
+++ b/ttg/ttg/parsec/devicefunc.h
@@ -11,6 +11,8 @@
 
 #if defined(PARSEC_HAVE_DEV_CUDA_SUPPORT)
 #include <parsec/mca/device/cuda/device_cuda.h>
+#elif defined(PARSEC_HAVE_DEV_HIP_SUPPORT)
+#include <parsec/mca/device/hip/device_hip.h>
 #endif // PARSEC_HAVE_DEV_CUDA_SUPPORT
 
 namespace ttg_parsec {
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 422e9f1ef..ea2063d56 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1352,7 +1352,7 @@ namespace ttg_parsec {
 
 #if defined(PARSEC_HAVE_DEV_HIP_SUPPORT) && defined(TTG_HAVE_HIP)
       {
-        parsec_cuda_exec_stream_t *hip_stream = (parsec_hip_exec_stream_t *)gpu_stream;
+        parsec_hip_exec_stream_t *hip_stream = (parsec_hip_exec_stream_t *)gpu_stream;
         int device = gpu_device->super.device_index - 2; // 0: host, 1: recursive, 2: first GPU
         ttg::device::detail::set_current(device, hip_stream->hip_stream);
       }

From 5b81ac4dc380af7473c33fecbbd6f2ef64f614f5 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 5 Oct 2023 15:12:41 +0000
Subject: [PATCH 136/259] buffer: cannot capture this pointer for deleter and
 move, use raw pointer instead

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/buffer.h | 60 ++++++++++++++++++++++++++++-------------
 1 file changed, 42 insertions(+), 18 deletions(-)

diff --git a/ttg/ttg/parsec/buffer.h b/ttg/ttg/parsec/buffer.h
index aac5de428..0309e7883 100644
--- a/ttg/ttg/parsec/buffer.h
+++ b/ttg/ttg/parsec/buffer.h
@@ -51,9 +51,10 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
 private:
   using delete_fn_t = std::function<void(element_type*)>;
 
-  using host_data_ptr   = std::unique_ptr<element_type[], delete_fn_t>;
-  host_data_ptr m_host_data;
+  using host_data_ptr   = std::add_pointer_t<element_type>;
+  host_data_ptr m_host_data = nullptr;
   std::size_t m_count = 0;
+  bool m_owned= false;
 
   static void delete_non_owned(element_type *ptr) {
     // nothing to be done, we don't own the memory
@@ -67,8 +68,8 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
     return allocator_traits::allocate(get_allocator_reference(), n);
   }
 
-  void deallocate(element_type *ptr, std::size_t n) {
-    allocator_traits::deallocate(get_allocator_reference(), ptr, n);
+  void deallocate() {
+    allocator_traits::deallocate(get_allocator_reference(), m_host_data, m_count);
   }
 
 public:
@@ -82,14 +83,15 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
   buffer(std::size_t n)
   : ttg_parsec_data_wrapper_t()
   , allocator_type()
-  , m_host_data(allocate(n), [this, n](element_type* ptr){ deallocate(ptr, n); })
+  , m_host_data(allocate(n))
   , m_count(n)
+  , m_owned(true)
   {
     //std::cout << "buffer " << this << " ctor count "
     //          << count << "(" << m_host_data.get() << ") ttg_copy "
     //          << m_ttg_copy
     //          << " parsec_data " << m_data.get() << std::endl;
-    this->reset_parsec_data(m_host_data.get(), n*sizeof(element_type));
+    this->reset_parsec_data(m_host_data, n*sizeof(element_type));
   }
 
   /* Constructing a buffer using application-managed memory.
@@ -98,13 +100,18 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
   buffer(element_type* ptr, std::size_t n = 1)
   : ttg_parsec_data_wrapper_t()
   , allocator_type()
-  , m_host_data(ptr, &delete_non_owned)
+  , m_host_data(ptr)
   , m_count(n)
+  , m_owned(false)
   {
-    this->reset_parsec_data(m_host_data.get(), n*sizeof(element_type));
+    this->reset_parsec_data(m_host_data, n*sizeof(element_type));
   }
 
   virtual ~buffer() {
+    if (m_owned) {
+      deallocate();
+      m_owned = false;
+    }
     unpin(); // make sure the copies are not pinned
   }
 
@@ -112,10 +119,13 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
   buffer(buffer&& db)
   : ttg_parsec_data_wrapper_t(std::move(db))
   , allocator_type(std::move(db))
-  , m_host_data(std::move(db.m_host_data))
+  , m_host_data(db.m_host_data)
   , m_count(db.m_count)
+  , m_owned(db.m_owned)
   {
+    db.m_host_data = nullptr;
     db.m_count = 0;
+    db.m_owned = false;
   }
 
   /* explicitly disable copying of buffers
@@ -127,9 +137,9 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
   buffer& operator=(buffer&& db) {
     ttg_parsec_data_wrapper_t::operator=(std::move(db));
     allocator_type::operator=(std::move(db));
-    m_host_data = std::move(db.m_host_data);
-    m_count = db.m_count;
-    db.m_count = 0;
+    std::swap(m_host_data, db.m_host_data);
+    std::swap(m_count, db.m_count);
+    std::swap(m_owned, db.m_owned);
     //std::cout << "buffer " << this << " other " << &db << " mv op ttg_copy " << m_ttg_copy << std::endl;
     //std::cout << "buffer::move-assign from " << &db << " ttg-copy " << db.m_ttg_copy
     //          << " to " << this << " ttg-copy " << m_ttg_copy
@@ -255,12 +265,20 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
     /* TODO: can we resize if count is smaller than m_count? */
     /* drop the current data and reallocate */
     reset();
+
+    if (m_owned) {
+      deallocate();
+      m_owned = false;
+    }
+
     if (n == 0) {
-      m_host_data = host_data_ptr(nullptr, &delete_non_owned);
+      m_host_data = nullptr;
+      m_owned = false;
     } else {
-      m_host_data = host_data_ptr(allocate(n), [this, n](element_type* ptr){ deallocate(ptr, n); });
+      m_host_data = allocate(n);
+      m_owned = true;
     }
-    reset_parsec_data(m_host_data.get(), n*sizeof(element_type));
+    reset_parsec_data(m_host_data, n*sizeof(element_type));
     //std::cout << "buffer::reset(" << count << ") ptr " << m_host_data.get()
     //          << " ttg_copy " << m_ttg_copy
     //          << " parsec_data " << m_data.get() << std::endl;
@@ -274,14 +292,20 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
       return;
     }
 
+    if (m_owned) {
+      deallocate();
+    }
+
     if (nullptr == ptr) {
-      m_host_data = host_data_ptr(nullptr, &delete_non_owned);
+      m_host_data = nullptr;
       m_count = 0;
+      m_owned = false;
     } else {
-      m_host_data = host_data_ptr(ptr, &delete_non_owned);
+      m_host_data = ptr;
       m_count = n;
+      m_owned = false;
     }
-    reset_parsec_data(m_host_data.get(), n*sizeof(element_type));
+    reset_parsec_data(m_host_data, n*sizeof(element_type));
     //std::cout << "buffer::reset(" << ptr << ", " << count << ") ptr " << m_host_data.get()
     //          << " ttg_copy " << m_ttg_copy
     //          << " parsec_data " << m_data.get() << std::endl;

From 403b06b575cbddb6de08c1880cf5acb823d8afca Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 5 Oct 2023 15:15:04 +0000
Subject: [PATCH 137/259] POTRF: construct whole TTG each iteration

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/potrf/testing_dpotrf.cc | 68 ++++++++++++++++----------------
 1 file changed, 35 insertions(+), 33 deletions(-)

diff --git a/examples/potrf/testing_dpotrf.cc b/examples/potrf/testing_dpotrf.cc
index 402f50dd9..4ea3b9167 100644
--- a/examples/potrf/testing_dpotrf.cc
+++ b/examples/potrf/testing_dpotrf.cc
@@ -96,45 +96,46 @@ int main(int argc, char **argv)
   parsec_data_collection_set_key((parsec_data_collection_t*)&dcA, (char*)"Matrix A");
 
   if(!check) {
-    ttg::Edge<Key2, void> startup("startup");
-    ttg::Edge<Key2, MatrixTile<double>> topotrf("To POTRF");
-    ttg::Edge<Key2, MatrixTile<double>> result("To result");
-
-    //Matrix<double>* A = new Matrix<double>(n_rows, n_cols, NB, NB);
-    MatrixT<double> A{&dcA};
-    /* TODO: initialize the matrix */
-    /* This works only with the parsec backend! */
-    int random_seed = 3872;
-
-    auto init_tt =  ttg::make_tt<void>([&](std::tuple<ttg::Out<Key2, void>>& out) {
-      for(int i = 0; i < A.rows(); i++) {
-        for(int j = 0; j <= i && j < A.cols(); j++) {
-          if(A.is_local(i, j)) {
-            if(ttg::tracing()) ttg::print("init(", Key2{i, j}, ")");
-            ttg::sendk<0>(Key2{i, j}, out);
+    for (int i = 0; i < niter; ++i) {
+      parsec_devices_release_memory();
+      ttg::Edge<Key2, void> startup("startup");
+      ttg::Edge<Key2, MatrixTile<double>> topotrf("To POTRF");
+      ttg::Edge<Key2, MatrixTile<double>> result("To result");
+
+      //Matrix<double>* A = new Matrix<double>(n_rows, n_cols, NB, NB);
+      MatrixT<double> A{&dcA};
+      /* TODO: initialize the matrix */
+      /* This works only with the parsec backend! */
+      int random_seed = 3872;
+
+      auto init_tt =  ttg::make_tt<void>([&](std::tuple<ttg::Out<Key2, void>>& out) {
+        for(int i = 0; i < A.rows(); i++) {
+          for(int j = 0; j <= i && j < A.cols(); j++) {
+            if(A.is_local(i, j)) {
+              if(ttg::tracing()) ttg::print("init(", Key2{i, j}, ")");
+              ttg::sendk<0>(Key2{i, j}, out);
+            }
           }
         }
-      }
-    }, ttg::edges(), ttg::edges(startup), "Startup Trigger", {}, {"startup"});
-    init_tt->set_keymap([&]() {return world.rank();});
+      }, ttg::edges(), ttg::edges(startup), "Startup Trigger", {}, {"startup"});
+      init_tt->set_keymap([&]() {return world.rank();});
 
-    auto plgsy_ttg = make_plgsy_ttg(A, N, random_seed, startup, topotrf, cow_hint);
-    auto potrf_ttg = potrf::make_potrf_ttg(A, topotrf, result, cow_hint);
-    auto result_ttg = make_result_ttg(A, result, cow_hint);
+      auto plgsy_ttg = make_plgsy_ttg(A, N, random_seed, startup, topotrf, cow_hint);
+      auto potrf_ttg = potrf::make_potrf_ttg(A, topotrf, result, cow_hint);
+      auto result_ttg = make_result_ttg(A, result, cow_hint);
 
-    auto connected = make_graph_executable(init_tt.get());
-    assert(connected);
-    TTGUNUSED(connected);
-    std::cout << "Graph is connected: " << connected << std::endl;
+      auto connected = make_graph_executable(init_tt.get());
+      assert(connected);
+      TTGUNUSED(connected);
+      std::cout << "Graph is connected: " << connected << std::endl;
 
-    if (world.rank() == 0) {
-      std::cout << "==== begin dot ====\n";
-      std::cout << ttg::Dot()(init_tt.get()) << std::endl;
-      std::cout << "==== end dot ====\n";
-      beg = std::chrono::high_resolution_clock::now();
-    }
+      if (world.rank() == 0) {
+        std::cout << "==== begin dot ====\n";
+        std::cout << ttg::Dot()(init_tt.get()) << std::endl;
+        std::cout << "==== end dot ====\n";
+        beg = std::chrono::high_resolution_clock::now();
+      }
 
-    for (int i = 0; i < niter; ++i) {
       if (world.rank() == 0) {
         beg = std::chrono::high_resolution_clock::now();
       }
@@ -150,6 +151,7 @@ int main(int argc, char **argv)
         std::cout << "TTG Execution Time (milliseconds) : "
                   << elapsed / 1E3 << " : Flops " << (potrf::FLOPS_DPOTRF(N)) << " " << (potrf::FLOPS_DPOTRF(N)/1e9)/(elapsed/1e6) << " GF/s" << std::endl;
       }
+      parsec_devices_reset_load(ttg::default_execution_context().impl().context());
     }
 
     world.dag_off();

From 1998655dfaafb564ec089f26fa2e31b38266d646 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 5 Oct 2023 15:19:22 +0000
Subject: [PATCH 138/259] Add TTG_ENABLE_ASAN CMake option

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 CMakeLists.txt | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b6b9c39fe..3a65a6335 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -54,6 +54,7 @@ option(TTG_PARSEC_USE_BOOST_SERIALIZATION "Whether to select Boost serialization
 option(TTG_ENABLE_CUDA "Whether to TTG will look for CUDA" OFF)
 option(TTG_ENABLE_HIP "Whether to TTG will look for HIP" OFF)
 option(TTG_EXAMPLES "Whether to build examples" OFF)
+option(TTG_ENABLE_ASAN "Whether to enable address sanitizer" OFF)
 
 option(TTG_FETCH_BOOST "Whether to fetch+build Boost, if missing" OFF)
 option(TTG_IGNORE_BUNDLED_EXTERNALS "Whether to skip installation and use of bundled external depenedencies (Boost.CallableTraits)" OFF)
@@ -68,6 +69,11 @@ if (FORCE_COLORED_OUTPUT)
     endif ()
 endif (FORCE_COLORED_OUTPUT)
 
+if (TTG_ENABLE_ASAN)
+    add_compile_options(-fsanitize=address)
+    add_link_options(-fsanitize=address)
+endif (TTG_ENABLE_ASAN)
+
 set(TTG_HIP_PLATFORM "__HIP_PLATFORM_AMD__" CACHE STRING "Which platform to use when compiling HIP-related code (default: __HIP_PLATFORM_AMD__)")
 ##########################
 #### prerequisites

From 548b65bd4664a0ff7faecc951eb9ed0c512be150 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 6 Oct 2023 14:33:17 +0000
Subject: [PATCH 139/259] Chain benchmark: Disable wait_kernel co_yield

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/task-benchmarks/chain-ttg-dev.cc | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/examples/task-benchmarks/chain-ttg-dev.cc b/examples/task-benchmarks/chain-ttg-dev.cc
index 5ccf1297d..ac96e16b3 100644
--- a/examples/task-benchmarks/chain-ttg-dev.cc
+++ b/examples/task-benchmarks/chain-ttg-dev.cc
@@ -11,7 +11,7 @@
 #error "Either CUDA OR HIP is required to build this test!"
 #endif // 0
 
-#define NUM_TASKS 10000
+#define NUM_TASKS 100000
 
 using namespace ttg;
 
@@ -56,7 +56,7 @@ auto make_ttg<1>(bool do_move) {
   auto next = make_tt<ES, int>([=](const int &key, auto&& value) -> ttg::device_task {
     //++task_counter;
     co_await ttg::to_device(value.b);
-    co_await ttg::wait_kernel(); // empty kernel
+    //co_await ttg::wait_kernel(); // empty kernel
     if (key < NUM_TASKS) {
       if (do_move) {
         co_await ttg::device::forward(ttg::device::send<0>(key+1, std::move(value)));
@@ -83,7 +83,7 @@ auto make_ttg<2>(bool do_move) {
 
   auto next = make_tt<ES, int>([=](const int &key, A&& v1, A&& v2) -> ttg::device_task {
     co_await ttg::to_device(v1.b, v2.b);
-    co_await ttg::wait_kernel(); // empty kernel
+    //co_await ttg::wait_kernel(); // empty kernel
     if (key < NUM_TASKS) {
       if (do_move) {
         co_await ttg::device::forward(ttg::device::send<0>(key+1, std::move(v1)),
@@ -114,7 +114,7 @@ auto make_ttg<4>(bool do_move) {
 
   auto next = make_tt<ES, int>([=](const int &key, A&& v1, A&& v2, A&& v3, A&& v4) -> ttg::device_task {
     co_await ttg::to_device(v1.b, v2.b, v3.b, v4.b);
-    co_await ttg::wait_kernel(); // empty kernel
+    //co_await ttg::wait_kernel(); // empty kernel
     if (key < NUM_TASKS) {
       if (do_move) {
         co_await ttg::device::forward(ttg::device::send<0>(key+1, std::move(v1)),
@@ -155,7 +155,7 @@ auto make_ttg<8>(bool do_move) {
 
   auto next = make_tt<ES, int>([=](const int &key, auto&& v1, auto&& v2, auto&& v3, auto&& v4, auto&& v5, auto&& v6, auto&& v7, auto&& v8) -> ttg::device_task {
     co_await ttg::to_device(v1.b, v2.b, v3.b, v4.b, v5.b, v6.b, v7.b, v8.b);
-    co_await ttg::wait_kernel(); // empty kernel
+    //co_await ttg::wait_kernel(); // empty kernel
     if (key < NUM_TASKS) {
       if (do_move) {
         co_await ttg::device::forward(ttg::device::send<0>(key+1, std::move(v1)),
@@ -193,7 +193,7 @@ auto make_ttg<0>(bool do_move) {
 
   auto next = make_tt<ES>([](const int& key) -> ttg::device_task {
     co_await ttg::to_device();
-    co_await ttg::wait_kernel();
+    //co_await ttg::wait_kernel();
     if (key < NUM_TASKS) {
       co_await ttg::device::forward(ttg::device::sendk<0>(key+1));
     }
@@ -211,6 +211,11 @@ void run_bench(bool do_move)
   assert(connected);
   std::cout << "Graph " << num_flows << " is connected.\n";
 
+  if (ttg::default_execution_context().rank() == 0) init->invoke();
+
+  ttg_execute(ttg_default_execution_context());
+  ttg_fence(ttg_default_execution_context());
+
   auto t0 = now();
   if (ttg::default_execution_context().rank() == 0) init->invoke();
 
@@ -218,8 +223,8 @@ void run_bench(bool do_move)
   ttg_fence(ttg_default_execution_context());
   auto t1 = now();
 
-  std::cout << "# of tasks = " << task_counter.load() << std::endl;
-  std::cout << "time elapsed (microseconds) = " << duration_in_mus(t0, t1) << std::endl;
+  std::cout << "# of tasks = " << NUM_TASKS << std::endl;
+  std::cout << "time elapsed (microseconds) = " << duration_in_mus(t0, t1) << ", avg " << duration_in_mus(t0, t1) / (double)NUM_TASKS << std::endl;
 }
 
 int main(int argc, char* argv[]) {
@@ -242,7 +247,7 @@ int main(int argc, char* argv[]) {
   case 2: run_bench<2>(do_move); break;
   case 4: run_bench<4>(do_move); break;
   case 8: run_bench<8>(do_move); break;
-  default: std::cout << "Unsupported number of flows: " << num_flows << std::endl;
+  default: std::cout << "Unsupported number of flows: " << NUM_TASKS << std::endl;
   }
 
   ttg_finalize();

From fdf6e2e1218ea81661606da390df38d6b36e459d Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 6 Oct 2023 14:33:53 +0000
Subject: [PATCH 140/259] PaRSEC: Increase max AM size to 1MB

---
 ttg/ttg/parsec/ttg.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index ea2063d56..c942940b1 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -240,7 +240,7 @@ namespace ttg_parsec {
     int parsec_ttg_profile_backend_allocate_datacopy, parsec_ttg_profile_backend_free_datacopy;
 #endif
 
-    static constexpr const int PARSEC_TTG_MAX_AM_SIZE = 4 * 1024;
+    static constexpr const int PARSEC_TTG_MAX_AM_SIZE = 1 * 1024*1024;
     WorldImpl(int *argc, char **argv[], int ncores, parsec_context_t *c = nullptr)
         : WorldImplBase(query_comm_size(), query_comm_rank())
         , ctx(c)

From 9d21225a774c461470cecac424f3006000bcac34 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 6 Oct 2023 14:39:58 +0000
Subject: [PATCH 141/259] Remove TTG cublas_helper, superseeded by devblas
 helper in examples

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/CMakeLists.txt               | 15 +-----
 ttg/ttg/device/cublas_helper.cpp | 81 --------------------------------
 ttg/ttg/device/cublas_helper.h   | 35 --------------
 ttg/ttg/parsec/ttg.h             | 19 --------
 4 files changed, 1 insertion(+), 149 deletions(-)
 delete mode 100644 ttg/ttg/device/cublas_helper.cpp
 delete mode 100644 ttg/ttg/device/cublas_helper.h

diff --git a/ttg/CMakeLists.txt b/ttg/CMakeLists.txt
index 476118524..427a3ffee 100644
--- a/ttg/CMakeLists.txt
+++ b/ttg/CMakeLists.txt
@@ -68,7 +68,6 @@ set(ttg-impl-headers
         ${CMAKE_CURRENT_BINARY_DIR}/ttg/config.h
     )
 set(ttg_device_headers
-        ${CMAKE_CURRENT_SOURCE_DIR}/ttg/device/cublas_helper.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/device/device.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/device/task.h
     )
@@ -117,23 +116,11 @@ if (TARGET std::coroutine)
           )
 endif ()
 if (TTG_HAVE_CUDA)
-  list(APPEND ttg-sources
-          ${CMAKE_CURRENT_SOURCE_DIR}/ttg/device/cublas_helper.cpp
-          )
-  list(APPEND ttg-deps CUDA::cudart CUDA::cublas)
+  list(APPEND ttg-deps CUDA::cudart)
 endif (TTG_HAVE_CUDA)
 
 if (TTG_HAVE_HIPBLAS)
-  # add the helper file if not added already
-  if (NOT TTG_HAVE_CUDA)
-    list(APPEND ttg-sources
-      ${CMAKE_CURRENT_SOURCE_DIR}/ttg/device/cublas_helper.cpp
-    )
-  endif (NOT TTG_HAVE_CUDA)
-  list(APPEND ttg-deps roc::hipblas)
-  find_package(hip REQUIRED)
   list(APPEND ttg-deps hip::host)
-
 endif (TTG_HAVE_HIPBLAS)
 
 set(ttg-public-headers ${ttg-headers};${ttg-impl-headers};${ttg-base-headers};${ttg-util-headers};${ttg_device_headers})
diff --git a/ttg/ttg/device/cublas_helper.cpp b/ttg/ttg/device/cublas_helper.cpp
deleted file mode 100644
index 768411282..000000000
--- a/ttg/ttg/device/cublas_helper.cpp
+++ /dev/null
@@ -1,81 +0,0 @@
-#include "ttg/config.h"
-#include "ttg/device/cublas_helper.h"
-
-#include <memory>
-#include <stdexcept>
-#include <optional>
-#include <map>
-#ifdef TTG_HAVE_CUDART
-#include <cuda_runtime.h>
-#endif // TTG_HAVE_CUDART
-
-#ifdef TTG_HAVE_HIPBLAS
-#include <hip/hip_runtime.h>
-#endif // TTG_HAVE_HIPBLAS
-
-namespace ttg::detail {
-
-#ifdef TTG_HAVE_CUDART
-/// \brief Returns the cuBLAS handle to be used for launching cuBLAS kernels from the current thread
-/// \return the cuBLAS handle for the current thread
-const cublasHandle_t& cublas_get_handle() {
-  static thread_local std::map<int, cublasHandle_t> handles;
-  int device;
-  if (cudaSuccess != cudaGetDevice(&device)){
-      throw std::runtime_error("cudaGetDevice failed");
-  }
-  std::map<int, cublasHandle_t>::iterator it;
-  if ((it = handles.find(device)) == handles.end()){
-    cublasHandle_t handle;
-    auto status = cublasCreate_v2(&handle);
-    if (CUBLAS_STATUS_SUCCESS != status) {
-      throw std::runtime_error("cublasCreate_v2 failed");
-    }
-    auto [iterator, success] = handles.insert({device, handle});
-    it = iterator;
-  }
-
-  return it->second;
-}
-
-void cublas_set_kernel_stream(cudaStream_t stream) {
-    cublasStatus_t status = cublasSetStream_v2(cublas_get_handle(), stream);
-    if (CUBLAS_STATUS_SUCCESS != status) {
-        throw std::runtime_error("cublasSetStream_v2 failed");
-    }
-}
-#endif // TTG_HAVE_CUDART
-
-#ifdef TTG_HAVE_HIPBLAS
-/// \brief Returns the rocBLAS handle to be used for launching rocBLAS kernels from the current thread
-/// \return the rocBLAS handle for the current thread
-const hipblasHandle_t& hipblas_get_handle() {
-  static thread_local std::map<int, hipblasHandle_t> handles;
-  int device;
-  if (hipSuccess != hipGetDevice(&device)){
-      throw std::runtime_error("hipGetDevice failed");
-  }
-  std::map<int, hipblasHandle_t>::iterator it;
-  if ((it = handles.find(device)) == handles.end()){
-    hipblasHandle_t handle;
-    auto status = hipblasCreate(&handle);
-    if (HIPBLAS_STATUS_SUCCESS != status) {
-      throw std::runtime_error("hipblasCreate failed");
-    }
-    auto [iterator, success] = handles.insert({device, handle});
-    it = iterator;
-  }
-  return it->second;
-}
-
-void hipblas_set_kernel_stream(hipStream_t stream) {
-    hipblasStatus_t status = hipblasSetStream(hipblas_get_handle(), stream);
-    if (HIPBLAS_STATUS_SUCCESS != status) {
-        throw std::runtime_error("hipblasSetStream failed");
-    }
-}
-
-#endif // TTG_HAVE_HIPBLAS
-
-
-} // namespace
\ No newline at end of file
diff --git a/ttg/ttg/device/cublas_helper.h b/ttg/ttg/device/cublas_helper.h
deleted file mode 100644
index 6ab106bfb..000000000
--- a/ttg/ttg/device/cublas_helper.h
+++ /dev/null
@@ -1,35 +0,0 @@
-#ifndef TTG_DEVICE_CUBLAS_HELPER_H
-#define TTG_DEVICE_CUBLAS_HELPER_H
-
-#include "ttg/config.h"
-
-#ifdef TTG_HAVE_CUDART
-#include <cublas_v2.h>
-
-namespace ttg::detail {
-
-/// \brief Returns the cuBLAS context used by this thread
-const cublasHandle_t& cublas_get_handle();
-
-/// associates the given CUDA stream with the cuBLAS context used by this thread
-void cublas_set_kernel_stream(cudaStream_t stream);
-
-} // namespace ttg::detail
-#endif // TTG_HAVE_CUDART
-
-
-#ifdef TTG_HAVE_HIPBLAS
-#include <hipblas/hipblas.h>
-
-namespace ttg::detail {
-
-const hipblasHandle_t& hipblas_get_handle();
-
-/// associates the given HIP stream with the hipblas context used by this thread
-void hipblas_set_kernel_stream(hipStream_t stream);
-
-} // namespace ttg::detail
-#endif // TTG_HAVE_HIPBLAS
-
-
-#endif // TTG_DEVICE_CUBLAS_HELPER_H
\ No newline at end of file
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index c942940b1..367d6c250 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -110,7 +110,6 @@
 #include "ttg/parsec/thread_local.h"
 #include "ttg/parsec/ptr.h"
 #include "ttg/parsec/task.h"
-#include "ttg/device/cublas_helper.h"
 #include "ttg/parsec/parsec-ext.h"
 
 #include "ttg/device/device.h"
@@ -1324,15 +1323,6 @@ namespace ttg_parsec {
       assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_TRANSFER ||
              dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_KERNEL);
 
-#if 0
-#if defined(PARSEC_HAVE_DEV_CUDA_SUPPORT) && defined(TTG_HAVE_CUDART)
-      {
-        parsec_cuda_exec_stream_t *cuda_stream = (parsec_cuda_exec_stream_t *)gpu_stream;
-        ttg::detail::cublas_set_kernel_stream(cuda_stream->cuda_stream);
-      }
-#endif // PARSEC_HAVE_DEV_CUDA_SUPPORT
-#endif //0
-
 #if defined(PARSEC_HAVE_DEV_CUDA_SUPPORT) && defined(TTG_HAVE_CUDA)
       {
         parsec_cuda_exec_stream_t *cuda_stream = (parsec_cuda_exec_stream_t *)gpu_stream;
@@ -1341,15 +1331,6 @@ namespace ttg_parsec {
       }
 #endif // defined(PARSEC_HAVE_DEV_CUDA_SUPPORT) && defined(TTG_HAVE_CUDA)
 
-#if 0
-#if defined(PARSEC_HAVE_DEV_HIP_SUPPORT) && defined(TTG_HAVE_HIPBLAS)
-      {
-        parsec_hip_exec_stream_t *hip_stream = (parsec_hip_exec_stream_t *)gpu_stream;
-        ttg::detail::hipblas_set_kernel_stream(hip_stream->hip_stream);
-      }
-#endif // PARSEC_HAVE_DEV_HIP_SUPPORT
-#endif // 0
-
 #if defined(PARSEC_HAVE_DEV_HIP_SUPPORT) && defined(TTG_HAVE_HIP)
       {
         parsec_hip_exec_stream_t *hip_stream = (parsec_hip_exec_stream_t *)gpu_stream;

From 24155bc6f0ad06c515a56b691d0944075a2344df Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Wed, 11 Oct 2023 08:12:43 -0400
Subject: [PATCH 142/259] [cmake] FindTBB can handle OneTBB

---
 cmake/modules/FindTBB.cmake | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/cmake/modules/FindTBB.cmake b/cmake/modules/FindTBB.cmake
index d9676221f..6fe1969d4 100644
--- a/cmake/modules/FindTBB.cmake
+++ b/cmake/modules/FindTBB.cmake
@@ -354,11 +354,17 @@ findpkg_finish(TBB_MALLOC_PROXY)
 #parse all the version numbers from tbb
 if(NOT TBB_VERSION)
 
- #only read the start of the file
- file(READ
-      "${TBB_INCLUDE_DIR}/tbb/tbb_stddef.h"
-      TBB_VERSION_CONTENTS
-      LIMIT 2048)
+  if (EXISTS "${TBB_INCLUDE_DIR}/oneapi/tbb/version.h")
+    file(STRINGS
+            "${TBB_INCLUDE_DIR}/oneapi/tbb/version.h"
+            TBB_VERSION_CONTENTS
+            REGEX "VERSION")
+  else()
+    file(STRINGS
+            "${TBB_INCLUDE_DIR}/tbb/tbb_stddef.h"
+            TBB_VERSION_CONTENTS
+            REGEX "VERSION")
+  endif()
 
   string(REGEX REPLACE
     ".*#define TBB_VERSION_MAJOR ([0-9]+).*" "\\1"

From ee82565b304d30bcb71e863136688258a7485e8e Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Wed, 11 Oct 2023 08:13:42 -0400
Subject: [PATCH 143/259] [cmake] introduce TTG_HAVE_DEVICE

---
 CMakeLists.txt      | 6 ++++++
 ttg/ttg/config.in.h | 2 ++
 2 files changed, 8 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 06169f4e8..7c75bc1ac 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -123,6 +123,12 @@ if (TTG_ENABLE_HIP)
   add_compile_definitions(${TTG_HIP_PLATFORM})
 endif(TTG_ENABLE_HIP)
 
+set(_ttg_have_device FALSE)
+if (TTG_HAVE_CUDA OR TTG_HAVE_HIP)
+    set(_ttg_have_device TRUE)
+endif()
+set(TTG_HAVE_DEVICE ${_ttg_have_device} CACHE BOOL "True if TTG has support for any device programming model")
+
 ##########################
 #### prerequisite runtimes
 ##########################
diff --git a/ttg/ttg/config.in.h b/ttg/ttg/config.in.h
index 22b92608d..ea9a20a63 100644
--- a/ttg/ttg/config.in.h
+++ b/ttg/ttg/config.in.h
@@ -19,6 +19,8 @@
 
 #cmakedefine TTG_HAVE_HIPBLAS
 
+#cmakedefine TTG_HAVE_DEVICE
+
 #cmakedefine TTG_HAVE_MPI
 #cmakedefine TTG_HAVE_MPIEXT
 

From 0993f7913d1b7a85ce9892dccb159704fe3e5cf3 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Wed, 11 Oct 2023 08:15:48 -0400
Subject: [PATCH 144/259] [cmake] introduce TTG_HAVE_DEVICE used to
 conditionally compile *portable* device-specific code

---
 ttg/ttg/make_tt.h    | 20 +++++++++++++++++---
 ttg/ttg/parsec/ttg.h | 17 ++++++++++++++---
 2 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/ttg/ttg/make_tt.h b/ttg/ttg/make_tt.h
index 5dd18402d..c2d8f2c72 100644
--- a/ttg/ttg/make_tt.h
+++ b/ttg/ttg/make_tt.h
@@ -152,9 +152,14 @@ class CallableWrapTTArgs
 #ifdef TTG_HAS_COROUTINE
       std::conditional_t<std::is_same_v<returnT, ttg::resumable_task>,
                          ttg::coroutine_handle<>,
+#ifdef TTG_HAVE_DEVICE
                          std::conditional_t<std::is_same_v<returnT, ttg::device_task>,
                                             ttg::device_task::base_type,
-                                            void>>;
+                                            void>
+#else  // TTG_HAVE_DEVICE
+                           void
+#endif  // TTG_HAVE_DEVICE
+                         >;
 #else   // TTG_HAS_COROUTINE
       void;
 #endif  // TTG_HAS_COROUTINE
@@ -180,11 +185,20 @@ class CallableWrapTTArgs
           coro_handle = ret;
         }
         return coro_handle;
-      } else if constexpr (std::is_same_v<returnT, ttg::device_task>) {
+      } else
+#ifdef TTG_HAVE_DEVICE
+          if constexpr (std::is_same_v<returnT, ttg::device_task>) {
         ttg::device_task::base_type coro_handle = ret;
         return coro_handle;
       }
-      if constexpr (!(std::is_same_v<returnT, ttg::resumable_task> || std::is_same_v<returnT, ttg::device_task>))
+#else  // TTG_HAVE_DEVICE
+        ttg::abort();  // should not happen
+#endif  // TTG_HAVE_DEVICE
+      if constexpr (!(std::is_same_v<returnT, ttg::resumable_task>
+#ifdef TTG_HAVE_DEVICE
+          || std::is_same_v<returnT, ttg::device_task>
+#endif  // TTG_HAVE_DEVICE
+              ))
 #endif
       {
         static_assert(std::tuple_size_v<std::remove_reference_t<decltype(out)>> == 1,
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 288545dbb..7b40162a4 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -33,7 +33,9 @@
 #include "ttg/util/print.h"
 #include "ttg/util/trace.h"
 #include "ttg/util/typelist.h"
+#ifdef TTG_HAVE_DEVICE
 #include "ttg/device/task.h"
+#endif  // TTG_HAVE_DEVICE
 
 #include "ttg/serialization/data_descriptor.h"
 
@@ -1299,6 +1301,7 @@ namespace ttg_parsec {
               task->copies[IS]->get_ptr()))...};
     }
 
+#ifdef TTG_HAVE_DEVICE
     /**
      * Submit callback called by PaRSEC once all input transfers have completed.
      */
@@ -1484,6 +1487,7 @@ namespace ttg_parsec {
       ttg::abort();
       return PARSEC_HOOK_RETURN_DONE; // will not be reacehed
     }
+#endif  // TTG_HAVE_DEVICE
 
     template <ttg::ExecutionSpace Space>
     static parsec_hook_return_t static_op(parsec_task_t *parsec_task) {
@@ -1525,6 +1529,7 @@ namespace ttg_parsec {
         detail::parsec_ttg_caller = nullptr;
       }
       else {  // resume the suspended coroutine
+#ifdef TTG_HAVE_DEVICE
         auto coro = static_cast<ttg::device_task>(ttg::device_task_handle_type::from_address(suspended_task_address));
         assert(detail::parsec_ttg_caller == nullptr);
         detail::parsec_ttg_caller = static_cast<detail::parsec_ttg_task_base_t*>(task);
@@ -1552,9 +1557,12 @@ namespace ttg_parsec {
         }
         task->suspended_task_address = suspended_task_address;
 #else
-#endif // 0
+#endif  // TTG_HAS_COROUTINE
         ttg::abort();  // should not happen
-#endif
+#endif  // 0
+#else  // TTG_HAVE_DEVICE
+ttg::abort();  // should not happen
+#endif  // TTG_HAVE_DEVICE
       }
       task->suspended_task_address = suspended_task_address;
 
@@ -3437,6 +3445,7 @@ namespace ttg_parsec {
 
       /* if we still have a coroutine handle we invoke it one more time to get the sends/broadcasts */
       if (task->suspended_task_address) {
+#ifdef TTG_HAVE_DEVICE
         // get the device task from the coroutine handle
         auto dev_task = ttg::device_task_handle_type::from_address(task->suspended_task_address);
 
@@ -3453,7 +3462,9 @@ namespace ttg_parsec {
           dev_data.do_sends();
           detail::parsec_ttg_caller = nullptr;
         }
-
+#else // TTG_HAVE_DEVICE
+        ttg::abort();  // should not happen
+#endif // TTG_HAVE_DEVICE
         /* the coroutine should have completed and we cannot access the promise anymore */
         task->suspended_task_address = nullptr;
       }

From 366e60e149a8c249db070afee1cab6beb057f40b Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Wed, 11 Oct 2023 08:20:28 -0400
Subject: [PATCH 145/259] bspmm: support other reals ... depends on
 https://github.com/ValeevGroup/BTAS/pull/164 so bump BTAS tag

---
 .../ExternalDependenciesVersions.cmake        |  2 +-
 examples/CMakeLists.txt                       |  2 +-
 examples/spmm/spmm.cc                         | 41 ++++++++++++-------
 examples/spmm/spmm_cuda.cc                    | 39 +++++++++++-------
 4 files changed, 52 insertions(+), 32 deletions(-)

diff --git a/cmake/modules/ExternalDependenciesVersions.cmake b/cmake/modules/ExternalDependenciesVersions.cmake
index b4aa2e8e1..725dac9b6 100644
--- a/cmake/modules/ExternalDependenciesVersions.cmake
+++ b/cmake/modules/ExternalDependenciesVersions.cmake
@@ -7,5 +7,5 @@ set(TTG_TRACKED_CATCH2_VERSION 2.13.1)
 set(TTG_TRACKED_CEREAL_VERSION 1.3.0)
 set(TTG_TRACKED_MADNESS_TAG e4ee892717e7f91f824a9652d53d4da2acf6920e)
 set(TTG_TRACKED_PARSEC_TAG e59bed9d4934775792a58980cf5500bd41381bc4)
-set(TTG_TRACKED_BTAS_TAG d73153ad9bc41a177e441ef04eceff7fab0c766d)
+set(TTG_TRACKED_BTAS_TAG a02be0d29fb4a788ecef43de711dcd6d6f1cb6b8)
 set(TTG_TRACKED_TILEDARRAY_TAG dfa3f76ebd58c05c64acf6e57cbdc85e90c880a8)
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 24b6f5897..74c1cb775 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -14,7 +14,7 @@ if (TARGET tiledarray)
     if (TARGET CUDA::cublas)
       add_ttg_executable(bspmm-cuda spmm/spmm_cuda.cc
                          LINK_LIBRARIES tiledarray TiledArray_Eigen BTAS Boost::boost CUDA::cublas
-                         COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2;TTG_HAVE_CUDA=1
+                         COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2
                          RUNTIMES "parsec")
     elseif (TARGET roc::hipblas)
       add_ttg_executable(bspmm-hip spmm/spmm_cuda.cc
diff --git a/examples/spmm/spmm.cc b/examples/spmm/spmm.cc
index c0e6dd881..b2ae2558a 100644
--- a/examples/spmm/spmm.cc
+++ b/examples/spmm/spmm.cc
@@ -10,7 +10,6 @@
 #include <vector>
 
 #if __has_include(<btas/features.h>)
-#pragma message("C Preprocessor got here!")
 #include <btas/features.h>
 #ifdef BTAS_IS_USABLE
 #include <btas/btas.h>
@@ -41,7 +40,8 @@ using namespace ttg;
 #include "ttg/util/bug.h"
 
 #if defined(BLOCK_SPARSE_GEMM) && defined(BTAS_IS_USABLE)
-using blk_t = btas::Tensor<double, btas::DEFAULT::range, btas::mohndle<btas::varray<double>, btas::Handle::shared_ptr>>;
+using scalar_t = double;
+using blk_t = btas::Tensor<scalar_t, btas::DEFAULT::range, btas::mohndle<btas::varray<scalar_t>, btas::Handle::shared_ptr>>;
 
 #if defined(TTG_USE_PARSEC)
 namespace ttg {
@@ -695,32 +695,43 @@ class Control : public TT<void, std::tuple<Out<Key<2>>>, Control> {
   }
 };
 
+std::tuple<float, float> norms(float t) { return std::make_tuple(t * t, std::abs(t)); }
+std::tuple<double, double> norms(double t) { return std::make_tuple(t * t, std::abs(t)); }
+
+template <typename T>
+std::tuple<T, T> norms(std::complex<T> t) {
+  auto abs_t = std::abs(t);
+  return std::make_tuple(abs_t * abs_t, abs_t);
+}
+
 #ifdef BTAS_IS_USABLE
 template <typename T_, class Range_, class Store_>
-std::tuple<T_, T_> norms(const btas::Tensor<T_, Range_, Store_> &t) {
-  T_ norm_2_square = 0.0;
-  T_ norm_inf = 0.0;
-  for (auto k : t) {
-    norm_2_square += k * k;
-    norm_inf = std::max(norm_inf, std::abs(k));
+auto norms(const btas::Tensor<T_, Range_, Store_> &t) {
+  using T = decltype(std::abs(std::declval<T_>()));
+  T norm_2_square = 0.0;
+  T norm_inf = 0.0;
+  for (auto elem : t) {
+    T elem_norm_2_square, elem_norm_inf;
+    std::tie(elem_norm_2_square, elem_norm_inf) = norms(elem);
+    norm_2_square += elem_norm_2_square;
+    norm_inf = std::max(norm_inf, elem_norm_inf);
   }
   return std::make_tuple(norm_2_square, norm_inf);
 }
 #endif
 
-std::tuple<double, double> norms(double t) { return std::make_tuple(t * t, std::abs(t)); }
-
-template <typename Blk = blk_t>
-std::tuple<double, double> norms(const SpMatrix<Blk> &A) {
-  double norm_2_square = 0.0;
-  double norm_inf = 0.0;
+template <typename Blk>
+auto norms(const SpMatrix<Blk> &A) {
+  using T = decltype(std::abs(std::declval<typename Blk::value_type>()));
+  T norm_2_square = 0.0;
+  T norm_inf = 0.0;
   for (int i = 0; i < A.outerSize(); ++i) {
     for (typename SpMatrix<Blk>::InnerIterator it(A, i); it; ++it) {
       //  cout << 1+it.row() << "\t"; // row index
       //  cout << 1+it.col() << "\t"; // col index (here it is equal to k)
       //  cout << it.value() << endl;
       auto elem = it.value();
-      double elem_norm_2_square, elem_norm_inf;
+      T elem_norm_2_square, elem_norm_inf;
       std::tie(elem_norm_2_square, elem_norm_inf) = norms(elem);
       norm_2_square += elem_norm_2_square;
       norm_inf = std::max(norm_inf, elem_norm_inf);
diff --git a/examples/spmm/spmm_cuda.cc b/examples/spmm/spmm_cuda.cc
index 05a6599b9..b9631a35a 100644
--- a/examples/spmm/spmm_cuda.cc
+++ b/examples/spmm/spmm_cuda.cc
@@ -246,13 +246,14 @@ struct DeviceTensor : public ttg::TTValue<DeviceTensor<_T, _Range, _Storage>>
 
 };
 
+using scalar_t = double;
 #if defined(TTG_HAVE_CUDA) || defined(TTG_HAVE_HIPBLAS)
-using blk_t = DeviceTensor<double, btas::DEFAULT::range,
-                           btas::mohndle<btas::varray<double, TiledArray::device_pinned_allocator<double>>,
+using blk_t = DeviceTensor<scalar_t, btas::DEFAULT::range,
+                           btas::mohndle<btas::varray<scalar_t, TiledArray::device_pinned_allocator<scalar_t>>,
                                          btas::Handle::shared_ptr>>;
 #else
-using blk_t = DeviceTensor<double, btas::DEFAULT::range,
-                           btas::mohndle<btas::varray<double>, btas::Handle::shared_ptr>>;
+using blk_t = DeviceTensor<scalar_t, btas::DEFAULT::range,
+                           btas::mohndle<btas::varray<scalar_t>, btas::Handle::shared_ptr>>;
 #endif
 
 
@@ -263,9 +264,13 @@ using blk_t = DeviceTensor<double, btas::DEFAULT::range,
 //}
 
 /* TODO: call CUDA gemm here */
-static void device_gemm(blk_t &C, const blk_t &A, const blk_t &B) {
-  static const double alpha = 1.0;
-  static const double beta  = 1.0;
+template <typename Blk>
+static void device_gemm(Blk &C, const Blk &A, const Blk &B) {
+  using blk_t = Blk;
+  using T = typename blk_t::value_type;
+  static_assert(std::is_same_v<T,double> || std::is_same_v<T,float>);
+  static const T alpha = 1.0;
+  static const T beta  = 1.0;
   // make sure all memory is on the device
   // TODO: A and B are read-only so the owner device will be 0. How to fix?
   //assert(A.b.get_current_device() != 0);
@@ -273,11 +278,16 @@ static void device_gemm(blk_t &C, const blk_t &A, const blk_t &B) {
   int device = C.b.get_current_device();
   assert(device != 0);
 #if defined(TTG_HAVE_CUDA)
-  cublasDgemm(ttg::detail::cublas_get_handle(),
-              CUBLAS_OP_N, CUBLAS_OP_N, C.extent(0), C.extent(1), A.extent(1), &alpha,
-              A.b.device_ptr_on(device), A.extent(0),
-              B.b.device_ptr_on(device), B.extent(0), &beta,
-              C.b.current_device_ptr(), C.extent(0));
+  if constexpr (std::is_same_v<T,double>) {
+      cublasDgemm(ttg::detail::cublas_get_handle(), CUBLAS_OP_N, CUBLAS_OP_N, C.extent(0), C.extent(1), A.extent(1),
+                  &alpha, A.b.device_ptr_on(device), A.extent(0), B.b.device_ptr_on(device), B.extent(0), &beta,
+                  C.b.current_device_ptr(), C.extent(0));
+  }
+  else if constexpr (std::is_same_v<T,float>) {
+      cublasSgemm(ttg::detail::cublas_get_handle(), CUBLAS_OP_N, CUBLAS_OP_N, C.extent(0), C.extent(1), A.extent(1),
+                  &alpha, A.b.device_ptr_on(device), A.extent(0), B.b.device_ptr_on(device), B.extent(0), &beta,
+                  C.b.current_device_ptr(), C.extent(0));
+  }
 #elif defined(TTG_HAVE_HIPBLAS)
   hipblasDgemm(ttg::detail::hipblas_get_handle(),
                 HIPBLAS_OP_N, HIPBLAS_OP_N,
@@ -288,8 +298,6 @@ static void device_gemm(blk_t &C, const blk_t &A, const blk_t &B) {
 #endif
 }
 
-//using blk_t = btas::Tensor<double, btas::DEFAULT::range, btas::mohndle<btas::varray<double>, btas::Handle::shared_ptr>>;
-
 #if defined(TTG_USE_PARSEC)
 namespace ttg {
   template <>
@@ -311,8 +319,9 @@ namespace ttg {
       return dim;
     }
     static auto get_data(blk_t &b) {
+      using T = typename blk_t::value_type;
       if (!b.empty())
-        return boost::container::small_vector<iovec, 1>(1, iovec{b.size() * sizeof(double), b.data()});
+        return boost::container::small_vector<iovec, 1>(1, iovec{b.size() * sizeof(T), b.data()});
       else
         return boost::container::small_vector<iovec, 1>{};
     }

From 148a3ad9d3f179474179f68b064e00ce69c4cb53 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 30 Oct 2023 01:59:27 +0000
Subject: [PATCH 146/259] Add debugging capabilities for POTRF benchmark

Set DEBUG_TILES_VALUES=1 to enable norm2 check of tiles.

This also fixes issues with the parsec data wrapper. This needs some
more thought and might come back to haunt us if we try to use data with
extrinsic tracking information.

Also adds ttg::device::Device to query the ID and type of an instance of
the current execution space (e.g., the currently assigned GPU).

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/CMakeLists.txt        |  20 ++---
 examples/devblas_helper.h      |  16 +++-
 examples/matrixtile.h          |  35 +++++++--
 examples/potrf/plgsy.h         |   8 +-
 examples/potrf/potrf.h         | 134 ++++++++++++++++++++++++++++++++-
 examples/potrf/result.h        |  10 ++-
 examples/potrf/util.h          |  10 +++
 ttg/ttg/device/device.h        |  76 +++++++++++++++++--
 ttg/ttg/execution.h            |  12 +++
 ttg/ttg/parsec/buffer.h        |  29 +++++--
 ttg/ttg/parsec/ttg.h           |   2 +
 ttg/ttg/parsec/ttg_data_copy.h |  10 +--
 12 files changed, 318 insertions(+), 44 deletions(-)
 create mode 100644 examples/potrf/util.h

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 4d8896444..647cdffaa 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -11,6 +11,12 @@ if (TARGET tiledarray)
     # block-sparse needs BTAS ... it's always provided by TA
     # since only need to use matrices, limit BTAS_TARGET_MAX_INDEX_RANK to 2
     add_ttg_executable(bspmm spmm/spmm.cc LINK_LIBRARIES tiledarray TiledArray_Eigen BTAS Boost::boost COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2)
+
+    add_ttg_executable(testing_dpotrf potrf/testing_dpotrf.cc LINK_LIBRARIES tiledarray lapackpp)
+    add_ttg_executable(testing_dtrtri potrf/testing_dtrtri.cc LINK_LIBRARIES tiledarray lapackpp)
+    add_ttg_executable(testing_dlauum potrf/testing_dlauum.cc LINK_LIBRARIES tiledarray lapackpp)
+    add_ttg_executable(testing_dpoinv potrf/testing_dpoinv.cc LINK_LIBRARIES tiledarray lapackpp)
+
     if (TARGET CUDA::cublas)
       add_ttg_executable(bspmm-cuda spmm/spmm_cuda.cc
                          LINK_LIBRARIES tiledarray TiledArray_Eigen BTAS Boost::boost CUDA::cublas
@@ -20,17 +26,18 @@ if (TARGET tiledarray)
       if (TARGET CUDA::cusolver)
         add_ttg_executable(testing_dpotrf_cuda potrf/testing_dpotrf.cc
                           LINK_LIBRARIES lapackpp tiledarray CUDA::cublas CUDA::cusolver
-                          COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2;TTG_ENABLE_CUDA=1
+                          COMPILE_DEFINITIONS TTG_ENABLE_CUDA=1 #;DEBUG_TILES_VALUES=1
                           RUNTIMES "parsec")
       endif(TARGET CUDA::cusolver)
     elseif (TARGET roc::hipblas)
       add_ttg_executable(bspmm-hip spmm/spmm_cuda.cc
-                        LINK_LIBRARIES tiledarray TiledArray_Eigen BTAS Boost::boost roc::hipblas
-                        COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2
-                        RUNTIMES "parsec")
+                          LINK_LIBRARIES tiledarray TiledArray_Eigen BTAS Boost::boost roc::hipblas
+                          COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2
+                          RUNTIMES "parsec")
       if (TARGET roc::hipsolver)
         add_ttg_executable(testing_dpotrf_hip potrf/testing_dpotrf.cc
                            LINK_LIBRARIES lapackpp tiledarray roc::hipblas roc::hipsolver
+                           COMPILE_DEFINITIONS TTG_ENABLE_HIP=1;DEBUG_TILES_VALUES=1
                            RUNTIMES "parsec")
       endif(TARGET roc::hipsolver)
     endif()
@@ -55,11 +62,6 @@ add_ttg_executable(fw-apsp floyd-warshall/floyd_warshall.cc LINK_LIBRARIES MADwo
 add_ttg_executable(helloworld helloworld/helloworld.cpp)
 add_ttg_executable(simplegenerator simplegenerator/simplegenerator.cc RUNTIMES "mad")
 
-add_ttg_executable(testing_dpotrf potrf/testing_dpotrf.cc LINK_LIBRARIES lapackpp)
-add_ttg_executable(testing_dtrtri potrf/testing_dtrtri.cc LINK_LIBRARIES lapackpp)
-add_ttg_executable(testing_dlauum potrf/testing_dlauum.cc LINK_LIBRARIES lapackpp)
-add_ttg_executable(testing_dpoinv potrf/testing_dpoinv.cc LINK_LIBRARIES lapackpp)
-
 add_ttg_executable(device_mock device_mock/device_mock.cc LINK_LIBRARIES MADworld)
 
 if (OpenMP_CXX_FOUND AND TARGET std::execution)
diff --git a/examples/devblas_helper.h b/examples/devblas_helper.h
index 349025209..589b48541 100644
--- a/examples/devblas_helper.h
+++ b/examples/devblas_helper.h
@@ -18,7 +18,12 @@ inline const cublasHandle_t& cublas_handle(T _ = 0) {
   using map_type = std::map<std::pair<int, cudaStream_t>, cublasHandle_t>;
   static thread_local map_type handles;
 
-  int device = ttg::device::current_device();
+  auto d = ttg::device::current_device();
+  int device = 0; // assume 0 if we don't have a device
+  if (d.is_device()) {
+    device = d;
+  }
+
   cudaStream_t stream = ttg::device::current_stream();
 
   map_type::iterator it;
@@ -46,12 +51,18 @@ inline const cusolverDnHandle_t& cusolver_handle(T _ = 0) {
   using map_type = std::map<std::pair<int, cudaStream_t>, cusolverDnHandle_t>;
   static thread_local map_type handles;
 
-  int device = ttg::device::current_device();
+  auto d = ttg::device::current_device();
+  int device = 0; // assume 0 if we don't have a device
+  if (d.is_device()) {
+    device = d;
+  }
   cudaStream_t stream = ttg::device::current_stream();
 
   map_type::iterator it;
   if ((it = handles.find({device, stream})) == handles.end()){
     cusolverDnHandle_t handle;
+    std::cout << "Creating cusolver handle " << handle << " for device " << device << " stream " << stream << std::endl;
+
     auto status = cusolverDnCreate(&handle);
     if (CUSOLVER_STATUS_SUCCESS != status) {
       std::cerr << "cusolverDnCreate failed: " << status << std::endl;
@@ -63,7 +74,6 @@ inline const cusolverDnHandle_t& cusolver_handle(T _ = 0) {
       throw std::runtime_error("cusolverDnSetStream failed");
     }
 
-    std::cout << "Creating cusolver handle " << handle << " for device " << device << " stream " << stream << std::endl;
     auto [iterator, success] = handles.insert({{device, stream}, handle});
     it = iterator;
   } else {
diff --git a/examples/matrixtile.h b/examples/matrixtile.h
index ba2fcf9cd..540aef53d 100644
--- a/examples/matrixtile.h
+++ b/examples/matrixtile.h
@@ -19,11 +19,17 @@ class MatrixTile : public ttg::TTValue<MatrixTile<T, Allocator>> {
  private:
   buffer_t _buffer;
   int _rows = 0, _cols = 0, _lda = 0;
+#ifdef DEBUG_TILES_VALUES
+  T _norm = 0.0;
+#endif // DEBUG_TILES_VALUES
 
   // (Re)allocate the tile memory
   void realloc() {
     // std::cout << "Reallocating new tile" << std::endl;
     _buffer.reset(_lda * _cols);
+#ifdef DEBUG_TILES_VALUES
+    std::fill(_buffer.host_ptr(), _lda * _cols, T{});
+#endif // DEBUG_TILES_VALUES
   }
 
  public:
@@ -53,6 +59,9 @@ class MatrixTile : public ttg::TTValue<MatrixTile<T, Allocator>> {
   , _rows(rows)
   , _cols(cols)
   , _lda(lda)
+#ifdef DEBUG_TILES_VALUES
+  , _norm(blas::nrm2(cols*rows, data, 1))
+#endif // DEBUG_TILES_VALUES
   { }
 
   MatrixTile(MatrixTile<T, Allocator>&& other) = default;
@@ -68,7 +77,11 @@ class MatrixTile : public ttg::TTValue<MatrixTile<T, Allocator>> {
   , _buffer(other._lda*other._cols)
   , _rows(other._rows)
   , _cols(other._cols)
-  , _lda(other._lda) {
+  , _lda(other._lda)
+#ifdef DEBUG_TILES_VALUES
+  , _norm(other._norm)
+#endif // DEBUG_TILES_VALUES
+  {
     std::copy_n(other.data(), _lda * _cols, this->data());
   }
 
@@ -117,21 +130,33 @@ class MatrixTile : public ttg::TTValue<MatrixTile<T, Allocator>> {
     return *this;
   }
 
+#ifdef DEBUG_TILES_VALUES
+  /* Only available if debugging is enabled. Norm must be
+   * set by application and is not computed automatically. */
+  T norm() const {
+    return _norm;
+  }
+
+  void set_norm(T norm) {
+    _norm = norm;
+  }
+#endif // DEBUG_TILES_VALUES
+
   friend std::ostream& operator<<(std::ostream& o, MatrixTile<T> const& tt) {
     auto ptr = tt.data();
     o << std::endl << " ";
     o << "MatrixTile<" << typeid(T).name() << ">{ rows=" << tt.rows() << " cols=" << tt.cols() << " ld=" << tt.lda();
 #if DEBUG_TILES_VALUES
-    o << " data=[ " for (int i = 0; i < tt.rows(); i++) {
+    o << " data=[ ";
+    for (int i = 0; i < tt.rows(); i++) {
       for (int j = 0; j < tt.cols(); j++) {
         o << ptr[i + j * tt.lda()] << " ";
       }
       o << std::endl << " ";
     }
-    o << " ] "
+    o << " ] ";
 #endif
-        o
-      << " } ";
+    o << " } ";
     return o;
   }
 };
diff --git a/examples/potrf/plgsy.h b/examples/potrf/plgsy.h
index cfbc4192a..a29336712 100644
--- a/examples/potrf/plgsy.h
+++ b/examples/potrf/plgsy.h
@@ -13,7 +13,8 @@ auto make_plgsy(MatrixT<T>& A, unsigned long bump, unsigned long random_seed, tt
     if(ttg::tracing()) ttg::print("PLGSY( ", key, ") on rank ", A.rank_of(key[0], key[1]));
     assert(A.is_local(I, J));
 
-    T *a = A(I, J).data();
+    auto tile = A(I, J);
+    T *a = tile.data();
     int tempmm, tempnn, ldam;
 
     tempmm = (I==A.rows()-1) ? A.rows_in_matrix()-I*A.rows_in_tile() : A.rows_in_tile();
@@ -22,8 +23,11 @@ auto make_plgsy(MatrixT<T>& A, unsigned long bump, unsigned long random_seed, tt
 
     CORE_plgsy((double)bump, tempmm, tempnn, a, ldam,
                A.rows_in_matrix(), I*A.rows_in_tile(), J*A.cols_in_tile(), random_seed);
+#ifdef DEBUG_TILES_VALUES
+    tile.set_norm(blas::nrm2(tile.size(), a, 1));
+#endif // DEBUG_TILES_VALUES
 
-    ttg::send<0>(key, std::move(A(I, J)), out);
+    ttg::send<0>(key, std::move(tile), out);
   };
 
   return ttg::make_tt(f, ttg::edges(input), ttg::edges(output), "PLGSY", {"startup"}, {"output"});
diff --git a/examples/potrf/potrf.h b/examples/potrf/potrf.h
index 24bdd44d2..2503e8a2b 100644
--- a/examples/potrf/potrf.h
+++ b/examples/potrf/potrf.h
@@ -4,9 +4,9 @@
 #include <ttg/config.h>
 #include "lapack.hh"
 #include "pmw.h"
+#include "util.h"
 #include "../devblas_helper.h"
 
-#undef DEBUG_TILES_VALUES
 
 #if defined(TTG_HAVE_CUDART)
 #define ES ttg::ExecutionSpace::CUDA
@@ -69,6 +69,19 @@ namespace potrf {
                       devInfo);
   #endif
   }
+
+  static void device_norm(const MatrixTile<double> &A, double *norm) {
+    auto size = A.size();
+    auto buffer = A.buffer().current_device_ptr();
+    std::cout << "device_norm ptr " << buffer << " device " << ttg::device::current_device() << std::endl;
+#if defined(TTG_HAVE_CUDA)
+    auto handle = cublas_handle();
+    //double n = 1.0;
+    cublasDnrm2(handle, size, buffer, 1, norm);
+  #elif defined(TTG_HAVE_HIPBLAS)
+    hipblasDnrm2(hipblas_handle(), size, buffer, 1, norm);
+  #endif
+  }
 #endif // defined(TTG_HAVE_CUDART) || defined(TTG_HAVE_HIP)
 
   template <typename MatrixT>
@@ -77,16 +90,16 @@ namespace potrf {
                   ttg::Edge<Key1, MatrixTile<typename MatrixT::element_type>>& input,
                   ttg::Edge<Key2, MatrixTile<typename MatrixT::element_type>>& output_trsm,
                   ttg::Edge<Key2, MatrixTile<typename MatrixT::element_type>>& output_result) {
-      using T = typename MatrixT::element_type;
-      auto iallocator = std::make_shared<TiledArray::device_pinned_allocator<int>>();
+    using T = typename MatrixT::element_type;
 #if defined(TTG_HAVE_CUDART) || defined(TTG_HAVE_HIP)
+    auto iallocator = std::make_shared<TiledArray::device_pinned_allocator<int>>();
     //std::cout << "Creating CUDA POTRF task " << std::endl;
     auto f_dev = [=, iallocator = std::move(iallocator)]
                     (const Key1& key, MatrixTile<T>&& tile_kk,
                      std::tuple<ttg::Out<Key2, MatrixTile<T>>, ttg::Out<Key2, MatrixTile<T>>>& out) TASKRET {
       const auto K = key[0];
 
-      /* compute successors before submitting the kernel running
+      /* compute successors before submitting the kernel
        * TODO: this is parsec specific since this code is still executing on the worker threads
        */
       std::vector<Key2> keylist;
@@ -116,8 +129,25 @@ namespace potrf {
 
       *hostInfo = -32;
 
+#ifdef DEBUG_TILES_VALUES
+      std::array<T, 2> norms;
+      //auto norms_s  = ttg::make_scratch(norms.data(), ttg::scope::Allocate, norms.size());
+      /* the workspace and the devInfo must be device-level pointers */
+      //co_await ttg::to_device(tile_kk.buffer(), devWS, devInfo, norms_s);
+      co_await ttg::to_device(tile_kk.buffer(), devWS, devInfo);
+
+      /* compute the norm at input */
+      static_assert(std::is_same_v<double, T>, "Norm debugging only implementation for T=double");
+      device_norm(tile_kk, &norms[0]);
+#else
       /* the workspace and the devInfo must be device-level pointers */
       co_await ttg::to_device(tile_kk.buffer(), devWS, devInfo);
+#endif // DEBUG_TILES_VALUES
+
+      int device = ttg::device::current_device();
+      std::cout << "POTRF [" << K << "] tile kk "
+                << tile_kk.buffer().device_ptr_on(device) << std::endl;
+
 
       //std::cout << "devWS host ptr " << hostWS << " device ptr " << devWS.device_ptr() << " size " <<  devWS.size()
       //          << " devInfo host ptr " << hostInfo << " device ptr " << devInfo.device_ptr() << "size "  << devInfo.size() << std::endl;
@@ -125,8 +155,20 @@ namespace potrf {
       /* everything is on the device, call the POTRF */
       device_potrf(tile_kk, devWS.device_ptr(), Lwork, devInfo.device_ptr());
 
+#ifdef DEBUG_TILES_VALUES
+      /* compute the norm at input */
+      static_assert(std::is_same_v<double, T>, "Verification only implementation for T=double");
+      device_norm(tile_kk, &norms[1]);
+      /* wait for the kernel to complete */
+      co_await ttg::wait_kernel(devInfo);
+      // check that we got the input tile we expected
+     assert(check_norm(tile_kk.norm(), norms[0]));
+      // set the new norm
+      tile_kk.set_norm(norms[1]);
+#else
       /* wait for the kernel to complete */
       co_await ttg::wait_kernel(devInfo);
+#endif // DEBUG_TILES_VALUES
 
       delete[] hostWS;
       int info = *hostInfo;
@@ -224,9 +266,28 @@ namespace potrf {
       }
 
 
+#ifdef DEBUG_TILES_VALUES
+      std::array<T, 3> norms; // input for tile_kk & tile_mk and output
+      //auto norms_s = ttg::make_scratch(norms.data(), ttg::scope::Allocate, norms.size());
+      co_await ttg::to_device(tile_kk.buffer(), tile_mk.buffer());
+#else
       co_await ttg::to_device(tile_kk.buffer(), tile_mk.buffer());
+#endif // DEBUG_TILES_VALUES
+
       int device = ttg::device::current_device();
       double alpha = 1.0;
+
+#ifdef DEBUG_TILES_VALUES
+      /* compute the norms at input */
+      device_norm(tile_kk, &norms[0]);
+      device_norm(tile_mk, &norms[1]);
+#endif // DEBUG_TILES_VALUES
+
+
+      std::cout << "TRSM [" << K << ", " << M << "] tile mk "
+                << tile_mk.buffer().device_ptr_on(device)
+                << " tile kk " << tile_kk.buffer().device_ptr_on(device) << std::endl;
+
 #if defined(TTG_HAVE_CUDA)
       cublasDtrsm(cublas_handle(),
                   CUBLAS_SIDE_RIGHT, CUBLAS_FILL_MODE_LOWER,
@@ -241,8 +302,21 @@ namespace potrf {
                    mb, nb, &alpha,
                    tile_kk.buffer().device_ptr_on(device), tile_kk.lda(),
                    tile_mk.buffer().device_ptr_on(device), tile_mk.lda());
+
 #endif
 
+#ifdef DEBUG_TILES_VALUES
+      /* compute the norms at input */
+      device_norm(tile_mk, &norms[2]);
+      /* wait for the kernel to complete */
+      co_await ttg::wait_kernel();
+      // check that we got the input tiles we expected
+      assert(check_norm(tile_kk.norm(), norms[0]));
+      assert(check_norm(tile_mk.norm(), norms[1]));
+      // set the new norm
+      tile_mk.set_norm(norms[2]);
+#endif // DEBUG_TILES_VALUES
+
       co_await ttg::device::forward(ttg::device::broadcast<0, 1, 2, 3>(std::make_tuple(key, Key2(K, M), keylist_row, keylist_col),
                                                                        std::move(tile_mk), out));
     };
@@ -326,12 +400,26 @@ namespace potrf {
 
       if (ttg::tracing()) ttg::print("SYRK(", key, ")");
 
+#ifdef DEBUG_TILES_VALUES
+      std::array<T, 3> norms; // input for tile_kk & tile_mk and output
+      //auto norms_s  = ttg::make_scratch(norms.data(), ttg::scope::Allocate, norms.size());
+      co_await ttg::to_device(tile_kk.buffer(), tile_mk.buffer());
+      /* compute the norms at input */
+      device_norm(tile_mk, &norms[0]);
+      device_norm(tile_kk, &norms[1]);
+#else
       co_await ttg::to_device(tile_kk.buffer(), tile_mk.buffer());
+#endif // DEBUG_TILES_VALUES
 
       int device = ttg::device::current_device();
 
       double alpha = -1.0;
       double beta  =  1.0;
+
+      std::cout << "SYRK [" << K << ", " << M << "] tile mk "
+                << tile_mk.buffer().device_ptr_on(device)
+                << " tile kk " << tile_kk.buffer().device_ptr_on(device) << std::endl;
+
 #if defined(TTG_HAVE_CUDA)
       cublasDsyrk(cublas_handle(),
                   CUBLAS_FILL_MODE_LOWER,
@@ -348,6 +436,18 @@ namespace potrf {
                    tile_kk.buffer().device_ptr_on(device), tile_kk.lda());
 #endif
 
+#ifdef DEBUG_TILES_VALUES
+      /* compute the norm at output */
+      device_norm(tile_kk, &norms[2]);
+      /* wait for the kernel to complete */
+      co_await ttg::wait_kernel();
+      // check that we got the input tiles we expected
+      assert(check_norm(tile_mk.norm(), norms[0]));
+      assert(check_norm(tile_kk.norm(), norms[1]));
+      // set the new norm
+      tile_kk.set_norm(norms[2]);
+#endif // DEBUG_TILES_VALUES
+
       if (M == K + 1) {
         /* send the tile to potrf */
         if (ttg::tracing()) ttg::print("SYRK(", key, "): sending output to POTRF(", Key1{K + 1}, ")");
@@ -428,11 +528,23 @@ namespace potrf {
                 << N << ") is " << tile_nk << " and A(" << M << ", " << N << ") is " << tile_mn;
 #endif
 
+#ifdef DEBUG_TILES_VALUES
+      std::array<T, 4> norms; // input for tile_mk & tile_nk & tile_mn and output
+      //auto norms_s  = ttg::make_scratch(norms.data(), ttg::scope::Allocate, norms.size());
+      co_await ttg::to_device(tile_mk.buffer(), tile_nk.buffer(), tile_mn.buffer());
+
+      /* compute the norms at input */
+      device_norm(tile_mk, &norms[0]);
+      device_norm(tile_nk, &norms[1]);
+      device_norm(tile_mn, &norms[2]);
+#else
       co_await ttg::to_device(tile_mk.buffer(), tile_nk.buffer(), tile_mn.buffer());
+#endif // DEBUG_TILES_VALUES
 
       int device = ttg::device::current_device();
       double alpha = -1.0;
       double beta  =  1.0;
+
 #if defined(TTG_HAVE_CUDA)
       cublasDgemm(cublas_handle(),
                   CUBLAS_OP_N, CUBLAS_OP_T,
@@ -451,6 +563,20 @@ namespace potrf {
                    tile_mn.buffer().device_ptr_on(device), tile_mn.lda());
 #endif
 
+
+#ifdef DEBUG_TILES_VALUES
+      /* compute the norm at output */
+      device_norm(tile_mn, &norms[3]);
+      /* wait for the kernel to complete */
+      co_await ttg::wait_kernel();
+      // check that we got the input tiles we expected
+      assert(check_norm(tile_mk.norm(), norms[0]));
+      assert(check_norm(tile_nk.norm(), norms[1]));
+      assert(check_norm(tile_mn.norm(), norms[2]));
+      // set the new norm
+      tile_mn.set_norm(norms[3]);
+#endif // DEBUG_TILES_VALUES
+
       if (N == K + 1) {
         /* send the tile to trsm */
         if (ttg::tracing()) ttg::print("GEMM(", key, "): sending output to TRSM(", Key2{M, N}, ")");
diff --git a/examples/potrf/result.h b/examples/potrf/result.h
index caafd5795..257f1fd5a 100644
--- a/examples/potrf/result.h
+++ b/examples/potrf/result.h
@@ -2,6 +2,7 @@
 
 #include <ttg.h>
 #include "pmw.h"
+#include "util.h"
 
 template <typename T>
 auto make_result(MatrixT<T>& A, ttg::Edge<Key2, MatrixTile<T>>& result) {
@@ -10,9 +11,14 @@ auto make_result(MatrixT<T>& A, ttg::Edge<Key2, MatrixTile<T>>& result) {
     const int I = key[0];
     const int J = key[1];
     if (ttg::tracing()) ttg::print("RESULT( ", key, ") on rank ", A.rank_of(key[0], key[1]));
-    if (A(I, J).data() != tile.data()) {
+#if defined(DEBUG_TILES_VALUES)
+    T norm = blas::nrm2(tile.size(), tile.data(), 1);
+    assert(check_norm(norm, tile.norm()));
+#endif //defined(DEBUG_TILES_VALUES)
+    auto atile = A(I, J);
+    if (atile.data() != tile.data()) {
       if (ttg::tracing()) ttg::print("Writing back tile {", I, ",", J, "} ");
-      std::copy_n(tile.data(), tile.rows() * tile.cols(), A(I, J).data());
+      std::copy_n(tile.data(), tile.rows() * tile.cols(), atile.data());
     }
 #ifdef TTG_USE_USER_TERMDET
     if (I == A.cols() - 1 && J == A.rows() - 1) {
diff --git a/examples/potrf/util.h b/examples/potrf/util.h
new file mode 100644
index 000000000..c76c2c59b
--- /dev/null
+++ b/examples/potrf/util.h
@@ -0,0 +1,10 @@
+#pragma once
+
+#include <cmath>
+
+inline auto check_norm(double expected, double actual) {
+  if (std::abs(expected - actual) <= std::max(std::abs(expected), std::abs(actual))*1E-12) {
+    return true;
+  }
+  return false;
+}
diff --git a/ttg/ttg/device/device.h b/ttg/ttg/device/device.h
index 17f8364f1..a8cf6e623 100644
--- a/ttg/ttg/device/device.h
+++ b/ttg/ttg/device/device.h
@@ -1,29 +1,87 @@
+#pragma once
 
 #include <ttg/config.h>
 
+
+namespace ttg::device {
+  class Device {
+    int m_id = 0;
+    ttg::ExecutionSpace m_space = ttg::ExecutionSpace::Host;
+
+  public:
+    Device() = default;
+    Device(int id, ttg::ExecutionSpace space)
+    : m_id(id)
+    , m_space(space)
+    { }
+
+    int id() const {
+      if (is_host()) {
+        throw std::runtime_error("No valid ID for Host execution space!");
+      }
+      if (is_invalid()) {
+        throw std::runtime_error("Invalid execution space!");
+      }
+      return m_id;
+    }
+
+    operator int() const {
+      return id();
+    }
+
+    ttg::ExecutionSpace space() const {
+      return m_space;
+    }
+
+    bool is_device() const {
+      return ((!is_invalid()) && (m_space != ttg::ExecutionSpace::Host));
+    }
+
+    bool is_host() const {
+      return (m_space == ttg::ExecutionSpace::Host);
+    }
+
+    bool is_invalid() const {
+      return (m_space == ttg::ExecutionSpace::Invalid);
+    }
+  };
+} // namespace ttg::device
+
+namespace std {
+  std::ostream& operator<<(std::ostream& os, ttg::device::Device device) {
+    os << ttg::detail::execution_space_name(device.space());
+    if (device.is_device()) {
+      os << "(" << device.id() << ")";
+    }
+    return os;
+  }
+} // namespace std
+
 #if defined(TTG_HAVE_CUDA)
 #include <cuda_runtime.h>
 
 namespace ttg::device {
   namespace detail {
-    inline thread_local int current_device_ts = 0;
+    inline thread_local ttg::device::Device current_device_ts = {};
     inline thread_local cudaStream_t current_stream_ts = 0; // default stream
 
     void reset_current() {
-      current_device_ts = 0;
+      current_device_ts = {};
       current_stream_ts = 0;
     }
 
     void set_current(int device, cudaStream_t stream) {
-      current_device_ts = device;
+      current_device_ts = ttg::device::Device(device, ttg::ExecutionSpace::CUDA);
       current_stream_ts = stream;
     }
   } // namespace detail
 
-  int current_device() {
+  inline
+  Device current_device() {
     return detail::current_device_ts;
   }
 
+  inline
   cudaStream_t current_stream() {
     return detail::current_stream_ts;
   }
@@ -35,24 +93,26 @@ namespace ttg::device {
 
 namespace ttg::device {
   namespace detail {
-    inline thread_local int current_device_ts = 0;
+    inline thread_local ttg::device::Device current_device_ts = {};
     inline thread_local hipStream_t current_stream_ts = 0; // default stream
 
     void reset_current() {
-      current_device_ts = 0;
+      current_device_ts = {};
       current_stream_ts = 0;
     }
 
     void set_current(int device, hipStream_t stream) {
-      current_device_ts = device;
+      current_device_ts = ttg::device::Device(device, ttg::ExecutionSpace::HIP);
       current_stream_ts = stream;
     }
   } // namespace detail
 
-  int current_device() {
+  inline
+  Device current_device() {
     return detail::current_device_ts;
   }
 
+  inline
   hipStream_t current_stream() {
     return detail::current_stream_ts;
   }
diff --git a/ttg/ttg/execution.h b/ttg/ttg/execution.h
index cbdd65ff1..4d99a3ae7 100644
--- a/ttg/ttg/execution.h
+++ b/ttg/ttg/execution.h
@@ -21,6 +21,18 @@ enum class ExecutionSpace {
   Invalid
 };
 
+namespace detail {
+  inline const char *execution_space_name(ExecutionSpace space) noexcept {
+    switch (space) {
+      case ExecutionSpace::Host: return "Host";
+      case ExecutionSpace::CUDA: return "CUDA";
+      case ExecutionSpace::HIP: return "HIP";
+      case ExecutionSpace::Invalid: return "INVALID";
+      default: return "UNKNOWN";
+    }
+  }
+} // namespace detail
+
 };
 
 #endif //TTG_EXECUTION_H
diff --git a/ttg/ttg/parsec/buffer.h b/ttg/ttg/parsec/buffer.h
index 0309e7883..552d0b52e 100644
--- a/ttg/ttg/parsec/buffer.h
+++ b/ttg/ttg/parsec/buffer.h
@@ -9,6 +9,7 @@
 #include "ttg/parsec/ttg_data_copy.h"
 #include "ttg/parsec/parsec-ext.h"
 #include "ttg/util/iovec.h"
+#include "ttg/device/device.h"
 
 #if defined(PARSEC_HAVE_DEV_CUDA_SUPPORT)
 #include <cuda_runtime.h>
@@ -88,7 +89,7 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
   , m_owned(true)
   {
     //std::cout << "buffer " << this << " ctor count "
-    //          << count << "(" << m_host_data.get() << ") ttg_copy "
+    //          << m_count << "(" << m_host_data << ") ttg_copy "
     //          << m_ttg_copy
     //          << " parsec_data " << m_data.get() << std::endl;
     this->reset_parsec_data(m_host_data, n*sizeof(element_type));
@@ -104,6 +105,10 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
   , m_count(n)
   , m_owned(false)
   {
+    //std::cout << "buffer " << this << " ctor ptr " << ptr << "count "
+    //          << m_count << "(" << m_host_data << ") ttg_copy "
+    //          << m_ttg_copy
+    //          << " parsec_data " << m_data.get() << std::endl;
     this->reset_parsec_data(m_host_data, n*sizeof(element_type));
   }
 
@@ -165,22 +170,36 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
     m_data->owner_device = device_id+2;
   }
 
-  /* get the current device ID, i.e., the last updated
+  /* Get the owner device ID, i.e., the last updated
    * device buffer. A value of -2 designates the host
    * as the current device. */
-  int get_current_device() const {
+  int get_owner_device() const {
     assert(is_valid());
     return m_data->owner_device - 2; // 0: host, 1: recursive, 2: first device
   }
 
-  /* get the current device pointer */
+  /* Get the pointer on the currently active device. */
   element_type* current_device_ptr() {
+    assert(is_valid());
+    return static_cast<element_type*>(m_data->device_copies[ttg::device::current_device()+2]->device_private);
+  }
+
+  /* Get the pointer on the currently active device. */
+  const element_type* current_device_ptr() const {
+    assert(is_valid());
+    return static_cast<element_type*>(m_data->device_copies[ttg::device::current_device()+2]->device_private);
+  }
+
+  /* Get the pointer on the owning device.
+   * @note: This may not be the device assigned to the currently executing task.
+   *        See \ref ttg::device::current_device for that. */
+  element_type* owner_device_ptr() {
     assert(is_valid());
     return static_cast<element_type*>(m_data->device_copies[m_data->owner_device]->device_private);
   }
 
   /* get the current device pointer */
-  const element_type* current_device_ptr() const {
+  const element_type* owner_device_ptr() const {
     assert(is_valid());
     return static_cast<element_type*>(m_data->device_copies[m_data->owner_device]->device_private);
   }
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 8398a9244..f32da330d 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1345,6 +1345,8 @@ namespace ttg_parsec {
       /* Here we call back into the coroutine again after the transfers have completed */
       static_op<Space>(&task->parsec_task);
 
+      ttg::device::detail::reset_current();
+
       /* we will come back into this function once the kernel and transfers are done */
       int rc = PARSEC_HOOK_RETURN_DONE;
       if (nullptr != task->suspended_task_address) {
diff --git a/ttg/ttg/parsec/ttg_data_copy.h b/ttg/ttg/parsec/ttg_data_copy.h
index 9d21cc2d1..5eb3d5b0e 100644
--- a/ttg/ttg/parsec/ttg_data_copy.h
+++ b/ttg/ttg/parsec/ttg_data_copy.h
@@ -303,8 +303,6 @@ namespace ttg_parsec {
       using iterator = ttg_parsec_data_wrapper_t**;
 
       void add_device_data(ttg_parsec_data_wrapper_t* data) {
-        // TODO: REMOVE AFTER DEBUG
-        //assert(m_num_dev_data == 0);
         switch (m_num_dev_data) {
           case 0:
             m_single_dev_data = data;
@@ -344,8 +342,6 @@ namespace ttg_parsec {
           m_single_dev_data = m_dev_data[0];
           m_dev_data.clear();
         }
-        // TODO: REMOVE AFTER DEBUG
-        //assert(m_num_dev_data == 0);
       }
 
       int num_dev_data() const {
@@ -553,7 +549,7 @@ namespace ttg_parsec {
     , m_ttg_copy(detail::ttg_data_copy_container())
     {
       /* the ttg_data_copy may have moved us already */
-      if (other.m_ttg_copy != m_ttg_copy) {
+      //if (other.m_ttg_copy != m_ttg_copy) {
         // try to remove the old buffer from the *old* ttg_copy
         other.remove_from_owner();
 
@@ -561,7 +557,9 @@ namespace ttg_parsec {
         if (nullptr != m_ttg_copy) {
           m_ttg_copy->add_device_data(this);
         }
-      }
+      //} else {
+      //  other.m_ttg_copy = nullptr;
+      //}
     }
 
     ttg_parsec_data_wrapper_t& ttg_parsec_data_wrapper_t::operator=(ttg_parsec_data_wrapper_t&& other) {

From 57f914e18901a2ab96b923fa0dff46973038ca50 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 30 Oct 2023 19:49:46 +0000
Subject: [PATCH 147/259] Cleanup devblas support in rocm support

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/devblas_helper.h | 62 ++++++++++++++++++++++++---------------
 1 file changed, 38 insertions(+), 24 deletions(-)

diff --git a/examples/devblas_helper.h b/examples/devblas_helper.h
index 589b48541..d9a3a37ea 100644
--- a/examples/devblas_helper.h
+++ b/examples/devblas_helper.h
@@ -34,7 +34,7 @@ inline const cublasHandle_t& cublas_handle(T _ = 0) {
       std::cerr << "cublasCreate_v2 failed: " << status << std::endl;
       throw std::runtime_error("cublasCreate_v2 failed");
     }
-    status = cublasSetStream_v2(handle, ttg::device::current_stream());
+    status = cublasSetStream_v2(handle, stream);
     if (CUBLAS_STATUS_SUCCESS != status) {
       std::cerr << "cublasSetStream_v2 failed: " << status << std::endl;
       throw std::runtime_error("cublasSetStream_v2 failed");
@@ -61,8 +61,6 @@ inline const cusolverDnHandle_t& cusolver_handle(T _ = 0) {
   map_type::iterator it;
   if ((it = handles.find({device, stream})) == handles.end()){
     cusolverDnHandle_t handle;
-    std::cout << "Creating cusolver handle " << handle << " for device " << device << " stream " << stream << std::endl;
-
     auto status = cusolverDnCreate(&handle);
     if (CUSOLVER_STATUS_SUCCESS != status) {
       std::cerr << "cusolverDnCreate failed: " << status << std::endl;
@@ -76,8 +74,6 @@ inline const cusolverDnHandle_t& cusolver_handle(T _ = 0) {
 
     auto [iterator, success] = handles.insert({{device, stream}, handle});
     it = iterator;
-  } else {
-    std::cout << "Found cusolver handle " << it->second << " for device " << device << " stream " << stream << std::endl;
   }
 
   return it->second;
@@ -93,46 +89,64 @@ inline const cusolverDnHandle_t& cusolver_handle(T _ = 0) {
 /// \brief Returns the rocBLAS handle to be used for launching rocBLAS kernels from the current thread
 /// \return the rocBLAS handle for the current thread
 template<typename T = int>
-const hipblasHandle_t& hipblas_handle(T _ = 0) {
-  static thread_local std::map<int, hipblasHandle_t> handles;
-  int device = ttg::device::current_device();
-  std::map<int, hipblasHandle_t>::iterator it;
-  if ((it = handles.find(device)) == handles.end()){
+inline const hipblasHandle_t& hipblas_handle(T _ = 0) {
+  using map_type = std::map<std::pair<int, hipStream_t>, hipblasHandle_t>;
+  static thread_local map_type handles;
+
+  auto d = ttg::device::current_device();
+  int device = 0; // assume 0 if we don't have a device
+  if (d.is_device()) {
+    device = d;
+  }
+
+  cudaStream_t stream = ttg::device::current_stream();
+
+  map_type::iterator it;
+  if ((it = handles.find({device, stream})) == handles.end()){
     hipblasHandle_t handle;
     auto status = hipblasCreate(&handle);
     if (HIPBLAS_STATUS_SUCCESS != status) {
       throw std::runtime_error("hipblasCreate failed");
     }
-    auto [iterator, success] = handles.insert({device, handle});
+    hipblasStatus_t status = hipblasSetStream(handle, stream);
+    if (HIPBLAS_STATUS_SUCCESS != status) {
+      throw std::runtime_error("hipblasSetStream failed");
+    }
+    auto [iterator, success] = handles.insert({{device, stream}, handle});
     it = iterator;
   }
-  hipblasStatus_t status = hipblasSetStream(it->second, ttg::device::current_stream());
-  if (HIPBLAS_STATUS_SUCCESS != status) {
-    throw std::runtime_error("hipblasSetStream failed");
-  }
+
   return it->second;
 }
 
 /// \brief Returns the hipsolver handle to be used for launching rocBLAS kernels from the current thread
 /// \return the hipsolver handle for the current thread
 template<typename T = int>
-const hipsolverDnHandle_t& hipsolver_handle(T _ = 0) {
-  static thread_local std::map<int, hipsolverDnHandle_t> handles;
-  int device = ttg::device::current_device();
+inline const hipsolverDnHandle_t& hipsolver_handle(T _ = 0) {
+  using map_type = std::map<std::pair<int, hipStream_t>, hipsolverDnHandle_t>;
+  static thread_local map_type handles;
+  auto d = ttg::device::current_device();
+  int device = 0; // assume 0 if we don't have a device
+  if (d.is_device()) {
+    device = d;
+  }
+
+  cudaStream_t stream = ttg::device::current_stream();
+
   std::map<int, hipsolverDnHandle_t>::iterator it;
-  if ((it = handles.find(device)) == handles.end()){
+  if ((it = handles.find({device, stream})) == handles.end()){
     hipsolverDnHandle_t handle;
     auto status = hipsolverDnCreate(&handle);
     if (HIPSOLVER_STATUS_SUCCESS != status) {
       throw std::runtime_error("hipsolverCreate failed");
     }
-    auto [iterator, success] = handles.insert({device, handle});
+    hipsolverStatus_t status = hipsolverDnSetStream(handle, stream);
+    if (HIPSOLVER_STATUS_SUCCESS != status) {
+      throw std::runtime_error("hipsolverSetStream failed");
+    }
+    auto [iterator, success] = handles.insert({{device, stream}, handle});
     it = iterator;
   }
-  hipsolverStatus_t status = hipsolverDnSetStream(it->second, ttg::device::current_stream());
-  if (HIPSOLVER_STATUS_SUCCESS != status) {
-    throw std::runtime_error("hipsolverSetStream failed");
-  }
   return it->second;
 }
 #endif // TTG_HAVE_HIPBLAS

From 162a0a1aa7a9a53020308b2cd90f4a43f7770802 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 30 Oct 2023 19:52:17 +0000
Subject: [PATCH 148/259] Lazily compute norm on MatrixTile

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/matrixtile.h | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/examples/matrixtile.h b/examples/matrixtile.h
index 540aef53d..bbacc64df 100644
--- a/examples/matrixtile.h
+++ b/examples/matrixtile.h
@@ -3,6 +3,7 @@
 
 #include <iostream>
 #include <memory>
+#include <optional>
 
 #include <ttg/serialization/splitmd_data_descriptor.h>
 
@@ -20,7 +21,7 @@ class MatrixTile : public ttg::TTValue<MatrixTile<T, Allocator>> {
   buffer_t _buffer;
   int _rows = 0, _cols = 0, _lda = 0;
 #ifdef DEBUG_TILES_VALUES
-  T _norm = 0.0;
+  mutable std::optional<T> _norm;
 #endif // DEBUG_TILES_VALUES
 
   // (Re)allocate the tile memory
@@ -59,9 +60,6 @@ class MatrixTile : public ttg::TTValue<MatrixTile<T, Allocator>> {
   , _rows(rows)
   , _cols(cols)
   , _lda(lda)
-#ifdef DEBUG_TILES_VALUES
-  , _norm(blas::nrm2(cols*rows, data, 1))
-#endif // DEBUG_TILES_VALUES
   { }
 
   MatrixTile(MatrixTile<T, Allocator>&& other) = default;
@@ -134,6 +132,7 @@ class MatrixTile : public ttg::TTValue<MatrixTile<T, Allocator>> {
   /* Only available if debugging is enabled. Norm must be
    * set by application and is not computed automatically. */
   T norm() const {
+    if (!_norm) _norm = blas::nrm2(size(), data(), 1);
     return _norm;
   }
 

From 9dbe64814df211917a376698eac296e612f39739 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 30 Oct 2023 19:55:06 +0000
Subject: [PATCH 149/259] Fix previous commits

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/devblas_helper.h | 8 ++++----
 examples/matrixtile.h     | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/examples/devblas_helper.h b/examples/devblas_helper.h
index d9a3a37ea..f93e3799a 100644
--- a/examples/devblas_helper.h
+++ b/examples/devblas_helper.h
@@ -99,7 +99,7 @@ inline const hipblasHandle_t& hipblas_handle(T _ = 0) {
     device = d;
   }
 
-  cudaStream_t stream = ttg::device::current_stream();
+  hipStream_t stream = ttg::device::current_stream();
 
   map_type::iterator it;
   if ((it = handles.find({device, stream})) == handles.end()){
@@ -108,7 +108,7 @@ inline const hipblasHandle_t& hipblas_handle(T _ = 0) {
     if (HIPBLAS_STATUS_SUCCESS != status) {
       throw std::runtime_error("hipblasCreate failed");
     }
-    hipblasStatus_t status = hipblasSetStream(handle, stream);
+    status = hipblasSetStream(handle, stream);
     if (HIPBLAS_STATUS_SUCCESS != status) {
       throw std::runtime_error("hipblasSetStream failed");
     }
@@ -131,7 +131,7 @@ inline const hipsolverDnHandle_t& hipsolver_handle(T _ = 0) {
     device = d;
   }
 
-  cudaStream_t stream = ttg::device::current_stream();
+  hipStream_t stream = ttg::device::current_stream();
 
   std::map<int, hipsolverDnHandle_t>::iterator it;
   if ((it = handles.find({device, stream})) == handles.end()){
@@ -140,7 +140,7 @@ inline const hipsolverDnHandle_t& hipsolver_handle(T _ = 0) {
     if (HIPSOLVER_STATUS_SUCCESS != status) {
       throw std::runtime_error("hipsolverCreate failed");
     }
-    hipsolverStatus_t status = hipsolverDnSetStream(handle, stream);
+    status = hipsolverDnSetStream(handle, stream);
     if (HIPSOLVER_STATUS_SUCCESS != status) {
       throw std::runtime_error("hipsolverSetStream failed");
     }
diff --git a/examples/matrixtile.h b/examples/matrixtile.h
index bbacc64df..21bfa5aa6 100644
--- a/examples/matrixtile.h
+++ b/examples/matrixtile.h
@@ -133,7 +133,7 @@ class MatrixTile : public ttg::TTValue<MatrixTile<T, Allocator>> {
    * set by application and is not computed automatically. */
   T norm() const {
     if (!_norm) _norm = blas::nrm2(size(), data(), 1);
-    return _norm;
+    return _norm.value();
   }
 
   void set_norm(T norm) {

From fdc3c95f7701bcfd42e8ed47c9487ff90a81dafb Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 30 Oct 2023 19:56:59 +0000
Subject: [PATCH 150/259] Fix previous commit

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/devblas_helper.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/devblas_helper.h b/examples/devblas_helper.h
index f93e3799a..f621c7e9c 100644
--- a/examples/devblas_helper.h
+++ b/examples/devblas_helper.h
@@ -133,7 +133,7 @@ inline const hipsolverDnHandle_t& hipsolver_handle(T _ = 0) {
 
   hipStream_t stream = ttg::device::current_stream();
 
-  std::map<int, hipsolverDnHandle_t>::iterator it;
+  map_type::iterator it;
   if ((it = handles.find({device, stream})) == handles.end()){
     hipsolverDnHandle_t handle;
     auto status = hipsolverDnCreate(&handle);

From 7200e1c99f7b28ee09f924043c48e26492549a82 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 30 Oct 2023 20:40:54 +0000
Subject: [PATCH 151/259] Fix bspmm-cuda after device and devblas helper
 changes

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/devblas_helper.h  |  4 ++++
 examples/spmm/spmm_cuda.cc | 24 ++++++++++++------------
 ttg/ttg.h                  |  2 ++
 ttg/ttg/device/device.h    |  4 +++-
 4 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/examples/devblas_helper.h b/examples/devblas_helper.h
index f621c7e9c..76781d9ec 100644
--- a/examples/devblas_helper.h
+++ b/examples/devblas_helper.h
@@ -1,5 +1,9 @@
+#pragma once
+
 #include "ttg/config.h"
 
+#include "ttg/device/device.h"
+
 #include <memory>
 #include <stdexcept>
 #include <optional>
diff --git a/examples/spmm/spmm_cuda.cc b/examples/spmm/spmm_cuda.cc
index e0fec5915..60fb969fa 100644
--- a/examples/spmm/spmm_cuda.cc
+++ b/examples/spmm/spmm_cuda.cc
@@ -20,7 +20,7 @@
 #include <btas/optimize/contract.h>
 #include <btas/util/mohndle.h>
 #include <TiledArray/device/allocators.h>
-#include <ttg/device/cublas_helper.h>
+#include "../devblas_helper.h"
 #include <madness/world/parsec.h>  // need to initialize MADNESS purely for the purposes of TA allocators
 #else
 #warning "found btas/features.h but Boost.Iterators is missing, hence BTAS is unusable ... add -I/path/to/boost"
@@ -277,17 +277,17 @@ static void device_gemm(Blk &C, const Blk &A, const Blk &B) {
   // TODO: A and B are read-only so the owner device will be 0. How to fix?
   //assert(A.b.get_current_device() != 0);
   //assert(B.b.get_current_device() != 0);
-  int device = C.b.get_current_device();
-  assert(device != 0);
+  auto device = ttg::device::current_device();
+  assert(device.is_device());
 #if defined(TTG_HAVE_CUDA)
   if constexpr (std::is_same_v<T,double>) {
-      cublasDgemm(ttg::detail::cublas_get_handle(), CUBLAS_OP_N, CUBLAS_OP_N, C.extent(0), C.extent(1), A.extent(1),
-                  &alpha, A.b.device_ptr_on(device), A.extent(0), B.b.device_ptr_on(device), B.extent(0), &beta,
+      cublasDgemm(cublas_handle(), CUBLAS_OP_N, CUBLAS_OP_N, C.extent(0), C.extent(1), A.extent(1),
+                  &alpha, A.b.current_device_ptr(), A.extent(0), B.b.current_device_ptr(), B.extent(0), &beta,
                   C.b.current_device_ptr(), C.extent(0));
   }
   else if constexpr (std::is_same_v<T,float>) {
-      cublasSgemm(ttg::detail::cublas_get_handle(), CUBLAS_OP_N, CUBLAS_OP_N, C.extent(0), C.extent(1), A.extent(1),
-                  &alpha, A.b.device_ptr_on(device), A.extent(0), B.b.device_ptr_on(device), B.extent(0), &beta,
+      cublasSgemm(cublas_handle(), CUBLAS_OP_N, CUBLAS_OP_N, C.extent(0), C.extent(1), A.extent(1),
+                  &alpha, A.b.current_device_ptr(), A.extent(0), B.b.current_device_ptr(), B.extent(0), &beta,
                   C.b.current_device_ptr(), C.extent(0));
   }
 #elif defined(TTG_HAVE_HIPBLAS)
@@ -295,18 +295,18 @@ static void device_gemm(Blk &C, const Blk &A, const Blk &B) {
     hipblasDgemm(hipblas_handle(),
                  HIPBLAS_OP_N, HIPBLAS_OP_N,
                  C.extent(0), C.extent(1), A.extent(1), &alpha,
-                 A.b.device_ptr_on(device), A.extent(0),
-                 B.b.device_ptr_on(device), B.extent(0), &beta,
+                 A.b.current_device_ptr(), A.extent(0),
+                 B.b.current_device_ptr(), B.extent(0), &beta,
                  C.b.current_device_ptr(), C.extent(0));
   } else if constexpr (std::is_same_v<T,float>) {
     hipblasSgemm(hipblas_handle(),
                  HIPBLAS_OP_N, HIPBLAS_OP_N,
                  C.extent(0), C.extent(1), A.extent(1), &alpha,
-                 A.b.device_ptr_on(device), A.extent(0),
-                 B.b.device_ptr_on(device), B.extent(0), &beta,
+                 A.b.current_device_ptr(), A.extent(0),
+                 B.b.current_device_ptr(), B.extent(0), &beta,
                  C.b.current_device_ptr(), C.extent(0));
   }
-  
+
 #endif
 }
 
diff --git a/ttg/ttg.h b/ttg/ttg.h
index c84af7854..1a88cea2a 100644
--- a/ttg/ttg.h
+++ b/ttg/ttg.h
@@ -32,6 +32,8 @@
 #include "ttg/buffer.h"
 #include "ttg/devicescratch.h"
 #include "ttg/devicescope.h"
+#include "ttg/device/device.h"
+#include "ttg/device/task.h"
 
 #if defined(TTG_USE_PARSEC)
 #include "ttg/parsec/ttg.h"
diff --git a/ttg/ttg/device/device.h b/ttg/ttg/device/device.h
index a8cf6e623..83845fb32 100644
--- a/ttg/ttg/device/device.h
+++ b/ttg/ttg/device/device.h
@@ -1,6 +1,8 @@
 #pragma once
 
-#include <ttg/config.h>
+#include "ttg/config.h"
+#include "ttg/execution.h"
+
 
 
 namespace ttg::device {

From 9d15ba1b4d0ca3756b8356c741063a5526cb243a Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Tue, 31 Oct 2023 23:05:32 +0000
Subject: [PATCH 152/259] POTRF: make tile argument to dispatcher const

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/potrf/lauum.h | 34 ++++++++++++++++++++--------------
 examples/potrf/potrf.h | 10 +++++-----
 2 files changed, 25 insertions(+), 19 deletions(-)

diff --git a/examples/potrf/lauum.h b/examples/potrf/lauum.h
index d30c011ce..dba40ef5b 100644
--- a/examples/potrf/lauum.h
+++ b/examples/potrf/lauum.h
@@ -44,7 +44,7 @@ auto make_lauum(const MatrixT<T>& A,
         ttg::send<1>(Key2{K, K}, std::move(tile_kk), out);
     }
   };
-  return ttg::make_tt(f, ttg::edges(input_disp), ttg::edges(to_syrk_C, output_result), "LAUUM", 
+  return ttg::make_tt(f, ttg::edges(input_disp), ttg::edges(to_syrk_C, output_result), "LAUUM",
                       {"tile_kk"}, {"to_syrk_C", "output_result"});
 }
 
@@ -85,7 +85,7 @@ auto make_trmm(const MatrixT<T>& A,
       ttg::send<1>(Key2{K, N}, std::move(tile_kn), out);
     }
   };
-  return ttg::make_tt(f, ttg::edges(input_kk, input_kn), ttg::edges(to_gemm_C, output_result), "TRMM", 
+  return ttg::make_tt(f, ttg::edges(input_kk, input_kn), ttg::edges(to_gemm_C, output_result), "TRMM",
                       {"tile_kk", "tile_kn"}, {"to_GEMM_C", "output_result"});
 }
 
@@ -93,7 +93,7 @@ auto make_trmm(const MatrixT<T>& A,
 template <typename T>
 auto make_syrk(const MatrixT<T>& A,
                ttg::Edge<Key2, MatrixTile<T>>& input_kn, // will from the dispatcher
-               ttg::Edge<Key2, MatrixTile<T>>& input_nn, 
+               ttg::Edge<Key2, MatrixTile<T>>& input_nn,
                ttg::Edge<Key2, MatrixTile<T>>& to_syrk_nn,
                ttg::Edge<Key2, MatrixTile<T>>& output_result)
 {
@@ -116,7 +116,7 @@ auto make_syrk(const MatrixT<T>& A,
     blas::syrk(blas::Layout::ColMajor,
                lapack::Uplo::Lower,
                blas::Op::Trans,
-               m, k, 
+               m, k,
                1.0, tile_kn.data(), tile_kn.rows(),
                1.0, tile_nn.data(), tile_nn.rows());
 
@@ -127,8 +127,8 @@ auto make_syrk(const MatrixT<T>& A,
         ttg::send<1>(Key2{N, N}, tile_kn, out);
     }
   };
-  return ttg::make_tt(f, ttg::edges(input_kn, input_nn), 
-                         ttg::edges(to_syrk_nn, output_result), "SYRK", 
+  return ttg::make_tt(f, ttg::edges(input_kn, input_nn),
+                         ttg::edges(to_syrk_nn, output_result), "SYRK",
                       {"tile_kn", "tile_nn"}, {"SYRK_nn", "output_result"});
 }
 
@@ -162,7 +162,7 @@ auto make_gemm(const MatrixT<T>& A,
                blas::Op::NoTrans,
                input_A.rows(), input_B.cols(), input_A.cols(),
                1.0, input_A.data(), input_A.rows(),
-                    input_B.data(), input_B.rows(), 
+                    input_B.data(), input_B.rows(),
                1.0, input_C.data(), input_C.rows());
 
     if(K < A.rows()-1) {
@@ -172,8 +172,8 @@ auto make_gemm(const MatrixT<T>& A,
         ttg::send<1>(Key2{M, N}, std::move(input_C), out);
     }
   };
-  return ttg::make_tt(f, ttg::edges(input_A, input_B, input_C), 
-                         ttg::edges(to_gemm_C, output_result), "GEMM", 
+  return ttg::make_tt(f, ttg::edges(input_A, input_B, input_C),
+                         ttg::edges(to_gemm_C, output_result), "GEMM",
                       {"A", "B", "C"}, {"GEMM_C", "output result"});
 }
 
@@ -189,7 +189,7 @@ auto make_dispatcher(const MatrixT<T>& A,
                      ttg::Edge<Key3, MatrixTile<T>>& to_gemm_B)
 {
   auto f = [=](const Key2& key,
-               MatrixTile<T>&&tile,
+               const MatrixTile<T>& tile,
                std::tuple<ttg::Out<Key1, MatrixTile<T>>,
                           ttg::Out<Key2, MatrixTile<T>>,
                           ttg::Out<Key2, MatrixTile<T>>,
@@ -215,7 +215,7 @@ auto make_dispatcher(const MatrixT<T>& A,
                 keylist_trmm_A.push_back(Key2{key[1], n});
             }
         }
-    } 
+    }
     if(key[0] > key[1]) {
         keylist_syrk.reserve(1);
         if(ttg::tracing()) ttg::print("LAUUM_Dispatch(", key, ") sending to SYRK(", key, ")");
@@ -227,7 +227,7 @@ auto make_dispatcher(const MatrixT<T>& A,
             keylist_gemm_A.reserve(key[1]);
             for(int n = 0; n < key[1]; n++) {
                 if(ttg::tracing()) ttg::print("LAUUM_Dispatch(", key, ") sending to A of GEMM(", Key3{key[0], n, key[1]}, ")");
-                keylist_gemm_A.push_back(Key3{key[0], n, key[1]});      
+                keylist_gemm_A.push_back(Key3{key[0], n, key[1]});
             }
         }
     }
@@ -238,10 +238,16 @@ auto make_dispatcher(const MatrixT<T>& A,
             keylist_gemm_B.push_back(Key3{key[0], key[1], n});
         }
     }
-    ttg::broadcast<0, 1, 2, 3, 4, 5>(std::make_tuple(std::move(keylist_lauum), std::move(keylist_syrk), std::move(keylist_trmm_A), std::move(keylist_trmm_B), std::move(keylist_gemm_A), std::move(keylist_gemm_B)), std::move(tile), out);
+    ttg::broadcast<0, 1, 2, 3, 4, 5>(std::make_tuple(std::move(keylist_lauum),
+                                                     std::move(keylist_syrk),
+                                                     std::move(keylist_trmm_A),
+                                                     std::move(keylist_trmm_B),
+                                                     std::move(keylist_gemm_A),
+                                                     std::move(keylist_gemm_B)),
+                                                     tile, out);
   };
 
-  return ttg::make_tt(f, ttg::edges(input), ttg::edges(to_lauum, to_syrk, to_trmm_A, to_trmm_B, to_gemm_A, to_gemm_B), 
+  return ttg::make_tt(f, ttg::edges(input), ttg::edges(to_lauum, to_syrk, to_trmm_A, to_trmm_B, to_gemm_A, to_gemm_B),
                       "LAUUM Dispatch", {"Input"}, {"LAUUM", "SYRK", "TRMM_A", "TRMM_B", "GEMM_A", "GEMM_B"});
 }
 
diff --git a/examples/potrf/potrf.h b/examples/potrf/potrf.h
index 2503e8a2b..50af884dd 100644
--- a/examples/potrf/potrf.h
+++ b/examples/potrf/potrf.h
@@ -636,20 +636,20 @@ namespace potrf {
   auto make_dispatcher(ttg::Edge<Key2, MatrixTile<T>>& input, ttg::Edge<Key1, MatrixTile<T>>& to_potrf,
                        ttg::Edge<Key2, MatrixTile<T>>& to_trsm, ttg::Edge<Key2, MatrixTile<T>>& to_syrk,
                        ttg::Edge<Key3, MatrixTile<T>>& to_gemm) {
-    auto f = [=](const Key2& key, MatrixTile<T>&& tile,
+    auto f = [=](const Key2& key, const MatrixTile<T>& tile,
                  std::tuple<ttg::Out<Key1, MatrixTile<T>>, ttg::Out<Key2, MatrixTile<T>>, ttg::Out<Key2, MatrixTile<T>>,
                             ttg::Out<Key3, MatrixTile<T>>>& out) {
       if (ttg::tracing()) ttg::print("POTRF_Dispatch(", key, ")");
       if (0 == key[0] && 0 == key[1]) {
         // First element goes to POTRF
         if (ttg::tracing()) ttg::print("POTRF_Dispatch(", key, ") sending to POTRF(", Key1{key[0]}, ")");
-        ttg::send<0>(Key1{key[0]}, std::move(tile), out);
+        ttg::send<0>(Key1{key[0]}, tile, out);
         return;
       }
       if (key[0] == key[1]) {
         // Other diagonal elements go to SYRK
         if (ttg::tracing()) ttg::print("POTRF_Dispatch(", key, ") sending to SYRK(", Key2{0, key[0]}, ")");
-        ttg::send<2>(Key2{0, key[0]}, std::move(tile), out);
+        ttg::send<2>(Key2{0, key[0]}, tile, out);
         return;
       }
       // We only consider the lower triangular
@@ -657,12 +657,12 @@ namespace potrf {
       if (0 == key[1]) {
         // First column goes to TRSM
         if (ttg::tracing()) ttg::print("POTRF_Dispatch(", key, ") sending to TRSM(", key, ")");
-        ttg::send<1>(key, std::move(tile), out);
+        ttg::send<1>(key, tile, out);
         return;
       }
       // Rest goes to GEMM
       if (ttg::tracing()) ttg::print("POTRF_Dispatch(", key, ") sending to GEMM(", Key3{key[0], key[1], 0}, ")");
-      ttg::send<3>(Key3{key[0], key[1], 0}, std::move(tile), out);
+      ttg::send<3>(Key3{key[0], key[1], 0}, tile, out);
     };
 
     return ttg::make_tt(f, ttg::edges(input), ttg::edges(to_potrf, to_trsm, to_syrk, to_gemm), "POTRF Dispatch",

From 166bdecfbe8a6cd1e4745c06f3ab04e39aaa221e Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 3 Nov 2023 14:06:41 +0000
Subject: [PATCH 153/259] Multi-GPU: Play nice with the PaRSEC accelerator

We need to a) push tiles back if they move from RW to RO tasks, and b)
create a fake task-class that contains all the flows we actually need
(i.e., that are part of to_device()). This way PaRSEC can select the
right device and we get all the transfers in multi-GPU settings.

This needs to be revisited in the PaRSEC backend. We should not need to
do the push-back and create a fake task-class to select the correct
device. That requires an overhaul of the device backend though...

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/potrf/potrf.h | 45 ++++++++++++++------------------
 ttg/ttg/parsec/task.h  |  3 ++-
 ttg/ttg/parsec/ttg.h   | 59 +++++++++++++++++++++++++++++++++++++++---
 3 files changed, 78 insertions(+), 29 deletions(-)

diff --git a/examples/potrf/potrf.h b/examples/potrf/potrf.h
index 50af884dd..369bc9a8d 100644
--- a/examples/potrf/potrf.h
+++ b/examples/potrf/potrf.h
@@ -55,7 +55,7 @@ namespace potrf {
 #if defined(TTG_HAVE_CUDA)
     //std::cout << "POTRF A " << A.buffer().device_ptr_on(device) << " device " << device << " cols " << A.cols() << " lda " << A.lda() << " Lwork " << Lwork << " WS " << workspace << " devInfo " << devInfo << std::endl;
     auto handle = cusolver_handle();
-    std::cout << "POTRF handle  " << handle << " device " << device << " stream " << ttg::device::current_stream() << std::endl;
+    //std::cout << "POTRF handle  " << handle << " device " << device << " stream " << ttg::device::current_stream() << std::endl;
     cusolverDnDpotrf(handle,
                       CUBLAS_FILL_MODE_LOWER, A.cols(),
                       A.buffer().device_ptr_on(device), A.lda(),
@@ -73,7 +73,7 @@ namespace potrf {
   static void device_norm(const MatrixTile<double> &A, double *norm) {
     auto size = A.size();
     auto buffer = A.buffer().current_device_ptr();
-    std::cout << "device_norm ptr " << buffer << " device " << ttg::device::current_device() << std::endl;
+    //std::cout << "device_norm ptr " << buffer << " device " << ttg::device::current_device() << std::endl;
 #if defined(TTG_HAVE_CUDA)
     auto handle = cublas_handle();
     //double n = 1.0;
@@ -145,8 +145,7 @@ namespace potrf {
 #endif // DEBUG_TILES_VALUES
 
       int device = ttg::device::current_device();
-      std::cout << "POTRF [" << K << "] tile kk "
-                << tile_kk.buffer().device_ptr_on(device) << std::endl;
+      //std::cout << "POTRF [" << K << "] on " << device << std::endl;
 
 
       //std::cout << "devWS host ptr " << hostWS << " device ptr " << devWS.device_ptr() << " size " <<  devWS.size()
@@ -162,7 +161,7 @@ namespace potrf {
       /* wait for the kernel to complete */
       co_await ttg::wait_kernel(devInfo);
       // check that we got the input tile we expected
-     assert(check_norm(tile_kk.norm(), norms[0]));
+      assert(check_norm(tile_kk.norm(), norms[0]));
       // set the new norm
       tile_kk.set_norm(norms[1]);
 #else
@@ -193,14 +192,14 @@ namespace potrf {
 
       if (ttg::tracing()) ttg::print("POTRF(", key, ")");
 #if defined(DEBUG_TILES_VALUES)
-      std::cout << "Before POTRF(" << key << "), A(" << K << ", " << K << ") is " << tile_kk;
+      //std::cout << "Before POTRF(" << key << "), A(" << K << ", " << K << ") is " << tile_kk;
 #endif
 
       auto info = lapack::potrf(lapack::Uplo::Lower, tile_kk.rows(), tile_kk.data(), tile_kk.lda());
       assert(info == 0);
 
 #if defined(DEBUG_TILES_VALUES)
-      std::cout << "After POTRF(" << key << "), A(" << K << ", " << K << ") is " << tile_kk << std::endl;
+      //std::cout << "After POTRF(" << key << "), A(" << K << ", " << K << ") is " << tile_kk << std::endl;
 #endif
 
       /* send the tile to outputs */
@@ -284,9 +283,7 @@ namespace potrf {
 #endif // DEBUG_TILES_VALUES
 
 
-      std::cout << "TRSM [" << K << ", " << M << "] tile mk "
-                << tile_mk.buffer().device_ptr_on(device)
-                << " tile kk " << tile_kk.buffer().device_ptr_on(device) << std::endl;
+      //std::cout << "TRSM [" << K << ", " << M << "] on " << device << std::endl;
 
 #if defined(TTG_HAVE_CUDA)
       cublasDtrsm(cublas_handle(),
@@ -338,15 +335,15 @@ namespace potrf {
 
       if (ttg::tracing()) ttg::print("TRSM(", key, ")");
 #if defined(DEBUG_TILES_VALUES)
-      std::cout << "Before TRSM(" << key << "), A(" << K << ", " << K << ") is " << tile_kk << " and A(" << M << ", "
-                << K << ") is " << tile_mk;
+      //std::cout << "Before TRSM(" << key << "), A(" << K << ", " << K << ") is " << tile_kk << " and A(" << M << ", "
+      //          << K << ") is " << tile_mk;
 #endif
 
       blas::trsm(blas::Layout::ColMajor, blas::Side::Right, lapack::Uplo::Lower, blas::Op::Trans, blas::Diag::NonUnit,
                  mb, nb, 1.0, tile_kk.data(), tile_kk.lda(), tile_mk.data(), tile_mk.lda());
 
 #if defined(DEBUG_TILES_VALUES)
-      std::cout << "After TRSM(" << key << "), A(" << K << ", " << K << ") is " << tile_mk << std::endl;
+      //std::cout << "After TRSM(" << key << "), A(" << K << ", " << K << ") is " << tile_mk << std::endl;
 #endif
 
       std::vector<Key3> keylist_row;
@@ -416,9 +413,7 @@ namespace potrf {
       double alpha = -1.0;
       double beta  =  1.0;
 
-      std::cout << "SYRK [" << K << ", " << M << "] tile mk "
-                << tile_mk.buffer().device_ptr_on(device)
-                << " tile kk " << tile_kk.buffer().device_ptr_on(device) << std::endl;
+      //std::cout << "SYRK [" << K << ", " << M << "] on " << device << std::endl;
 
 #if defined(TTG_HAVE_CUDA)
       cublasDsyrk(cublas_handle(),
@@ -475,15 +470,15 @@ namespace potrf {
 
       if (ttg::tracing()) ttg::print("SYRK(", key, ")");
 #if defined(DEBUG_TILES_VALUES)
-      std::cout << "Before SYRK(" << key << "), A(" << M << ", " << K << ") is " << tile_mk << " and A(" << K << ", "
-                << K << ") is " << tile_kk;
+      //std::cout << "Before SYRK(" << key << "), A(" << M << ", " << K << ") is " << tile_mk << " and A(" << K << ", "
+      //          << K << ") is " << tile_kk;
 #endif
 
       blas::syrk(blas::Layout::ColMajor, lapack::Uplo::Lower, blas::Op::NoTrans, mb, nb, -1.0, tile_mk.data(),
                  tile_mk.lda(), 1.0, tile_kk.data(), tile_kk.lda());
 
 #if defined(DEBUG_TILES_VALUES)
-      std::cout << "After SYRK(" << key << "), A(" << K << ", " << K << ") is " << tile_kk << std::endl;
+      //std::cout << "After SYRK(" << key << "), A(" << K << ", " << K << ") is " << tile_kk << std::endl;
 #endif
 
       if (M == K + 1) {
@@ -523,9 +518,9 @@ namespace potrf {
       assert(tile_nk.rows() == tile_mn.cols());
 
       if (ttg::tracing()) ttg::print("GEMM(", key, ")");
-#if defined(DEBUG_TILES_VALUES)
-      std::cout << "Before GEMM(" << key << "), A(" << M << ", " << K << ") is " << tile_mk << " and A(" << K << ", "
-                << N << ") is " << tile_nk << " and A(" << M << ", " << N << ") is " << tile_mn;
+#if defined(DEBUG_TILES_VALUES) && 0
+      //std::cout << "Before GEMM(" << key << "), A(" << M << ", " << K << ") is " << tile_mk << " and A(" << K << ", "
+      //          << N << ") is " << tile_nk << " and A(" << M << ", " << N << ") is " << tile_mn;
 #endif
 
 #ifdef DEBUG_TILES_VALUES
@@ -604,8 +599,8 @@ namespace potrf {
 
       if (ttg::tracing()) ttg::print("GEMM(", key, ")");
 #if defined(DEBUG_TILES_VALUES)
-      std::cout << "Before GEMM(" << key << "), A(" << M << ", " << K << ") is " << tile_mk << " and A(" << K << ", "
-                << N << ") is " << tile_nk << " and A(" << M << ", " << N << ") is " << tile_mn;
+      //std::cout << "Before GEMM(" << key << "), A(" << M << ", " << K << ") is " << tile_mk << " and A(" << K << ", "
+      //          << N << ") is " << tile_nk << " and A(" << M << ", " << N << ") is " << tile_mn;
 #endif
 
       blas::gemm(blas::Layout::ColMajor, blas::Op::NoTrans, blas::Op::Trans, tile_mk.rows(), tile_nk.rows(),
@@ -613,7 +608,7 @@ namespace potrf {
                  tile_mn.data(), tile_mn.lda());
 
 #if defined(DEBUG_TILES_VALUES)
-      std::cout << "After GEMM(" << key << "), A(" << M << ", " << N << ") is " << tile_mn << std::endl;
+      //std::cout << "After GEMM(" << key << "), A(" << M << ", " << N << ") is " << tile_mn << std::endl;
 #endif
 
       if (N == K + 1) {
diff --git a/ttg/ttg/parsec/task.h b/ttg/ttg/parsec/task.h
index 0d4c890b2..917b63c9b 100644
--- a/ttg/ttg/parsec/task.h
+++ b/ttg/ttg/parsec/task.h
@@ -45,7 +45,8 @@ namespace ttg_parsec {
       MULTIPLE_READER   = 1 << 1,
       SINGLE_WRITER  = 1 << 2,
       MULTIPLE_WRITER   = 1 << 3,
-      MARKED_PUSHOUT = 1 << 4
+      IS_MODIFIED    = 1 << 4,
+      MARKED_PUSHOUT = 1 << 5
     };
 
     inline
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index f32da330d..71ef55e0f 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -716,7 +716,7 @@ namespace ttg_parsec {
 
     template<typename TT, std::size_t I>
     inline void transfer_ownership_impl(ttg_data_copy_t *copy, int device) {
-      if constexpr(!std::is_const_v<std::tuple_element_t<I, typename TT::input_args_type>>) {
+      if constexpr(!std::is_const_v<std::tuple_element_t<I, typename TT::input_values_tuple_type>>) {
         copy->transfer_ownership(PARSEC_FLOW_ACCESS_RW, device);
       }
     }
@@ -732,7 +732,7 @@ namespace ttg_parsec {
     inline parsec_hook_return_t hook(struct parsec_execution_stream_s *es, parsec_task_t *parsec_task) {
       parsec_ttg_task_t<TT> *me = (parsec_ttg_task_t<TT> *)parsec_task;
       if constexpr(std::tuple_size_v<typename TT::input_values_tuple_type> > 0) {
-        transfer_ownership<TT>(me, 0, std::index_sequence_for<typename TT::input_values_tuple_type>{});
+        transfer_ownership<TT>(me, 0, std::make_index_sequence<std::tuple_size_v<typename TT::input_values_tuple_type>>{});
       }
       return me->template invoke_op<ttg::ExecutionSpace::Host>();
     }
@@ -1454,11 +1454,30 @@ namespace ttg_parsec {
       /* for now make sure we're waiting for transfers and the coro hasn't skipped this step */
       assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_TRANSFER);
 
+      /* set up a temporary task-class to correctly specify the flows */
+      parsec_task_class_t tc = *task->parsec_task.task_class;
+
+      tc.name = task->parsec_task.task_class->name;
+      // input flows are set up during register_device_memory as part of the first invocation above
+      for (int i = 0; i < MAX_PARAM_COUNT; ++i) {
+        tc.in[i]  = gpu_task->flow[i];
+        tc.out[i] = gpu_task->flow[i];
+      }
+      tc.nb_flows = MAX_PARAM_COUNT;
+
+      /* swap in the new task class */
+      const parsec_task_class_t* tmp = task->parsec_task.task_class;
+      *const_cast<parsec_task_class_t**>(&task->parsec_task.task_class) = &tc;
+
       /* TODO: is this the right place to set the mask? */
       task->parsec_task.chore_mask = PARSEC_DEV_ALL;
       /* get a device and come back if we need another one */
       int64_t task_load = 1;
       dev_index = parsec_get_best_device(parsec_task, &task_load);
+
+      /* swap back the original task class */
+      task->parsec_task.task_class = tmp;
+
       gpu_task->load = task_load;
       assert(dev_index >= 0);
       if (dev_index < 2) {
@@ -3157,6 +3176,12 @@ ttg::abort();  // should not happen
       }
 
       if constexpr (value_is_const) {
+        if (caller->data_flags & detail::ttg_parsec_data_flags::IS_MODIFIED) {
+          /* The data has been modified previously. PaRSEC requires us to pushout
+           * data if we transition from a writer to one or more readers. */
+          need_pushout = true;
+        }
+
         /* check for multiple readers */
         if (caller->data_flags & detail::ttg_parsec_data_flags::SINGLE_READER) {
           caller->data_flags |= detail::ttg_parsec_data_flags::MULTIPLE_READER;
@@ -3341,6 +3366,7 @@ ttg::abort();  // should not happen
       junk[0]++;
     }
 
+#if 0
     template <typename input_terminals_tupleT, std::size_t... IS, typename flowsT>
     void _initialize_flows(std::index_sequence<IS...>, flowsT &&flows) {
       int junk[] = {0,
@@ -3356,6 +3382,7 @@ ttg::abort();  // should not happen
       _initialize_flows<input_terminals_tupleT>(
           std::make_index_sequence<std::tuple_size<input_terminals_tupleT>::value>{}, flows);
     }
+#endif // 0
 
     void fence() override { ttg::default_execution_context().impl().fence(); }
 
@@ -3446,15 +3473,39 @@ ttg::abort();  // should not happen
 
     parsec_key_fn_t tasks_hash_fcts = {key_equal, key_print, key_hash};
 
+    template<std::size_t I>
+    inline static void increment_data_version_impl(task_t *task) {
+      if constexpr (!std::is_const_v<std::tuple_element_t<I, typename TT::input_values_tuple_type>>) {
+        if (task->copies[I] != nullptr){
+          task->copies[I]->inc_current_version();
+        }
+      }
+    }
+
+    template<std::size_t... Is>
+    inline static void increment_data_versions(task_t *task, std::index_sequence<Is...>) {
+      /* increment version of each mutable data */
+      int junk[] = {0, (increment_data_version_impl<Is>(task), 0)...};
+      junk[0]++;
+    }
+
     static parsec_hook_return_t complete_task_and_release(parsec_execution_stream_t *es, parsec_task_t *parsec_task) {
 
       //std::cout << "complete_task_and_release: task " << parsec_task << std::endl;
 
       task_t *task = (task_t*)parsec_task;
 
+      if constexpr (derived_has_device_op()) {
+        assert(nullptr != task->suspended_task_address);
+      }
+
       /* if we still have a coroutine handle we invoke it one more time to get the sends/broadcasts */
       if (task->suspended_task_address) {
 #ifdef TTG_HAVE_DEVICE
+        /* increment versions of all data we might have modified
+         * this must happen before we issue the sends */
+        //increment_data_versions(task, std::make_index_sequence<std::tuple_size_v<typename TT::input_values_tuple_type>>{});
+
         // get the device task from the coroutine handle
         auto dev_task = ttg::device_task_handle_type::from_address(task->suspended_task_address);
 
@@ -4085,7 +4136,9 @@ struct ttg::detail::value_copy_handler<ttg::Runtime::PaRSEC> {
       /* the value was potentially changed, so increment version */
       copy->inc_current_version();
     }
-    caller->data_flags = ttg_parsec::detail::ttg_parsec_data_flags::NONE;
+    /* We're coming from a writer so mark the data as modified.
+     * That way we can force a pushout in prepare_send if we move to read-only tasks (needed by PaRSEC). */
+    caller->data_flags = ttg_parsec::detail::ttg_parsec_data_flags::IS_MODIFIED;
     return *value_ptr;
   }
 

From 33a5a8d1129eeace495869ba3162d2b9b823ed69 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 3 Nov 2023 14:13:49 +0000
Subject: [PATCH 154/259] Bump PaRSEC and MADNESS tags for proper tear-down

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 cmake/modules/ExternalDependenciesVersions.cmake | 4 ++--
 cmake/modules/FindOrFetchMADNESS.cmake           | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmake/modules/ExternalDependenciesVersions.cmake b/cmake/modules/ExternalDependenciesVersions.cmake
index 725dac9b6..3b76b30da 100644
--- a/cmake/modules/ExternalDependenciesVersions.cmake
+++ b/cmake/modules/ExternalDependenciesVersions.cmake
@@ -5,7 +5,7 @@
 set(TTG_TRACKED_BOOST_VERSION 1.66)
 set(TTG_TRACKED_CATCH2_VERSION 2.13.1)
 set(TTG_TRACKED_CEREAL_VERSION 1.3.0)
-set(TTG_TRACKED_MADNESS_TAG e4ee892717e7f91f824a9652d53d4da2acf6920e)
-set(TTG_TRACKED_PARSEC_TAG e59bed9d4934775792a58980cf5500bd41381bc4)
+set(TTG_TRACKED_MADNESS_TAG cb195817d7807c4aead10ba200cd20649036cbae)
+set(TTG_TRACKED_PARSEC_TAG 6fd959e4a8d2dad8b701d0b83cd85b372f1a306b)
 set(TTG_TRACKED_BTAS_TAG a02be0d29fb4a788ecef43de711dcd6d6f1cb6b8)
 set(TTG_TRACKED_TILEDARRAY_TAG dfa3f76ebd58c05c64acf6e57cbdc85e90c880a8)
diff --git a/cmake/modules/FindOrFetchMADNESS.cmake b/cmake/modules/FindOrFetchMADNESS.cmake
index 24f0bc798..f112e4ff7 100644
--- a/cmake/modules/FindOrFetchMADNESS.cmake
+++ b/cmake/modules/FindOrFetchMADNESS.cmake
@@ -10,7 +10,7 @@ if (NOT TARGET MADworld)
   set(MADNESS_TASK_BACKEND PaRSEC CACHE STRING "The task backend to use for MADNESS tasks")
   FetchContent_Declare(
           MADNESS
-          GIT_REPOSITORY https://github.com/m-a-d-n-e-s-s/madness.git
+          GIT_REPOSITORY https://github.com/therault/madness.git
           GIT_TAG ${TTG_TRACKED_MADNESS_TAG}
   )
   FetchContent_MakeAvailable(MADNESS)

From f9375c8bf7174829095bf420eb83143b53c59a8c Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 6 Nov 2023 00:03:09 +0000
Subject: [PATCH 155/259] Bump PaRSEC reference for TTG-specific fixes

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 cmake/modules/ExternalDependenciesVersions.cmake | 2 +-
 cmake/modules/FindOrFetchPARSEC.cmake            | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmake/modules/ExternalDependenciesVersions.cmake b/cmake/modules/ExternalDependenciesVersions.cmake
index 3b76b30da..9c364d730 100644
--- a/cmake/modules/ExternalDependenciesVersions.cmake
+++ b/cmake/modules/ExternalDependenciesVersions.cmake
@@ -6,6 +6,6 @@ set(TTG_TRACKED_BOOST_VERSION 1.66)
 set(TTG_TRACKED_CATCH2_VERSION 2.13.1)
 set(TTG_TRACKED_CEREAL_VERSION 1.3.0)
 set(TTG_TRACKED_MADNESS_TAG cb195817d7807c4aead10ba200cd20649036cbae)
-set(TTG_TRACKED_PARSEC_TAG 6fd959e4a8d2dad8b701d0b83cd85b372f1a306b)
+set(TTG_TRACKED_PARSEC_TAG 9a9f76e3cba2ae006d5946a009ce617ecf53588b)
 set(TTG_TRACKED_BTAS_TAG a02be0d29fb4a788ecef43de711dcd6d6f1cb6b8)
 set(TTG_TRACKED_TILEDARRAY_TAG dfa3f76ebd58c05c64acf6e57cbdc85e90c880a8)
diff --git a/cmake/modules/FindOrFetchPARSEC.cmake b/cmake/modules/FindOrFetchPARSEC.cmake
index b3fd5faa3..7b164019f 100644
--- a/cmake/modules/FindOrFetchPARSEC.cmake
+++ b/cmake/modules/FindOrFetchPARSEC.cmake
@@ -17,7 +17,7 @@ if (NOT TARGET PaRSEC::parsec)
 
   FetchContent_Declare(
           PARSEC
-          GIT_REPOSITORY      https://github.com/ICLDisco/parsec.git
+          GIT_REPOSITORY      https://github.com/devreal/parsec-1.git
           GIT_TAG             ${TTG_TRACKED_PARSEC_TAG}
   )
   FetchContent_MakeAvailable(PARSEC)

From 9903f253b7c7b1eb0d72d8b8e48a4dd87cb226db Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 15 Nov 2023 16:33:48 -0700
Subject: [PATCH 156/259] Bump PaRSEC ref for reworked GPU backend

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 cmake/modules/ExternalDependenciesVersions.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/modules/ExternalDependenciesVersions.cmake b/cmake/modules/ExternalDependenciesVersions.cmake
index 9c364d730..1b2538595 100644
--- a/cmake/modules/ExternalDependenciesVersions.cmake
+++ b/cmake/modules/ExternalDependenciesVersions.cmake
@@ -6,6 +6,6 @@ set(TTG_TRACKED_BOOST_VERSION 1.66)
 set(TTG_TRACKED_CATCH2_VERSION 2.13.1)
 set(TTG_TRACKED_CEREAL_VERSION 1.3.0)
 set(TTG_TRACKED_MADNESS_TAG cb195817d7807c4aead10ba200cd20649036cbae)
-set(TTG_TRACKED_PARSEC_TAG 9a9f76e3cba2ae006d5946a009ce617ecf53588b)
+set(TTG_TRACKED_PARSEC_TAG 6a5a986d6ec1f9cace09f3ad475c553632077a24)
 set(TTG_TRACKED_BTAS_TAG a02be0d29fb4a788ecef43de711dcd6d6f1cb6b8)
 set(TTG_TRACKED_TILEDARRAY_TAG dfa3f76ebd58c05c64acf6e57cbdc85e90c880a8)

From e9271ff36985a0ac02352d0d67df359b7c89856a Mon Sep 17 00:00:00 2001
From: Thomas Herault <herault@icl.utk.edu>
Date: Thu, 16 Nov 2023 17:22:55 +0000
Subject: [PATCH 157/259] Add a LEVEL_ZERO implementation for the GEMM kernel
 in spmm_cuda.cc

---
 examples/spmm/spmm_cuda.cc | 41 ++++++++++++++++++++++++++++++++++++--
 1 file changed, 39 insertions(+), 2 deletions(-)

diff --git a/examples/spmm/spmm_cuda.cc b/examples/spmm/spmm_cuda.cc
index 60fb969fa..2c0dbffc5 100644
--- a/examples/spmm/spmm_cuda.cc
+++ b/examples/spmm/spmm_cuda.cc
@@ -49,6 +49,11 @@ using namespace ttg;
 
 #include "ttg/serialization/std/pair.h"
 
+#if defined(TTG_HAVE_LEVEL_ZERO)
+#include <oneapi/mkl.hpp>
+#include <sys/time.h>
+#endif
+
 #if defined(BLOCK_SPARSE_GEMM) && defined(BTAS_IS_USABLE)
 
 template <typename _T, class _Range, class _Storage>
@@ -249,7 +254,7 @@ struct DeviceTensor : public ttg::TTValue<DeviceTensor<_T, _Range, _Storage>>
 };
 
 using scalar_t = double;
-#if defined(TTG_HAVE_CUDA) || defined(TTG_HAVE_HIPBLAS)
+#if defined(TTG_HAVE_CUDA) || defined(TTG_HAVE_HIPBLAS) || defined(TTG_HAVE_LEVEL_ZERO)
 using blk_t = DeviceTensor<scalar_t, btas::DEFAULT::range,
                            btas::mohndle<btas::varray<scalar_t, TiledArray::device_pinned_allocator<scalar_t>>,
                                          btas::Handle::shared_ptr>>;
@@ -306,7 +311,36 @@ static void device_gemm(Blk &C, const Blk &A, const Blk &B) {
                  B.b.current_device_ptr(), B.extent(0), &beta,
                  C.b.current_device_ptr(), C.extent(0));
   }
-
+#elif defined(TTG_HAVE_LEVEL_ZERO)
+
+#if defined(DEBUG_SYNCHRONOUS)
+  try {
+#endif /* DEBUG_SYNCHRONOUS */
+    cl::sycl::event gemm_event;
+    gemm_event = oneapi::mkl::blas::gemm(lz_queue(),
+         oneapi::mkl::transpose::N, oneapi::mkl::transpose::N,
+         C.extent(0), C.extent(1), A.extent(1),
+         alpha, A.b.current_device_ptr(), A.extent(0),
+                B.b.current_device_ptr(), B.extent(0),
+         beta,  C.b.current_device_ptr(), C.extent(0));
+#if defined(DEBUG_SYNCHRONOUS)
+    gemm_event.wait();
+  } catch (const oneapi::mkl::invalid_argument &e) {
+    std::cerr << "OneAPI MKL BLAS GEMM throws invalid argument exception" << std::endl;
+  } catch (const oneapi::mkl::unsupported_device &e) {
+    std::cerr << "OneAPI MKL BLAS GEMM throws unsuported device exception" << std::endl;
+  } catch (const oneapi::mkl::host_bad_alloc &e) {
+    std::cerr << "OneAPI MKL BLAS GEMM throws host bad allocation exception" << std::endl;
+  } catch (const oneapi::mkl::device_bad_alloc &e) {
+    std::cerr << "OneAPI MKL BLAS GEMM throws device bad allocation exception" << std::endl;
+  } catch (const oneapi::mkl::unimplemented &e) {
+    std::cerr << "OneAPI MKL BLAS GEMM throws unimplemented exception" << std::endl;
+  } catch (const std::exception& e) {
+    std::cerr << "OneAPI MKL BLAS GEMM throws unexpected exception" << std::endl;
+  } catch (...) {
+    std::cerr << "OneAPI MKL BLAS GEMM throws unexpected exception that is also badly formatted..." << std::endl;
+  }
+#endif /* DEBUG_SYNCHRONOUS */
 #endif
 }
 
@@ -734,6 +768,9 @@ class SpMM25D {
 #elif defined(TTG_HAVE_HIPBLAS)
     static constexpr bool have_hip_op  = true;
 #warning SPMM using HIP implementation
+#elif defined(TTG_HAVE_LEVEL_ZERO)
+    static constexpr bool have_level_zero_op = true;
+#warning SPMM using LEVEL_ZERO implementation
 #else
 #error No valid device implementation found!
 #endif

From e20758ac75039d86384b03f5d4a1d03676a05e8c Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 16 Nov 2023 17:45:10 +0000
Subject: [PATCH 158/259] Add level zero support and refactor device naming

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/device/device.h | 44 +++++++++++++++++++++++++++--
 ttg/ttg/execution.h     |  1 +
 ttg/ttg/parsec/buffer.h | 45 +++++++++++++++---------------
 ttg/ttg/parsec/device.h | 40 ++++++++++++++++++++++++++
 ttg/ttg/parsec/ttg.h    | 62 ++++++++++++++++++++++++++---------------
 5 files changed, 145 insertions(+), 47 deletions(-)
 create mode 100644 ttg/ttg/parsec/device.h

diff --git a/ttg/ttg/device/device.h b/ttg/ttg/device/device.h
index 83845fb32..95e1e0fc8 100644
--- a/ttg/ttg/device/device.h
+++ b/ttg/ttg/device/device.h
@@ -6,6 +6,14 @@
 
 
 namespace ttg::device {
+
+#if defined(TTG_HAVE_CUDA)
+  constexpr ttg::ExecutionSpace available_execution_space = ttg::ExecutionSpace::CUDA;
+#elif defined(TTG_HAVE_HIP)
+  constexpr ttg::ExecutionSpace available_execution_space = ttg::ExecutionSpace::HIP;
+#elif defined(TTG_HAVE_LEVEL_ZERO)
+  constexpr ttg::ExecutionSpace available_execution_space = ttg::ExecutionSpace::L0;
+#endif
   class Device {
     int m_id = 0;
     ttg::ExecutionSpace m_space = ttg::ExecutionSpace::Host;
@@ -36,11 +44,11 @@ namespace ttg::device {
     }
 
     bool is_device() const {
-      return ((!is_invalid()) && (m_space != ttg::ExecutionSpace::Host));
+      return !is_host();
     }
 
     bool is_host() const {
-      return (m_space == ttg::ExecutionSpace::Host);
+      return !is_invalid() && (m_space == ttg::ExecutionSpace::Host);
     }
 
     bool is_invalid() const {
@@ -120,4 +128,36 @@ namespace ttg::device {
   }
 } // namespace ttg
 
+#elif defined(TTG_HAVE_LEVEL_ZERO)
+
+#include <CL/sycl.hpp>
+
+namespace ttg::device {
+  namespace detail {
+    inline thread_local ttg::device::Device current_device_ts = {};
+    inline thread_local sycl::queue* current_stream_ts = nullptr; // default stream
+
+    void reset_current() {
+      current_device_ts = {};
+      current_stream_ts = nullptr;
+    }
+
+    void set_current(int device, sycl::queue& stream) {
+      current_device_ts = ttg::device::Device(device, ttg::ExecutionSpace::HIP);
+      current_stream_ts = &stream;
+    }
+  } // namespace detail
+
+  inline
+  Device current_device() {
+    return detail::current_device_ts;
+  }
+
+  inline
+  const sycl::queue& current_stream() {
+    return *detail::current_stream_ts;
+  }
+} // namespace ttg
+
+
 #endif // defined(TTG_HAVE_HIP)
diff --git a/ttg/ttg/execution.h b/ttg/ttg/execution.h
index 4d99a3ae7..495d0e248 100644
--- a/ttg/ttg/execution.h
+++ b/ttg/ttg/execution.h
@@ -18,6 +18,7 @@ enum class ExecutionSpace {
   Host,   // a CPU
   CUDA,   // an NVIDIA CUDA device
   HIP,    // an AMD HIP device
+  L0,     // an Intel L0 device
   Invalid
 };
 
diff --git a/ttg/ttg/parsec/buffer.h b/ttg/ttg/parsec/buffer.h
index 552d0b52e..e3d7a84b3 100644
--- a/ttg/ttg/parsec/buffer.h
+++ b/ttg/ttg/parsec/buffer.h
@@ -10,6 +10,7 @@
 #include "ttg/parsec/parsec-ext.h"
 #include "ttg/util/iovec.h"
 #include "ttg/device/device.h"
+#include "ttg/parsec/device.h"
 
 #if defined(PARSEC_HAVE_DEV_CUDA_SUPPORT)
 #include <cuda_runtime.h>
@@ -75,9 +76,6 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
 
 public:
 
-  /* The device ID of the CPU. */
-  static constexpr int cpu_device = -2;
-
   buffer() : buffer(nullptr, 0)
   { }
 
@@ -161,33 +159,35 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
 
   /* set the current device, useful when a device
    * buffer was modified outside of a TTG */
-  void set_current_device(int device_id) {
+  void set_current_device(const ttg::device::Device& device) {
     assert(is_valid());
     /* make sure it's a valid device */
     assert(parsec_nb_devices > device_id);
     /* make sure it's a valid copy */
-    assert(m_data->device_copies[device_id+2] != nullptr);
-    m_data->owner_device = device_id+2;
+    int parsec_id = detail::device_to_parsec_device(device);
+    assert(m_data->device_copies[parsec_id] != nullptr);
+    m_data->owner_device = parsec_id;
   }
 
   /* Get the owner device ID, i.e., the last updated
-   * device buffer. A value of -2 designates the host
-   * as the current device. */
-  int get_owner_device() const {
+   * device buffer. */
+  ttg::device::Device get_owner_device() const {
     assert(is_valid());
-    return m_data->owner_device - 2; // 0: host, 1: recursive, 2: first device
+    return detail::parsec_device_to_device(m_data->owner_device);
   }
 
   /* Get the pointer on the currently active device. */
   element_type* current_device_ptr() {
     assert(is_valid());
-    return static_cast<element_type*>(m_data->device_copies[ttg::device::current_device()+2]->device_private);
+    int device_id = ttg::device::current_device()+detail::first_device_id;
+    return static_cast<element_type*>(m_data->device_copies[device_id]->device_private);
   }
 
   /* Get the pointer on the currently active device. */
   const element_type* current_device_ptr() const {
     assert(is_valid());
-    return static_cast<element_type*>(m_data->device_copies[ttg::device::current_device()+2]->device_private);
+    int device_id = ttg::device::current_device()+detail::first_device_id;
+    return static_cast<element_type*>(m_data->device_copies[device_id]->device_private);
   }
 
   /* Get the pointer on the owning device.
@@ -205,19 +205,19 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
   }
 
   /* get the device pointer at the given device
-   * \sa cpu_device
    */
-  element_type* device_ptr_on(int device_id) {
+  element_type* device_ptr_on(const ttg::device::Device& device) {
     assert(is_valid());
-    return static_cast<element_type*>(parsec_data_get_ptr(m_data.get(), device_id + 2));
+    int device_id = detail::device_to_parsec_device(device);
+    return static_cast<element_type*>(parsec_data_get_ptr(m_data.get(), device_id));
   }
 
   /* get the device pointer at the given device
-   * \sa cpu_device
    */
-  const element_type* device_ptr_on(int device_id) const {
+  const element_type* device_ptr_on(const ttg::device::Device& device) const {
     assert(is_valid());
-    return static_cast<element_type*>(parsec_data_get_ptr(m_data.get(), device_id + 2)); // GPUs start at 2
+    int device_id = detail::device_to_parsec_device(device);
+    return static_cast<element_type*>(parsec_data_get_ptr(m_data.get(), device_id));
   }
 
   element_type* host_ptr() {
@@ -228,12 +228,13 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
     return static_cast<element_type*>(parsec_data_get_ptr(m_data.get(), 0));
   }
 
-  bool is_valid_on(int device_id) const {
+  bool is_valid_on(const ttg::device::Device& device) const {
     assert(is_valid());
-    return (parsec_data_get_ptr(m_data.get(), device_id+2) != nullptr);
+    int device_id = detail::device_to_parsec_device(device);
+    return (parsec_data_get_ptr(m_data.get(), device_id) != nullptr);
   }
 
-  void allocate_on(int device_id) {
+  void allocate_on(const ttg::device::Device& device_id) {
     /* TODO: need exposed PaRSEC memory allocator */
     throw std::runtime_error("not implemented yet");
   }
@@ -252,7 +253,7 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
   /* Unpin the memory on all devices we currently track. */
   void unpin() {
     if (!is_valid()) return;
-    for (int i = 0; i < parsec_nb_devices-2; ++i) {
+    for (int i = 0; i < parsec_nb_devices-detail::first_device_id; ++i) {
       unpin_on(i);
     }
   }
diff --git a/ttg/ttg/parsec/device.h b/ttg/ttg/parsec/device.h
new file mode 100644
index 000000000..8f46453ac
--- /dev/null
+++ b/ttg/ttg/parsec/device.h
@@ -0,0 +1,40 @@
+#ifndef TTG_PARSEC_DEVICE_H
+#define TTG_PARSEC_DEVICE_H
+
+#include "ttg/device/device.h"
+
+namespace ttg_parsec {
+
+  namespace detail {
+
+    // the first ID of an accelerator in the parsec ID-space
+    inline int first_device_id = 0;
+
+    /**
+     * map from TTG ID-space to parsec ID-space
+     */
+    inline
+    int device_to_parsec_device(const ttg::device::Device& device) {
+      if (device.is_host()) {
+        return 0;
+      } else {
+        return device.id() + first_device_id;
+      }
+    }
+
+    /**
+     * map from parsec ID-space to TTG ID-space
+     */
+    inline
+    ttg::device::Device parsec_device_to_device(int parsec_id) {
+      if (parsec_id < first_device_id) {
+        return ttg::device::Device(parsec_id, ttg::ExecutionSpace::Host);
+      }
+      return ttg::device::Device(parsec_id - first_device_id,
+                                ttg::device::available_execution_space);
+    }
+  } // namespace detail
+
+} // namespace ttg_parsec
+
+#endif // TTG_PARSEC_DEVICE_H
\ No newline at end of file
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 71ef55e0f..cf0cd33b2 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -974,6 +974,14 @@ namespace ttg_parsec {
     std::shared_ptr<ttg::base::WorldImplBase> world_sptr{static_cast<ttg::base::WorldImplBase *>(world_ptr)};
     ttg::World world{std::move(world_sptr)};
     ttg::detail::set_default_world(std::move(world));
+
+    // query the first device ID
+    for (int i = 0; i < parsec_nb_devices; ++i) {
+      if (parsec_mca_device_is_gpu(i)) {
+        detail::first_device_id = i;
+        break;
+      }
+    }
   }
   inline void ttg_finalize() {
     // We need to notify the current taskpool of termination if we are in user termination detection mode
@@ -1329,7 +1337,7 @@ namespace ttg_parsec {
 #if defined(PARSEC_HAVE_DEV_CUDA_SUPPORT) && defined(TTG_HAVE_CUDA)
       {
         parsec_cuda_exec_stream_t *cuda_stream = (parsec_cuda_exec_stream_t *)gpu_stream;
-        int device = gpu_device->super.device_index - 2; // 0: host, 1: recursive, 2: first GPU
+        int device = gpu_device->super.device_index - detail::first_device_id;
         ttg::device::detail::set_current(device, cuda_stream->cuda_stream);
       }
 #endif // defined(PARSEC_HAVE_DEV_CUDA_SUPPORT) && defined(TTG_HAVE_CUDA)
@@ -1337,11 +1345,20 @@ namespace ttg_parsec {
 #if defined(PARSEC_HAVE_DEV_HIP_SUPPORT) && defined(TTG_HAVE_HIP)
       {
         parsec_hip_exec_stream_t *hip_stream = (parsec_hip_exec_stream_t *)gpu_stream;
-        int device = gpu_device->super.device_index - 2; // 0: host, 1: recursive, 2: first GPU
+        int device = gpu_device->super.device_index - detail::first_device_id;
         ttg::device::detail::set_current(device, hip_stream->hip_stream);
       }
 #endif // defined(PARSEC_HAVE_DEV_CUDA_SUPPORT) && defined(TTG_HAVE_CUDA)
 
+#if defined(PARSEC_HAVE_DEV_LEVEL_ZERO_SUPPORT) && defined(TTG_HAVE_LEVEL_ZERO)
+      {
+        parsec_level_zero_exec_stream_t *stream;
+        stream = (parsec_level_zero_exec_stream_t *)gpu_stream;
+        int device = gpu_device->super.device_index - detail::first_device_id;
+        ttg::device::detail::set_current(device, stream->swq->queue);
+      }
+#endif // defined(PARSEC_HAVE_DEV_CUDA_SUPPORT) && defined(TTG_HAVE_CUDA)
+
       /* Here we call back into the coroutine again after the transfers have completed */
       static_op<Space>(&task->parsec_task);
 
@@ -1394,24 +1411,14 @@ namespace ttg_parsec {
         }
       }
     }
-#if defined(PARSEC_HAVE_DEV_CUDA_SUPPORT)
-    static int
-    static_cuda_stage_in(parsec_gpu_task_t        *gtask,
-                         uint32_t                  flow_mask,
-                         parsec_gpu_exec_stream_t *gpu_stream) {
-      static_device_stage_in(gtask, flow_mask, gpu_stream);
-      return parsec_default_cuda_stage_in(gtask, flow_mask, gpu_stream);
-    }
-#endif // PARSEC_HAVE_DEV_CUDA_SUPPORT
-#if defined(PARSEC_HAVE_DEV_HIP_SUPPORT)
+
     static int
-    static_hip_stage_in(parsec_gpu_task_t        *gtask,
-                         uint32_t                  flow_mask,
-                         parsec_gpu_exec_stream_t *gpu_stream) {
+    static_device_stage_in_hook(parsec_gpu_task_t        *gtask,
+                                uint32_t                  flow_mask,
+                                parsec_gpu_exec_stream_t *gpu_stream) {
       static_device_stage_in(gtask, flow_mask, gpu_stream);
-      return parsec_default_hip_stage_in(gtask, flow_mask, gpu_stream);
+      return parsec_default_gpu_stage_in(gtask, flow_mask, gpu_stream);
     }
-#endif
 
     template <ttg::ExecutionSpace Space>
     static parsec_hook_return_t device_static_op(parsec_task_t* parsec_task) {
@@ -1493,21 +1500,30 @@ namespace ttg_parsec {
           if constexpr (Space == ttg::ExecutionSpace::CUDA) {
             /* TODO: we need custom staging functions because PaRSEC looks at the
              *       task-class to determine the number of flows. */
-            gpu_task->stage_in  = static_cuda_stage_in;
-            gpu_task->stage_out = parsec_default_cuda_stage_out;
-            return parsec_cuda_kernel_scheduler(es, gpu_task, dev_index);
+            gpu_task->stage_in  = static_device_stage_in_hook;
+            gpu_task->stage_out = parsec_default_gpu_stage_out;
+            return parsec_device_kernel_scheduler(device, es, gpu_task);
           }
           break;
 #endif
 #if defined(PARSEC_HAVE_DEV_HIP_SUPPORT)
         case PARSEC_DEV_HIP:
           if constexpr (Space == ttg::ExecutionSpace::HIP) {
-            gpu_task->stage_in  = static_hip_stage_in;
-            gpu_task->stage_out = parsec_default_hip_stage_out;
-            return parsec_hip_kernel_scheduler(es, gpu_task, dev_index);
+            gpu_task->stage_in  = static_device_stage_in_hook;
+            gpu_task->stage_out = parsec_default_gpu_stage_out;
+            return parsec_device_kernel_scheduler(device, es, gpu_task);
           }
           break;
 #endif // PARSEC_HAVE_DEV_HIP_SUPPORT
+#if defined(PARSEC_HAVE_DEV_LEVEL_ZERO_SUPPORT)
+        case PARSEC_DEV_LEVEL_ZERO:
+          if constexpr (Space == ttg::ExecutionSpace::L0) {
+            gpu_task->stage_in  = static_device_stage_in_hook;
+            gpu_task->stage_out = parsec_default_gpu_stage_out;
+            return parsec_device_kernel_scheduler(device, es, gpu_task);
+          }
+          break;
+#endif // PARSEC_HAVE_DEV_LEVEL_ZERO_SUPPORT
         default:
           break;
       }

From 97d0bf6ef7422750486b79873148e0c47108dfb0 Mon Sep 17 00:00:00 2001
From: Thomas Herault <herault@icl.utk.edu>
Date: Thu, 16 Nov 2023 17:49:05 +0000
Subject: [PATCH 159/259] Attempt at cmakery for level-zero

---
 CMakeLists.txt                     | 19 ++++++-
 cmake/modules/FindDPCPP.cmake      | 43 ++++++++++++++++
 cmake/modules/Findlevel-zero.cmake | 81 ++++++++++++++++++++++++++++++
 ttg/CMakeLists.txt                 |  4 ++
 4 files changed, 146 insertions(+), 1 deletion(-)
 create mode 100644 cmake/modules/FindDPCPP.cmake
 create mode 100644 cmake/modules/Findlevel-zero.cmake

diff --git a/CMakeLists.txt b/CMakeLists.txt
index db311f42e..cfc449528 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -53,6 +53,7 @@ set(CMAKE_INSTALL_CMAKEDIR "lib/cmake/ttg"
 option(TTG_PARSEC_USE_BOOST_SERIALIZATION "Whether to select Boost serialization methods in PaRSEC backend" ON)
 option(TTG_ENABLE_CUDA "Whether to TTG will look for CUDA" OFF)
 option(TTG_ENABLE_HIP "Whether to TTG will look for HIP" OFF)
+option(TTG_ENABLE_LEVEL_ZERO "Whether to TTG will look for LEVEL_ZERO" OFF)
 option(TTG_EXAMPLES "Whether to build examples" OFF)
 option(TTG_ENABLE_ASAN "Whether to enable address sanitizer" OFF)
 
@@ -134,8 +135,24 @@ if (TTG_ENABLE_HIP)
   add_compile_definitions(${TTG_HIP_PLATFORM})
 endif(TTG_ENABLE_HIP)
 
+if (TTG_ENABLE_LEVEL_ZERO)
+  find_package(level-zero)
+  set(TTG_HAVE_LEVEL_ZERO ${LEVEL_ZERO_FOUND} CACHE BOOL "True if TTG provide support for Intel Level Zero")
+  if(TTG_HAVE_LEVEL_ZERO)
+    include_directories("${LEVEL_ZERO_INCLUDE_DIR}/level_zero/")
+    find_package(DPCPP)
+    if(DPCPP_EXECUTABLE)
+      set(TTG_HAVE_DPCPP TRUE CACHE BOOL "True if TTG knows how to compile DPCPP code")
+      message(STATUS "Found Intel level-zero ${LEVEL_ZERO_VERSION} in -I${LEVEL_ZERO_INCLUDE_DIR} / -L${LEVEL_ZERO_LIBRARY_DIR}")
+      message(STATUS "Found dpcpp in ${DPCPP_EXECUTABLE}")
+    else(DPCPP_EXECUTABLE)
+      set(TTG_HAVE_DPCPP FALSE CACHE BOOL "True if TTG knows how to compile DPCPP code")
+    endif(DPCPP_EXECUTABLE)
+  endif(TTG_HAVE_LEVEL_ZERO)
+endif(TTG_ENABLE_LEVEL_ZERO)
+
 set(_ttg_have_device FALSE)
-if (TTG_HAVE_CUDA OR TTG_HAVE_HIP)
+if (TTG_HAVE_CUDA OR TTG_HAVE_HIP OR TTG_HAVE_LEVEL_ZERO)
     set(_ttg_have_device TRUE)
 endif()
 set(TTG_HAVE_DEVICE ${_ttg_have_device} CACHE BOOL "True if TTG has support for any device programming model")
diff --git a/cmake/modules/FindDPCPP.cmake b/cmake/modules/FindDPCPP.cmake
new file mode 100644
index 000000000..42ebe2135
--- /dev/null
+++ b/cmake/modules/FindDPCPP.cmake
@@ -0,0 +1,43 @@
+if(DPCPP_EXECUTABLE)
+    get_filename_component( _dpcpp_path ${DPCPP_EXECUTABLE} PATH )
+else(DPCPP_EXECUTABLE)
+    set(_dpcpp_path "")
+endif(DPCPP_EXECUTABLE)
+find_program( DPCPP_EXECUTABLE NAMES dpcpp HINTS ${_dpcpp_path} ENV PATH )
+mark_as_advanced( DPCPP_EXECUTABLE )
+
+if( DPCPP_EXECUTABLE )
+    execute_process(COMMAND ${DPCPP_EXECUTABLE} --version
+            RESULT_VARIABLE _res
+            OUTPUT_VARIABLE _out
+            ERROR_VARIABLE _err)
+    if( _res EQUAL 0 )
+        string(REGEX MATCH "([^\n]+)" _ ${_out})
+        message(STATUS "Found dpcpp: ${DPCPP_EXECUTABLE} version ${CMAKE_MATCH_1}")
+
+        get_filename_component( _dpcpp_dir ${DPCPP_EXECUTABLE} DIRECTORY )
+        get_filename_component( _dpcpp_lib_dir "${_dpcpp_dir}/../lib" ABSOLUTE )
+
+        find_library(SYCL_LIBRARY sycl HINTS "${_dpcpp_lib_dir}" "${SYCL_LIBRARY_DIR}" ENV "SYCL_LIBRARY_DIR")
+        if( SYCL_LIBRARY )
+            message(STATUS "SYCL library: ${SYCL_LIBRARY}")
+
+            get_filename_component( _dpcpp_inc_dir "${_dpcpp_dir}/../include" ABSOLUTE )
+            find_file(_sycl_include_file "sycl.hpp" HINTS "${_dpcpp_inc_dir}" "${_dpcpp_inc_dir}/sycl/CL" "${SYCL_INCLUDE_DIR}" ENV "SYCL_INCLUDE_DIR")
+            if(_sycl_include_file)
+                get_filename_component(SYCL_INCLUDE_DIR "${_sycl_include_file}" DIRECTORY)
+                message(STATUS "SYCL include dir: ${SYCL_INCLUDE_DIR}")
+            else(_sycl_include_file)
+                message(WARNING "Found DPC++, and SYCL library, but could not find SYCL include directory. Define SYCL_INCLUDE_DIR to enable DPC++ support")
+		set(DPCPP_EXECUTABLE OFF)
+            endif(_sycl_include_file)
+        else( SYCL_LIBRARY )
+            message(WARNING "Found DPC++, but did not find SYCL library. Define SYCL_LIBRARY_DIR to enable DPC++ support")
+	    set(DPCPP_EXECUTABLE OFF)
+        endif( SYCL_LIBRARY )
+    else( _res EQUAL 0 )
+	set(DPCPP_EXECUTABLE OFF)
+        message(WARNING "${DPCPP_EXECUTABLE} does not work: 'dpcpp -v' returned ${_res}, with error ${_err}")
+        message(WARNING "dpc++ support is disabled, set DPCPP_EXECUTABLE to the path of dpcpp to enable DPC++ support, and/or fix your environment to run dpcpp")
+    endif( _res EQUAL 0 )
+endif( DPCPP_EXECUTABLE )
diff --git a/cmake/modules/Findlevel-zero.cmake b/cmake/modules/Findlevel-zero.cmake
new file mode 100644
index 000000000..0a13a32fe
--- /dev/null
+++ b/cmake/modules/Findlevel-zero.cmake
@@ -0,0 +1,81 @@
+if(LEVEL_ZERO_FOUND)
+    if(TARGET level_zero::ze_loader)
+        message(STATUS "level-zero is found and TARGET level_zero::ze_loader is defined")
+        return()
+    endif(TARGET level_zero::ze_loader)
+
+    # If the user defines LEVEL_ZERO_INCLUDE_DIR and LEVEL_ZERO_LIBRARY_DIR, CMake's find_package declares that LEVEL_ZERO_FOUND is 1, but does not define the target.
+    check_library_exists("ze_loader" "zeInit" "${LEVEL_ZERO_LIBRARY_DIR}" LEVEL_ZERO_HAVE_ZE_LOADER)
+    check_include_file("${LEVEL_ZERO_INCLUDE_DIR}/level_zero/ze_api.h" LEVEL_ZERO_HAVE_ZE_API_H)
+    if(LEVEL_ZERO_HAVE_ZE_LOADER AND LEVEL_ZERO_HAVE_ZE_API_H)
+        message(STATUS "Defining level_zero::ze_loader target with interface ${LEVEL_ZERO_INCLUDE_DIR}/level_zero/ze_api.h and library ${LEVEL_ZERO_LIBRARY_DIR}/libze_loader.so")
+        add_library(level_zero::ze_loader UNKNOWN IMPORTED GLOBAL)
+        set_property(TARGET level_zero::ze_loader PROPERTY IMPORTED_LOCATION "${LEVEL_ZERO_LIBRARY_DIR}/libze_loader.so")
+        set_property(TARGET level_zero::ze_loader PROPERTY INTERFACE "${LEVEL_ZERO_INCLUDE_DIR}/level_zero/ze_api.h")
+        include_directories("${LEVEL_ZERO_INCLUDE_DIR}/") 
+        return()
+    else(LEVEL_ZERO_HAVE_ZE_LOADER AND LEVEL_ZERO_HAVE_ZE_API_H)
+        if(NOT LEVEL_ZERO_HAVE_ZE_LOADER)
+            message(STATUS "LEVEL_ZERO_FOUND is set, but could not find ze_loader library in ${LEVEL_ZERO_LIBRARY_DIR}")
+        endif(NOT LEVEL_ZERO_HAVE_ZE_LOADER)
+        if(NOT LEVEL_ZERO_HAVE_ZE_API_H)
+            message(STATUS "LEVEL_ZERO_FOUND is set, but could not find level_zero/ze_api.h in ${LEVEL_ZERO_INCLUDE_DIR}")
+        endif(NOT LEVEL_ZERO_HAVE_ZE_API_H)
+    endif(LEVEL_ZERO_HAVE_ZE_LOADER AND LEVEL_ZERO_HAVE_ZE_API_H)
+endif(LEVEL_ZERO_FOUND)
+
+if(LEVEL_ZERO_ROOT_DIR)
+    message(STATUS "Trying to locate level-zero library and headers under ${LEVEL_ZERO_ROOT_DIR}")
+    find_library(ZE_LOADER_LIBRARY "ze_loader" HINTS "${LEVEL_ZERO_ROOT_DIR}/lib" "${LEVEL_ZERO_ROOT_DIR}/lib64" NO_DEFAULT_PATH)
+    find_path (LEVEL_ZERO_INCLUDE_DIR NAMES "level_zero/ze_api.h" PATHS "${LEVEL_ZERO_ROOT_DIR}/include" NO_DEFAULT_PATH)
+
+    if(ZE_LOADER_LIBRARY AND LEVEL_ZERO_INCLUDE_DIR)
+	    get_filename_component(ZE_LOADER_LIBRARY_DIR ${ZE_LOADER_LIBRARY} DIRECTORY)
+	    include(CheckLibraryExists)
+	    check_library_exists("ze_loader" "zeInit" ${ZE_LOADER_LIBRARY_DIR} ZE_LOADER_HAVE_ZEINIT)
+
+        if(ZE_LOADER_HAVE_ZEINIT)
+            message(STATUS "Found ze_loader library in ${ZE_LOADER_LIBRARY} and level_zero/ze_api.h in ${LEVEL_ZERO_INCLUDE_DIR}")
+            add_library(level_zero::ze_loader UNKNOWN IMPORTED GLOBAL)
+            set_property(TARGET level_zero::ze_loader PROPERTY IMPORTED_LOCATION "${ZE_LOADER_LIBRARY}")
+            set_property(TARGET level_zero::ze_loader PROPERTY INTERFACE "${LEVEL_ZERO_INCLUDE_DIR}/level_zero/ze_api.h")
+            include_directories("${LEVEL_ZERO_INCLUDE_DIR}/") 
+            set(LEVEL_ZERO_FOUND TRUE)
+        else(ZE_LOADER_HAVE_ZEINIT)
+            if(NOT ZE_LOADER_HAVE_ZEINIT)
+                message(WARNING "Found ze_loader library under ${ZE_LOADER_LIBRARY}, but could not find symbol zeInit in this library -- falling back to package config search")
+            endif(NOT ZE_LOADER_HAVE_ZEINIT)
+        endif(ZE_LOADER_HAVE_ZEINIT)
+    else(ZE_LOADER_LIBRARY AND LEVEL_ZERO_INCLUDE_DIR)
+	    if(NOT ZE_LOADER_LIBRARY)
+		    message(WARNING "Could not find ze_loader library under provided LEVEL_ZERO_ROOT_DIR='${LEVEL_ZERO_ROOT_DIR}' (tried subdirectories lib/ and lib64/) -- falling back to package config search")
+	    endif(NOT ZE_LOADER_LIBRARY)
+	    if(NOT LEVEL_ZERO_INCLUDE_DIR)
+		    message(WARNING "Cound not find level_zero/ze_api.h under provided LEVEL_ZERO_ROOT_DIR=${LEVEL_ZERO_ROOT_DIR}' (tried subdirectory include) -- falling back to package config search")
+	    endif(NOT LEVEL_ZERO_INCLUDE_DIR)
+    endif(ZE_LOADER_LIBRARY AND LEVEL_ZERO_INCLUDE_DIR)
+endif(LEVEL_ZERO_ROOT_DIR)
+
+if(NOT LEVEL_ZERO_FOUND)
+    find_package(PkgConfig QUIET)
+
+    if(PKG_CONFIG_FOUND)
+        pkg_check_modules(LEVEL_ZERO level-zero)
+        if(LEVEL_ZERO_FOUND)
+            pkg_get_variable(LEVEL_ZERO_LIBRARY_DIR level-zero libdir)
+            pkg_get_variable(LEVEL_ZERO_INCLUDE_DIR level-zero includedir)
+            #We double-check that the level-zero library is indeed where find_package claims it is
+            find_library(_PARSEC_ZE_LOADER_LIBRARY_FOUND ze_loader PATHS "${LEVEL_ZERO_LIBRARY_DIR}/libze_loader.so" 
+                         NO_DEFAULT_PATH NO_PACKAGE_ROOT_PATH NO_CMAKE_PATH NO_CMAKE_ENVIRONMENT_PATH NO_SYSTEM_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH NO_CMAKE_INSTALL_PREFIX)
+            if( _PARSEC_ZE_LOADER_LIBRARY_FOUND )
+                add_library(level_zero::ze_loader UNKNOWN IMPORTED GLOBAL)
+                set_property(TARGET level_zero::ze_loader PROPERTY IMPORTED_LOCATION "${LEVEL_ZERO_LIBRARY_DIR}/libze_loader.so")
+                set_property(TARGET level_zero::ze_loader PROPERTY INTERFACE "${LEVEL_ZERO_INCLUDE_DIR}/level_zero/ze_api.h")
+                include_directories("${LEVEL_ZERO_INCLUDE_DIR}/")
+            else()
+                message(WARNING "level-zero was found in `${LEVEL_ZERO_LIBRARY_DIR}` and `${LEVEL_ZERO_INCLUDE_DIR}` according to pkg-config, but ze_loader library is not in `${LEVEL_ZERO_LIBRARY_DIR}/libze_loader.so`. Deactivating level-zero. Provide -DLEVEL_ZERO_ROOT_DIR to specify the location of level zero manually.")
+                set(LEVEL_ZERO_FOUND OFF CACHE BOOL "if level-zero was found" FORCE)
+            endif()
+        endif(LEVEL_ZERO_FOUND)
+    endif(PKG_CONFIG_FOUND)
+endif(NOT LEVEL_ZERO_FOUND)
diff --git a/ttg/CMakeLists.txt b/ttg/CMakeLists.txt
index 427a3ffee..a2e1471ab 100644
--- a/ttg/CMakeLists.txt
+++ b/ttg/CMakeLists.txt
@@ -123,6 +123,10 @@ if (TTG_HAVE_HIPBLAS)
   list(APPEND ttg-deps hip::host)
 endif (TTG_HAVE_HIPBLAS)
 
+if (TTG_HAVE_LEVEL_ZERO)
+  list(APPEND ttg-deps level_zero::ze_loader)
+endif (TTG_HAVE_LEVEL_ZERO)
+
 set(ttg-public-headers ${ttg-headers};${ttg-impl-headers};${ttg-base-headers};${ttg-util-headers};${ttg_device_headers})
 if (NOT TTG_IGNORE_BUNDLED_EXTERNALS)
   list(APPEND ttg-sources ${ttg-external-headers})

From 6af1828faf22f58fdaf7005c8a49264ae529824b Mon Sep 17 00:00:00 2001
From: Thomas Herault <herault@icl.utk.edu>
Date: Thu, 16 Nov 2023 18:09:44 +0000
Subject: [PATCH 160/259] Missing declaration for level zero cmakery

---
 ttg/ttg/config.in.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ttg/ttg/config.in.h b/ttg/ttg/config.in.h
index ea9a20a63..f04d089de 100644
--- a/ttg/ttg/config.in.h
+++ b/ttg/ttg/config.in.h
@@ -17,6 +17,8 @@
 
 #cmakedefine TTG_HAVE_HIP
 
+#cmakedefine TTG_HAVE_LEVEL_ZERO
+
 #cmakedefine TTG_HAVE_HIPBLAS
 
 #cmakedefine TTG_HAVE_DEVICE

From 0be12126d60e738305fbfe6d44ebbeecdee12a0d Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 16 Nov 2023 18:16:54 +0000
Subject: [PATCH 161/259] Agnostic async copy and non-device support

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/device/device.h     | 17 +++++++++++++++++
 ttg/ttg/parsec/devicefunc.h | 21 ++++++---------------
 ttg/ttg/parsec/task.h       |  3 ++-
 ttg/ttg/parsec/ttg.h        |  3 +++
 4 files changed, 28 insertions(+), 16 deletions(-)

diff --git a/ttg/ttg/device/device.h b/ttg/ttg/device/device.h
index 95e1e0fc8..fad3226a6 100644
--- a/ttg/ttg/device/device.h
+++ b/ttg/ttg/device/device.h
@@ -13,6 +13,8 @@ namespace ttg::device {
   constexpr ttg::ExecutionSpace available_execution_space = ttg::ExecutionSpace::HIP;
 #elif defined(TTG_HAVE_LEVEL_ZERO)
   constexpr ttg::ExecutionSpace available_execution_space = ttg::ExecutionSpace::L0;
+#else
+  constexpr ttg::ExecutionSpace available_execution_space = ttg::ExecutionSpace::Invalid;
 #endif
   class Device {
     int m_id = 0;
@@ -159,5 +161,20 @@ namespace ttg::device {
   }
 } // namespace ttg
 
+#else
 
+namespace ttg::device {
+  inline
+  Device current_device() {
+    return {};
+  }
+
+  template<ttg::ExecutionSpace Space = ttg::ExecutionSpace::Invalid>
+  inline
+  const void* current_stream() {
+    static_assert(ttg::ExecutionSpace Space != ttg::ExecutionSpace::Invalid,
+                  "TTG was built without any known device support so we cannot provide a current stream!");
+    return nullptr;
+  }
+} // namespace ttg
 #endif // defined(TTG_HAVE_HIP)
diff --git a/ttg/ttg/parsec/devicefunc.h b/ttg/ttg/parsec/devicefunc.h
index 325750e04..c0710776d 100644
--- a/ttg/ttg/parsec/devicefunc.h
+++ b/ttg/ttg/parsec/devicefunc.h
@@ -116,22 +116,13 @@ namespace ttg_parsec {
       parsec_data_t* data = detail::get_parsec_data(view);
       parsec_gpu_task_t *gpu_task = detail::parsec_ttg_caller->dev_ptr->gpu_task;
       parsec_gpu_exec_stream_t *stream = detail::parsec_ttg_caller->dev_ptr->stream;
-      /* enqueue the transfer into the compute stream to come back once the compute and transfer are complete */
 
-#if defined(TTG_HAVE_CUDART) && defined(PARSEC_HAVE_DEV_CUDA_SUPPORT)
-      //std::cout << "cudaMemcpyAsync of " << data->nb_elts << "B from " << data->device_copies[data->owner_device]->device_private << " device " << (int)data->owner_device << " to " << data->device_copies[0]->device_private << std::endl;
-      parsec_cuda_exec_stream_t *cuda_stream = (parsec_cuda_exec_stream_t *)stream;
-      cudaMemcpyAsync(data->device_copies[0]->device_private,
-                      data->device_copies[data->owner_device]->device_private,
-                      data->nb_elts, cudaMemcpyDeviceToHost, cuda_stream->cuda_stream);
-#elif defined(TTG_HAVE_HIP) && defined(PARSEC_HAVE_DEV_HIP_SUPPORT)
-      parsec_hip_exec_stream_t *hip_stream = (parsec_hip_exec_stream_t *)stream;
-      hipMemcpyAsync(data->device_copies[0]->device_private,
-                      data->device_copies[data->owner_device]->device_private,
-                      data->nb_elts, hipMemcpyDeviceToHost, hip_stream->hip_stream);
-#else
-      static_assert(DeviceAvail, "No device implementation detected!");
-#endif // defined(PARSEC_HAVE_DEV_CUDA_SUPPORT)
+      /* enqueue the transfer into the compute stream to come back once the compute and transfer are complete */
+      parsec_device_gpu_module_t *device_module = detail::parsec_ttg_caller->dev_ptr->device;
+      device_module->memcpy_async(device_module, stream,
+                                  data->device_copies[0]->device_private,
+                                  data->device_copies[data->owner_device]->device_private,
+                                  data->nb_elts, parsec_device_gpu_transfer_direction_d2h);
 
       if constexpr (sizeof...(Is) > 0) {
         // recursion
diff --git a/ttg/ttg/parsec/task.h b/ttg/ttg/parsec/task.h
index 917b63c9b..7a8ed524a 100644
--- a/ttg/ttg/parsec/task.h
+++ b/ttg/ttg/parsec/task.h
@@ -14,6 +14,7 @@ namespace ttg_parsec {
       parsec_gpu_task_t* gpu_task = nullptr;
       parsec_flow_t* flows = nullptr;
       parsec_gpu_exec_stream_t* stream = nullptr;
+      parsec_device_module_t* device = nullptr;
     };
 
     template<bool SupportDevice>
@@ -33,7 +34,7 @@ namespace ttg_parsec {
       static constexpr bool support_device = false;
       static constexpr size_t num_flows = MAX_PARAM_COUNT;
       parsec_flow_t m_flows[num_flows];
-      device_ptr_t m_dev_ptr = {nullptr, &m_flows[0], nullptr}; // gpu_task will be allocated in each task
+      device_ptr_t m_dev_ptr = {nullptr, &m_flows[0], nullptr, nullptr}; // gpu_task will be allocated in each task
       device_ptr_t* dev_ptr() {
         return &m_dev_ptr;
       }
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index cf0cd33b2..b74970835 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1493,6 +1493,9 @@ namespace ttg_parsec {
 
       parsec_device_module_t *device = parsec_mca_device_get(dev_index);
       assert(NULL != device);
+
+      task->dev_ptr->device = device;
+
       switch(device->type) {
 
 #if defined(PARSEC_HAVE_DEV_CUDA_SUPPORT)

From a1198283ed44bd96623ab2e621f86a83c073355c Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 16 Nov 2023 18:21:31 +0000
Subject: [PATCH 162/259] Add missing parsec backend header file to CMake

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ttg/CMakeLists.txt b/ttg/CMakeLists.txt
index a2e1471ab..307572f10 100644
--- a/ttg/CMakeLists.txt
+++ b/ttg/CMakeLists.txt
@@ -225,6 +225,7 @@ endif(TARGET MADworld)
 if (TARGET PaRSEC::parsec)
   set(ttg-parsec-headers
           ${CMAKE_CURRENT_SOURCE_DIR}/ttg/parsec/buffer.h
+          ${CMAKE_CURRENT_SOURCE_DIR}/ttg/parsec/device.h
           ${CMAKE_CURRENT_SOURCE_DIR}/ttg/parsec/devicefunc.h
           ${CMAKE_CURRENT_SOURCE_DIR}/ttg/parsec/devicescratch.h
           ${CMAKE_CURRENT_SOURCE_DIR}/ttg/parsec/fwd.h

From 4f125ad1856470afd2df4d6c2491c88f93784810 Mon Sep 17 00:00:00 2001
From: Thomas Herault <herault@icl.utk.edu>
Date: Thu, 16 Nov 2023 18:29:15 +0000
Subject: [PATCH 163/259] Add MKL dependency to spmm-lz

---
 CMakeLists.txt          | 2 ++
 examples/CMakeLists.txt | 5 +++++
 2 files changed, 7 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index cfc449528..684d3a58c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -145,6 +145,8 @@ if (TTG_ENABLE_LEVEL_ZERO)
       set(TTG_HAVE_DPCPP TRUE CACHE BOOL "True if TTG knows how to compile DPCPP code")
       message(STATUS "Found Intel level-zero ${LEVEL_ZERO_VERSION} in -I${LEVEL_ZERO_INCLUDE_DIR} / -L${LEVEL_ZERO_LIBRARY_DIR}")
       message(STATUS "Found dpcpp in ${DPCPP_EXECUTABLE}")
+
+      find_package(MKL)
     else(DPCPP_EXECUTABLE)
       set(TTG_HAVE_DPCPP FALSE CACHE BOOL "True if TTG knows how to compile DPCPP code")
     endif(DPCPP_EXECUTABLE)
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 647cdffaa..14b041ce9 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -40,6 +40,11 @@ if (TARGET tiledarray)
                            COMPILE_DEFINITIONS TTG_ENABLE_HIP=1;DEBUG_TILES_VALUES=1
                            RUNTIMES "parsec")
       endif(TARGET roc::hipsolver)
+    elseif (TARGET MKL::MKL)
+      add_ttg_executable(bspmm-lz spmm/spmm_cuda.cc
+	                  LINK_LIBRARIES tiledarray TiledArray_Eigen BTAS Boost::boost MKL::MKL
+                          COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2
+                          RUNTIMES "parsec")
     endif()
 
     add_ttg_executable(chain-ttg-dev task-benchmarks/chain-ttg-dev.cc LINK_LIBRARIES tiledarray RUNTIMES "parsec")

From 03b54bb19c3f003d708277b5f0f85d1899a06f2a Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 16 Nov 2023 18:39:25 +0000
Subject: [PATCH 164/259] Work on device_gpu_module instead of device_module

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/task.h | 2 +-
 ttg/ttg/parsec/ttg.h  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ttg/ttg/parsec/task.h b/ttg/ttg/parsec/task.h
index 7a8ed524a..12b7587c9 100644
--- a/ttg/ttg/parsec/task.h
+++ b/ttg/ttg/parsec/task.h
@@ -14,7 +14,7 @@ namespace ttg_parsec {
       parsec_gpu_task_t* gpu_task = nullptr;
       parsec_flow_t* flows = nullptr;
       parsec_gpu_exec_stream_t* stream = nullptr;
-      parsec_device_module_t* device = nullptr;
+      parsec_device_gpu_module_t* device = nullptr;
     };
 
     template<bool SupportDevice>
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index b74970835..b198d8729 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1491,7 +1491,7 @@ namespace ttg_parsec {
           return PARSEC_HOOK_RETURN_NEXT; /* Fall back */
       }
 
-      parsec_device_module_t *device = parsec_mca_device_get(dev_index);
+      parsec_device_gpu_module_t *device = (parsec_device_gpu_module_t*)parsec_mca_device_get(dev_index);
       assert(NULL != device);
 
       task->dev_ptr->device = device;

From dde416cce92fe63c9c592d9fb5afa7355019ae06 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 16 Nov 2023 18:41:12 +0000
Subject: [PATCH 165/259] Get sycl queue in spmm

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/spmm/spmm_cuda.cc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/examples/spmm/spmm_cuda.cc b/examples/spmm/spmm_cuda.cc
index 2c0dbffc5..0c6b8fa23 100644
--- a/examples/spmm/spmm_cuda.cc
+++ b/examples/spmm/spmm_cuda.cc
@@ -317,7 +317,7 @@ static void device_gemm(Blk &C, const Blk &A, const Blk &B) {
   try {
 #endif /* DEBUG_SYNCHRONOUS */
     cl::sycl::event gemm_event;
-    gemm_event = oneapi::mkl::blas::gemm(lz_queue(),
+    gemm_event = oneapi::mkl::blas::gemm(ttg::device::current_stream(),
          oneapi::mkl::transpose::N, oneapi::mkl::transpose::N,
          C.extent(0), C.extent(1), A.extent(1),
          alpha, A.b.current_device_ptr(), A.extent(0),
@@ -1852,13 +1852,13 @@ int main(int argc, char **argv) {
         std::string Mstr(getCmdOption(argv, argv + argc, "-M"));
         M = parseOption(Mstr, 1200);
         std::string Nstr(getCmdOption(argv, argv + argc, "-N"));
-        N = parseOption(Nstr, 1200);
+        N = parseOption(Nstr, M);
         std::string Kstr(getCmdOption(argv, argv + argc, "-K"));
-        K = parseOption(Kstr, 1200);
+        K = parseOption(Kstr, N);
         std::string minTsStr(getCmdOption(argv, argv + argc, "-t"));
-        minTs = parseOption(minTsStr, 32);
+        minTs = parseOption(minTsStr, 64);
         std::string maxTsStr(getCmdOption(argv, argv + argc, "-T"));
-        maxTs = parseOption(maxTsStr, 256);
+        maxTs = parseOption(maxTsStr, minTs);
         std::string avgStr(getCmdOption(argv, argv + argc, "-a"));
         double avg = parseOption(avgStr, 0.3);
         timing = (check == 0);

From 5569b57207e049ada495aa3e5c889969a16ef655 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 16 Nov 2023 18:54:50 +0000
Subject: [PATCH 166/259] Add parsec level-zero include

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/ttg.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index b198d8729..9be7906c7 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -92,6 +92,9 @@
 #ifdef PARSEC_HAVE_DEV_HIP_SUPPORT
 #include <parsec/mca/device/hip/device_hip.h>
 #endif // PARSEC_HAVE_DEV_HIP_SUPPORT
+#ifdef PARSEC_HAVE_DEV_LEVEL_ZERO_SUPPORT
+#include <parsec/mca/device/level_zero/device_level_zero.h>
+#endif //PARSEC_HAVE_DEV_LEVEL_ZERO_SUPPORT
 
 #include <parsec/mca/device/device_gpu.h>
 #if defined(PARSEC_PROF_TRACE)

From 86b92db7e8da4b6d2d40746e3dcf6d83c23a63ad Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 16 Nov 2023 19:05:17 +0000
Subject: [PATCH 167/259] Return sycl queue as non-const

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/device/device.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ttg/ttg/device/device.h b/ttg/ttg/device/device.h
index fad3226a6..1102ef185 100644
--- a/ttg/ttg/device/device.h
+++ b/ttg/ttg/device/device.h
@@ -156,7 +156,7 @@ namespace ttg::device {
   }
 
   inline
-  const sycl::queue& current_stream() {
+  sycl::queue& current_stream() {
     return *detail::current_stream_ts;
   }
 } // namespace ttg

From 5aac5c23ebd5b302dd68a2ebb4aaa463916d6390 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 16 Nov 2023 19:05:44 +0000
Subject: [PATCH 168/259] Use the right parsec device module types

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/ttg.h | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 9be7906c7..ac6e38d92 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1499,7 +1499,7 @@ namespace ttg_parsec {
 
       task->dev_ptr->device = device;
 
-      switch(device->type) {
+      switch(device->super.type) {
 
 #if defined(PARSEC_HAVE_DEV_CUDA_SUPPORT)
         case PARSEC_DEV_CUDA:
@@ -1508,7 +1508,7 @@ namespace ttg_parsec {
              *       task-class to determine the number of flows. */
             gpu_task->stage_in  = static_device_stage_in_hook;
             gpu_task->stage_out = parsec_default_gpu_stage_out;
-            return parsec_device_kernel_scheduler(device, es, gpu_task);
+            return parsec_device_kernel_scheduler(&device->super, es, gpu_task);
           }
           break;
 #endif
@@ -1517,7 +1517,7 @@ namespace ttg_parsec {
           if constexpr (Space == ttg::ExecutionSpace::HIP) {
             gpu_task->stage_in  = static_device_stage_in_hook;
             gpu_task->stage_out = parsec_default_gpu_stage_out;
-            return parsec_device_kernel_scheduler(device, es, gpu_task);
+            return parsec_device_kernel_scheduler(&device->super, es, gpu_task);
           }
           break;
 #endif // PARSEC_HAVE_DEV_HIP_SUPPORT
@@ -1526,14 +1526,14 @@ namespace ttg_parsec {
           if constexpr (Space == ttg::ExecutionSpace::L0) {
             gpu_task->stage_in  = static_device_stage_in_hook;
             gpu_task->stage_out = parsec_default_gpu_stage_out;
-            return parsec_device_kernel_scheduler(device, es, gpu_task);
+            return parsec_device_kernel_scheduler(&device->super, es, gpu_task);
           }
           break;
 #endif // PARSEC_HAVE_DEV_LEVEL_ZERO_SUPPORT
         default:
           break;
       }
-      ttg::print_error(task->tt->get_name(), " : received mismatching device type ", (int)device->type, " from PaRSEC");
+      ttg::print_error(task->tt->get_name(), " : received mismatching device type ", (int)device->super.type, " from PaRSEC");
       ttg::abort();
       return PARSEC_HOOK_RETURN_DONE; // will not be reacehed
     }

From afeda4622c6f0b5e026b2e13d37d4f52e45a85ef Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 16 Nov 2023 19:09:35 +0000
Subject: [PATCH 169/259] Add inline in device.h

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/device/device.h | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/ttg/ttg/device/device.h b/ttg/ttg/device/device.h
index 1102ef185..0ff4e64cf 100644
--- a/ttg/ttg/device/device.h
+++ b/ttg/ttg/device/device.h
@@ -60,6 +60,7 @@ namespace ttg::device {
 } // namespace ttg::device
 
 namespace std {
+  inline
   std::ostream& operator<<(std::ostream& os, ttg::device::Device device) {
     os << ttg::detail::execution_space_name(device.space());
     if (device.is_device()) {
@@ -77,12 +78,12 @@ namespace ttg::device {
     inline thread_local ttg::device::Device current_device_ts = {};
     inline thread_local cudaStream_t current_stream_ts = 0; // default stream
 
-    void reset_current() {
+    inline void reset_current() {
       current_device_ts = {};
       current_stream_ts = 0;
     }
 
-    void set_current(int device, cudaStream_t stream) {
+    inline void set_current(int device, cudaStream_t stream) {
       current_device_ts = ttg::device::Device(device, ttg::ExecutionSpace::CUDA);
       current_stream_ts = stream;
     }
@@ -108,12 +109,12 @@ namespace ttg::device {
     inline thread_local ttg::device::Device current_device_ts = {};
     inline thread_local hipStream_t current_stream_ts = 0; // default stream
 
-    void reset_current() {
+    inline void reset_current() {
       current_device_ts = {};
       current_stream_ts = 0;
     }
 
-    void set_current(int device, hipStream_t stream) {
+    inline void set_current(int device, hipStream_t stream) {
       current_device_ts = ttg::device::Device(device, ttg::ExecutionSpace::HIP);
       current_stream_ts = stream;
     }
@@ -139,12 +140,13 @@ namespace ttg::device {
     inline thread_local ttg::device::Device current_device_ts = {};
     inline thread_local sycl::queue* current_stream_ts = nullptr; // default stream
 
-    void reset_current() {
+
+    inline void reset_current() {
       current_device_ts = {};
       current_stream_ts = nullptr;
     }
 
-    void set_current(int device, sycl::queue& stream) {
+    inline void set_current(int device, sycl::queue& stream) {
       current_device_ts = ttg::device::Device(device, ttg::ExecutionSpace::HIP);
       current_stream_ts = &stream;
     }
@@ -164,14 +166,12 @@ namespace ttg::device {
 #else
 
 namespace ttg::device {
-  inline
-  Device current_device() {
+  inline Device current_device() {
     return {};
   }
 
   template<ttg::ExecutionSpace Space = ttg::ExecutionSpace::Invalid>
-  inline
-  const void* current_stream() {
+  inline const void* current_stream() {
     static_assert(ttg::ExecutionSpace Space != ttg::ExecutionSpace::Invalid,
                   "TTG was built without any known device support so we cannot provide a current stream!");
     return nullptr;

From d1fb80fb9a0e7cc5c43e63167c42c63e4c84a705 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 16 Nov 2023 19:17:10 +0000
Subject: [PATCH 170/259] PaRSEC: Add have_level_zero_op()

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/make_tt.h    |  1 +
 ttg/ttg/parsec/ttg.h | 32 +++++++++++++++++++++++++++++++-
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/ttg/ttg/make_tt.h b/ttg/ttg/make_tt.h
index c2d8f2c72..e4d5d8039 100644
--- a/ttg/ttg/make_tt.h
+++ b/ttg/ttg/make_tt.h
@@ -167,6 +167,7 @@ class CallableWrapTTArgs
 public:
   static constexpr bool have_cuda_op = (space == ttg::ExecutionSpace::CUDA);
   static constexpr bool have_hip_op  = (space == ttg::ExecutionSpace::HIP);
+  static constexpr bool have_level_zero_op = (space == ttg::ExecutionSpace::L0);
 
 protected:
 
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index ac6e38d92..03186a17e 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -760,6 +760,16 @@ namespace ttg_parsec {
       }
     }
 
+    template<typename TT>
+    inline parsec_hook_return_t hook_level_zero(struct parsec_execution_stream_s *es, parsec_task_t *parsec_task) {
+      if constexpr(TT::derived_has_level_zero_op()) {
+        parsec_ttg_task_t<TT> *me = (parsec_ttg_task_t<TT> *)parsec_task;
+        return me->template invoke_op<ttg::ExecutionSpace::L0>();
+      } else {
+        throw std::runtime_error("PaRSEC HIP hook invoked on a TT that does not support HIP operations!");
+      }
+    }
+
     template <typename KeyT, typename ActivationCallbackT>
     class rma_delayed_activate {
       std::vector<KeyT> _keylist;
@@ -1129,9 +1139,18 @@ namespace ttg_parsec {
       }
     }
 
+    /// @return true if derivedT::have_hip_op exists and is defined to true
+    static constexpr bool derived_has_level_zero_op() {
+      if constexpr (ttg::meta::is_detected_v<have_level_zero_op_non_type_t, derivedT>) {
+        return derivedT::have_level_zero_op;
+      } else {
+        return false;
+      }
+    }
+
     /// @return true if the TT supports device execution
     static constexpr bool derived_has_device_op() {
-      return (derived_has_cuda_op() || derived_has_hip_op());
+      return (derived_has_cuda_op() || derived_has_hip_op() || derived_has_level_zero_op());
     }
 
     using ttT = TT;
@@ -3239,6 +3258,8 @@ ttg::abort();  // should not happen
           device_supported = !world.impl().mpi_support(ttg::ExecutionSpace::CUDA);
         } else if constexpr (derived_has_hip_op()) {
           device_supported = !world.impl().mpi_support(ttg::ExecutionSpace::HIP);
+        } else if constexpr (derived_has_level_zero_op()) {
+          device_supported = !world.impl().mpi_support(ttg::ExecutionSpace::L0);
         }
         /* if MPI supports the device we don't care whether we have remote peers
          * because we can send from the device directly */
@@ -3642,6 +3663,15 @@ ttg::abort();  // should not happen
         ((__parsec_chore_t *)self.incarnations)[0].evaluate = NULL;
         ((__parsec_chore_t *)self.incarnations)[0].hook = &detail::hook_hip<TT>;
 
+        ((__parsec_chore_t *)self.incarnations)[1].type = PARSEC_DEV_NONE;
+        ((__parsec_chore_t *)self.incarnations)[1].evaluate = NULL;
+        ((__parsec_chore_t *)self.incarnations)[1].hook = NULL;
+      } else if (derived_has_level_zero_op()) {
+        self.incarnations = (__parsec_chore_t *)malloc(3 * sizeof(__parsec_chore_t));
+        ((__parsec_chore_t *)self.incarnations)[0].type = PARSEC_DEV_LEVEL_ZERO;
+        ((__parsec_chore_t *)self.incarnations)[0].evaluate = NULL;
+        ((__parsec_chore_t *)self.incarnations)[0].hook = &detail::hook_level_zero<TT>;
+
         ((__parsec_chore_t *)self.incarnations)[1].type = PARSEC_DEV_NONE;
         ((__parsec_chore_t *)self.incarnations)[1].evaluate = NULL;
         ((__parsec_chore_t *)self.incarnations)[1].hook = NULL;

From bfa3fde6a294e005df4d43d649ca31d993b796a0 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 16 Nov 2023 19:20:54 +0000
Subject: [PATCH 171/259] Add missing have_level_zero_op_non_type_t

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/ttg.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 03186a17e..58e73671e 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1113,6 +1113,9 @@ namespace ttg_parsec {
     template <typename T>
     using have_hip_op_non_type_t = decltype(T::have_hip_op);
 
+    template <typename T>
+    using have_level_zero_op_non_type_t = decltype(T::have_hip_op);
+
     bool alive = true;
 
     static constexpr int numinedges = std::tuple_size_v<input_tuple_type>;     // number of input edges

From 68e482547636c4cf34fbe8b75f08c86de38997a3 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 16 Nov 2023 19:31:17 +0000
Subject: [PATCH 172/259] Fix bug in level zero detection of derived TT

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/ttg.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 58e73671e..49f03837b 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1114,7 +1114,7 @@ namespace ttg_parsec {
     using have_hip_op_non_type_t = decltype(T::have_hip_op);
 
     template <typename T>
-    using have_level_zero_op_non_type_t = decltype(T::have_hip_op);
+    using have_level_zero_op_non_type_t = decltype(T::have_level_zero_op);
 
     bool alive = true;
 

From 9afb6f8c282d1401ba31934b93c518b4d3f2e448 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 16 Nov 2023 23:37:26 +0000
Subject: [PATCH 173/259] Don't ever assume that 2 is the first device

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/ttg.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 49f03837b..241b723f5 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1512,7 +1512,7 @@ namespace ttg_parsec {
 
       gpu_task->load = task_load;
       assert(dev_index >= 0);
-      if (dev_index < 2) {
+      if (dev_index < detail::first_device_id) {
           return PARSEC_HOOK_RETURN_NEXT; /* Fall back */
       }
 
@@ -1997,8 +1997,8 @@ ttg::abort();  // should not happen
             // TODO: first attempt at sending directly to the device
             parsec_gpu_data_copy_t* gpu_elem;
             gpu_elem = PARSEC_DATA_GET_COPY(master, gpu_device->super.device_index);
-            int i = 2; // 0: cpu, 1: recursive -> start with 2 (first accelerator)
-            int devid = 2;
+            int i = detail::first_device_id;ß
+            int devid = detail::first_device_id;
             while (i < parsec_nb_devices) {
               if (nullptr == gpu_elem) {
                 gpu_elem = PARSEC_OBJ_NEW(parsec_data_copy_t);

From 961b0d3491afe8552c56d06f65e1b7af9165355e Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Mon, 20 Nov 2023 10:48:34 -0500
Subject: [PATCH 174/259] [cmake] cleanup FindOrFetchTiledArray.cmake

---
 cmake/modules/FindOrFetchTiledArray.cmake | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/cmake/modules/FindOrFetchTiledArray.cmake b/cmake/modules/FindOrFetchTiledArray.cmake
index 314ddff9f..27ead4479 100644
--- a/cmake/modules/FindOrFetchTiledArray.cmake
+++ b/cmake/modules/FindOrFetchTiledArray.cmake
@@ -28,13 +28,6 @@ else (TARGET tiledarray)
   if (NOT DEFINED TA_ASSUMES_ASLR_DISABLED)
     set(TA_ASSUMES_ASLR_DISABLED ${MPQC_ASSUMES_ASLR_DISABLED} CACHE BOOL "TA assumes the Address Space Layout Randomization (ASLR) to be disabled")
   endif(NOT DEFINED TA_ASSUMES_ASLR_DISABLED)
-  if (NOT DEFINED TA_ASSERT_POLICY)
-    string(REPLACE "MPQC_" "TA_" TA_ASSERT_POLICY "${MPQC_ASSERT_POLICY}")
-    set(TA_ASSERT_POLICY ${TA_ASSERT_POLICY} CACHE STRING "Controls the behavior of TA_ASSERT")
-  endif (NOT DEFINED TA_ASSERT_POLICY)
-  if (NOT DEFINED TA_BUILD_UNITTEST)
-    set(TA_BUILD_UNITTEST FALSE CACHE BOOL "Whether to build TA unit tests")
-  endif (NOT DEFINED TA_BUILD_UNITTEST)
 
   include(FetchContent)
   FetchContent_Declare(

From 78876e5dc5b76139fe52dbe970d3c509894121d7 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Mon, 20 Nov 2023 10:49:21 -0500
Subject: [PATCH 175/259] [cmake] bump TA tag to pull in
 https://github.com/ValeevGroup/tiledarray/pull/434

---
 cmake/modules/ExternalDependenciesVersions.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/modules/ExternalDependenciesVersions.cmake b/cmake/modules/ExternalDependenciesVersions.cmake
index 1b2538595..9e17ab80a 100644
--- a/cmake/modules/ExternalDependenciesVersions.cmake
+++ b/cmake/modules/ExternalDependenciesVersions.cmake
@@ -8,4 +8,4 @@ set(TTG_TRACKED_CEREAL_VERSION 1.3.0)
 set(TTG_TRACKED_MADNESS_TAG cb195817d7807c4aead10ba200cd20649036cbae)
 set(TTG_TRACKED_PARSEC_TAG 6a5a986d6ec1f9cace09f3ad475c553632077a24)
 set(TTG_TRACKED_BTAS_TAG a02be0d29fb4a788ecef43de711dcd6d6f1cb6b8)
-set(TTG_TRACKED_TILEDARRAY_TAG dfa3f76ebd58c05c64acf6e57cbdc85e90c880a8)
+set(TTG_TRACKED_TILEDARRAY_TAG f0115e9e4a3f988224afbfb3c241e92171e916b8)

From cf757ee2ab58030dff41f539b3cf93e7e37e611d Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 27 Nov 2023 15:18:08 +0000
Subject: [PATCH 176/259] Proper use of device ID conversion functions

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/potrf/potrf.h  | 32 ++++++++++++++++----------------
 ttg/ttg/parsec/buffer.h | 14 +++++++-------
 ttg/ttg/parsec/device.h |  6 +++---
 ttg/ttg/parsec/ttg.h    | 17 ++++++++++-------
 4 files changed, 36 insertions(+), 33 deletions(-)

diff --git a/examples/potrf/potrf.h b/examples/potrf/potrf.h
index 369bc9a8d..00452f78f 100644
--- a/examples/potrf/potrf.h
+++ b/examples/potrf/potrf.h
@@ -58,13 +58,13 @@ namespace potrf {
     //std::cout << "POTRF handle  " << handle << " device " << device << " stream " << ttg::device::current_stream() << std::endl;
     cusolverDnDpotrf(handle,
                       CUBLAS_FILL_MODE_LOWER, A.cols(),
-                      A.buffer().device_ptr_on(device), A.lda(),
+                      A.buffer().current_device_ptr(), A.lda(),
                       workspace, Lwork,
                       devInfo);
   #elif defined(TTG_HAVE_HIPBLAS)
     hipsolverDpotrf(hipsolver_handle(),
                       HIPSOLVER_FILL_MODE_LOWER, A.cols(),
-                      A.buffer().device_ptr_on(device), A.lda(),
+                      A.buffer().current_device_ptr(), A.lda(),
                       workspace, Lwork,
                       devInfo);
   #endif
@@ -290,15 +290,15 @@ namespace potrf {
                   CUBLAS_SIDE_RIGHT, CUBLAS_FILL_MODE_LOWER,
                   CUBLAS_OP_T, CUBLAS_DIAG_NON_UNIT,
                   mb, nb, &alpha,
-                  tile_kk.buffer().device_ptr_on(device), tile_kk.lda(),
-                  tile_mk.buffer().device_ptr_on(device), tile_mk.lda());
+                  tile_kk.buffer().current_device_ptr(), tile_kk.lda(),
+                  tile_mk.buffer().current_device_ptr(), tile_mk.lda());
 #elif defined(TTG_HAVE_HIPBLAS)
       hipblasDtrsm(hipblas_handle(),
                    HIPBLAS_SIDE_RIGHT, HIPBLAS_FILL_MODE_LOWER,
                    HIPBLAS_OP_T, HIPBLAS_DIAG_NON_UNIT,
                    mb, nb, &alpha,
-                   tile_kk.buffer().device_ptr_on(device), tile_kk.lda(),
-                   tile_mk.buffer().device_ptr_on(device), tile_mk.lda());
+                   tile_kk.buffer().current_device_ptr(), tile_kk.lda(),
+                   tile_mk.buffer().current_device_ptr(), tile_mk.lda());
 
 #endif
 
@@ -420,15 +420,15 @@ namespace potrf {
                   CUBLAS_FILL_MODE_LOWER,
                   CUBLAS_OP_N,
                   mb, nb, &alpha,
-                  tile_mk.buffer().device_ptr_on(device), tile_mk.lda(), &beta,
-                  tile_kk.buffer().device_ptr_on(device), tile_kk.lda());
+                  tile_mk.buffer().current_device_ptr(), tile_mk.lda(), &beta,
+                  tile_kk.buffer().current_device_ptr(), tile_kk.lda());
 #elif defined(TTG_HAVE_HIPBLAS)
       hipblasDsyrk(hipblas_handle(),
                    HIPBLAS_FILL_MODE_LOWER,
                    HIPBLAS_OP_N,
                    mb, nb, &alpha,
-                   tile_mk.buffer().device_ptr_on(device), tile_mk.lda(), &beta,
-                   tile_kk.buffer().device_ptr_on(device), tile_kk.lda());
+                   tile_mk.buffer().current_device_ptr(), tile_mk.lda(), &beta,
+                   tile_kk.buffer().current_device_ptr(), tile_kk.lda());
 #endif
 
 #ifdef DEBUG_TILES_VALUES
@@ -545,17 +545,17 @@ namespace potrf {
                   CUBLAS_OP_N, CUBLAS_OP_T,
                   tile_mk.rows(), tile_nk.rows(),
                   tile_nk.cols(), &alpha,
-                  tile_mk.buffer().device_ptr_on(device), tile_mk.lda(),
-                  tile_nk.buffer().device_ptr_on(device), tile_nk.lda(), &beta,
-                  tile_mn.buffer().device_ptr_on(device), tile_mn.lda());
+                  tile_mk.buffer().current_device_ptr(), tile_mk.lda(),
+                  tile_nk.buffer().current_device_ptr(), tile_nk.lda(), &beta,
+                  tile_mn.buffer().current_device_ptr(), tile_mn.lda());
 #elif defined(TTG_HAVE_HIPBLAS)
       hipblasDgemm(hipblas_handle(),
                    HIPBLAS_OP_N, HIPBLAS_OP_T,
                    tile_mk.rows(), tile_nk.rows(),
                    tile_nk.cols(), &alpha,
-                   tile_mk.buffer().device_ptr_on(device), tile_mk.lda(),
-                   tile_nk.buffer().device_ptr_on(device), tile_nk.lda(), &beta,
-                   tile_mn.buffer().device_ptr_on(device), tile_mn.lda());
+                   tile_mk.buffer().current_device_ptr(), tile_mk.lda(),
+                   tile_nk.buffer().current_device_ptr(), tile_nk.lda(), &beta,
+                   tile_mn.buffer().current_device_ptr(), tile_mn.lda());
 #endif
 
 
diff --git a/ttg/ttg/parsec/buffer.h b/ttg/ttg/parsec/buffer.h
index e3d7a84b3..bd1acf338 100644
--- a/ttg/ttg/parsec/buffer.h
+++ b/ttg/ttg/parsec/buffer.h
@@ -164,7 +164,7 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
     /* make sure it's a valid device */
     assert(parsec_nb_devices > device_id);
     /* make sure it's a valid copy */
-    int parsec_id = detail::device_to_parsec_device(device);
+    int parsec_id = detail::ttg_device_to_parsec_device(device);
     assert(m_data->device_copies[parsec_id] != nullptr);
     m_data->owner_device = parsec_id;
   }
@@ -173,20 +173,20 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
    * device buffer. */
   ttg::device::Device get_owner_device() const {
     assert(is_valid());
-    return detail::parsec_device_to_device(m_data->owner_device);
+    return detail::parsec_device_to_ttg_device(m_data->owner_device);
   }
 
   /* Get the pointer on the currently active device. */
   element_type* current_device_ptr() {
     assert(is_valid());
-    int device_id = ttg::device::current_device()+detail::first_device_id;
+    int device_id = detail::ttg_device_to_parsec_device(ttg::device::current_device());
     return static_cast<element_type*>(m_data->device_copies[device_id]->device_private);
   }
 
   /* Get the pointer on the currently active device. */
   const element_type* current_device_ptr() const {
     assert(is_valid());
-    int device_id = ttg::device::current_device()+detail::first_device_id;
+    int device_id = detail::ttg_device_to_parsec_device(ttg::device::current_device());
     return static_cast<element_type*>(m_data->device_copies[device_id]->device_private);
   }
 
@@ -208,7 +208,7 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
    */
   element_type* device_ptr_on(const ttg::device::Device& device) {
     assert(is_valid());
-    int device_id = detail::device_to_parsec_device(device);
+    int device_id = detail::ttg_device_to_parsec_device(device);
     return static_cast<element_type*>(parsec_data_get_ptr(m_data.get(), device_id));
   }
 
@@ -216,7 +216,7 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
    */
   const element_type* device_ptr_on(const ttg::device::Device& device) const {
     assert(is_valid());
-    int device_id = detail::device_to_parsec_device(device);
+    int device_id = detail::ttg_device_to_parsec_device(device);
     return static_cast<element_type*>(parsec_data_get_ptr(m_data.get(), device_id));
   }
 
@@ -230,7 +230,7 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
 
   bool is_valid_on(const ttg::device::Device& device) const {
     assert(is_valid());
-    int device_id = detail::device_to_parsec_device(device);
+    int device_id = detail::ttg_device_to_parsec_device(device);
     return (parsec_data_get_ptr(m_data.get(), device_id) != nullptr);
   }
 
diff --git a/ttg/ttg/parsec/device.h b/ttg/ttg/parsec/device.h
index 8f46453ac..77722b1c1 100644
--- a/ttg/ttg/parsec/device.h
+++ b/ttg/ttg/parsec/device.h
@@ -8,13 +8,13 @@ namespace ttg_parsec {
   namespace detail {
 
     // the first ID of an accelerator in the parsec ID-space
-    inline int first_device_id = 0;
+    inline int first_device_id = -1;
 
     /**
      * map from TTG ID-space to parsec ID-space
      */
     inline
-    int device_to_parsec_device(const ttg::device::Device& device) {
+    int ttg_device_to_parsec_device(const ttg::device::Device& device) {
       if (device.is_host()) {
         return 0;
       } else {
@@ -26,7 +26,7 @@ namespace ttg_parsec {
      * map from parsec ID-space to TTG ID-space
      */
     inline
-    ttg::device::Device parsec_device_to_device(int parsec_id) {
+    ttg::device::Device parsec_device_to_ttg_device(int parsec_id) {
       if (parsec_id < first_device_id) {
         return ttg::device::Device(parsec_id, ttg::ExecutionSpace::Host);
       }
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 241b723f5..694a13917 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -989,10 +989,13 @@ namespace ttg_parsec {
     ttg::detail::set_default_world(std::move(world));
 
     // query the first device ID
+    detail::first_device_id = -1;
     for (int i = 0; i < parsec_nb_devices; ++i) {
-      if (parsec_mca_device_is_gpu(i)) {
+      bool is_gpu = parsec_mca_device_is_gpu(i);
+      if (detail::first_device_id == -1 && is_gpu) {
         detail::first_device_id = i;
-        break;
+      } else if (detail::first_device_id > -1 && !is_gpu) {
+        throw std::runtime_error("PaRSEC: Found non-GPU device in GPU ID range!");
       }
     }
   }
@@ -1362,7 +1365,7 @@ namespace ttg_parsec {
 #if defined(PARSEC_HAVE_DEV_CUDA_SUPPORT) && defined(TTG_HAVE_CUDA)
       {
         parsec_cuda_exec_stream_t *cuda_stream = (parsec_cuda_exec_stream_t *)gpu_stream;
-        int device = gpu_device->super.device_index - detail::first_device_id;
+        int device = detail::parsec_device_to_ttg_device(gpu_device->super.device_index);
         ttg::device::detail::set_current(device, cuda_stream->cuda_stream);
       }
 #endif // defined(PARSEC_HAVE_DEV_CUDA_SUPPORT) && defined(TTG_HAVE_CUDA)
@@ -1370,7 +1373,7 @@ namespace ttg_parsec {
 #if defined(PARSEC_HAVE_DEV_HIP_SUPPORT) && defined(TTG_HAVE_HIP)
       {
         parsec_hip_exec_stream_t *hip_stream = (parsec_hip_exec_stream_t *)gpu_stream;
-        int device = gpu_device->super.device_index - detail::first_device_id;
+        int device = detail::parsec_device_to_ttg_device(gpu_device->super.device_index);
         ttg::device::detail::set_current(device, hip_stream->hip_stream);
       }
 #endif // defined(PARSEC_HAVE_DEV_CUDA_SUPPORT) && defined(TTG_HAVE_CUDA)
@@ -1379,7 +1382,7 @@ namespace ttg_parsec {
       {
         parsec_level_zero_exec_stream_t *stream;
         stream = (parsec_level_zero_exec_stream_t *)gpu_stream;
-        int device = gpu_device->super.device_index - detail::first_device_id;
+        int device = detail::parsec_device_to_ttg_device(gpu_device->super.device_index);
         ttg::device::detail::set_current(device, stream->swq->queue);
       }
 #endif // defined(PARSEC_HAVE_DEV_CUDA_SUPPORT) && defined(TTG_HAVE_CUDA)
@@ -1512,7 +1515,7 @@ namespace ttg_parsec {
 
       gpu_task->load = task_load;
       assert(dev_index >= 0);
-      if (dev_index < detail::first_device_id) {
+      if (!parsec_mca_device_is_gpu(dev_index)) {
           return PARSEC_HOOK_RETURN_NEXT; /* Fall back */
       }
 
@@ -1997,7 +2000,7 @@ ttg::abort();  // should not happen
             // TODO: first attempt at sending directly to the device
             parsec_gpu_data_copy_t* gpu_elem;
             gpu_elem = PARSEC_DATA_GET_COPY(master, gpu_device->super.device_index);
-            int i = detail::first_device_id;ß
+            int i = detail::first_device_id;
             int devid = detail::first_device_id;
             while (i < parsec_nb_devices) {
               if (nullptr == gpu_elem) {

From d660b07e0821fca4723b3fcb7e67251bb12b4694 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 27 Nov 2023 16:20:45 +0000
Subject: [PATCH 177/259] Bump parsec commit to rebased fork

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 cmake/modules/ExternalDependenciesVersions.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/modules/ExternalDependenciesVersions.cmake b/cmake/modules/ExternalDependenciesVersions.cmake
index 9e17ab80a..6187ec0fd 100644
--- a/cmake/modules/ExternalDependenciesVersions.cmake
+++ b/cmake/modules/ExternalDependenciesVersions.cmake
@@ -6,6 +6,6 @@ set(TTG_TRACKED_BOOST_VERSION 1.66)
 set(TTG_TRACKED_CATCH2_VERSION 2.13.1)
 set(TTG_TRACKED_CEREAL_VERSION 1.3.0)
 set(TTG_TRACKED_MADNESS_TAG cb195817d7807c4aead10ba200cd20649036cbae)
-set(TTG_TRACKED_PARSEC_TAG 6a5a986d6ec1f9cace09f3ad475c553632077a24)
+set(TTG_TRACKED_PARSEC_TAG 8690b1eb6080500af1ad32ffcd4b95aa38f65c22)
 set(TTG_TRACKED_BTAS_TAG a02be0d29fb4a788ecef43de711dcd6d6f1cb6b8)
 set(TTG_TRACKED_TILEDARRAY_TAG f0115e9e4a3f988224afbfb3c241e92171e916b8)

From 0ab8f47c41712e70f5d356494eafb81361d2f2cc Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 27 Nov 2023 16:22:34 +0000
Subject: [PATCH 178/259] There is no allocators for level zero in TiledArray
 yet

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/spmm/spmm_cuda.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/spmm/spmm_cuda.cc b/examples/spmm/spmm_cuda.cc
index 0c6b8fa23..f9cd1dad3 100644
--- a/examples/spmm/spmm_cuda.cc
+++ b/examples/spmm/spmm_cuda.cc
@@ -254,7 +254,7 @@ struct DeviceTensor : public ttg::TTValue<DeviceTensor<_T, _Range, _Storage>>
 };
 
 using scalar_t = double;
-#if defined(TTG_HAVE_CUDA) || defined(TTG_HAVE_HIPBLAS) || defined(TTG_HAVE_LEVEL_ZERO)
+#if defined(TTG_HAVE_CUDA) || defined(TTG_HAVE_HIPBLAS)
 using blk_t = DeviceTensor<scalar_t, btas::DEFAULT::range,
                            btas::mohndle<btas::varray<scalar_t, TiledArray::device_pinned_allocator<scalar_t>>,
                                          btas::Handle::shared_ptr>>;

From fb5f111ec8b7cf70c40a0263e78d6aecde65d301 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 27 Nov 2023 16:32:29 +0000
Subject: [PATCH 179/259] Fix misplaced assert

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/buffer.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ttg/ttg/parsec/buffer.h b/ttg/ttg/parsec/buffer.h
index bd1acf338..72ec386f4 100644
--- a/ttg/ttg/parsec/buffer.h
+++ b/ttg/ttg/parsec/buffer.h
@@ -161,10 +161,10 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
    * buffer was modified outside of a TTG */
   void set_current_device(const ttg::device::Device& device) {
     assert(is_valid());
+    int parsec_id = detail::ttg_device_to_parsec_device(device);
     /* make sure it's a valid device */
-    assert(parsec_nb_devices > device_id);
+    assert(parsec_nb_devices > device);
     /* make sure it's a valid copy */
-    int parsec_id = detail::ttg_device_to_parsec_device(device);
     assert(m_data->device_copies[parsec_id] != nullptr);
     m_data->owner_device = parsec_id;
   }

From 383d36d27fc0a9cf2ee15cad71f7dab318acf74d Mon Sep 17 00:00:00 2001
From: Aurelien Bouteiller <bouteill@icl.utk.edu>
Date: Mon, 27 Nov 2023 11:37:33 -0500
Subject: [PATCH 180/259] ze: add the missing link-libraries to spmm

Signed-off-by: Aurelien Bouteiller <bouteill@icl.utk.edu>
---
 examples/CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 14b041ce9..0ae64c831 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -40,9 +40,9 @@ if (TARGET tiledarray)
                            COMPILE_DEFINITIONS TTG_ENABLE_HIP=1;DEBUG_TILES_VALUES=1
                            RUNTIMES "parsec")
       endif(TARGET roc::hipsolver)
-    elseif (TARGET MKL::MKL)
+    elseif (TARGET MKL::MKL_DPCPP)
       add_ttg_executable(bspmm-lz spmm/spmm_cuda.cc
-	                  LINK_LIBRARIES tiledarray TiledArray_Eigen BTAS Boost::boost MKL::MKL
+                          LINK_LIBRARIES tiledarray TiledArray_Eigen BTAS Boost::boost MKL::MKL_DPCPP level_zero::ze_loader m
                           COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2
                           RUNTIMES "parsec")
     endif()

From f4b482adf2fa1001dcbfae8657363666c9d4aee4 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 27 Nov 2023 17:15:17 +0000
Subject: [PATCH 181/259] Use std::size_t in parsec matrix wrapper

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/potrf/pmw.h | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/examples/potrf/pmw.h b/examples/potrf/pmw.h
index 076f1fffc..000e5693b 100644
--- a/examples/potrf/pmw.h
+++ b/examples/potrf/pmw.h
@@ -49,7 +49,7 @@ class PaRSECMatrixWrapper {
     //}
   }
 
-  MatrixTile<ValueT> operator()(int row, int col) const {
+  MatrixTile<ValueT> operator()(std::size_t row, std::size_t col) const {
     ValueT* ptr = static_cast<ValueT*>(parsec_data_copy_get_ptr(
                       parsec_data_get_copy(pm->super.super.data_of(&pm->super.super, row, col), 0)));
     auto mb = (row < pm->super.mt - 1) ? pm->super.mb : pm->super.m - row * pm->super.mb;
@@ -58,45 +58,45 @@ class PaRSECMatrixWrapper {
   }
 
   /** Number of tiled rows **/
-  int rows(void) const {
+  std::size_t rows(void) const {
     return pm->super.mt;
   }
 
   /** Number of rows in tile */
-  int rows_in_tile(void) const {
+  std::size_t rows_in_tile(void) const {
     return pm->super.mb;
   }
 
   /** Number of rows in the matrix */
-  int rows_in_matrix(void) const {
+  std::size_t rows_in_matrix(void) const {
     return pm->super.m;
   }
 
   /** Number of tiled columns **/
-  int cols(void) const {
+  std::size_t cols(void) const {
     return pm->super.nt;
   }
 
   /** Number of columns in tile */
-  int cols_in_tile(void) const {
+  std::size_t cols_in_tile(void) const {
     return pm->super.nb;
   }
 
   /** Number of columns in the matrix */
-  int cols_in_matrix(void) const {
+  std::size_t cols_in_matrix(void) const {
     return pm->super.n;
   }
 
   /* The rank storing the tile at {row, col} */
-  int rank_of(int row, int col) const {
+  std::size_t rank_of(std::size_t row, std::size_t col) const {
     return pm->super.super.rank_of(&pm->super.super, row, col);
   }
 
-  bool is_local(int row, int col) const {
+  bool is_local(std::size_t row, std::size_t col) const {
     return ttg::default_execution_context().rank() == rank_of(row, col);
   }
 
-  bool in_matrix(int row, int col) const {
+  bool in_matrix(std::size_t row, std::size_t col) const {
     return (pm->uplo == PARSEC_MATRIX_LOWER && col <= row) ||
            (pm->uplo == PARSEC_MATRIX_UPPER && col >= row);
   }
@@ -112,8 +112,8 @@ class PaRSECMatrixWrapper {
   /* Copy entire input matrix (which is local) into a single LAPACK format matrix */
   ValueT *getLAPACKMatrix() const {
     ValueT *ret = new ValueT[rows_in_matrix()*cols_in_matrix()];
-    for(auto i = 0; i < rows_in_matrix(); i++) {
-      for(auto j = 0; j < cols_in_matrix(); j++) {
+    for(std::size_t i = 0; i < rows_in_matrix(); i++) {
+      for(std::size_t j = 0; j < cols_in_matrix(); j++) {
         if( in_matrix(i/rows_in_tile(), j/cols_in_tile()) ) {
           auto m = i/rows_in_tile();
           auto n = j/cols_in_tile();
@@ -136,8 +136,8 @@ using MatrixT = PaRSECMatrixWrapper<sym_two_dim_block_cyclic_t, ValueT>;
 static auto make_load_tt(MatrixT<double> &A, ttg::Edge<Key2, MatrixTile<double>> &toop, bool defer_write)
 {
   auto load_tt = ttg::make_tt<void>([&](std::tuple<ttg::Out<Key2, MatrixTile<double>>>& out) {
-      for(int i = 0; i < A.rows(); i++) {
-        for(int j = 0; j < A.cols() && A.in_matrix(i, j); j++) {
+      for(std::size_t i = 0; i < A.rows(); i++) {
+        for(std::size_t j = 0; j < A.cols() && A.in_matrix(i, j); j++) {
           if(A.is_local(i, j)) {
             if(ttg::tracing()) ttg::print("load(", Key2{i, j}, ")");
             ttg::send<0>(Key2{i, j}, std::move(A(i, j)), out);
@@ -154,9 +154,9 @@ static auto make_load_tt(MatrixT<double> &A, ttg::Edge<Key2, MatrixTile<double>>
 static void print_LAPACK_matrix( const double *A, int N, const char *label)
 {
   std::cout << label << std::endl;
-  for(int i = 0; i < N; i++) {
+  for(std::size_t i = 0; i < N; i++) {
     std::cout << " ";
-    for(int j = 0; j < N; j++) {
+    for(std::size_t j = 0; j < N; j++) {
       std::cout << std::setw(11) << std::setprecision(5) << A[i+j*N] << " ";
     }
     std::cout << std::endl;

From 0b1325e3b11bd3e760e82c74de3dd84ff797ce81 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 27 Nov 2023 17:21:50 +0000
Subject: [PATCH 182/259] MatrixTile: use std::size_t consistently

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/matrixtile.h | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/examples/matrixtile.h b/examples/matrixtile.h
index 21bfa5aa6..51f384226 100644
--- a/examples/matrixtile.h
+++ b/examples/matrixtile.h
@@ -12,14 +12,14 @@
 template <typename T, class Allocator = TiledArray::device_pinned_allocator<double>>
 class MatrixTile : public ttg::TTValue<MatrixTile<T, Allocator>> {
  public:
-  using metadata_t = typename std::tuple<int, int, int>;
+  using metadata_t = typename std::tuple<std::size_t, std::size_t, std::size_t>;
 
   using buffer_t  = typename ttg::buffer<T, Allocator>;
   using ttvalue_type = ttg::TTValue<MatrixTile<T, Allocator>>;
 
  private:
   buffer_t _buffer;
-  int _rows = 0, _cols = 0, _lda = 0;
+  std::size_t _rows = 0, _cols = 0, _lda = 0;
 #ifdef DEBUG_TILES_VALUES
   mutable std::optional<T> _norm;
 #endif // DEBUG_TILES_VALUES
@@ -36,7 +36,7 @@ class MatrixTile : public ttg::TTValue<MatrixTile<T, Allocator>> {
  public:
   MatrixTile() {}
 
-  MatrixTile(int rows, int cols, int lda)
+  MatrixTile(std::size_t rows, std::size_t cols, std::size_t lda)
   : ttvalue_type()
   , _buffer(lda*cols)
   , _rows(rows)
@@ -54,7 +54,7 @@ class MatrixTile : public ttg::TTValue<MatrixTile<T, Allocator>> {
    * Constructor with outside memory. The tile will *not* delete this memory
    * upon destruction.
    */
-  MatrixTile(int rows, int cols, T* data, int lda)
+  MatrixTile(std::size_t rows, std::size_t cols, T* data, std::size_t lda)
   : ttvalue_type()
   , _buffer(data, lda*cols)
   , _rows(rows)
@@ -108,11 +108,11 @@ class MatrixTile : public ttg::TTValue<MatrixTile<T, Allocator>> {
 
   size_t size() const { return _cols * _lda; }
 
-  int rows() const { return _rows; }
+  std::size_t rows() const { return _rows; }
 
-  int cols() const { return _cols; }
+  std::size_t cols() const { return _cols; }
 
-  int lda() const { return _lda; }
+  std::size_t lda() const { return _lda; }
 
   buffer_t& buffer() {
     return _buffer;
@@ -145,10 +145,10 @@ class MatrixTile : public ttg::TTValue<MatrixTile<T, Allocator>> {
     auto ptr = tt.data();
     o << std::endl << " ";
     o << "MatrixTile<" << typeid(T).name() << ">{ rows=" << tt.rows() << " cols=" << tt.cols() << " ld=" << tt.lda();
-#if DEBUG_TILES_VALUES
+#if DEBUG_TILES_VALUES && 0
     o << " data=[ ";
-    for (int i = 0; i < tt.rows(); i++) {
-      for (int j = 0; j < tt.cols(); j++) {
+    for (std::size_t i = 0; i < tt.rows(); i++) {
+      for (std::size_t j = 0; j < tt.cols(); j++) {
         o << ptr[i + j * tt.lda()] << " ";
       }
       o << std::endl << " ";
@@ -187,7 +187,7 @@ namespace madness {
     template <class Archive, typename T>
     struct ArchiveLoadImpl<Archive, MatrixTile<T>> {
       static inline void load(const Archive& ar, MatrixTile<T>& tile) {
-        int rows, cols, lda;
+        std::size_t rows, cols, lda;
         ar >> rows >> cols >> lda;
         tile = MatrixTile<T>(rows, cols, lda);
         ar >> wrap(tile.data(), tile.rows() * tile.cols());  // MatrixTile<T>(bm.rows(), bm.cols());

From 1f3619a03dcb6487f6005ccb70925d3ac3cd2de7 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 27 Nov 2023 17:30:07 +0000
Subject: [PATCH 183/259] Fun with signed integers in parsec matrix wrapper

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/potrf/pmw.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/examples/potrf/pmw.h b/examples/potrf/pmw.h
index 000e5693b..0f8d75d7d 100644
--- a/examples/potrf/pmw.h
+++ b/examples/potrf/pmw.h
@@ -54,7 +54,8 @@ class PaRSECMatrixWrapper {
                       parsec_data_get_copy(pm->super.super.data_of(&pm->super.super, row, col), 0)));
     auto mb = (row < pm->super.mt - 1) ? pm->super.mb : pm->super.m - row * pm->super.mb;
     auto nb = (col < pm->super.nt - 1) ? pm->super.nb : pm->super.n - col * pm->super.nb;
-    return MatrixTile<ValueT>{mb, nb, ptr, pm->super.mb};
+    std::size_t lda = pm->super.mb;
+    return MatrixTile<ValueT>{mb, nb, ptr, lda};
   }
 
   /** Number of tiled rows **/

From 81c25e3cf796608495ccd2f515d147a2608b0c64 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 29 Nov 2023 20:02:57 +0000
Subject: [PATCH 184/259] Fix is_dummy name collision in parsec_ttg_task_base_t

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/task.h | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ttg/ttg/parsec/task.h b/ttg/ttg/parsec/task.h
index 12b7587c9..99f683278 100644
--- a/ttg/ttg/parsec/task.h
+++ b/ttg/ttg/parsec/task.h
@@ -122,7 +122,7 @@ namespace ttg_parsec {
       release_task_fn* release_task_cb = nullptr;
       device_ptr_t* dev_ptr = nullptr;
       bool remove_from_hash = true;
-      bool is_dummy = false;
+      bool dummy = false;
       bool defer_writer = TTG_PARSEC_DEFER_WRITER; // whether to defer writer instead of creating a new copy
       ttg_parsec_data_flags data_flags; // HACKY: flags set by prepare_send and reset by the copy_handler
 
@@ -171,8 +171,8 @@ namespace ttg_parsec {
       }
 
     public:
-      void set_dummy(bool d) { is_dummy = d; }
-      bool dummy() { return is_dummy; }
+      void set_dummy(bool d) { dummy = d; }
+      bool is_dummy() { return dummy; }
     };
 
     template <typename TT, bool KeyIsVoid = ttg::meta::is_void_v<typename TT::key_type>>
@@ -318,4 +318,4 @@ namespace ttg_parsec {
 
 } // namespace ttg_parsec
 
-#endif // TTG_PARSEC_TASK_H
\ No newline at end of file
+#endif // TTG_PARSEC_TASK_H

From bf821a53eaed68d90e299bd3dd269bd2fb621d9d Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 4 Dec 2023 10:04:10 +0000
Subject: [PATCH 185/259] Bump PaRSEC version to get first set of eviction
 fixes

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 cmake/modules/ExternalDependenciesVersions.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/modules/ExternalDependenciesVersions.cmake b/cmake/modules/ExternalDependenciesVersions.cmake
index 6187ec0fd..fe38dc1c7 100644
--- a/cmake/modules/ExternalDependenciesVersions.cmake
+++ b/cmake/modules/ExternalDependenciesVersions.cmake
@@ -6,6 +6,6 @@ set(TTG_TRACKED_BOOST_VERSION 1.66)
 set(TTG_TRACKED_CATCH2_VERSION 2.13.1)
 set(TTG_TRACKED_CEREAL_VERSION 1.3.0)
 set(TTG_TRACKED_MADNESS_TAG cb195817d7807c4aead10ba200cd20649036cbae)
-set(TTG_TRACKED_PARSEC_TAG 8690b1eb6080500af1ad32ffcd4b95aa38f65c22)
+set(TTG_TRACKED_PARSEC_TAG 542a7d88105d660213e386adc5291172c2b1476a)
 set(TTG_TRACKED_BTAS_TAG a02be0d29fb4a788ecef43de711dcd6d6f1cb6b8)
 set(TTG_TRACKED_TILEDARRAY_TAG f0115e9e4a3f988224afbfb3c241e92171e916b8)

From 6727faffe1ce22e5c2a47c2363b54763ff35f74a Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Tue, 5 Dec 2023 09:10:29 +0000
Subject: [PATCH 186/259] task dummy() -> is_dummy()

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/ttg.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 694a13917..41d7d0349 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -2323,7 +2323,7 @@ ttg::abort();  // should not happen
 
       if( world_impl.dag_profiling() ) {
 #if defined(PARSEC_PROF_GRAPHER)
-        if(NULL != detail::parsec_ttg_caller && !detail::parsec_ttg_caller->dummy()) {
+        if(NULL != detail::parsec_ttg_caller && !detail::parsec_ttg_caller->is_dummy()) {
           int orig_index = detail::find_index_of_copy_in_task(detail::parsec_ttg_caller, &value);
           char orig_str[32];
           char dest_str[32];
@@ -2667,7 +2667,7 @@ ttg::abort();  // should not happen
       }
 #endif
 #if defined(PARSEC_PROF_GRAPHER)
-      if(NULL != detail::parsec_ttg_caller && !detail::parsec_ttg_caller->dummy()) {
+      if(NULL != detail::parsec_ttg_caller && !detail::parsec_ttg_caller->is_dummy()) {
         int orig_index = detail::find_index_of_copy_in_task(detail::parsec_ttg_caller, value_ptr);
         char orig_str[32];
         char dest_str[32];
@@ -3545,7 +3545,7 @@ ttg::abort();  // should not happen
       task_t *task = (task_t*)parsec_task;
 
       if constexpr (derived_has_device_op()) {
-        assert(nullptr != task->suspended_task_address);
+        assert(task->is_dummy() || nullptr != task->suspended_task_address);
       }
 
       /* if we still have a coroutine handle we invoke it one more time to get the sends/broadcasts */

From 049297e41a8d285622b31344ffd8605b4c6e5421 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Tue, 5 Dec 2023 09:14:34 +0000
Subject: [PATCH 187/259] Fix usage of wrong device ID in buffer

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/buffer.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ttg/ttg/parsec/buffer.h b/ttg/ttg/parsec/buffer.h
index 72ec386f4..3102bf025 100644
--- a/ttg/ttg/parsec/buffer.h
+++ b/ttg/ttg/parsec/buffer.h
@@ -163,7 +163,7 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
     assert(is_valid());
     int parsec_id = detail::ttg_device_to_parsec_device(device);
     /* make sure it's a valid device */
-    assert(parsec_nb_devices > device);
+    assert(parsec_nb_devices > parsec_id);
     /* make sure it's a valid copy */
     assert(m_data->device_copies[parsec_id] != nullptr);
     m_data->owner_device = parsec_id;

From 8146481f460c78eb33362d7153054343124ce0de Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Tue, 5 Dec 2023 09:21:53 +0000
Subject: [PATCH 188/259] Bump PaRSEC tag for more memory oversubscription
 fixes

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 cmake/modules/ExternalDependenciesVersions.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/modules/ExternalDependenciesVersions.cmake b/cmake/modules/ExternalDependenciesVersions.cmake
index fe38dc1c7..1e6dd1fef 100644
--- a/cmake/modules/ExternalDependenciesVersions.cmake
+++ b/cmake/modules/ExternalDependenciesVersions.cmake
@@ -6,6 +6,6 @@ set(TTG_TRACKED_BOOST_VERSION 1.66)
 set(TTG_TRACKED_CATCH2_VERSION 2.13.1)
 set(TTG_TRACKED_CEREAL_VERSION 1.3.0)
 set(TTG_TRACKED_MADNESS_TAG cb195817d7807c4aead10ba200cd20649036cbae)
-set(TTG_TRACKED_PARSEC_TAG 542a7d88105d660213e386adc5291172c2b1476a)
+set(TTG_TRACKED_PARSEC_TAG 25d1931e863b6741e453112d2117d85ad32e7fba)
 set(TTG_TRACKED_BTAS_TAG a02be0d29fb4a788ecef43de711dcd6d6f1cb6b8)
 set(TTG_TRACKED_TILEDARRAY_TAG f0115e9e4a3f988224afbfb3c241e92171e916b8)

From b9727a437ab1f61c7396e8f48ff47b46f1c3137c Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 13 Dec 2023 16:10:59 +0000
Subject: [PATCH 189/259] Map blas/solver handles to streams, not threads and
 streams

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/devblas_helper.h | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/examples/devblas_helper.h b/examples/devblas_helper.h
index 76781d9ec..5f2ad3e89 100644
--- a/examples/devblas_helper.h
+++ b/examples/devblas_helper.h
@@ -8,6 +8,7 @@
 #include <stdexcept>
 #include <optional>
 #include <map>
+#include <mutex>
 
 #ifdef TTG_HAVE_CUDART
 
@@ -20,7 +21,8 @@
 template<typename T = int>
 inline const cublasHandle_t& cublas_handle(T _ = 0) {
   using map_type = std::map<std::pair<int, cudaStream_t>, cublasHandle_t>;
-  static thread_local map_type handles;
+  static map_type handles;
+  static std::mutex handle_mtx;
 
   auto d = ttg::device::current_device();
   int device = 0; // assume 0 if we don't have a device
@@ -30,6 +32,7 @@ inline const cublasHandle_t& cublas_handle(T _ = 0) {
 
   cudaStream_t stream = ttg::device::current_stream();
 
+  std::lock_guard g(handle_mtx);
   map_type::iterator it;
   if ((it = handles.find({device, stream})) == handles.end()){
     cublasHandle_t handle;
@@ -53,7 +56,8 @@ template<typename T = int>
 inline const cusolverDnHandle_t& cusolver_handle(T _ = 0) {
 
   using map_type = std::map<std::pair<int, cudaStream_t>, cusolverDnHandle_t>;
-  static thread_local map_type handles;
+  static map_type handles;
+  static std::mutex handle_mtx;
 
   auto d = ttg::device::current_device();
   int device = 0; // assume 0 if we don't have a device
@@ -62,6 +66,7 @@ inline const cusolverDnHandle_t& cusolver_handle(T _ = 0) {
   }
   cudaStream_t stream = ttg::device::current_stream();
 
+  std::lock_guard g(handle_mtx);
   map_type::iterator it;
   if ((it = handles.find({device, stream})) == handles.end()){
     cusolverDnHandle_t handle;
@@ -95,7 +100,8 @@ inline const cusolverDnHandle_t& cusolver_handle(T _ = 0) {
 template<typename T = int>
 inline const hipblasHandle_t& hipblas_handle(T _ = 0) {
   using map_type = std::map<std::pair<int, hipStream_t>, hipblasHandle_t>;
-  static thread_local map_type handles;
+  static map_type handles;
+  static std::mutex handle_mtx;
 
   auto d = ttg::device::current_device();
   int device = 0; // assume 0 if we don't have a device
@@ -105,6 +111,7 @@ inline const hipblasHandle_t& hipblas_handle(T _ = 0) {
 
   hipStream_t stream = ttg::device::current_stream();
 
+  std::lock_guard g(handle_mtx);
   map_type::iterator it;
   if ((it = handles.find({device, stream})) == handles.end()){
     hipblasHandle_t handle;
@@ -128,7 +135,8 @@ inline const hipblasHandle_t& hipblas_handle(T _ = 0) {
 template<typename T = int>
 inline const hipsolverDnHandle_t& hipsolver_handle(T _ = 0) {
   using map_type = std::map<std::pair<int, hipStream_t>, hipsolverDnHandle_t>;
-  static thread_local map_type handles;
+  static map_type handles;
+  static std::mutex handle_mtx;
   auto d = ttg::device::current_device();
   int device = 0; // assume 0 if we don't have a device
   if (d.is_device()) {
@@ -137,6 +145,7 @@ inline const hipsolverDnHandle_t& hipsolver_handle(T _ = 0) {
 
   hipStream_t stream = ttg::device::current_stream();
 
+  std::lock_guard g(handle_mtx);
   map_type::iterator it;
   if ((it = handles.find({device, stream})) == handles.end()){
     hipsolverDnHandle_t handle;

From 603af4e1d6407382b570ce00ee651706e9f6a190 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 18 Dec 2023 21:21:31 +0000
Subject: [PATCH 190/259] Naming convetions: rename ttg::Buffer

Objects in TTG start with a capital letter so rename ttg::buffer to
ttg::Buffer to be consistent.

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/matrixtile.h      |  2 +-
 examples/spmm/spmm_cuda.cc |  6 +++---
 ttg/ttg/buffer.h           |  8 ++++----
 ttg/ttg/device/task.h      |  4 ++--
 ttg/ttg/parsec/buffer.h    | 28 ++++++++++++++--------------
 ttg/ttg/parsec/fwd.h       |  2 +-
 6 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/examples/matrixtile.h b/examples/matrixtile.h
index 51f384226..a6e3a02c2 100644
--- a/examples/matrixtile.h
+++ b/examples/matrixtile.h
@@ -14,7 +14,7 @@ class MatrixTile : public ttg::TTValue<MatrixTile<T, Allocator>> {
  public:
   using metadata_t = typename std::tuple<std::size_t, std::size_t, std::size_t>;
 
-  using buffer_t  = typename ttg::buffer<T, Allocator>;
+  using buffer_t  = typename ttg::Buffer<T, Allocator>;
   using ttvalue_type = ttg::TTValue<MatrixTile<T, Allocator>>;
 
  private:
diff --git a/examples/spmm/spmm_cuda.cc b/examples/spmm/spmm_cuda.cc
index f9cd1dad3..5386d7da5 100644
--- a/examples/spmm/spmm_cuda.cc
+++ b/examples/spmm/spmm_cuda.cc
@@ -61,7 +61,7 @@ struct DeviceTensor : public ttg::TTValue<DeviceTensor<_T, _Range, _Storage>>
                     , public btas::Tensor<_T, _Range, _Storage> {
   using tensor_type = typename btas::Tensor<_T, _Range, _Storage>;
   using ttvalue_type = typename ttg::TTValue<DeviceTensor<_T, _Range, _Storage>>;
-  ttg::buffer<_T> b; // does not own the host buffer
+  ttg::Buffer<_T> b; // does not own the host buffer
 
   using value_type = typename tensor_type::value_type;
   using size_type = typename tensor_type::size_type;
@@ -193,7 +193,7 @@ struct DeviceTensor : public ttg::TTValue<DeviceTensor<_T, _Range, _Storage>>
     /* Grrrr, moving a Tensor does not guarantee to move the pointer */
     , b((this->size() == 0 ||
          this->data() == x.b.host_ptr()) ? std::move(x.b)
-                                         : ttg::buffer<_T>(this->size() ? this->data()
+                                         : ttg::Buffer<_T>(this->size() ? this->data()
                                                                         : nullptr,
                                                            this->size()))
     {
@@ -239,7 +239,7 @@ struct DeviceTensor : public ttg::TTValue<DeviceTensor<_T, _Range, _Storage>>
       if (this->size() == 0 || this->data() == x.b.host_ptr()){
         b = std::move(x.b);
       } else  {
-        b = ttg::buffer<_T>(this->size() ? this->data() : nullptr, this->size());
+        b = ttg::Buffer<_T>(this->size() ? this->data() : nullptr, this->size());
       }
       //std::swap(x.b, b);
       //std::cout << "DeviceTensor move ctor" << std::endl;
diff --git a/ttg/ttg/buffer.h b/ttg/ttg/buffer.h
index 6d252cb09..5c8bbfed5 100644
--- a/ttg/ttg/buffer.h
+++ b/ttg/ttg/buffer.h
@@ -8,7 +8,7 @@
 namespace ttg {
 
 template<typename T, typename Allocator = std::allocator<T>>
-using buffer = TTG_IMPL_NS::buffer<T, Allocator>;
+using Buffer = TTG_IMPL_NS::Buffer<T, Allocator>;
 
 namespace detail {
   template<typename T>
@@ -16,14 +16,14 @@ namespace detail {
   { };
 
   template<typename T, typename A>
-  struct is_buffer<ttg::buffer<T, A>> : std::true_type
+  struct is_buffer<ttg::Buffer<T, A>> : std::true_type
   { };
 
   template<typename T>
   constexpr bool is_buffer_v = is_buffer<T>::value;
 
-  static_assert(is_buffer_v<ttg::buffer<double>>);
-  static_assert(is_buffer_v<TTG_IMPL_NS::buffer<double>>);
+  static_assert(is_buffer_v<ttg::Buffer<double>>);
+  static_assert(is_buffer_v<TTG_IMPL_NS::Buffer<double>>);
 } // namespace detail
 
 } // namespace ttg
diff --git a/ttg/ttg/device/task.h b/ttg/ttg/device/task.h
index c51990d99..1b099b87f 100644
--- a/ttg/ttg/device/task.h
+++ b/ttg/ttg/device/task.h
@@ -59,14 +59,14 @@ namespace ttg {
     };
   } // namespace detail
 
-  /* Wait for kernel to complete and provided ttg::buffer and ttg::devicescratch
+  /* Wait for kernel to complete and provided ttg::Buffer and ttg::devicescratch
    * to be transferred back to host */
   template<typename... Buffers>
   [[nodiscard]]
   inline auto wait_kernel(Buffers&&... args) {
     static_assert(((ttg::detail::is_buffer_v<std::decay_t<Buffers>>
                     ||ttg::detail::is_devicescratch_v<std::decay_t<Buffers>>)&&...),
-                  "Only ttg::buffer and ttg::devicescratch can be waited on!");
+                  "Only ttg::Buffer and ttg::devicescratch can be waited on!");
     return detail::wait_kernel_t<std::remove_reference_t<Buffers>...>{std::tie(std::forward<Buffers>(args)...)};
   }
 
diff --git a/ttg/ttg/parsec/buffer.h b/ttg/ttg/parsec/buffer.h
index 3102bf025..f9f185de3 100644
--- a/ttg/ttg/parsec/buffer.h
+++ b/ttg/ttg/parsec/buffer.h
@@ -22,7 +22,7 @@ namespace ttg_parsec {
 namespace detail {
   // fwd decl
   template<typename T, typename A>
-  parsec_data_t* get_parsec_data(const ttg_parsec::buffer<T, A>& db);
+  parsec_data_t* get_parsec_data(const ttg_parsec::Buffer<T, A>& db);
 } // namespace detail
 
 /**
@@ -37,7 +37,7 @@ namespace detail {
  * tracking of the containing object.
  */
 template<typename T, typename Allocator>
-struct buffer : public detail::ttg_parsec_data_wrapper_t
+struct Buffer : public detail::ttg_parsec_data_wrapper_t
               , private Allocator {
 
   using element_type = std::decay_t<T>;
@@ -62,7 +62,7 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
     // nothing to be done, we don't own the memory
   }
 
-  friend parsec_data_t* detail::get_parsec_data<T, Allocator>(const ttg_parsec::buffer<T, Allocator>&);
+  friend parsec_data_t* detail::get_parsec_data<T, Allocator>(const ttg_parsec::Buffer<T, Allocator>&);
 
   allocator_type& get_allocator_reference() { return static_cast<allocator_type&>(*this); }
 
@@ -76,10 +76,10 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
 
 public:
 
-  buffer() : buffer(nullptr, 0)
+  Buffer() : Buffer(nullptr, 0)
   { }
 
-  buffer(std::size_t n)
+  Buffer(std::size_t n)
   : ttg_parsec_data_wrapper_t()
   , allocator_type()
   , m_host_data(allocate(n))
@@ -96,7 +96,7 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
   /* Constructing a buffer using application-managed memory.
    * The memory pointed to by ptr must be accessible during
    * the life-time of the buffer. */
-  buffer(element_type* ptr, std::size_t n = 1)
+  Buffer(element_type* ptr, std::size_t n = 1)
   : ttg_parsec_data_wrapper_t()
   , allocator_type()
   , m_host_data(ptr)
@@ -110,7 +110,7 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
     this->reset_parsec_data(m_host_data, n*sizeof(element_type));
   }
 
-  virtual ~buffer() {
+  virtual ~Buffer() {
     if (m_owned) {
       deallocate();
       m_owned = false;
@@ -119,7 +119,7 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
   }
 
   /* allow moving device buffers */
-  buffer(buffer&& db)
+  Buffer(Buffer&& db)
   : ttg_parsec_data_wrapper_t(std::move(db))
   , allocator_type(std::move(db))
   , m_host_data(db.m_host_data)
@@ -134,10 +134,10 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
   /* explicitly disable copying of buffers
    * TODO: should we allow this? What data to use?
    */
-  buffer(const buffer& db) = delete;
+  Buffer(const Buffer& db) = delete;
 
   /* allow moving device buffers */
-  buffer& operator=(buffer&& db) {
+  Buffer& operator=(Buffer&& db) {
     ttg_parsec_data_wrapper_t::operator=(std::move(db));
     allocator_type::operator=(std::move(db));
     std::swap(m_host_data, db.m_host_data);
@@ -155,7 +155,7 @@ struct buffer : public detail::ttg_parsec_data_wrapper_t
   /* explicitly disable copying of buffers
    * TODO: should we allow this? What data to use?
    */
-  buffer& operator=(const buffer& db) = delete;
+  Buffer& operator=(const Buffer& db) = delete;
 
   /* set the current device, useful when a device
    * buffer was modified outside of a TTG */
@@ -408,11 +408,11 @@ struct is_buffer : std::false_type
 { };
 
 template<typename T, typename A>
-struct is_buffer<buffer<T, A>> : std::true_type
+struct is_buffer<Buffer<T, A>> : std::true_type
 { };
 
 template<typename T, typename A>
-struct is_buffer<const buffer<T, A>> : std::true_type
+struct is_buffer<const Buffer<T, A>> : std::true_type
 { };
 
 template<typename T>
@@ -420,7 +420,7 @@ constexpr static const bool is_buffer_v = is_buffer<T>::value;
 
 namespace detail {
   template<typename T, typename A>
-  parsec_data_t* get_parsec_data(const ttg_parsec::buffer<T, A>& db) {
+  parsec_data_t* get_parsec_data(const ttg_parsec::Buffer<T, A>& db) {
     return const_cast<parsec_data_t*>(db.m_data.get());
   }
 } // namespace detail
diff --git a/ttg/ttg/parsec/fwd.h b/ttg/ttg/parsec/fwd.h
index 098f51220..3a7dadb5b 100644
--- a/ttg/ttg/parsec/fwd.h
+++ b/ttg/ttg/parsec/fwd.h
@@ -17,7 +17,7 @@ namespace ttg_parsec {
   struct ptr;
 
   template<typename T, typename Allocator = std::allocator<T>>
-  struct buffer;
+  struct Buffer;
   template<typename T>
   struct devicescratch;
 

From 9afe0c09cf477b7b447ba80b377d38f2b5052d3b Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 18 Dec 2023 21:44:06 +0000
Subject: [PATCH 191/259] Remove legacy device implementation files

Both view.h and make_device_tt.h were the initial implementation and are
not used anymore.

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/CMakeLists.txt       |   3 -
 ttg/ttg/madness/ttg.h    |   1 -
 ttg/ttg/make_device_tt.h | 281 ----------------------------
 ttg/ttg/make_tt.h        |   2 -
 ttg/ttg/parsec/ttg.h     |   2 -
 ttg/ttg/view.h           | 389 ---------------------------------------
 6 files changed, 678 deletions(-)
 delete mode 100644 ttg/ttg/make_device_tt.h
 delete mode 100644 ttg/ttg/view.h

diff --git a/ttg/CMakeLists.txt b/ttg/CMakeLists.txt
index 307572f10..874e8d5d4 100644
--- a/ttg/CMakeLists.txt
+++ b/ttg/CMakeLists.txt
@@ -62,9 +62,6 @@ set(ttg-impl-headers
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/traverse.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/world.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/make_tt.h
-        ${CMAKE_CURRENT_SOURCE_DIR}/ttg/make_device_tt.h
-        ${CMAKE_CURRENT_SOURCE_DIR}/ttg/view.h
-        ${CMAKE_CURRENT_SOURCE_DIR}/ttg/device/task.h
         ${CMAKE_CURRENT_BINARY_DIR}/ttg/config.h
     )
 set(ttg_device_headers
diff --git a/ttg/ttg/madness/ttg.h b/ttg/ttg/madness/ttg.h
index d283d5541..7f2ffab08 100644
--- a/ttg/ttg/madness/ttg.h
+++ b/ttg/ttg/madness/ttg.h
@@ -22,7 +22,6 @@
 #include "ttg/util/meta.h"
 #include "ttg/util/meta/callable.h"
 #include "ttg/util/void.h"
-#include "ttg/view.h"
 #include "ttg/world.h"
 #ifdef TTG_HAS_COROUTINE
 #include "ttg/util/coroutine.h"
diff --git a/ttg/ttg/make_device_tt.h b/ttg/ttg/make_device_tt.h
deleted file mode 100644
index 09f096d83..000000000
--- a/ttg/ttg/make_device_tt.h
+++ /dev/null
@@ -1,281 +0,0 @@
-// clang-format off
-#ifndef TTG_MAKE_DEVICE_TT_H
-#define TTG_MAKE_DEVICE_TT_H
-
-// to be #include'd within runtime::ttg namespace
-
-
-namespace detail {
-#ifdef TTG_HAVE_CUDA
-  inline thread_local cudaStream_t* ts_stream = nullptr;
-#endif // TTG_HAVE_CUDA
-
-  template<typename FuncT, typename keyT, typename... Args, std::size_t... Is>
-  inline void invoke_with_unpacked_views(FuncT&& func, const keyT& key, std::tuple<Args...>& views, std::index_sequence<Is...>) {
-#ifdef TTG_HAVE_CUDA
-    func(key, std::get<Is>(views)..., ts_stream);
-#else // TTG_HAVE_CUDA
-    func(key, std::get<Is>(views)...);
-#endif // TTG_HAVE_CUDA
-  }
-
-  /* TODO: extract host objects from views */
-  template<typename ViewTupleT>
-  struct host_obj_type;
-
-  template<typename... ViewTs>
-  struct host_obj_type<std::tuple<ViewTs...>> {
-    using type = std::tuple<typename ViewTs::host_type...>;
-  };
-
-  template<typename ViewTupleT>
-  using host_obj_type_t = typename host_obj_type<ViewTupleT>::type;
-
-  template<typename FuncT, typename keyT, typename... ViewTs, typename  std::size_t... Is>
-  inline void invoke_out_with_unpacked_views(FuncT&& func, const keyT& key, std::tuple<ViewTs...> views, std::index_sequence<Is...>) {
-    func(key, std::get<Is>(views).get_host_object()...);
-  }
-
-  template<typename HostT, typename... DevTs, std::size_t I, std::size_t... Is>
-  inline void create_view_on_device(const ttg::View<HostT, DevTs...>& view,
-                                    std::tuple<ttg::ViewSpan<DevTs>...>& dev_spans,
-                                    std::index_sequence<I, Is...>) {
-
-    /* fill in pointers for the device -- we're relying on managed memory for this simple wrapper */
-    typename std::tuple_element_t<I, typename ttg::View<HostT, DevTs...>::span_tuple_type>::element_type *ptr;
-    size_t size;
-    ptr = view.template get_device_ptr<I>();
-    size = view.template get_device_size<I>();
-    //cudaMalloc(&ptr, span.size_bytes());
-    std::get<I>(dev_spans) = ttg::ViewSpan(ptr, size, view.template get_scope<I>());
-
-    /* copy data to device */
-    //cudaMemcpy(ptr, span.data(), span.size_bytes(), cudaMemcpyHostToDevice);
-    if (view.template get_span<I>().is_sync_in()) {
-#if defined(TTG_HAVE_CUDA) && defined(TTG_USE_CUDA_PREFETCH)
-      cudaMemPrefetchAsync(span.data(), span.size_bytes(), 0, *ts_stream);
-#endif // TTG_USE_CUDA_PREFETCH
-    }
-
-    if constexpr(sizeof...(Is) > 0) {
-      create_view_on_device(view, dev_spans, std::index_sequence<Is...>{});
-    }
-  }
-
-  template<typename HostT, typename... ViewSpanTs, std::size_t... Is>
-  auto make_view_from_tuple(HostT& obj, std::tuple<ttg::ViewSpan<ViewSpanTs>...>& spans, std::index_sequence<Is...>) {
-    return ttg::make_view(obj, std::get<Is>(spans)...);
-  }
-
-  template<typename... ViewTs, std::size_t I, std::size_t... Is>
-  inline void create_on_device(std::tuple<ViewTs...>& views, std::tuple<ViewTs...>& dev_views, std::index_sequence<I, Is...>) {
-
-    using view_tuple_t = typename std::tuple<ViewTs...>;
-    auto& view = std::get<I>(views);
-    typename std::tuple_element_t<I, view_tuple_t>::span_tuple_type dev_spans;
-    create_view_on_device(view, dev_spans, std::make_index_sequence<std::tuple_element_t<I, view_tuple_t>::size()>());
-
-    /* set the view for the device */
-    std::get<I>(dev_views) = make_view_from_tuple(view.get_host_object(), dev_spans, std::make_index_sequence<std::tuple_size_v<decltype(dev_spans)>>{});
-    if constexpr(sizeof...(Is) > 0) {
-      create_on_device(views, dev_views, std::index_sequence<Is...>{});
-    }
-  }
-
-  template<typename HostT, typename... DevTs, std::size_t I, std::size_t... Is>
-  inline void sync_view_to_host(ttg::View<HostT, DevTs...>& dev_view, std::index_sequence<I, Is...>) {
-    /* prefetch back to host */
-    auto span = dev_view.template get_span<I>();
-
-    /* prefetch data from device */
-    if (span.is_sync_out()) {
-#if defined(TTG_HAVE_CUDA) && defined(TTG_USE_CUDA_PREFETCH)
-      cudaMemPrefetchAsync(span.data(), span.size_bytes(), cudaCpuDeviceId, *ts_stream);
-#endif // TTG_USE_CUDA_PREFETCH
-    }
-
-    if constexpr(sizeof...(Is) > 0) {
-      sync_view_to_host(dev_view, std::index_sequence<Is...>{});
-    }
-  }
-
-  template<typename... ViewTs, std::size_t I, std::size_t... Is>
-  inline void sync_back_to_host(std::tuple<ViewTs...>& dev_views, std::index_sequence<I, Is...>) {
-
-    sync_view_to_host(std::get<I>(dev_views), std::make_index_sequence<std::tuple_element_t<I, std::tuple<ViewTs...>>::size()>());
-
-    if constexpr(sizeof...(Is) > 0) {
-      sync_back_to_host(dev_views, std::index_sequence<Is...>{});
-    }
-  }
-
-  template<typename keyT,
-           bool have_outterm_tuple,
-           typename DevViewFuncT,
-           typename DevKernelFuncT,
-           typename DevOutFuncT,
-           typename... input_edge_valuesT,
-           typename... output_edgesT,
-           typename... Args>
-  auto make_device_tt_helper(DevViewFuncT &&view_func,
-                             DevKernelFuncT &&kernel_func,
-                             DevOutFuncT &&out_func,
-                             ttg::ExecutionSpace space,
-                             const std::tuple<ttg::Edge<keyT, input_edge_valuesT>...> &inedges,
-                             const std::tuple<output_edgesT...> &outedges,
-                             const std::string &name,
-                             const std::vector<std::string> &innames,
-                             const std::vector<std::string> &outnames,
-                             const ttg::typelist<Args...>& full_input_args) {
-
-    using output_terminals_type = typename ttg::edges_to_output_terminals<std::tuple<output_edgesT...>>::type;
-
-    auto taskfn = [=](const keyT& key, Args... args) mutable {
-
-#ifdef TTG_HAVE_CUDA
-      if (nullptr == ts_stream) {
-        ts_stream = new cudaStream_t();
-        cudaStreamCreate(ts_stream);
-      }
-#endif // TTG_HAVE_CUDA
-
-      auto views = view_func(key, std::forward<Args>(args)...);
-      using view_tuple_t = std::remove_reference_t<decltype(views)>;
-      constexpr std::size_t view_tuple_size = std::tuple_size_v<view_tuple_t>;
-      /* 1) allocate memory on device */
-      auto device_views = views;
-      /* 2) move data from views to device */
-      if constexpr(std::tuple_size_v<view_tuple_t> > 0) {
-        create_on_device(views, device_views, std::make_index_sequence<view_tuple_size>());
-      }
-      /* 3) call kernel function */
-      detail::invoke_with_unpacked_views(kernel_func, key, device_views, std::make_index_sequence<view_tuple_size>());
-      /* 4) move data back out into host objects */
-      if constexpr(std::tuple_size_v<view_tuple_t> > 0) {
-        sync_back_to_host(device_views, std::make_index_sequence<view_tuple_size>());
-      }
-  #ifdef TTG_HAVE_CUDA
-      /* wait for the */
-      cudaStreamSynchronize(*ts_stream);
-  #endif // TTG_HAVE_CUDA
-      /* 5) call output function */
-      detail::invoke_out_with_unpacked_views(out_func, key, views, std::make_index_sequence<view_tuple_size>());
-    };
-
-  using wrapT = typename CallableWrapTTArgsAsTypelist<decltype(taskfn), void, have_outterm_tuple, keyT, output_terminals_type,
-                                                      ttg::typelist<Args...>>::type;
-
-  return std::make_unique<wrapT>(std::move(taskfn), inedges, outedges, name, innames, outnames);
-
-  }
-
-
-} // namespace detail
-
-
-template <typename keyT,
-          typename DevViewFuncT,
-          typename DevKernelFuncT,
-          typename DevOutFuncT,
-          typename... input_edge_valuesT,
-          typename... output_edgesT>
-auto make_device_tt(DevViewFuncT &&view_func,
-                    DevKernelFuncT &&kernel_func,
-                    DevOutFuncT &&out_func,
-                    ttg::ExecutionSpace space,
-                    const std::tuple<ttg::Edge<keyT, input_edge_valuesT>...> &inedges,
-                    const std::tuple<output_edgesT...> &outedges, const std::string &name = "wrapper",
-                    const std::vector<std::string> &innames = std::vector<std::string>(sizeof...(input_edge_valuesT), "input"),
-                    const std::vector<std::string> &outnames = std::vector<std::string>(sizeof...(output_edgesT), "output")) {
-
-  using output_terminals_type = typename ttg::edges_to_output_terminals<std::tuple<output_edgesT...>>::type;
-
-  constexpr auto void_key = ttg::meta::is_void_v<keyT>;
-
-  // gross list of candidate argument types
-  using gross_candidate_func_args_t = ttg::meta::typelist<
-      ttg::meta::candidate_argument_bindings_t<std::add_const_t<keyT>>,
-      ttg::meta::candidate_argument_bindings_t<typename ttg::Edge<keyT, input_edge_valuesT>::value_type>...,
-      ttg::meta::typelist<output_terminals_type &, void>>;
-
-  // net list of candidate argument types excludes the empty typelists for void arguments
-  using candidate_func_args_t = ttg::meta::filter_t<gross_candidate_func_args_t, ttg::meta::typelist_is_not_empty>;
-
-  // gross argument typelist for invoking func, can include void for optional args
-  constexpr static auto func_is_generic = ttg::meta::is_generic_callable_v<DevViewFuncT>;
-  using gross_func_args_t = decltype(ttg::meta::compute_arg_binding_types_r<void>(view_func, candidate_func_args_t{}));
-  constexpr auto DETECTED_HOW_TO_INVOKE_GENERIC_FUNC =
-      func_is_generic ? !std::is_same_v<gross_func_args_t, ttg::typelist<>> : true;
-  static_assert(DETECTED_HOW_TO_INVOKE_GENERIC_FUNC,
-                "ttd::make_tt(func, inedges, ...): could not detect how to invoke generic callable func, either the "
-                "signature of func "
-                "is faulty, or inedges does match the expected list of types, or both");
-
-  // net argument typelist
-  using func_args_t = ttg::meta::drop_void_t<gross_func_args_t>;
-  constexpr auto num_args = std::tuple_size_v<func_args_t>;
-
-  // if given task id, make sure it's passed via const lvalue ref
-  constexpr bool TASK_ID_PASSED_AS_CONST_LVALUE_REF =
-      !void_key ? ttg::meta::probe_first_v<ttg::meta::is_const_lvalue_reference, true, func_args_t> : true;
-  static_assert(TASK_ID_PASSED_AS_CONST_LVALUE_REF,
-                "ttg::make_tt(func, ...): if given to func, the task id must be passed by const lvalue ref");
-
-  // if given out-terminal tuple, make sure it's passed via nonconst lvalue ref
-  constexpr bool have_outterm_tuple =
-      func_is_generic ? !ttg::meta::is_last_void_v<gross_func_args_t>
-                      : ttg::meta::probe_last_v<ttg::meta::decays_to_output_terminal_tuple, false, gross_func_args_t>;
-  constexpr bool OUTTERM_TUPLE_PASSED_AS_NONCONST_LVALUE_REF =
-      have_outterm_tuple ? ttg::meta::probe_last_v<ttg::meta::is_nonconst_lvalue_reference, false, func_args_t> : true;
-  static_assert(
-      OUTTERM_TUPLE_PASSED_AS_NONCONST_LVALUE_REF,
-      "ttg::make_tt(func, ...): if given to func, the output terminal tuple must be passed by nonconst lvalue ref");
-
-  // TT needs actual types of arguments to func ... extract them and pass to CallableWrapTTArgs
-  using input_edge_value_types = ttg::meta::typelist<std::decay_t<input_edge_valuesT>...>;
-  // input_args_t = {input_valuesT&&...}
-  using input_args_t = typename ttg::meta::take_first_n<
-      typename ttg::meta::drop_first_n<func_args_t, std::size_t(void_key ? 0 : 1)>::type,
-      std::tuple_size_v<func_args_t> - (void_key ? 0 : 1) - (have_outterm_tuple ? 1 : 0)>::type;
-  constexpr auto NO_ARGUMENTS_PASSED_AS_NONCONST_LVALUE_REF =
-      !ttg::meta::is_any_nonconst_lvalue_reference_v<input_args_t>;
-  static_assert(
-      NO_ARGUMENTS_PASSED_AS_NONCONST_LVALUE_REF,
-      "ttg::make_tt(func, inedges, outedges): one or more arguments to func can only be passed by nonconst lvalue "
-      "ref; this is illegal, should only pass arguments as const lavlue ref or (nonconst) rvalue ref");
-  using decayed_input_args_t = ttg::meta::decayed_typelist_t<input_args_t>;
-  // 3. full_input_args_t = edge-types with non-void types replaced by input_args_t
-  using full_input_args_t = ttg::meta::replace_nonvoid_t<input_edge_value_types, input_args_t>;
-
-  return detail::make_device_tt_helper<keyT, have_outterm_tuple>(std::forward<DevViewFuncT>(view_func),
-                                                                 std::forward<DevKernelFuncT>(kernel_func),
-                                                                 std::forward<DevOutFuncT>(out_func),
-                                                                 space, inedges, outedges, name, innames, outnames,
-                                                                 full_input_args_t{});
-}
-
-#if 0
-template <typename keyT,
-          typename HostFuncT,
-          typename DevViewFuncT,
-          typename DevKernelFuncT,
-          typename DevOutFuncT,
-          typename... input_edge_valuesT,
-          typename... output_edgesT>
-auto make_device_tt(HostFuncT &&host_func,
-                    DevViewFuncT &&view_func,
-                    DevKernelFuncT &&kernel_func,
-                    DevOutFuncT &&out_func,
-                    ttg::ExecutionSpace space,
-                    const std::tuple<ttg::Edge<keyT, input_edge_valuesT>...> &inedges,
-                    const std::tuple<output_edgesT...> &outedges, const std::string &name = "wrapper",
-                    const std::vector<std::string> &innames = std::vector<std::string>(sizeof...(input_edge_valuesT), "input"),
-                    const std::vector<std::string> &outnames = std::vector<std::string>(sizeof...(output_edgesT), "output")) {
-
-  /* drop the host function */
-  return make_device_tt(view_func, kernel_func, out_func, space, inedges, outedges, name, innames, outnames);
-}
-#endif // 0
-#endif // TTG_MAKE_DEVICE_TT_H
-
-// clang-format on
diff --git a/ttg/ttg/make_tt.h b/ttg/ttg/make_tt.h
index e4d5d8039..8ba7f39b0 100644
--- a/ttg/ttg/make_tt.h
+++ b/ttg/ttg/make_tt.h
@@ -664,6 +664,4 @@ template <typename keyT, typename funcT, typename... input_edge_valuesT, typenam
   return make_tt<keyT>(std::forward<funcT>(func), inedges, outedges, name, innames, outnames);
 }
 
-//#include "ttg/make_device_tt.h"
-
 #endif  // TTG_MAKE_TT_H
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 41d7d0349..3355f648a 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -39,8 +39,6 @@
 
 #include "ttg/serialization/data_descriptor.h"
 
-//#include "ttg/view.h"
-
 #include "ttg/parsec/fwd.h"
 
 #include "ttg/parsec/buffer.h"
diff --git a/ttg/ttg/view.h b/ttg/ttg/view.h
deleted file mode 100644
index 6bb99a633..000000000
--- a/ttg/ttg/view.h
+++ /dev/null
@@ -1,389 +0,0 @@
-#ifndef TTG_VIEW_H
-#define TTG_VIEW_H
-
-#include <array>
-#include <type_traits>
-#include <span>
-
-#include "ttg/fwd.h"
-#include "ttg/impl_selector.h"
-#include "ttg/ptr.h"
-
-namespace ttg {
-
-  enum class ViewScope {
-    Allocate     = 0x0,  //< memory allocated as scratch, but not moved in or out
-    Available    = 0x1,  //< data will be reused on device if available, transferred to device otherwise
-    SyncIn       = 0x2,  //< data will be allocated on and transferred to device
-                         //< if latest version resides on the device (no previous sync-out) the data will
-                         //< not be transferred again
-    SyncOut      = 0x4,  //< value will be transferred from device to host after kernel completes
-    SyncInOut    = 0x8,  //< data will be moved in and synchronized back out after the kernel completes
-    AvailableOut = 0x16, //< similar to Available and data is transferred back to device after kernel completes
-  };
-
-  /**
-   * A view span that can be type-punned.
-   * We use it instead of a std::span to be able
-   * to remove the type and convert to void pointers instead.
-   */
-  template<typename T, typename = void>
-  struct ViewPart;
-
-  template<>
-  struct ViewPart<void, void> {
-
-    using element_type  = void;
-    using value_type    = void;
-    using view_part_type = ViewPart<value_type>;
-
-    constexpr ViewPart() = default;
-    constexpr ViewPart(void* ptr, std::size_t size, ViewScope scope = ViewScope::SyncIn)
-    : m_data_ptr(ptr)
-    , m_size(size)
-    , m_scope(scope)
-    { }
-
-    constexpr std::size_t size() const {
-      return m_size;
-    }
-
-    constexpr void* data() const {
-      return m_data_ptr;
-    }
-
-    void set_data(void *ptr) {
-      m_data_ptr = ptr;
-    }
-
-    constexpr bool is_allocate() const {
-      return (m_scope == ViewScope::Allocate);
-    }
-
-    constexpr bool is_sync_in() const {
-      return !!(static_cast<int>(m_scope) & static_cast<int>(ViewScope::SyncIn));
-    }
-
-    constexpr bool is_sync_out() const {
-      return !!(static_cast<int>(m_scope) & static_cast<int>(ViewScope::SyncOut));
-    }
-
-    constexpr ViewScope scope() const {
-      return m_scope;
-    }
-
-  private:
-    void *m_data_ptr = nullptr;
-    std::size_t m_size = 0;
-    ViewScope m_scope = ViewScope::Allocate;
-  };
-
-
-  template<typename T>
-  struct ViewPart<T, void> : public ViewPart<void, void> {
-
-    using element_type  = T;
-    using value_type    = std::remove_cv_t<T>;
-    using view_part_type = ViewPart<value_type>;
-
-    constexpr ViewPart() = default;
-    constexpr ViewPart(T* ptr, std::size_t size, ViewScope scope = ViewScope::SyncIn)
-    : ViewPart<void, void>(ptr, size, scope)
-    { }
-
-    constexpr T* data() const {
-      return static_cast<T*>(m_data_ptr);
-    }
-
-  };
-
-  namespace detail {
-    template<typename T>
-    struct view_trait
-    {
-      static constexpr bool is_view = false;
-      static constexpr bool is_persistent = false;
-    };
-  } // namespace detail
-
-  template<typename HostT, typename DevT = HostT, typename... DevTypeTs>
-  struct View {
-
-    using span_tuple_type  = std::tuple<ttg::ViewPart<DevT>, ttg::ViewPart<DevTypeTs>...>;
-    using host_type = HostT;
-
-    using view_type = View<HostT, DevT, DevTypeTs...>;
-
-    constexpr static std::size_t num_spans = std::tuple_size_v<span_tuple_type>;
-
-  private:
-    template<std::size_t... Is>
-    View(HostT& obj, span_tuple_type& spans, std::index_sequence<Is...>)
-    : m_obj(&obj)
-    , m_spans({std::get<Is>(spans)...})
-    { }
-
-    View(HostT& obj, span_tuple_type spans)
-    : View(obj, spans, std::make_index_sequence<sizeof...(DevTypeTs)+1>{})
-    {
-      /* TODO: let the runtime handle the view */
-      //ttg::detail::register_view(*this);
-    }
-
-    /* hidden so that users cannot create views outside of a task */
-    template<typename T, typename... ArgsT>
-    friend auto make_view(T& obj, ViewPart<ArgsT>... spans);
-
-  public:
-
-    constexpr View() = delete;
-
-    /* move ctor deleted to prevent moving out of a task */
-    View(view_type&&) = delete;
-
-    /* copy ctor deleted to prevent copying out of a task */
-    View(const view_type&) = delete;
-
-    ~View() {
-      /* TODO: let the runtime remove the view */
-      //ttg::detail::drop_view(*this);
-    }
-
-    /* move operator deleted to prevent moving out of a task */
-    view_type& operator=(view_type&&) = delete;
-    /* copy operator deleted to prevent moving out of a task */
-    view_type& operator=(const view_type&) = delete;
-
-    template<std::size_t i>
-    auto get_device_ptr() {
-      return static_cast<typename std::tuple_element_t<i, span_tuple_type>::value_type*>(std::get<i>(m_spans).data());
-    }
-
-    template<std::size_t i>
-    const auto get_device_ptr() const {
-      return static_cast<typename std::tuple_element_t<i, span_tuple_type>::value_type>(std::get<i>(m_spans).data());
-    }
-
-    template<std::size_t i>
-    std::size_t get_device_size() const {
-      return std::get<i>(m_spans).size();
-    }
-
-    template<std::size_t i>
-    auto& get_ViewPart() const {
-      return std::get<i>(m_spans);
-    }
-
-    HostT& get_host_object() {
-      return *m_obj;
-    }
-
-    const HostT& get_host_object() const {
-      return *m_obj;
-    }
-
-    template<std::size_t i>
-    ViewScope get_scope() const {
-      return std::get<i>(m_spans).scope();
-    }
-
-    constexpr static std::size_t size() {
-      return num_spans;
-    }
-
-    /* return a std::span of type-punned ViewParts */
-    std::span<ViewPart<void>> view_spans() {
-      return {m_spans.begin(), m_spans.end()};
-    }
-
-  private:
-    HostT* m_obj = nullptr;
-    /* type-punned storage, cast to actual types in get_device_ptr */
-    std::array<ViewPart<void>, num_spans> m_spans;
-    //span_tuple_type m_spans{};
-  };
-
-  template<typename HostT, typename... ViewPartTs>
-  auto make_view(HostT& obj, ViewPart<ViewPartTs>... spans) {
-    return View(obj, std::make_tuple(std::move(spans)...));
-  }
-
-  /* overload for trivially-copyable host objects */
-  template<typename HostT, typename = std::enable_if_t<std::is_trivially_copyable_v<HostT>>>
-  auto make_view(HostT& obj, ViewScope scope = ViewScope::SyncIn) {
-    return make_view(obj, ViewPart<HostT>(&obj, sizeof(HostT), scope));
-  }
-
-  namespace detail {
-    template<typename HostT, typename... DevTs>
-    struct view_trait<View<HostT, DevTs...>>
-    {
-      static constexpr bool is_view = true;
-      static constexpr bool is_persistent = false;
-    };
-  } // namespace detail
-
-  /* yielded when waiting on a kernel to complete */
-  struct device_op_wait_kernel
-  { };
-
-  enum ttg_device_coro_state {
-    TTG_DEVICE_CORO_STATE_NONE,
-    TTG_DEVICE_CORO_INIT,
-    TTG_DEVICE_CORO_WAIT_TRANSFER,
-    TTG_DEVICE_CORO_WAIT_KERNEL,
-    TTG_DEVICE_CORO_SENDOUT,
-    TTG_DEVICE_CORO_COMPLETE
-  };
-
-  /* type-punned version of the View, providing access to the object
-   * pointer and a std::span over the views of that object */
-  struct device_obj_view {
-    using span_type = std::span<ViewPart<void>>;
-    using iterator = typename span_type::iterator;
-
-    device_obj_view(void *obj, span_type&& span)
-    : m_obj(obj)
-    , m_span(std::forward<span_type>(span))
-    { }
-
-    void *host_obj() {
-      return m_obj;
-    }
-
-    auto begin() {
-      return m_span.begin();
-    }
-
-    auto end() {
-      return m_span.end();
-    }
-
-  private:
-    void *m_obj;
-    span_type m_span;
-  };
-
-
-  /**
-   * A view that is persistent and can be copied in and out of the TTG
-   */
-  template<typename HostT, typename DevT = HostT, typename... DevTypeTs>
-  struct PersistentView {
-
-    using span_tuple_type  = std::tuple<ttg::ViewPart<DevT>, ttg::ViewPart<DevTypeTs>...>;
-    using host_type = HostT;
-
-    using view_type = PersistentView<HostT, DevT, DevTypeTs...>;
-
-    using ptr_type = ttg::ptr<host_type>;
-
-    constexpr static std::size_t num_spans = std::tuple_size_v<span_tuple_type>;
-
-  private:
-    template<std::size_t... Is>
-    PersistentView(ptr_type&& ptr, span_tuple_type& spans, std::index_sequence<Is...>)
-    : m_ptr(std::forward<ptr_type>(ptr))
-    , m_spans({std::get<Is>(spans)...})
-    { }
-
-  public:
-
-    constexpr PersistentView() = default;
-
-    PersistentView(ptr_type ptr, span_tuple_type spans)
-    : PersistentView(std::move(ptr), spans, std::make_index_sequence<sizeof...(DevTypeTs)+1>{})
-    {
-      /* TODO: let the runtime handle the view */
-      //ttg::detail::register_view(*this);
-    }
-
-    PersistentView(view_type&&) = default;
-
-    PersistentView(const view_type&) = default;
-
-    ~PersistentView() {
-      /* TODO: let the runtime remove the view */
-      //ttg::detail::drop_view(*this);
-    }
-
-    view_type& operator=(view_type&&) = default;
-    view_type& operator=(const view_type&) = default;
-
-    template<std::size_t i>
-    auto get_device_ptr() {
-      return static_cast<typename std::tuple_element_t<i, span_tuple_type>::value_type*>(std::get<i>(m_spans).data());
-    }
-
-    template<std::size_t i>
-    const auto get_device_ptr() const {
-      return static_cast<typename std::tuple_element_t<i, span_tuple_type>::value_type>(std::get<i>(m_spans).data());
-    }
-
-    template<std::size_t i>
-    std::size_t get_device_size() const {
-      return std::get<i>(m_spans).size();
-    }
-
-    template<std::size_t i>
-    auto& get_ViewPart() const {
-      return std::get<i>(m_spans);
-    }
-
-    HostT& get_host_object() {
-      return *m_ptr;
-    }
-
-    const HostT& get_host_object() const {
-      return *m_ptr;
-    }
-
-    template<std::size_t i>
-    ViewScope get_scope() const {
-      return std::get<i>(m_spans).scope();
-    }
-
-    constexpr static std::size_t size() {
-      return num_spans;
-    }
-
-    ptr_type get_ptr() const {
-      return m_ptr;
-    }
-
-    /* return a std::span of type-punned ViewParts */
-    std::span<ViewPart<void>> view_spans() {
-      return {m_spans.begin(), m_spans.end()};
-    }
-
-  private:
-    ptr_type m_ptr;
-    /* type-punned storage, cast to actual types in get_device_ptr */
-    std::array<ViewPart<void>, num_spans> m_spans;
-    //span_tuple_type m_spans{};
-  };
-
-  template<typename HostT, typename... ViewPartTs>
-  auto make_persistent_view(HostT& obj, ViewPart<ViewPartTs>... spans) {
-    return PersistentView(obj, std::make_tuple(std::move(spans)...));
-  }
-
-  /* overload for trivially-copyable host objects */
-  template<typename HostT, typename = std::enable_if_t<std::is_trivially_copyable_v<HostT>>>
-  auto make_persistent_view(HostT& obj, ViewScope scope = ViewScope::SyncIn) {
-    return PersistentView<HostT, HostT>(obj, std::make_tuple(ViewPart<HostT>(&obj, sizeof(HostT), scope)));
-  }
-
-
-  namespace detail {
-    template<typename HostT, typename... DevTs>
-    struct view_trait<PersistentView<HostT, DevTs...>>
-    {
-      static constexpr bool is_view = true;
-      static constexpr bool is_persistent = true;
-    };
-  } // namespace detail
-
-} // namespace ttg
-
-#endif // TTG_VIEW_H

From f424bcd519b643798575e3daff18f5eb20440235 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 18 Dec 2023 22:40:05 +0000
Subject: [PATCH 192/259] Put all device functions into ttg::device namespace

Except for ttg::Buffer, which is separate and may be used by non-device
tasks/data structures.

The following were renamed:
- to_device -> select
- wait_kernel -> wait

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/potrf/potrf.h     |  30 +-
 examples/spmm/spmm_cuda.cc |   6 +-
 ttg/ttg/device/task.h      | 872 +++++++++++++++++--------------------
 ttg/ttg/make_tt.h          |  10 +-
 ttg/ttg/parsec/ttg.h       |  32 +-
 5 files changed, 443 insertions(+), 507 deletions(-)

diff --git a/examples/potrf/potrf.h b/examples/potrf/potrf.h
index 00452f78f..fb0c4f9b4 100644
--- a/examples/potrf/potrf.h
+++ b/examples/potrf/potrf.h
@@ -10,11 +10,11 @@
 
 #if defined(TTG_HAVE_CUDART)
 #define ES ttg::ExecutionSpace::CUDA
-#define TASKRET -> ttg::device_task
+#define TASKRET -> ttg::device::Task
 #include <cusolverDn.h>
 #elif defined(TTG_HAVE_HIP)
 #define ES ttg::ExecutionSpace::HIP
-#define TASKRET -> ttg::device_task
+#define TASKRET -> ttg::device::Task
 #include <hipsolver/hipsolver.h>
 #include <hipblas/hipblas.h>
 #else
@@ -134,14 +134,14 @@ namespace potrf {
       //auto norms_s  = ttg::make_scratch(norms.data(), ttg::scope::Allocate, norms.size());
       /* the workspace and the devInfo must be device-level pointers */
       //co_await ttg::to_device(tile_kk.buffer(), devWS, devInfo, norms_s);
-      co_await ttg::to_device(tile_kk.buffer(), devWS, devInfo);
+      co_await ttg::device::select(tile_kk.buffer(), devWS, devInfo);
 
       /* compute the norm at input */
       static_assert(std::is_same_v<double, T>, "Norm debugging only implementation for T=double");
       device_norm(tile_kk, &norms[0]);
 #else
       /* the workspace and the devInfo must be device-level pointers */
-      co_await ttg::to_device(tile_kk.buffer(), devWS, devInfo);
+      co_await ttg::device::select(tile_kk.buffer(), devWS, devInfo);
 #endif // DEBUG_TILES_VALUES
 
       int device = ttg::device::current_device();
@@ -159,14 +159,14 @@ namespace potrf {
       static_assert(std::is_same_v<double, T>, "Verification only implementation for T=double");
       device_norm(tile_kk, &norms[1]);
       /* wait for the kernel to complete */
-      co_await ttg::wait_kernel(devInfo);
+      co_await ttg::device::wait(devInfo);
       // check that we got the input tile we expected
       assert(check_norm(tile_kk.norm(), norms[0]));
       // set the new norm
       tile_kk.set_norm(norms[1]);
 #else
       /* wait for the kernel to complete */
-      co_await ttg::wait_kernel(devInfo);
+      co_await ttg::device::wait(devInfo);
 #endif // DEBUG_TILES_VALUES
 
       delete[] hostWS;
@@ -268,9 +268,9 @@ namespace potrf {
 #ifdef DEBUG_TILES_VALUES
       std::array<T, 3> norms; // input for tile_kk & tile_mk and output
       //auto norms_s = ttg::make_scratch(norms.data(), ttg::scope::Allocate, norms.size());
-      co_await ttg::to_device(tile_kk.buffer(), tile_mk.buffer());
+      co_await ttg::device::select(tile_kk.buffer(), tile_mk.buffer());
 #else
-      co_await ttg::to_device(tile_kk.buffer(), tile_mk.buffer());
+      co_await ttg::device::select(tile_kk.buffer(), tile_mk.buffer());
 #endif // DEBUG_TILES_VALUES
 
       int device = ttg::device::current_device();
@@ -306,7 +306,7 @@ namespace potrf {
       /* compute the norms at input */
       device_norm(tile_mk, &norms[2]);
       /* wait for the kernel to complete */
-      co_await ttg::wait_kernel();
+      co_await ttg::device::wait();
       // check that we got the input tiles we expected
       assert(check_norm(tile_kk.norm(), norms[0]));
       assert(check_norm(tile_mk.norm(), norms[1]));
@@ -400,12 +400,12 @@ namespace potrf {
 #ifdef DEBUG_TILES_VALUES
       std::array<T, 3> norms; // input for tile_kk & tile_mk and output
       //auto norms_s  = ttg::make_scratch(norms.data(), ttg::scope::Allocate, norms.size());
-      co_await ttg::to_device(tile_kk.buffer(), tile_mk.buffer());
+      co_await ttg::device::select(tile_kk.buffer(), tile_mk.buffer());
       /* compute the norms at input */
       device_norm(tile_mk, &norms[0]);
       device_norm(tile_kk, &norms[1]);
 #else
-      co_await ttg::to_device(tile_kk.buffer(), tile_mk.buffer());
+      co_await ttg::device::select(tile_kk.buffer(), tile_mk.buffer());
 #endif // DEBUG_TILES_VALUES
 
       int device = ttg::device::current_device();
@@ -435,7 +435,7 @@ namespace potrf {
       /* compute the norm at output */
       device_norm(tile_kk, &norms[2]);
       /* wait for the kernel to complete */
-      co_await ttg::wait_kernel();
+      co_await ttg::device::wait();
       // check that we got the input tiles we expected
       assert(check_norm(tile_mk.norm(), norms[0]));
       assert(check_norm(tile_kk.norm(), norms[1]));
@@ -526,14 +526,14 @@ namespace potrf {
 #ifdef DEBUG_TILES_VALUES
       std::array<T, 4> norms; // input for tile_mk & tile_nk & tile_mn and output
       //auto norms_s  = ttg::make_scratch(norms.data(), ttg::scope::Allocate, norms.size());
-      co_await ttg::to_device(tile_mk.buffer(), tile_nk.buffer(), tile_mn.buffer());
+      co_await ttg::device::select(tile_mk.buffer(), tile_nk.buffer(), tile_mn.buffer());
 
       /* compute the norms at input */
       device_norm(tile_mk, &norms[0]);
       device_norm(tile_nk, &norms[1]);
       device_norm(tile_mn, &norms[2]);
 #else
-      co_await ttg::to_device(tile_mk.buffer(), tile_nk.buffer(), tile_mn.buffer());
+      co_await ttg::device::select(tile_mk.buffer(), tile_nk.buffer(), tile_mn.buffer());
 #endif // DEBUG_TILES_VALUES
 
       int device = ttg::device::current_device();
@@ -563,7 +563,7 @@ namespace potrf {
       /* compute the norm at output */
       device_norm(tile_mn, &norms[3]);
       /* wait for the kernel to complete */
-      co_await ttg::wait_kernel();
+      co_await ttg::device::wait();
       // check that we got the input tiles we expected
       assert(check_norm(tile_mk.norm(), norms[0]));
       assert(check_norm(tile_nk.norm(), norms[1]));
diff --git a/examples/spmm/spmm_cuda.cc b/examples/spmm/spmm_cuda.cc
index 5386d7da5..6235554b2 100644
--- a/examples/spmm/spmm_cuda.cc
+++ b/examples/spmm/spmm_cuda.cc
@@ -812,7 +812,7 @@ class SpMM25D {
       }
     }
 
-    ttg::device_task op(const Key<3> &ijk, typename baseT::input_refs_tuple_type &&_ijk,
+    ttg::device::Task op(const Key<3> &ijk, typename baseT::input_refs_tuple_type &&_ijk,
             std::tuple<Out<Key<2>, Blk>, Out<Key<3>, Blk>> &result) {
       const auto i = ijk[0];
       const auto j = ijk[1];
@@ -830,7 +830,7 @@ class SpMM25D {
       }
 
       /* pull all buffers onto the device */
-      co_await ttg::to_device(A.b, B.b, C.b);
+      co_await ttg::device::select(A.b, B.b, C.b);
 
       /* everything is on the device, call the gemm */
       device_gemm(C, A, B);
@@ -844,7 +844,7 @@ class SpMM25D {
                  (have_next_k ? std::to_string(next_k) : "does not exist"));
 
       /* wait for the kernel to complete */
-      co_await ttg::wait_kernel();
+      co_await ttg::device::wait();
 
 
       // compute the contrib, pass the running total to the next flow, if needed
diff --git a/ttg/ttg/device/task.h b/ttg/ttg/device/task.h
index 1b099b87f..48ded0b4b 100644
--- a/ttg/ttg/device/task.h
+++ b/ttg/ttg/device/task.h
@@ -9,22 +9,7 @@
 #include "ttg/impl_selector.h"
 #include "ttg/ptr.h"
 
-namespace ttg {
-
-
-  /* yielded when waiting on a kernel to complete */
-  struct device_op_wait_kernel
-  { };
-
-  enum ttg_device_coro_state {
-    TTG_DEVICE_CORO_STATE_NONE,
-    TTG_DEVICE_CORO_INIT,
-    TTG_DEVICE_CORO_WAIT_TRANSFER,
-    TTG_DEVICE_CORO_WAIT_KERNEL,
-    TTG_DEVICE_CORO_SENDOUT,
-    TTG_DEVICE_CORO_COMPLETE
-  };
-
+namespace ttg::device {
 
   namespace detail {
     template<typename... Ts>
@@ -33,13 +18,31 @@ namespace ttg {
     };
   } // namespace detail
 
+  /**
+   * Select a device to execute on based on the provided buffer and scratchspace objects.
+   * Returns an object that should be awaited on using \c co_await.
+   * Upon resume, the device is selected (i.e., \sa ttg::device::current_device and
+   * \sa ttg::device::current_stream are available) and the buffers are available on the
+   * selected device.
+   */
   template<typename... Args>
   [[nodiscard]]
-  inline auto to_device(Args&&... args) {
+  inline auto select(Args&&... args) {
     return detail::to_device_t<std::remove_reference_t<Args>...>{std::tie(std::forward<Args>(args)...)};
   }
 
   namespace detail {
+
+    enum ttg_device_coro_state {
+      TTG_DEVICE_CORO_STATE_NONE,
+      TTG_DEVICE_CORO_INIT,
+      TTG_DEVICE_CORO_WAIT_TRANSFER,
+      TTG_DEVICE_CORO_WAIT_KERNEL,
+      TTG_DEVICE_CORO_SENDOUT,
+      TTG_DEVICE_CORO_COMPLETE
+    };
+
+
     template<typename... Ts>
     struct wait_kernel_t {
       std::tuple<Ts&...> ties;
@@ -59,374 +62,369 @@ namespace ttg {
     };
   } // namespace detail
 
-  /* Wait for kernel to complete and provided ttg::Buffer and ttg::devicescratch
-   * to be transferred back to host */
+  /**
+   * Wait for previously submitted kernels to complete and provided
+   * ttg::Buffer and ttg::devicescratch to be transferred back to host.
+   * Must only be called after awaiting \sa ttg::device::select has resumed.
+   */
   template<typename... Buffers>
   [[nodiscard]]
-  inline auto wait_kernel(Buffers&&... args) {
+  inline auto wait(Buffers&&... args) {
     static_assert(((ttg::detail::is_buffer_v<std::decay_t<Buffers>>
                     ||ttg::detail::is_devicescratch_v<std::decay_t<Buffers>>)&&...),
                   "Only ttg::Buffer and ttg::devicescratch can be waited on!");
     return detail::wait_kernel_t<std::remove_reference_t<Buffers>...>{std::tie(std::forward<Buffers>(args)...)};
   }
 
+  /******************************
+   * Send/Broadcast handling
+   * We pass the value returned by the backend's copy handler into a coroutine
+   * and execute the first part (prepare), before suspending it.
+   * The second part (send/broadcast) is executed after the task completed.
+   ******************************/
 
-  /* TODO: move all device code into ttg::device */
-  namespace device {
-
-    /******************************
-     * Send/Broadcast handling
-     * We pass the value returned by the backend's copy handler into a coroutine
-     * and execute the first part (prepare), before suspending it.
-     * The second part (send/broadcast) is executed after the task completed.
-     ******************************/
+  namespace detail {
+    struct send_coro_promise_type;
 
-    namespace detail {
-      struct send_coro_promise_type;
+    using send_coro_handle_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<send_coro_promise_type>;
 
-      using send_coro_handle_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<send_coro_promise_type>;
+    /// task that can be resumed after some events occur
+    struct send_coro_state : public send_coro_handle_type {
+      using base_type = send_coro_handle_type;
 
-      /// task that can be resumed after some events occur
-      struct send_coro_state : public send_coro_handle_type {
-        using base_type = send_coro_handle_type;
+      /// these are members mandated by the promise_type concept
+      ///@{
 
-        /// these are members mandated by the promise_type concept
-        ///@{
+      using promise_type = send_coro_promise_type;
 
-        using promise_type = send_coro_promise_type;
+      ///@}
 
-        ///@}
+      send_coro_state(base_type base) : base_type(std::move(base)) {}
 
-        send_coro_state(base_type base) : base_type(std::move(base)) {}
+      base_type& handle() { return *this; }
 
-        base_type& handle() { return *this; }
+      /// @return true if ready to resume
+      inline bool ready() {
+        return true;
+      }
 
-        /// @return true if ready to resume
-        inline bool ready() {
-          return true;
-        }
+      /// @return true if task completed and can be destroyed
+      inline bool completed();
+    };
 
-        /// @return true if task completed and can be destroyed
-        inline bool completed();
-      };
+    struct send_coro_promise_type {
 
-      struct send_coro_promise_type {
+      /* do not suspend the coroutine on first invocation, we want to run
+      * the coroutine immediately and suspend only once.
+      */
+      TTG_CXX_COROUTINE_NAMESPACE::suspend_never initial_suspend() {
+        return {};
+      }
 
-        /* do not suspend the coroutine on first invocation, we want to run
-        * the coroutine immediately and suspend only once.
-        */
-        TTG_CXX_COROUTINE_NAMESPACE::suspend_never initial_suspend() {
-          return {};
-        }
+      /* we don't suspend the coroutine at the end.
+      * it can be destroyed once the send/broadcast is done
+      */
+      TTG_CXX_COROUTINE_NAMESPACE::suspend_never final_suspend() noexcept {
+        return {};
+      }
 
-        /* we don't suspend the coroutine at the end.
-        * it can be destroyed once the send/broadcast is done
-        */
-        TTG_CXX_COROUTINE_NAMESPACE::suspend_never final_suspend() noexcept {
-          return {};
-        }
+      send_coro_state get_return_object() { return send_coro_state{send_coro_handle_type::from_promise(*this)}; }
 
-        send_coro_state get_return_object() { return send_coro_state{send_coro_handle_type::from_promise(*this)}; }
+      /* the send coros only have an empty co_await */
+      TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(ttg::Void) {
+        return {};
+      }
 
-        /* the send coros only have an empty co_await */
-        TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(ttg::Void) {
-          return {};
-        }
+      void unhandled_exception() {
+        std::cerr << "Send coroutine caught an unhandled exception!" << std::endl;
+        throw; // fwd
+      }
 
-        void unhandled_exception() {
-          std::cerr << "Send coroutine caught an unhandled exception!" << std::endl;
-          throw; // fwd
-        }
+    };
 
-      };
+    template<typename Key, typename Value, ttg::Runtime Runtime = ttg::ttg_runtime>
+    inline send_coro_state send_coro (const Key& key, Value&& value, ttg::Out<Key, std::decay_t<Value>> &t,
+                                      ttg::detail::value_copy_handler<Runtime>& ch) {
+      ttg::detail::value_copy_handler<Runtime> copy_handler = std::move(ch); // destroyed at the end of the coro
+      Key k = key;
+      t.prepare_send(k, std::forward<Value>(value));
+      co_await ttg::Void{}; // we'll come back once the task is done
+      t.send(k, std::forward<Value>(value));
+    };
 
-      template<typename Key, typename Value, ttg::Runtime Runtime = ttg::ttg_runtime>
-      inline send_coro_state send_coro (const Key& key, Value&& value, ttg::Out<Key, std::decay_t<Value>> &t,
+    template<typename Value, ttg::Runtime Runtime = ttg::ttg_runtime>
+    inline send_coro_state sendv_coro (Value&& value, ttg::Out<void, std::decay_t<Value>> &t,
                                         ttg::detail::value_copy_handler<Runtime>& ch) {
-        ttg::detail::value_copy_handler<Runtime> copy_handler = std::move(ch); // destroyed at the end of the coro
-        Key k = key;
-        t.prepare_send(k, std::forward<Value>(value));
-        co_await ttg::Void{}; // we'll come back once the task is done
-        t.send(k, std::forward<Value>(value));
-      };
-
-      template<typename Value, ttg::Runtime Runtime = ttg::ttg_runtime>
-      inline send_coro_state sendv_coro (Value&& value, ttg::Out<void, std::decay_t<Value>> &t,
-                                         ttg::detail::value_copy_handler<Runtime>& ch) {
-        ttg::detail::value_copy_handler<Runtime> copy_handler = std::move(ch); // destroyed at the end of the coro
-        t.prepare_send(std::forward<Value>(value));
-        co_await ttg::Void{}; // we'll come back once the task is done
-        t.sendv(std::forward<Value>(value));
-      };
+      ttg::detail::value_copy_handler<Runtime> copy_handler = std::move(ch); // destroyed at the end of the coro
+      t.prepare_send(std::forward<Value>(value));
+      co_await ttg::Void{}; // we'll come back once the task is done
+      t.sendv(std::forward<Value>(value));
+    };
 
-      template<typename Key, ttg::Runtime Runtime = ttg::ttg_runtime>
-      inline send_coro_state sendk_coro (const Key& key, ttg::Out<Key, void> &t) {
-        // no need to prepare the send but we have to suspend once
-        Key k = key;
-        co_await ttg::Void{}; // we'll come back once the task is done
-        t.sendk(k);
-      };
+    template<typename Key, ttg::Runtime Runtime = ttg::ttg_runtime>
+    inline send_coro_state sendk_coro (const Key& key, ttg::Out<Key, void> &t) {
+      // no need to prepare the send but we have to suspend once
+      Key k = key;
+      co_await ttg::Void{}; // we'll come back once the task is done
+      t.sendk(k);
+    };
 
-      template<ttg::Runtime Runtime = ttg::ttg_runtime>
-      inline send_coro_state send_coro (ttg::Out<void, void> &t) {
-        // no need to prepare the send but we have to suspend once
-        co_await ttg::Void{}; // we'll come back once the task is done
-        t.send();
-      };
-    } // namespace detail
+    template<ttg::Runtime Runtime = ttg::ttg_runtime>
+    inline send_coro_state send_coro (ttg::Out<void, void> &t) {
+      // no need to prepare the send but we have to suspend once
+      co_await ttg::Void{}; // we'll come back once the task is done
+      t.send();
+    };
 
+    struct send_t {
+      send_coro_state coro;
+    };
+  } // namespace detail
 
-    /* functionality to deal with sending/broadcasting */
-    namespace detail {
-      struct send_t {
-        send_coro_state coro;
-      };
-    } // namespace detail
+  template <size_t i, typename keyT, typename valueT, typename... out_keysT, typename... out_valuesT,
+            ttg::Runtime Runtime = ttg::ttg_runtime>
+  inline detail::send_t send(const keyT &key, valueT &&value,
+                                      std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
+    ttg::detail::value_copy_handler<Runtime> copy_handler;
+    return detail::send_t{detail::send_coro(key, copy_handler(std::forward<valueT>(value)), std::get<i>(t), copy_handler)};
+  }
 
-    template <size_t i, typename keyT, typename valueT, typename... out_keysT, typename... out_valuesT,
-              ttg::Runtime Runtime = ttg::ttg_runtime>
-    inline detail::send_t send(const keyT &key, valueT &&value,
-                                        std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
-      ttg::detail::value_copy_handler<Runtime> copy_handler;
-      return detail::send_t{detail::send_coro(key, copy_handler(std::forward<valueT>(value)), std::get<i>(t), copy_handler)};
-    }
+  template <size_t i, typename valueT, typename... out_keysT, typename... out_valuesT,
+            ttg::Runtime Runtime = ttg::ttg_runtime>
+  inline detail::send_t sendv(
+      valueT &&value, std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
+    ttg::detail::value_copy_handler<Runtime> copy_handler;
+    return detail::send_t{detail::sendv_coro(copy_handler(std::forward<valueT>(value)), std::get<i>(t), copy_handler)};
+  }
 
-    template <size_t i, typename valueT, typename... out_keysT, typename... out_valuesT,
-              ttg::Runtime Runtime = ttg::ttg_runtime>
-    inline detail::send_t sendv(
-        valueT &&value, std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
-      ttg::detail::value_copy_handler<Runtime> copy_handler;
-      return detail::send_t{detail::sendv_coro(copy_handler(std::forward<valueT>(value)), std::get<i>(t), copy_handler)};
-    }
+  template <size_t i, typename Key, typename... out_keysT, typename... out_valuesT,
+            ttg::Runtime Runtime = ttg::ttg_runtime>
+  inline detail::send_t sendk(
+      const Key& key, std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
+    return detail::send_t{detail::sendk_coro(key, std::get<i>(t))};
+  }
 
-    template <size_t i, typename Key, typename... out_keysT, typename... out_valuesT,
-              ttg::Runtime Runtime = ttg::ttg_runtime>
-    inline detail::send_t sendk(
-        const Key& key, std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
-      return detail::send_t{detail::sendk_coro(key, std::get<i>(t))};
-    }
+  // clang-format off
+  /// \brief Sends a task id and a value to the template tasks attached to the output terminal of this template task
+  /// \param[in] i Identifies which output terminal of this template task to select for sending
+  /// \param[in] key: the id of the task(s) receiving the value
+  /// \param[in] value: the value to send to the receiving task(s)
+  // clang-format on
+  template <typename keyT, typename valueT, ttg::Runtime Runtime = ttg::ttg_runtime>
+  inline detail::send_t send(size_t i, const keyT &key, valueT &&value) {
+    ttg::detail::value_copy_handler<Runtime> copy_handler;
+    auto *terminal_ptr = ttg::detail::get_out_terminal<keyT, valueT>(i, "ttg::device::send(i, key, value)");
+    return detail::send_t{detail::send_coro(key, copy_handler(std::forward<valueT>(value)), *terminal_ptr, copy_handler)};
+  }
 
-    // clang-format off
-    /// \brief Sends a task id and a value to the template tasks attached to the output terminal of this template task
-    /// \param[in] i Identifies which output terminal of this template task to select for sending
-    /// \param[in] key: the id of the task(s) receiving the value
-    /// \param[in] value: the value to send to the receiving task(s)
-    // clang-format on
-    template <typename keyT, typename valueT, ttg::Runtime Runtime = ttg::ttg_runtime>
-    inline detail::send_t send(size_t i, const keyT &key, valueT &&value) {
-      ttg::detail::value_copy_handler<Runtime> copy_handler;
-      auto *terminal_ptr = ttg::detail::get_out_terminal<keyT, valueT>(i, "ttg::device::send(i, key, value)");
-      return detail::send_t{detail::send_coro(key, copy_handler(std::forward<valueT>(value)), *terminal_ptr, copy_handler)};
-    }
+  // clang-format off
+  /// \brief Sends a task id and a value to the template tasks attached to the output terminal of this template task
+  /// \note this is provided to support `send<i>` with and without explicitly-passed terminal tuple
+  /// \tparam <i> Identifies which output terminal of this template task to select for sending
+  /// \param[in] key: the id of the task(s) receiving the value
+  /// \param[in] value: the value to send to the receiving task(s)
+  // clang-format on
+  template <size_t i, typename keyT, typename valueT>
+  inline auto send(const keyT &key, valueT &&value) {
+    return ttg::device::send(i, key, std::forward<valueT>(value));
+  }
 
-    // clang-format off
-    /// \brief Sends a task id and a value to the template tasks attached to the output terminal of this template task
-    /// \note this is provided to support `send<i>` with and without explicitly-passed terminal tuple
-    /// \tparam <i> Identifies which output terminal of this template task to select for sending
-    /// \param[in] key: the id of the task(s) receiving the value
-    /// \param[in] value: the value to send to the receiving task(s)
-    // clang-format on
-    template <size_t i, typename keyT, typename valueT>
-    inline auto send(const keyT &key, valueT &&value) {
-      return ttg::device::send(i, key, std::forward<valueT>(value));
-    }
 
+  template <typename valueT, ttg::Runtime Runtime = ttg::ttg_runtime>
+  inline detail::send_t sendv(std::size_t i, valueT &&value) {
+    auto *terminal_ptr = ttg::detail::get_out_terminal<void, valueT>(i, "ttg::device::send(i, key, value)");
+    ttg::detail::value_copy_handler<Runtime> copy_handler;
+    return detail::send_t{detail::sendv_coro(copy_handler(std::forward<valueT>(value)), *terminal_ptr, copy_handler)};
+  }
 
-    template <typename valueT, ttg::Runtime Runtime = ttg::ttg_runtime>
-    inline detail::send_t sendv(std::size_t i, valueT &&value) {
-      auto *terminal_ptr = ttg::detail::get_out_terminal<void, valueT>(i, "ttg::device::send(i, key, value)");
-      ttg::detail::value_copy_handler<Runtime> copy_handler;
-      return detail::send_t{detail::sendv_coro(copy_handler(std::forward<valueT>(value)), *terminal_ptr, copy_handler)};
-    }
+  template <typename Key, ttg::Runtime Runtime = ttg::ttg_runtime>
+  inline detail::send_t sendk(std::size_t i, const Key& key) {
+    auto *terminal_ptr = ttg::detail::get_out_terminal<Key, void>(i, "ttg::device::send(i, key, value)");
+    return detail::send_t{detail::sendk_coro(key, *terminal_ptr)};
+  }
 
-    template <typename Key, ttg::Runtime Runtime = ttg::ttg_runtime>
-    inline detail::send_t sendk(std::size_t i, const Key& key) {
-      auto *terminal_ptr = ttg::detail::get_out_terminal<Key, void>(i, "ttg::device::send(i, key, value)");
-      return detail::send_t{detail::sendk_coro(key, *terminal_ptr)};
-    }
+  template <ttg::Runtime Runtime = ttg::ttg_runtime>
+  inline detail::send_t send(std::size_t i) {
+    auto *terminal_ptr = ttg::detail::get_out_terminal<void, void>(i, "ttg::device::send(i, key, value)");
+    return detail::send_t{detail::send_coro(*terminal_ptr)};
+  }
 
-    template <ttg::Runtime Runtime = ttg::ttg_runtime>
-    inline detail::send_t send(std::size_t i) {
-      auto *terminal_ptr = ttg::detail::get_out_terminal<void, void>(i, "ttg::device::send(i, key, value)");
-      return detail::send_t{detail::send_coro(*terminal_ptr)};
-    }
 
+  template <std::size_t i, typename valueT, typename... out_keysT, typename... out_valuesT,
+            ttg::Runtime Runtime = ttg::ttg_runtime>
+  inline detail::send_t sendv(valueT &&value) {
+    return sendv(i, std::forward<valueT>(value));
+  }
 
-    template <std::size_t i, typename valueT, typename... out_keysT, typename... out_valuesT,
-              ttg::Runtime Runtime = ttg::ttg_runtime>
-    inline detail::send_t sendv(valueT &&value) {
-      return sendv(i, std::forward<valueT>(value));
-    }
+  template <size_t i, typename Key, ttg::Runtime Runtime = ttg::ttg_runtime>
+  inline detail::send_t sendk(const Key& key) {
+    return sendk(i, key);
+  }
 
-    template <size_t i, typename Key, ttg::Runtime Runtime = ttg::ttg_runtime>
-    inline detail::send_t sendk(const Key& key) {
-      return sendk(i, key);
-    }
+  template <size_t i, ttg::Runtime Runtime = ttg::ttg_runtime>
+  inline detail::send_t sendk() {
+    return send(i);
+  }
 
-    template <size_t i, ttg::Runtime Runtime = ttg::ttg_runtime>
-    inline detail::send_t sendk() {
-      return send(i);
-    }
+  namespace detail {
 
-    namespace detail {
-
-      template<typename T, typename Enabler = void>
-      struct broadcast_keylist_trait {
-        using type = T;
-      };
-
-      /* overload for iterable types that extracts the type of the first element */
-      template<typename T>
-      struct broadcast_keylist_trait<T, std::enable_if_t<ttg::meta::is_iterable_v<T>>> {
-        using key_type = decltype(*std::begin(std::get<0>(std::declval<T>())));
-      };
-
-      template <size_t KeyId, size_t I, size_t... Is, typename... RangesT, typename valueT,
-                typename... out_keysT, typename... out_valuesT>
-      inline void prepare_broadcast(const std::tuple<RangesT...> &keylists, valueT &&value,
-                                    std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
-        std::get<I>(t).prepare_send(std::get<KeyId>(keylists), std::forward<valueT>(value));
-        if constexpr (sizeof...(Is) > 0) {
-          prepare_broadcast<KeyId+1, Is...>(keylists, std::forward<valueT>(value), t);
-        }
-      }
+    template<typename T, typename Enabler = void>
+    struct broadcast_keylist_trait {
+      using type = T;
+    };
 
-      template <size_t KeyId, size_t I, size_t... Is, typename... RangesT, typename valueT,
-                typename... out_keysT, typename... out_valuesT>
-      inline void prepare_broadcast(const std::tuple<RangesT...> &keylists, valueT &&value) {
-        using key_t = typename broadcast_keylist_trait<
-                        std::tuple_element_t<KeyId, std::tuple<std::remove_reference_t<RangesT>...>>
-                      >::key_type;
-        auto *terminal_ptr = ttg::detail::get_out_terminal<key_t, valueT>(I, "ttg::device::broadcast(keylists, value)");
-        terminal_ptr->prepare_send(std::get<KeyId>(keylists), value);
-        if constexpr (sizeof...(Is) > 0) {
-          prepare_broadcast<KeyId+1, Is...>(keylists, std::forward<valueT>(value));
-        }
-      }
+    /* overload for iterable types that extracts the type of the first element */
+    template<typename T>
+    struct broadcast_keylist_trait<T, std::enable_if_t<ttg::meta::is_iterable_v<T>>> {
+      using key_type = decltype(*std::begin(std::get<0>(std::declval<T>())));
+    };
 
-      template <size_t KeyId, size_t I, size_t... Is, typename... RangesT, typename valueT,
-                typename... out_keysT, typename... out_valuesT>
-      inline void broadcast(const std::tuple<RangesT...> &keylists, valueT &&value,
-                                    std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
-        std::get<I>(t).broadcast(std::get<KeyId>(keylists), std::forward<valueT>(value));
-        if constexpr (sizeof...(Is) > 0) {
-          detail::broadcast<KeyId+1, Is...>(keylists, std::forward<valueT>(value), t);
-        }
+    template <size_t KeyId, size_t I, size_t... Is, typename... RangesT, typename valueT,
+              typename... out_keysT, typename... out_valuesT>
+    inline void prepare_broadcast(const std::tuple<RangesT...> &keylists, valueT &&value,
+                                  std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
+      std::get<I>(t).prepare_send(std::get<KeyId>(keylists), std::forward<valueT>(value));
+      if constexpr (sizeof...(Is) > 0) {
+        prepare_broadcast<KeyId+1, Is...>(keylists, std::forward<valueT>(value), t);
       }
+    }
 
-      template <size_t KeyId, size_t I, size_t... Is, typename... RangesT, typename valueT,
-                typename... out_keysT, typename... out_valuesT>
-      inline void broadcast(const std::tuple<RangesT...> &keylists, valueT &&value) {
-        using key_t = typename broadcast_keylist_trait<
-                        std::tuple_element_t<KeyId, std::tuple<std::remove_reference_t<RangesT>...>>
-                      >::key_type;
-        auto *terminal_ptr = ttg::detail::get_out_terminal<key_t, valueT>(I, "ttg::device::broadcast(keylists, value)");
-        terminal_ptr->broadcast(std::get<KeyId>(keylists), value);
-        if constexpr (sizeof...(Is) > 0) {
-          ttg::device::detail::broadcast<KeyId+1, Is...>(keylists, std::forward<valueT>(value));
-        }
+    template <size_t KeyId, size_t I, size_t... Is, typename... RangesT, typename valueT,
+              typename... out_keysT, typename... out_valuesT>
+    inline void prepare_broadcast(const std::tuple<RangesT...> &keylists, valueT &&value) {
+      using key_t = typename broadcast_keylist_trait<
+                      std::tuple_element_t<KeyId, std::tuple<std::remove_reference_t<RangesT>...>>
+                    >::key_type;
+      auto *terminal_ptr = ttg::detail::get_out_terminal<key_t, valueT>(I, "ttg::device::broadcast(keylists, value)");
+      terminal_ptr->prepare_send(std::get<KeyId>(keylists), value);
+      if constexpr (sizeof...(Is) > 0) {
+        prepare_broadcast<KeyId+1, Is...>(keylists, std::forward<valueT>(value));
       }
+    }
 
-      /* overload with explicit terminals */
-      template <size_t I, size_t... Is, typename RangesT, typename valueT,
-                typename... out_keysT, typename... out_valuesT,
-                ttg::Runtime Runtime = ttg::ttg_runtime>
-      inline send_coro_state
-      broadcast_coro(RangesT &&keylists, valueT &&value,
-                     std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t,
-                     ttg::detail::value_copy_handler<Runtime>&& ch) {
-        ttg::detail::value_copy_handler<Runtime> copy_handler = std::move(ch); // destroyed at the end of the coro
-        RangesT kl = std::forward<RangesT>(keylists); // capture the keylist(s)
-        if constexpr (ttg::meta::is_tuple_v<RangesT>) {
-          // treat as tuple
-          prepare_broadcast<0, I, Is...>(kl, std::forward<std::decay_t<decltype(value)>>(value), t);
-          co_await ttg::Void{}; // we'll come back once the task is done
-          ttg::device::detail::broadcast<0, I, Is...>(kl, std::forward<std::decay_t<decltype(value)>>(value), t);
-        } else if constexpr (!ttg::meta::is_tuple_v<RangesT>) {
-          // create a tie to the captured keylist
-          prepare_broadcast<0, I, Is...>(std::tie(kl), std::forward<std::decay_t<decltype(value)>>(value), t);
-          co_await ttg::Void{}; // we'll come back once the task is done
-          ttg::device::detail::broadcast<0, I, Is...>(std::tie(kl), std::forward<std::decay_t<decltype(value)>>(value), t);
-        }
+    template <size_t KeyId, size_t I, size_t... Is, typename... RangesT, typename valueT,
+              typename... out_keysT, typename... out_valuesT>
+    inline void broadcast(const std::tuple<RangesT...> &keylists, valueT &&value,
+                                  std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
+      std::get<I>(t).broadcast(std::get<KeyId>(keylists), std::forward<valueT>(value));
+      if constexpr (sizeof...(Is) > 0) {
+        detail::broadcast<KeyId+1, Is...>(keylists, std::forward<valueT>(value), t);
       }
+    }
 
-      /* overload with implicit terminals */
-      template <size_t I, size_t... Is, typename RangesT, typename valueT,
-                ttg::Runtime Runtime = ttg::ttg_runtime>
-      inline send_coro_state
-      broadcast_coro(RangesT &&keylists, valueT &&value,
-                     ttg::detail::value_copy_handler<Runtime>&& ch) {
-        ttg::detail::value_copy_handler<Runtime> copy_handler = std::move(ch); // destroyed at the end of the coro
-        RangesT kl = std::forward<RangesT>(keylists); // capture the keylist(s)
-        if constexpr (ttg::meta::is_tuple_v<RangesT>) {
-          // treat as tuple
-          static_assert(sizeof...(Is)+1 == std::tuple_size_v<RangesT>,
-                        "Size of keylist tuple must match the number of output terminals");
-          prepare_broadcast<0, I, Is...>(kl, std::forward<std::decay_t<decltype(value)>>(value));
-          co_await ttg::Void{}; // we'll come back once the task is done
-          ttg::device::detail::broadcast<0, I, Is...>(kl, std::forward<std::decay_t<decltype(value)>>(value));
-        } else if constexpr (!ttg::meta::is_tuple_v<RangesT>) {
-          // create a tie to the captured keylist
-          prepare_broadcast<0, I, Is...>(std::tie(kl), std::forward<std::decay_t<decltype(value)>>(value));
-          co_await ttg::Void{}; // we'll come back once the task is done
-          ttg::device::detail::broadcast<0, I, Is...>(std::tie(kl), std::forward<std::decay_t<decltype(value)>>(value));
-        }
+    template <size_t KeyId, size_t I, size_t... Is, typename... RangesT, typename valueT,
+              typename... out_keysT, typename... out_valuesT>
+    inline void broadcast(const std::tuple<RangesT...> &keylists, valueT &&value) {
+      using key_t = typename broadcast_keylist_trait<
+                      std::tuple_element_t<KeyId, std::tuple<std::remove_reference_t<RangesT>...>>
+                    >::key_type;
+      auto *terminal_ptr = ttg::detail::get_out_terminal<key_t, valueT>(I, "ttg::device::broadcast(keylists, value)");
+      terminal_ptr->broadcast(std::get<KeyId>(keylists), value);
+      if constexpr (sizeof...(Is) > 0) {
+        ttg::device::detail::broadcast<KeyId+1, Is...>(keylists, std::forward<valueT>(value));
       }
-    }  // namespace detail
+    }
 
-    /* overload with explicit terminals and keylist passed by const reference */
-    template <size_t I, size_t... Is, typename rangeT, typename valueT, typename... out_keysT, typename... out_valuesT,
+    /* overload with explicit terminals */
+    template <size_t I, size_t... Is, typename RangesT, typename valueT,
+              typename... out_keysT, typename... out_valuesT,
               ttg::Runtime Runtime = ttg::ttg_runtime>
-    [[nodiscard]]
-    inline detail::send_t broadcast(rangeT &&keylist,
-                                    valueT &&value,
-                                    std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
-      ttg::detail::value_copy_handler<Runtime> copy_handler;
-      return detail::send_t{
-             detail::broadcast_coro<I, Is...>(std::forward<rangeT>(keylist),
-                                              copy_handler(std::forward<valueT>(value)),
-                                              t, std::move(copy_handler))};
+    inline send_coro_state
+    broadcast_coro(RangesT &&keylists, valueT &&value,
+                    std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t,
+                    ttg::detail::value_copy_handler<Runtime>&& ch) {
+      ttg::detail::value_copy_handler<Runtime> copy_handler = std::move(ch); // destroyed at the end of the coro
+      RangesT kl = std::forward<RangesT>(keylists); // capture the keylist(s)
+      if constexpr (ttg::meta::is_tuple_v<RangesT>) {
+        // treat as tuple
+        prepare_broadcast<0, I, Is...>(kl, std::forward<std::decay_t<decltype(value)>>(value), t);
+        co_await ttg::Void{}; // we'll come back once the task is done
+        ttg::device::detail::broadcast<0, I, Is...>(kl, std::forward<std::decay_t<decltype(value)>>(value), t);
+      } else if constexpr (!ttg::meta::is_tuple_v<RangesT>) {
+        // create a tie to the captured keylist
+        prepare_broadcast<0, I, Is...>(std::tie(kl), std::forward<std::decay_t<decltype(value)>>(value), t);
+        co_await ttg::Void{}; // we'll come back once the task is done
+        ttg::device::detail::broadcast<0, I, Is...>(std::tie(kl), std::forward<std::decay_t<decltype(value)>>(value), t);
+      }
     }
 
-    /* overload with implicit terminals and keylist passed by const reference */
-    template <size_t i, typename rangeT, typename valueT,
+    /* overload with implicit terminals */
+    template <size_t I, size_t... Is, typename RangesT, typename valueT,
               ttg::Runtime Runtime = ttg::ttg_runtime>
-    inline detail::send_t broadcast(rangeT &&keylist, valueT &&value) {
-      ttg::detail::value_copy_handler<Runtime> copy_handler;
-      return detail::send_t{broadcast_coro<i>(std::tie(keylist), copy_handler(std::forward<valueT>(value)),
-                                              std::move(copy_handler))};
+    inline send_coro_state
+    broadcast_coro(RangesT &&keylists, valueT &&value,
+                    ttg::detail::value_copy_handler<Runtime>&& ch) {
+      ttg::detail::value_copy_handler<Runtime> copy_handler = std::move(ch); // destroyed at the end of the coro
+      RangesT kl = std::forward<RangesT>(keylists); // capture the keylist(s)
+      if constexpr (ttg::meta::is_tuple_v<RangesT>) {
+        // treat as tuple
+        static_assert(sizeof...(Is)+1 == std::tuple_size_v<RangesT>,
+                      "Size of keylist tuple must match the number of output terminals");
+        prepare_broadcast<0, I, Is...>(kl, std::forward<std::decay_t<decltype(value)>>(value));
+        co_await ttg::Void{}; // we'll come back once the task is done
+        ttg::device::detail::broadcast<0, I, Is...>(kl, std::forward<std::decay_t<decltype(value)>>(value));
+      } else if constexpr (!ttg::meta::is_tuple_v<RangesT>) {
+        // create a tie to the captured keylist
+        prepare_broadcast<0, I, Is...>(std::tie(kl), std::forward<std::decay_t<decltype(value)>>(value));
+        co_await ttg::Void{}; // we'll come back once the task is done
+        ttg::device::detail::broadcast<0, I, Is...>(std::tie(kl), std::forward<std::decay_t<decltype(value)>>(value));
+      }
     }
+  }  // namespace detail
 
-    template<typename... Args, ttg::Runtime Runtime = ttg::ttg_runtime>
-    [[nodiscard]]
-    std::vector<device::detail::send_t> forward(Args&&... args) {
-      // TODO: check the cost of this!
-      return std::vector{std::forward<Args>(args)...};
-    }
+  /* overload with explicit terminals and keylist passed by const reference */
+  template <size_t I, size_t... Is, typename rangeT, typename valueT, typename... out_keysT, typename... out_valuesT,
+            ttg::Runtime Runtime = ttg::ttg_runtime>
+  [[nodiscard]]
+  inline detail::send_t broadcast(rangeT &&keylist,
+                                  valueT &&value,
+                                  std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
+    ttg::detail::value_copy_handler<Runtime> copy_handler;
+    return detail::send_t{
+            detail::broadcast_coro<I, Is...>(std::forward<rangeT>(keylist),
+                                            copy_handler(std::forward<valueT>(value)),
+                                            t, std::move(copy_handler))};
+  }
 
-  } // namespace device
+  /* overload with implicit terminals and keylist passed by const reference */
+  template <size_t i, typename rangeT, typename valueT,
+            ttg::Runtime Runtime = ttg::ttg_runtime>
+  inline detail::send_t broadcast(rangeT &&keylist, valueT &&value) {
+    ttg::detail::value_copy_handler<Runtime> copy_handler;
+    return detail::send_t{broadcast_coro<i>(std::tie(keylist), copy_handler(std::forward<valueT>(value)),
+                                            std::move(copy_handler))};
+  }
+
+  template<typename... Args, ttg::Runtime Runtime = ttg::ttg_runtime>
+  [[nodiscard]]
+  std::vector<device::detail::send_t> forward(Args&&... args) {
+    // TODO: check the cost of this!
+    return std::vector{std::forward<Args>(args)...};
+  }
 
   /*******************************************
    * Device task promise and coroutine handle
    *******************************************/
 
-
-  struct device_task_promise_type;
-
-  using device_task_handle_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<device_task_promise_type>;
+  namespace detail {
+    // fwd-decl
+    struct device_task_promise_type;
+    // base type for ttg::device::Task
+    using device_task_handle_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<device_task_promise_type>;
+  } // namespace detail
 
   /// task that can be resumed after some events occur
-  struct device_task : public device_task_handle_type {
-    using base_type = device_task_handle_type;
+  struct Task : public detail::device_task_handle_type {
+    using base_type = detail::device_task_handle_type;
 
     /// these are members mandated by the promise_type concept
     ///@{
 
-    using promise_type = device_task_promise_type;
+    using promise_type = detail::device_task_promise_type;
 
     ///@}
 
-    device_task(base_type base) : base_type(std::move(base)) {}
+    Task(base_type base) : base_type(std::move(base)) {}
 
     base_type& handle() { return *this; }
 
@@ -439,176 +437,113 @@ namespace ttg {
     inline bool completed();
   };
 
-  /* The promise type that stores the views provided by the
-   * application task coroutine on the first co_yield. It subsequently
-   * tracks the state of the task when it moves from waiting for transfers
-   * to waiting for the submitted kernel to complete. */
-  struct device_task_promise_type {
-
-    /* do not suspend the coroutine on first invocation, we want to run
-     * the coroutine immediately and suspend when we get the device transfers.
-     */
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_never initial_suspend() {
-      m_state = TTG_DEVICE_CORO_INIT;
-      return {};
-    }
-
-    /* suspend the coroutine at the end of the execution
-     * so we can access the promise.
-     * TODO: necessary? maybe we can save one suspend here
-     */
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_always final_suspend() noexcept {
-      m_state = TTG_DEVICE_CORO_COMPLETE;
-      return {};
-    }
+  namespace detail {
 
-#if 0
-    /* waiting for transfers to complete should always suspend
-     * TODO: as an optimization, we could check here if all data
-     *       is already available and avoid suspending...
-     */
-    template<typename... Views>
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_always yield_value(std::tuple<Views&...> &views) {
-      /* gather all the views (host object + view spans, type-punned) into a vector */
-      constexpr static std::size_t view_count = std::tuple_size_v<std::tuple<Views...>>;
-      std::cout << "yield_value: views" << std::endl;
-      m_spans.clear(); // in case we ever come back here
-      m_spans.reserve(view_count);
-      if constexpr(view_count > 0) {
-        auto unpack_lambda = [&](Views&... view){
-                                ((m_spans.push_back(device_obj_view(&view.get_host_object(),
-                                                                    view.view_spans()))),
-                                  ...);
-                              };
-        std::apply(unpack_lambda, views);
+    /* The promise type that stores the views provided by the
+    * application task coroutine on the first co_yield. It subsequently
+    * tracks the state of the task when it moves from waiting for transfers
+    * to waiting for the submitted kernel to complete. */
+    struct device_task_promise_type {
+
+      /* do not suspend the coroutine on first invocation, we want to run
+      * the coroutine immediately and suspend when we get the device transfers.
+      */
+      TTG_CXX_COROUTINE_NAMESPACE::suspend_never initial_suspend() {
+        m_state = ttg::device::detail::TTG_DEVICE_CORO_INIT;
+        return {};
       }
-      m_state = TTG_DEVICE_CORO_WAIT_TRANSFER;
-      return {};
-    }
-
-    /* convenience-function to yield a single view */
-    template<typename HostT, typename... DeviceViewTs>
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_always yield_value(View<HostT, DeviceViewTs...> &view) {
-      auto tmp_tuple = std::tie(view);
-      return yield_value(tmp_tuple);
-    }
-
-    /* convenience-function to yield a single view */
-    template<typename HostT, typename... DeviceViewTs>
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_always yield_value(PersistentView<HostT, DeviceViewTs...> &view) {
-      auto tmp_tuple = std::tie(view);
-      return yield_value(tmp_tuple);
-    }
-#endif // 0
 
-    /* waiting for the kernel to complete should always suspend */
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_always yield_value(device_op_wait_kernel) {
-      std::cout << "yield_value: device_op_wait_kernel" << std::endl;
-      m_state = TTG_DEVICE_CORO_WAIT_KERNEL;
-      return {};
-    }
-
-    /* Allow co_await on a tuple */
-    template<typename... Views>
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(std::tuple<Views&...> &views) {
-      return yield_value(views);
-    }
-
-#if 0
-    /* convenience-function to await a single view */
-    template<typename HostT, typename... DeviceViewTs>
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(View<HostT, DeviceViewTs...> &view) {
-      auto tmp_tuple = std::tie(view);
-      return yield_value(tmp_tuple);
-    }
+      /* suspend the coroutine at the end of the execution
+      * so we can access the promise.
+      * TODO: necessary? maybe we can save one suspend here
+      */
+      TTG_CXX_COROUTINE_NAMESPACE::suspend_always final_suspend() noexcept {
+        m_state = ttg::device::detail::TTG_DEVICE_CORO_COMPLETE;
+        return {};
+      }
 
-    /* convenience-function to await a single view */
-    template<typename HostT, typename... DeviceViewTs>
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(PersistentView<HostT, DeviceViewTs...> &view) {
-      auto tmp_tuple = std::tie(view);
-      return yield_value(tmp_tuple);
-    }
-#endif // 0
+      /* Allow co_await on a tuple */
+      template<typename... Views>
+      TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(std::tuple<Views&...> &views) {
+        return yield_value(views);
+      }
 
-    /* co_await for the kernel to complete should always suspend */
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(device_op_wait_kernel) {
-      std::cout << "yield_value: device_op_wait_kernel" << std::endl;
-      m_state = TTG_DEVICE_CORO_WAIT_KERNEL;
-      return {};
-    }
+      template<typename... Ts>
+      TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(detail::to_device_t<Ts...>&& a) {
+        bool need_transfer = !(TTG_IMPL_NS::register_device_memory(a.ties));
+        /* TODO: are we allowed to not suspend here and launch the kernel directly? */
+        m_state = ttg::device::detail::TTG_DEVICE_CORO_WAIT_TRANSFER;
+        return {};
+      }
 
-    template<typename... Ts>
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(detail::to_device_t<Ts...>&& a) {
-      bool need_transfer = !(TTG_IMPL_NS::register_device_memory(a.ties));
-      /* TODO: are we allowed to not suspend here and launch the kernel directly? */
-      m_state = TTG_DEVICE_CORO_WAIT_TRANSFER;
-      return {};
-    }
+      template<typename... Ts>
+      auto await_transform(detail::wait_kernel_t<Ts...>&& a) {
+        //std::cout << "yield_value: wait_kernel_t" << std::endl;
+        if constexpr (sizeof...(Ts) > 0) {
+          TTG_IMPL_NS::mark_device_out(a.ties);
+        }
+        m_state = ttg::device::detail::TTG_DEVICE_CORO_WAIT_KERNEL;
+        return a;
+      }
 
-    template<typename... Ts>
-    auto await_transform(detail::wait_kernel_t<Ts...>&& a) {
-      //std::cout << "yield_value: wait_kernel_t" << std::endl;
-      if constexpr (sizeof...(Ts) > 0) {
-        TTG_IMPL_NS::mark_device_out(a.ties);
+      TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(std::vector<device::detail::send_t>&& v) {
+        m_sends = std::forward<std::vector<device::detail::send_t>>(v);
+        m_state = ttg::device::detail::TTG_DEVICE_CORO_SENDOUT;
+        return {};
       }
-      m_state = TTG_DEVICE_CORO_WAIT_KERNEL;
-      return a;
-    }
 
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(std::vector<device::detail::send_t>&& v) {
-      m_sends = std::forward<std::vector<device::detail::send_t>>(v);
-      m_state = TTG_DEVICE_CORO_SENDOUT;
-      return {};
-    }
+      TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(device::detail::send_t&& v) {
+        m_sends.clear();
+        m_sends.push_back(std::forward<device::detail::send_t>(v));
+        m_state = ttg::device::detail::TTG_DEVICE_CORO_SENDOUT;
+        return {};
+      }
 
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(device::detail::send_t&& v) {
-      m_sends.clear();
-      m_sends.push_back(std::forward<device::detail::send_t>(v));
-      m_state = TTG_DEVICE_CORO_SENDOUT;
-      return {};
-    }
+      void return_void() {
+        m_state = ttg::device::detail::TTG_DEVICE_CORO_COMPLETE;
+      }
 
-    void return_void() {
-      m_state = TTG_DEVICE_CORO_COMPLETE;
-    }
+      bool complete() const {
+        return m_state == ttg::device::detail::TTG_DEVICE_CORO_COMPLETE;
+      }
 
-    bool complete() const {
-      return m_state == TTG_DEVICE_CORO_COMPLETE;
-    }
+      ttg::device::Task get_return_object() { return {detail::device_task_handle_type::from_promise(*this)}; }
 
-    device_task get_return_object() { return device_task{device_task_handle_type::from_promise(*this)}; }
+      void unhandled_exception() {
+        std::cerr << "Task coroutine caught an unhandled exception!" << std::endl;
+        throw; // fwd
+      }
 
-    void unhandled_exception() {
-      std::cerr << "Task coroutine caught an unhandled exception!" << std::endl;
-      throw; // fwd
-    }
+      //using iterator = std::vector<device_obj_view>::iterator;
 
-    //using iterator = std::vector<device_obj_view>::iterator;
+      /* execute all pending send and broadcast operations */
+      void do_sends() {
+        for (auto& send : m_sends) {
+          send.coro();
+        }
+        m_sends.clear();
+      }
 
-    /* execute all pending send and broadcast operations */
-    void do_sends() {
-      for (auto& send : m_sends) {
-        send.coro();
+      auto state() {
+        return m_state;
       }
-      m_sends.clear();
-    }
 
-    auto state() {
-      return m_state;
-    }
+    private:
+      std::vector<device::detail::send_t> m_sends;
+      ttg_device_coro_state m_state = ttg::device::detail::TTG_DEVICE_CORO_STATE_NONE;
 
-  private:
-    std::vector<device::detail::send_t> m_sends;
-    ttg_device_coro_state m_state = TTG_DEVICE_CORO_STATE_NONE;
+    };
 
-  };
+  } // namespace detail
 
-  bool device_task::completed() { return base_type::promise().state() == TTG_DEVICE_CORO_COMPLETE; }
+  bool Task::completed() { return base_type::promise().state() == ttg::device::detail::TTG_DEVICE_CORO_COMPLETE; }
 
   struct device_wait_kernel
   { };
 
 
+  /* NOTE: below is preliminary for reductions on the device, which is not available yet */
+#if 0
   /**************************
    * Device reduction coros *
    **************************/
@@ -643,25 +578,25 @@ namespace ttg {
 
 
   /* The promise type that stores the views provided by the
-   * application task coroutine on the first co_yield. It subsequently
-   * tracks the state of the task when it moves from waiting for transfers
-   * to waiting for the submitted kernel to complete. */
+  * application task coroutine on the first co_yield. It subsequently
+  * tracks the state of the task when it moves from waiting for transfers
+  * to waiting for the submitted kernel to complete. */
   struct device_reducer_promise_type {
 
     /* do not suspend the coroutine on first invocation, we want to run
-     * the coroutine immediately and suspend when we get the device transfers.
-     */
+    * the coroutine immediately and suspend when we get the device transfers.
+    */
     TTG_CXX_COROUTINE_NAMESPACE::suspend_never initial_suspend() {
-      m_state = TTG_DEVICE_CORO_INIT;
+      m_state = ttg::device::detail::TTG_DEVICE_CORO_INIT;
       return {};
     }
 
     /* suspend the coroutine at the end of the execution
-     * so we can access the promise.
-     * TODO: necessary? maybe we can save one suspend here
-     */
+    * so we can access the promise.
+    * TODO: necessary? maybe we can save one suspend here
+    */
     TTG_CXX_COROUTINE_NAMESPACE::suspend_always final_suspend() noexcept {
-      m_state = TTG_DEVICE_CORO_COMPLETE;
+      m_state = ttg::device::detail::TTG_DEVICE_CORO_COMPLETE;
       return {};
     }
 
@@ -669,16 +604,16 @@ namespace ttg {
     TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(detail::to_device_t<Ts...>&& a) {
       bool need_transfer = !(TTG_IMPL_NS::register_device_memory(a.ties));
       /* TODO: are we allowed to not suspend here and launch the kernel directly? */
-      m_state = TTG_DEVICE_CORO_WAIT_TRANSFER;
+      m_state = ttg::device::detail::TTG_DEVICE_CORO_WAIT_TRANSFER;
       return {};
     }
 
     void return_void() {
-      m_state = TTG_DEVICE_CORO_COMPLETE;
+      m_state = ttg::device::detail::TTG_DEVICE_CORO_COMPLETE;
     }
 
     bool complete() const {
-      return m_state == TTG_DEVICE_CORO_COMPLETE;
+      return m_state == ttg::device::detail::TTG_DEVICE_CORO_COMPLETE;
     }
 
     device_reducer get_return_object() { return device_reducer{device_reducer_handle_type::from_promise(*this)}; }
@@ -691,12 +626,13 @@ namespace ttg {
 
 
   private:
-    ttg_device_coro_state m_state = TTG_DEVICE_CORO_STATE_NONE;
+    ttg::device::detail::ttg_device_coro_state m_state = ttg::device::detail::TTG_DEVICE_CORO_STATE_NONE;
 
   };
 
-  bool device_reducer::completed() { return base_type::promise().state() == TTG_DEVICE_CORO_COMPLETE; }
+  bool device_reducer::completed() { return base_type::promise().state() == ttg::device::detail::TTG_DEVICE_CORO_COMPLETE; }
+#endif // 0
 
-}  // namespace ttg
+}  // namespace ttg::devie
 
 #endif // TTG_DEVICE_TASK_H
\ No newline at end of file
diff --git a/ttg/ttg/make_tt.h b/ttg/ttg/make_tt.h
index 8ba7f39b0..4d1f060b4 100644
--- a/ttg/ttg/make_tt.h
+++ b/ttg/ttg/make_tt.h
@@ -153,8 +153,8 @@ class CallableWrapTTArgs
       std::conditional_t<std::is_same_v<returnT, ttg::resumable_task>,
                          ttg::coroutine_handle<>,
 #ifdef TTG_HAVE_DEVICE
-                         std::conditional_t<std::is_same_v<returnT, ttg::device_task>,
-                                            ttg::device_task::base_type,
+                         std::conditional_t<std::is_same_v<returnT, ttg::device::Task>,
+                                            ttg::device::Task::base_type,
                                             void>
 #else  // TTG_HAVE_DEVICE
                            void
@@ -188,8 +188,8 @@ class CallableWrapTTArgs
         return coro_handle;
       } else
 #ifdef TTG_HAVE_DEVICE
-          if constexpr (std::is_same_v<returnT, ttg::device_task>) {
-        ttg::device_task::base_type coro_handle = ret;
+          if constexpr (std::is_same_v<returnT, ttg::device::Task>) {
+        ttg::device::Task::base_type coro_handle = ret;
         return coro_handle;
       }
 #else  // TTG_HAVE_DEVICE
@@ -197,7 +197,7 @@ class CallableWrapTTArgs
 #endif  // TTG_HAVE_DEVICE
       if constexpr (!(std::is_same_v<returnT, ttg::resumable_task>
 #ifdef TTG_HAVE_DEVICE
-          || std::is_same_v<returnT, ttg::device_task>
+          || std::is_same_v<returnT, ttg::device::Task>
 #endif  // TTG_HAVE_DEVICE
               ))
 #endif
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 3355f648a..bed06d535 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1347,7 +1347,7 @@ namespace ttg_parsec {
 
       task_t *task = (task_t*)gpu_task->ec;
       // get the device task from the coroutine handle
-      ttg::device_task dev_task = ttg::device_task_handle_type::from_address(task->suspended_task_address);
+      ttg::device::Task dev_task = ttg::device::detail::device_task_handle_type::from_address(task->suspended_task_address);
 
       task->dev_ptr->stream = gpu_stream;
 
@@ -1357,8 +1357,8 @@ namespace ttg_parsec {
       auto dev_data = dev_task.promise();
 
       /* we should still be waiting for the transfer to complete */
-      assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_TRANSFER ||
-             dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_KERNEL);
+      assert(dev_data.state() == ttg::device::detail::TTG_DEVICE_CORO_WAIT_TRANSFER ||
+             dev_data.state() == ttg::device::detail::TTG_DEVICE_CORO_WAIT_KERNEL);
 
 #if defined(PARSEC_HAVE_DEV_CUDA_SUPPORT) && defined(TTG_HAVE_CUDA)
       {
@@ -1394,15 +1394,15 @@ namespace ttg_parsec {
       int rc = PARSEC_HOOK_RETURN_DONE;
       if (nullptr != task->suspended_task_address) {
         /* Get a new handle for the promise*/
-        dev_task = ttg::device_task_handle_type::from_address(task->suspended_task_address);
+        dev_task = ttg::device::detail::device_task_handle_type::from_address(task->suspended_task_address);
         dev_data = dev_task.promise();
 
-        assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_KERNEL ||
-               dev_data.state() == ttg::TTG_DEVICE_CORO_SENDOUT     ||
-               dev_data.state() == ttg::TTG_DEVICE_CORO_COMPLETE);
+        assert(dev_data.state() == ttg::device::detail::TTG_DEVICE_CORO_WAIT_KERNEL ||
+               dev_data.state() == ttg::device::detail::TTG_DEVICE_CORO_SENDOUT     ||
+               dev_data.state() == ttg::device::detail::TTG_DEVICE_CORO_COMPLETE);
 
-        if (ttg::TTG_DEVICE_CORO_SENDOUT == dev_data.state() ||
-            ttg::TTG_DEVICE_CORO_COMPLETE == dev_data.state()) {
+        if (ttg::device::detail::TTG_DEVICE_CORO_SENDOUT == dev_data.state() ||
+            ttg::device::detail::TTG_DEVICE_CORO_COMPLETE == dev_data.state()) {
           /* the task started sending so we won't come back here */
           //std::cout << "device_static_submit task " << task << " complete" << std::endl;
         } else {
@@ -1479,13 +1479,13 @@ namespace ttg_parsec {
       /* when we come back here, the flows in gpu_task are set (see register_device_memory) */
 
       // get the device task from the coroutine handle
-      auto dev_task = ttg::device_task_handle_type::from_address(task->suspended_task_address);
+      auto dev_task = ttg::device::detail::device_task_handle_type::from_address(task->suspended_task_address);
 
       // get the promise which contains the views
-      ttg::device_task_promise_type& dev_data = dev_task.promise();
+      ttg::device::detail::device_task_promise_type& dev_data = dev_task.promise();
 
       /* for now make sure we're waiting for transfers and the coro hasn't skipped this step */
-      assert(dev_data.state() == ttg::TTG_DEVICE_CORO_WAIT_TRANSFER);
+      assert(dev_data.state() == ttg::device::detail::TTG_DEVICE_CORO_WAIT_TRANSFER);
 
       /* set up a temporary task-class to correctly specify the flows */
       parsec_task_class_t tc = *task->parsec_task.task_class;
@@ -1603,7 +1603,7 @@ namespace ttg_parsec {
       }
       else {  // resume the suspended coroutine
 #ifdef TTG_HAVE_DEVICE
-        auto coro = static_cast<ttg::device_task>(ttg::device_task_handle_type::from_address(suspended_task_address));
+        ttg::device::Task coro = ttg::device::detail::device_task_handle_type::from_address(suspended_task_address);
         assert(detail::parsec_ttg_caller == nullptr);
         detail::parsec_ttg_caller = static_cast<detail::parsec_ttg_task_base_t*>(task);
         // TODO: unify the outputs tls handling
@@ -3554,16 +3554,16 @@ ttg::abort();  // should not happen
         //increment_data_versions(task, std::make_index_sequence<std::tuple_size_v<typename TT::input_values_tuple_type>>{});
 
         // get the device task from the coroutine handle
-        auto dev_task = ttg::device_task_handle_type::from_address(task->suspended_task_address);
+        auto dev_task = ttg::device::detail::device_task_handle_type::from_address(task->suspended_task_address);
 
         // get the promise which contains the views
         auto dev_data = dev_task.promise();
 
         /* for now make sure we're waiting for the kernel to complete and the coro hasn't skipped this step */
-        assert(dev_data.state() == ttg::TTG_DEVICE_CORO_SENDOUT);
+        assert(dev_data.state() == ttg::device::detail::TTG_DEVICE_CORO_SENDOUT);
 
         /* execute the sends we stored */
-        if (dev_data.state() == ttg::TTG_DEVICE_CORO_SENDOUT) {
+        if (dev_data.state() == ttg::device::detail::TTG_DEVICE_CORO_SENDOUT) {
           /* set the current task, needed inside the sends */
           detail::parsec_ttg_caller = task;
           dev_data.do_sends();

From 3bf1e1605e2d14c41f1de62701b5da18d533b916 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Tue, 19 Dec 2023 21:24:33 +0000
Subject: [PATCH 193/259] Add device-related fwd-decl to madness backend

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/buffer.h               | 18 ++----------
 ttg/ttg/device/task.h          |  4 +--
 ttg/ttg/devicescratch.h        | 15 ----------
 ttg/ttg/madness/fwd.h          | 26 ++++++++++++++++++
 ttg/ttg/parsec/buffer.h        | 15 ----------
 ttg/ttg/parsec/devicefunc.h    |  4 +--
 ttg/ttg/parsec/devicescratch.h | 15 ----------
 ttg/ttg/parsec/fwd.h           |  6 ++--
 ttg/ttg/parsec/import.h        |  1 +
 ttg/ttg/parsec/ptr.h           | 50 +++++++++++++++++-----------------
 ttg/ttg/parsec/ttg.h           |  6 ++--
 ttg/ttg/ptr.h                  | 16 +----------
 ttg/ttg/util/meta.h            | 50 ++++++++++++++++++++++++++++++++++
 13 files changed, 115 insertions(+), 111 deletions(-)

diff --git a/ttg/ttg/buffer.h b/ttg/ttg/buffer.h
index 5c8bbfed5..3067968ff 100644
--- a/ttg/ttg/buffer.h
+++ b/ttg/ttg/buffer.h
@@ -4,28 +4,14 @@
 #include <memory>
 #include "ttg/impl_selector.h"
 
+#if defined(TTG_IMPL_DEVICE_SUPPORT)
 
 namespace ttg {
 
 template<typename T, typename Allocator = std::allocator<T>>
 using Buffer = TTG_IMPL_NS::Buffer<T, Allocator>;
 
-namespace detail {
-  template<typename T>
-  struct is_buffer : std::false_type
-  { };
-
-  template<typename T, typename A>
-  struct is_buffer<ttg::Buffer<T, A>> : std::true_type
-  { };
-
-  template<typename T>
-  constexpr bool is_buffer_v = is_buffer<T>::value;
-
-  static_assert(is_buffer_v<ttg::Buffer<double>>);
-  static_assert(is_buffer_v<TTG_IMPL_NS::Buffer<double>>);
-} // namespace detail
-
 } // namespace ttg
 
+#endif // TTG_IMPL_DEVICE_SUPPORT
 #endif // TTG_buffer_H
\ No newline at end of file
diff --git a/ttg/ttg/device/task.h b/ttg/ttg/device/task.h
index 48ded0b4b..8f63ea564 100644
--- a/ttg/ttg/device/task.h
+++ b/ttg/ttg/device/task.h
@@ -70,8 +70,8 @@ namespace ttg::device {
   template<typename... Buffers>
   [[nodiscard]]
   inline auto wait(Buffers&&... args) {
-    static_assert(((ttg::detail::is_buffer_v<std::decay_t<Buffers>>
-                    ||ttg::detail::is_devicescratch_v<std::decay_t<Buffers>>)&&...),
+    static_assert(((ttg::meta::is_buffer_v<std::decay_t<Buffers>>
+                    ||ttg::meta::is_devicescratch_v<std::decay_t<Buffers>>)&&...),
                   "Only ttg::Buffer and ttg::devicescratch can be waited on!");
     return detail::wait_kernel_t<std::remove_reference_t<Buffers>...>{std::tie(std::forward<Buffers>(args)...)};
   }
diff --git a/ttg/ttg/devicescratch.h b/ttg/ttg/devicescratch.h
index 729361fa2..3e528d01c 100644
--- a/ttg/ttg/devicescratch.h
+++ b/ttg/ttg/devicescratch.h
@@ -14,21 +14,6 @@ auto make_scratch(T* val, ttg::scope scope, std::size_t count = 1) {
   return devicescratch<T>(val, scope, count);
 }
 
-namespace detail {
-
-  template<typename T>
-  struct is_devicescratch : std::false_type
-  { };
-
-  template<typename T>
-  struct is_devicescratch<ttg::devicescratch<T>> : std::true_type
-  { };
-
-  template<typename T>
-  constexpr bool is_devicescratch_v = is_devicescratch<T>::value;
-
-} // namespace detail
-
 } // namespace ttg
 
 #endif // TTG_DEVICESCRATCH_H
\ No newline at end of file
diff --git a/ttg/ttg/madness/fwd.h b/ttg/ttg/madness/fwd.h
index af050f9a1..66c6bacba 100644
--- a/ttg/ttg/madness/fwd.h
+++ b/ttg/ttg/madness/fwd.h
@@ -47,6 +47,32 @@ namespace ttg_madness {
   template <typename T>
   inline void ttg_broadcast(ttg::World world, T &data, int source_rank);
 
+
+  /* device definitions, not currently provided by this impl */
+  template<typename T, typename Allocator>
+  struct Buffer;
+
+  template<typename T>
+  struct Ptr;
+
+  template<typename T>
+  struct devicescratch;
+
+  template<typename T, typename... Args>
+  Ptr<T> make_ptr(Args&&... args);
+
+  template<typename T>
+  auto get_ptr(T&& obj);
+
+  template<typename... Views>
+  inline bool register_device_memory(std::tuple<Views&...> &views);
+
+  template<typename... Buffer>
+  inline void post_device_out(std::tuple<Buffer&...> &b);
+
+  template<typename... Buffer>
+  inline void mark_device_out(std::tuple<Buffer&...> &b);
+
 }  // namespace ttg_madness
 
 #endif  // TTG_MADNESS_FWD_H
diff --git a/ttg/ttg/parsec/buffer.h b/ttg/ttg/parsec/buffer.h
index f9f185de3..7325c0214 100644
--- a/ttg/ttg/parsec/buffer.h
+++ b/ttg/ttg/parsec/buffer.h
@@ -403,21 +403,6 @@ struct Buffer : public detail::ttg_parsec_data_wrapper_t
 
 };
 
-template<typename T>
-struct is_buffer : std::false_type
-{ };
-
-template<typename T, typename A>
-struct is_buffer<Buffer<T, A>> : std::true_type
-{ };
-
-template<typename T, typename A>
-struct is_buffer<const Buffer<T, A>> : std::true_type
-{ };
-
-template<typename T>
-constexpr static const bool is_buffer_v = is_buffer<T>::value;
-
 namespace detail {
   template<typename T, typename A>
   parsec_data_t* get_parsec_data(const ttg_parsec::Buffer<T, A>& db) {
diff --git a/ttg/ttg/parsec/devicefunc.h b/ttg/ttg/parsec/devicefunc.h
index c0710776d..d4a488673 100644
--- a/ttg/ttg/parsec/devicefunc.h
+++ b/ttg/ttg/parsec/devicefunc.h
@@ -30,7 +30,7 @@ namespace ttg_parsec {
 
       auto& view = std::get<I>(views);
       bool is_current = false;
-      static_assert(ttg::is_buffer_v<view_type> || ttg_parsec::is_devicescratch_v<view_type>);
+      static_assert(ttg::meta::is_buffer_v<view_type> || ttg::meta::is_devicescratch_v<view_type>);
       /* get_parsec_data is overloaded for buffer and devicescratch */
       parsec_data_t* data = detail::get_parsec_data(view);
       /* TODO: check whether the device is current */
@@ -41,7 +41,7 @@ namespace ttg_parsec {
         //if (flows[I].flow_flags != PARSEC_FLOW_ACCESS_RW) {
           access = PARSEC_FLOW_ACCESS_READ;
         //}
-      } else if constexpr (ttg_parsec::is_devicescratch_v<view_type>) {
+      } else if constexpr (ttg::meta::is_devicescratch_v<view_type>) {
         if (view.scope() == ttg::scope::Allocate) {
           access = PARSEC_FLOW_ACCESS_WRITE;
         }
diff --git a/ttg/ttg/parsec/devicescratch.h b/ttg/ttg/parsec/devicescratch.h
index 9e36757b5..e2c3743aa 100644
--- a/ttg/ttg/parsec/devicescratch.h
+++ b/ttg/ttg/parsec/devicescratch.h
@@ -133,21 +133,6 @@ struct devicescratch {
 
 };
 
-template<typename T>
-struct is_devicescratch : std::false_type
-{ };
-
-template<typename T>
-struct is_devicescratch<devicescratch<T>> : std::true_type
-{ };
-
-template<typename T>
-struct is_devicescratch<const devicescratch<T>> : std::true_type
-{ };
-
-template<typename T>
-constexpr static const bool is_devicescratch_v = is_devicescratch<T>::value;
-
 namespace detail {
   template<typename T>
   parsec_data_t* get_parsec_data(const ttg_parsec::devicescratch<T>& scratch) {
diff --git a/ttg/ttg/parsec/fwd.h b/ttg/ttg/parsec/fwd.h
index 3a7dadb5b..7900249cc 100644
--- a/ttg/ttg/parsec/fwd.h
+++ b/ttg/ttg/parsec/fwd.h
@@ -14,7 +14,7 @@ namespace ttg_parsec {
   class TT;
 
   template<typename T>
-  struct ptr;
+  struct Ptr;
 
   template<typename T, typename Allocator = std::allocator<T>>
   struct Buffer;
@@ -79,10 +79,10 @@ namespace ttg_parsec {
   inline std::pair<bool, std::tuple<ptr<std::decay_t<Args>>...>> get_ptr(Args&&... args);
 #endif
   template<typename T>
-  inline ptr<std::decay_t<T>> get_ptr(T&& obj);
+  inline Ptr<std::decay_t<T>> get_ptr(T&& obj);
 
   template<typename T, typename... Args>
-  inline ptr<T> make_ptr(Args&&... args);
+  inline Ptr<T> make_ptr(Args&&... args);
 
 
 }  // namespace ttg_parsec
diff --git a/ttg/ttg/parsec/import.h b/ttg/ttg/parsec/import.h
index 7a5a05108..50b720083 100644
--- a/ttg/ttg/parsec/import.h
+++ b/ttg/ttg/parsec/import.h
@@ -10,6 +10,7 @@
 #define TTG_SELECTED_DEFAULT_IMPL parsec
 #define TTG_PARSEC_IMPORTED 1
 #define TTG_IMPL_NS ttg_parsec
+#define TTG_IMPL_DEVICE_SUPPORT 1
 
 namespace ttg_parsec {}
 
diff --git a/ttg/ttg/parsec/ptr.h b/ttg/ttg/parsec/ptr.h
index 6ab48345d..c4f1200fd 100644
--- a/ttg/ttg/parsec/ptr.h
+++ b/ttg/ttg/parsec/ptr.h
@@ -19,11 +19,11 @@ namespace ttg_parsec {
     template <typename Value>
     inline ttg_data_copy_t *create_new_datacopy(Value &&value);
 
-    struct ptr {
+    struct ptr_impl {
       using copy_type = detail::ttg_data_copy_t;
 
     private:
-      static inline std::unordered_map<ptr*, bool> m_ptr_map;
+      static inline std::unordered_map<ptr_impl*, bool> m_ptr_map;
       static inline std::mutex m_ptr_map_mtx;
 
       copy_type *m_copy = nullptr;
@@ -51,7 +51,7 @@ namespace ttg_parsec {
       }
 
     public:
-      ptr(copy_type *copy)
+      ptr_impl(copy_type *copy)
       : m_copy(copy)
       {
         register_self();
@@ -63,7 +63,7 @@ namespace ttg_parsec {
         return m_copy;
       }
 
-      ptr(const ptr& p)
+      ptr_impl(const ptr_impl& p)
       : m_copy(p.m_copy)
       {
         register_self();
@@ -71,7 +71,7 @@ namespace ttg_parsec {
         std::cout << "ptr cpy " << m_copy << " ref " << m_copy->num_ref() << std::endl;
       }
 
-      ptr(ptr&& p)
+      ptr_impl(ptr_impl&& p)
       : m_copy(p.m_copy)
       {
         register_self();
@@ -79,12 +79,12 @@ namespace ttg_parsec {
         std::cout << "ptr mov " << m_copy << " ref " << m_copy->num_ref() << std::endl;
       }
 
-      ~ptr() {
+      ~ptr_impl() {
         deregister_self();
         drop_copy();
       }
 
-      ptr& operator=(const ptr& p)
+      ptr_impl& operator=(const ptr_impl& p)
       {
         drop_copy();
         m_copy = p.m_copy;
@@ -93,7 +93,7 @@ namespace ttg_parsec {
         return *this;
       }
 
-      ptr& operator=(ptr&& p) {
+      ptr_impl& operator=(ptr_impl&& p) {
         drop_copy();
         m_copy = p.m_copy;
         p.m_copy = nullptr;
@@ -128,34 +128,34 @@ namespace ttg_parsec {
 
   // fwd decl
   template<typename T, typename... Args>
-  ptr<T> make_ptr(Args&&... args);
+  Ptr<T> make_ptr(Args&&... args);
 
   // fwd decl
   template<typename T>
-  ptr<std::decay_t<T>> get_ptr(T&& obj);
+  Ptr<std::decay_t<T>> get_ptr(T&& obj);
 
   template<typename T>
-  struct ptr {
+  struct Ptr {
 
     using value_type = std::decay_t<T>;
 
   private:
     using copy_type = detail::ttg_data_value_copy_t<value_type>;
 
-    std::unique_ptr<detail::ptr> m_ptr;
+    std::unique_ptr<detail::ptr_impl> m_ptr;
 
     /* only PaRSEC backend functions are allowed to touch our private parts */
     template<typename... Args>
-    friend ptr<T> make_ptr(Args&&... args);
+    friend Ptr<T> make_ptr(Args&&... args);
     template<typename S>
-    friend ptr<std::decay_t<S>> get_ptr(S&& obj);
+    friend Ptr<std::decay_t<S>> get_ptr(S&& obj);
     template<typename S>
-    friend detail::ttg_data_copy_t* detail::get_copy(ptr<S>& p);
+    friend detail::ttg_data_copy_t* detail::get_copy(Ptr<S>& p);
     friend ttg::detail::value_copy_handler<ttg::Runtime::PaRSEC>;
 
     /* only accessible by get_ptr and make_ptr */
-    ptr(detail::ptr::copy_type *copy)
-    : m_ptr(new detail::ptr(copy))
+    Ptr(detail::ptr_impl::copy_type *copy)
+    : m_ptr(new detail::ptr_impl(copy))
     { }
 
     copy_type* get_copy() const {
@@ -164,22 +164,22 @@ namespace ttg_parsec {
 
   public:
 
-    ptr() = default;
+    Ptr() = default;
 
-    ptr(const ptr& p)
-    : ptr(p.get_copy())
+    Ptr(const Ptr& p)
+    : Ptr(p.get_copy())
     { }
 
-    ptr(ptr&& p) = default;
+    Ptr(Ptr&& p) = default;
 
-    ~ptr() = default;
+    ~Ptr() = default;
 
-    ptr& operator=(const ptr& p) {
-      m_ptr.reset(new detail::ptr(p.get_copy()));
+    Ptr& operator=(const Ptr& p) {
+      m_ptr.reset(new detail::ptr_impl(p.get_copy()));
       return *this;
     }
 
-    ptr& operator=(ptr&& p) = default;
+    Ptr& operator=(Ptr&& p) = default;
 
     value_type& operator*() const {
       return **static_cast<copy_type*>(m_ptr->get_copy());
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index bed06d535..97acaa839 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1003,7 +1003,7 @@ namespace ttg_parsec {
     if(0 == ttg::default_execution_context().rank())
       ttg::default_execution_context().impl().final_task();
     ttg::detail::set_default_world(ttg::World{});  // reset the default world
-    detail::ptr::drop_all_ptr();
+    detail::ptr_impl::drop_all_ptr();
     ttg::detail::destroy_worlds<ttg_parsec::WorldImpl>();
     if (detail::initialized_mpi()) MPI_Finalize();
   }
@@ -4004,7 +4004,7 @@ ttg::abort();  // should not happen
     template<typename Key, typename Arg, typename... Args, std::size_t I, std::size_t... Is>
     void invoke_arglist(std::index_sequence<I, Is...>, const Key& key, Arg&& arg, Args&&... args) {
       using arg_type = std::decay_t<Arg>;
-      if constexpr (ttg::detail::is_ptr_v<arg_type>) {
+      if constexpr (ttg::meta::is_ptr_v<arg_type>) {
         /* add a reference to the object */
         auto copy = ttg_parsec::detail::get_copy(arg);
         copy->add_ref();
@@ -4017,7 +4017,7 @@ ttg::abort();  // should not happen
           /* if the ptr was moved in we reset it */
           arg.reset();
         }
-      } else if constexpr (!ttg::detail::is_ptr_v<arg_type>) {
+      } else if constexpr (!ttg::meta::is_ptr_v<arg_type>) {
         set_arg<I>(key, std::forward<Arg>(arg));
       }
       if constexpr (sizeof...(Is) > 0) {
diff --git a/ttg/ttg/ptr.h b/ttg/ttg/ptr.h
index 3cc699f60..da1a1abfe 100644
--- a/ttg/ttg/ptr.h
+++ b/ttg/ttg/ptr.h
@@ -6,7 +6,7 @@
 namespace ttg {
 
 template<typename T>
-using Ptr = TTG_IMPL_NS::ptr<T>;
+using Ptr = TTG_IMPL_NS::Ptr<T>;
 
 template<typename T, typename... Args>
 Ptr<T> make_ptr(Args&&... args) {
@@ -18,20 +18,6 @@ auto get_ptr(T&& obj) {
   return TTG_IMPL_NS::get_ptr(std::forward<T>(obj));
 }
 
-namespace detail {
-    template<typename T>
-    struct is_ptr : std::false_type
-    { };
-
-    template<typename T>
-    struct is_ptr<ttg::Ptr<T>> : std::true_type
-    { };
-
-    template<typename T>
-    constexpr bool is_ptr_v = is_ptr<T>::value;
-
-} // namespace detail
-
 #if 0
 namespace detail {
 
diff --git a/ttg/ttg/util/meta.h b/ttg/ttg/util/meta.h
index 389a9c8d5..b7bb31690 100644
--- a/ttg/ttg/util/meta.h
+++ b/ttg/ttg/util/meta.h
@@ -6,6 +6,9 @@
 
 #include "ttg/util/span.h"
 #include "ttg/util/typelist.h"
+#include "ttg/buffer.h"
+#include "ttg/ptr.h"
+#include "ttg/devicescratch.h"
 
 namespace ttg {
 
@@ -290,6 +293,53 @@ namespace ttg {
     template <typename... Ts>
     constexpr bool is_any_nonconst_lvalue_reference_v<std::tuple<Ts...>> = is_any_nonconst_lvalue_reference_v<Ts...>;
 
+
+    ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+    // device type traits
+    ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+    template<typename T>
+    struct is_ptr : std::false_type
+    { };
+
+    template<typename T>
+    struct is_ptr<ttg::Ptr<T>> : std::true_type
+    { };
+
+    template<typename T>
+    constexpr bool is_ptr_v = is_ptr<T>::value;
+
+    template<typename T>
+    struct is_buffer : std::false_type
+    { };
+
+    template<typename T, typename A>
+    struct is_buffer<ttg::Buffer<T, A>> : std::true_type
+    { };
+
+    template<typename T, typename A>
+    struct is_buffer<const ttg::Buffer<T, A>> : std::true_type
+    { };
+
+    template<typename T>
+    constexpr bool is_buffer_v = is_buffer<T>::value;
+
+    template<typename T>
+    struct is_devicescratch : std::false_type
+    { };
+
+    template<typename T>
+    struct is_devicescratch<ttg::devicescratch<T>> : std::true_type
+    { };
+
+    template<typename T>
+    struct is_devicescratch<const ttg::devicescratch<T>> : std::true_type
+    { };
+
+    template<typename T>
+    constexpr bool is_devicescratch_v = is_devicescratch<T>::value;
+
+
     ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
     // typelist metafunctions
     ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

From 913f792ecda1b4a0d42233cedab67673dc011019 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 20 Dec 2023 01:10:07 +0000
Subject: [PATCH 194/259] Include fwd.h instead of impl_selector.h

fwd.h provides the symbols from the implementation namespace

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/buffer.h        |  6 +----
 ttg/ttg/devicescratch.h |  2 +-
 ttg/ttg/parsec/fwd.h    | 51 ++++++++++++++++++++---------------------
 ttg/ttg/ptr.h           |  2 +-
 4 files changed, 28 insertions(+), 33 deletions(-)

diff --git a/ttg/ttg/buffer.h b/ttg/ttg/buffer.h
index 3067968ff..4d998c29a 100644
--- a/ttg/ttg/buffer.h
+++ b/ttg/ttg/buffer.h
@@ -1,10 +1,7 @@
 #ifndef TTG_BUFFER_H
 #define TTG_BUFFER_H
 
-#include <memory>
-#include "ttg/impl_selector.h"
-
-#if defined(TTG_IMPL_DEVICE_SUPPORT)
+#include "ttg/fwd.h"
 
 namespace ttg {
 
@@ -13,5 +10,4 @@ using Buffer = TTG_IMPL_NS::Buffer<T, Allocator>;
 
 } // namespace ttg
 
-#endif // TTG_IMPL_DEVICE_SUPPORT
 #endif // TTG_buffer_H
\ No newline at end of file
diff --git a/ttg/ttg/devicescratch.h b/ttg/ttg/devicescratch.h
index 3e528d01c..9ccb60bce 100644
--- a/ttg/ttg/devicescratch.h
+++ b/ttg/ttg/devicescratch.h
@@ -2,7 +2,7 @@
 #define TTG_DEVICESCRATCH_H
 
 #include "ttg/devicescope.h"
-#include "ttg/impl_selector.h"
+#include "ttg/fwd.h"
 
 namespace ttg {
 
diff --git a/ttg/ttg/parsec/fwd.h b/ttg/ttg/parsec/fwd.h
index 7900249cc..d6f072a9c 100644
--- a/ttg/ttg/parsec/fwd.h
+++ b/ttg/ttg/parsec/fwd.h
@@ -13,26 +13,6 @@ namespace ttg_parsec {
   template <typename keyT, typename output_terminalsT, typename derivedT, typename input_valueTs = ttg::typelist<>>
   class TT;
 
-  template<typename T>
-  struct Ptr;
-
-  template<typename T, typename Allocator = std::allocator<T>>
-  struct Buffer;
-  template<typename T>
-  struct devicescratch;
-
-  template<typename... Views>
-  inline bool register_device_memory(std::tuple<Views&...> &views);
-
-  template<typename... Buffer>
-  inline void mark_device_out(std::tuple<Buffer&...> &b);
-
-  template<typename... Buffer>
-  inline void post_device_out(std::tuple<Buffer&...> &b);
-
-  /* the query of the parsec backend only returns whether the data should be marked for pushout */
-  using query_result_type = bool;
-
   /// \internal the OG name
   template <typename keyT, typename output_terminalsT, typename derivedT, typename... input_valueTs>
   using Op [[deprecated("use TT instead")]] = TT<keyT, output_terminalsT, derivedT, ttg::typelist<input_valueTs...>>;
@@ -74,16 +54,35 @@ namespace ttg_parsec {
   template <typename T>
   static void ttg_broadcast(ttg::World world, T &data, int source_rank);
 
-#if 0
-  template<typename... Args>
-  inline std::pair<bool, std::tuple<ptr<std::decay_t<Args>>...>> get_ptr(Args&&... args);
-#endif
+  /* device definitions */
+  template<typename T, typename Allocator>
+  struct Buffer;
+
+  template<typename T>
+  struct Ptr;
+
   template<typename T>
-  inline Ptr<std::decay_t<T>> get_ptr(T&& obj);
+  struct devicescratch;
 
   template<typename T, typename... Args>
-  inline Ptr<T> make_ptr(Args&&... args);
+  Ptr<T> make_ptr(Args&&... args);
+
+  template<typename T>
+  auto get_ptr(T&& obj);
+
+  template<typename... Views>
+  inline bool register_device_memory(std::tuple<Views&...> &views);
 
+  template<typename... Buffer>
+  inline void post_device_out(std::tuple<Buffer&...> &b);
+
+  template<typename... Buffer>
+  inline void mark_device_out(std::tuple<Buffer&...> &b);
+
+#if 0
+  template<typename... Args>
+  inline std::pair<bool, std::tuple<ptr<std::decay_t<Args>>...>> get_ptr(Args&&... args);
+#endif
 
 }  // namespace ttg_parsec
 
diff --git a/ttg/ttg/ptr.h b/ttg/ttg/ptr.h
index da1a1abfe..999b6124f 100644
--- a/ttg/ttg/ptr.h
+++ b/ttg/ttg/ptr.h
@@ -1,7 +1,7 @@
 #ifndef TTG_PTR_H
 #define TTG_PTR_H
 
-#include "ttg/impl_selector.h"
+#include "ttg/fwd.h"
 
 namespace ttg {
 

From 7c6044aad5005b6014737e728dcbdb477e39552a Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 20 Dec 2023 01:11:20 +0000
Subject: [PATCH 195/259] Bump catch2 version to latest release

This is needed to get the fix for https://github.com/catchorg/Catch2/issues/2421

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 .../modules/ExternalDependenciesVersions.cmake  |  2 +-
 tests/unit/CMakeLists.txt                       | 17 ++++++++++-------
 tests/unit/serialization.cc                     |  2 +-
 tests/unit/unit_main.cpp                        |  2 +-
 4 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/cmake/modules/ExternalDependenciesVersions.cmake b/cmake/modules/ExternalDependenciesVersions.cmake
index 1e6dd1fef..8d763f5db 100644
--- a/cmake/modules/ExternalDependenciesVersions.cmake
+++ b/cmake/modules/ExternalDependenciesVersions.cmake
@@ -3,7 +3,7 @@
 
 # need Boost.CallableTraits (header only, part of Boost 1.66 released in Dec 2017) for wrap.h to work
 set(TTG_TRACKED_BOOST_VERSION 1.66)
-set(TTG_TRACKED_CATCH2_VERSION 2.13.1)
+set(TTG_TRACKED_CATCH2_VERSION 3.5.0)
 set(TTG_TRACKED_CEREAL_VERSION 1.3.0)
 set(TTG_TRACKED_MADNESS_TAG cb195817d7807c4aead10ba200cd20649036cbae)
 set(TTG_TRACKED_PARSEC_TAG 25d1931e863b6741e453112d2117d85ad32e7fba)
diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt
index 154f03183..6b524010f 100644
--- a/tests/unit/CMakeLists.txt
+++ b/tests/unit/CMakeLists.txt
@@ -20,12 +20,14 @@ set(ut_libs Catch2::Catch2)
 add_ttg_executable(core-unittests-ttg "${ut_src}" LINK_LIBRARIES "${ut_libs}" COMPILE_DEFINITIONS "CATCH_CONFIG_NO_POSIX_SIGNALS=1" )
 
 # serialization test: probes serialization via all supported serialization methods (MADNESS, Boost::serialization, cereal) that are available
-add_executable(serialization "serialization.cc;unit_main.cpp")
-target_link_libraries(serialization "Catch2::Catch2;ttg-serialization")
-if (TARGET BTAS::BTAS)
-    target_link_libraries(serialization BTAS::BTAS)
-    target_compile_definitions(serialization PRIVATE TTG_HAS_BTAS=1)
-endif (TARGET BTAS::BTAS)
+add_ttg_executable(serialization serialization.cc unit_main.cpp
+                   LINK_LIBRARIES Catch2::Catch2 ttg-serialization  $<TARGET_NAME_IF_EXISTS:BTAS::BTAS>
+                   COMPILE_DEFINITIONS $<$<TARGET_EXISTS:BTAS::BTAS>:TTG_HAS_BTAS=1>)
+#target_link_libraries(serialization "Catch2::Catch2;ttg-serialization")
+#if (TARGET BTAS::BTAS)
+#    target_link_libraries(serialization BTAS::BTAS)
+#    target_compile_definitions(serialization PRIVATE TTG_HAS_BTAS=1)
+#endif (TARGET BTAS::BTAS)
 
 # TODO: convert into unit test
 #if (TARGET MADworld)
@@ -33,4 +35,5 @@ endif (TARGET BTAS::BTAS)
 #endif(TARGET MADworld)
 
 
-catch_discover_tests(serialization TEST_PREFIX "ttg/test/unit/")
+catch_discover_tests(serialization-parsec TEST_PREFIX "ttg/test/unit/")
+catch_discover_tests(serialization-mad TEST_PREFIX "ttg/test/unit/")
diff --git a/tests/unit/serialization.cc b/tests/unit/serialization.cc
index 647f7d1ec..0c14cfaca 100644
--- a/tests/unit/serialization.cc
+++ b/tests/unit/serialization.cc
@@ -463,7 +463,7 @@ namespace freestanding::symmetric::bc_v {
 
 #include "ttg/serialization/data_descriptor.h"
 
-#include <catch2/catch.hpp>
+#include <catch2/catch_all.hpp>
 
 static_assert(ttg::detail::is_madness_buffer_serializable_v<int>);
 static_assert(!ttg::detail::is_madness_user_buffer_serializable_v<int>);
diff --git a/tests/unit/unit_main.cpp b/tests/unit/unit_main.cpp
index 7ca861716..e4a923cd3 100644
--- a/tests/unit/unit_main.cpp
+++ b/tests/unit/unit_main.cpp
@@ -1,6 +1,6 @@
 #define CATCH_CONFIG_RUNNER
 
-#include <catch2/catch.hpp>
+#include <catch2/catch_all.hpp>
 
 #include <clocale>
 #include <iostream>

From ed055fb97d41894ad6bf6f576a21397c43ea07e7 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 20 Dec 2023 01:44:21 +0000
Subject: [PATCH 196/259] Fix clang compile issues

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/device/device.h | 2 +-
 ttg/ttg/device/task.h   | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/ttg/ttg/device/device.h b/ttg/ttg/device/device.h
index 0ff4e64cf..fbe036612 100644
--- a/ttg/ttg/device/device.h
+++ b/ttg/ttg/device/device.h
@@ -172,7 +172,7 @@ namespace ttg::device {
 
   template<ttg::ExecutionSpace Space = ttg::ExecutionSpace::Invalid>
   inline const void* current_stream() {
-    static_assert(ttg::ExecutionSpace Space != ttg::ExecutionSpace::Invalid,
+    static_assert(Space != ttg::ExecutionSpace::Invalid,
                   "TTG was built without any known device support so we cannot provide a current stream!");
     return nullptr;
   }
diff --git a/ttg/ttg/device/task.h b/ttg/ttg/device/task.h
index 8f63ea564..ea7c6e9a0 100644
--- a/ttg/ttg/device/task.h
+++ b/ttg/ttg/device/task.h
@@ -140,6 +140,8 @@ namespace ttg::device {
         throw; // fwd
       }
 
+      void return_void() {}
+
     };
 
     template<typename Key, typename Value, ttg::Runtime Runtime = ttg::ttg_runtime>
@@ -635,4 +637,4 @@ namespace ttg::device {
 
 }  // namespace ttg::devie
 
-#endif // TTG_DEVICE_TASK_H
\ No newline at end of file
+#endif // TTG_DEVICE_TASK_H

From 0c7d2919b54a513be186ef084f9af11b55615ef6 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 20 Dec 2023 02:34:08 +0000
Subject: [PATCH 197/259] Forward keyT in make_tt overload

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/make_tt.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ttg/ttg/make_tt.h b/ttg/ttg/make_tt.h
index 4d1f060b4..1e03b3ad5 100644
--- a/ttg/ttg/make_tt.h
+++ b/ttg/ttg/make_tt.h
@@ -643,7 +643,7 @@ auto make_tt(funcT &&func, const std::tuple<ttg::Edge<keyT, input_edge_valuesT>.
              const std::tuple<output_edgesT...> &outedges = std::tuple<>{}, const std::string &name = "wrapper",
              const std::vector<std::string> &innames = std::vector<std::string>(sizeof...(input_edge_valuesT), "input"),
              const std::vector<std::string> &outnames = std::vector<std::string>(sizeof...(output_edgesT), "output")) {
-  return make_tt<ttg::ExecutionSpace::Host>(std::forward<funcT>(func), inedges, outedges, name, innames, outnames);
+  return make_tt<ttg::ExecutionSpace::Host, keyT>(std::forward<funcT>(func), inedges, outedges, name, innames, outnames);
 }
 
 template <typename keyT, typename funcT, typename... input_valuesT, typename... output_edgesT>

From b486a2f5fa7b32b191d906d6578699031bb2693c Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 20 Dec 2023 03:13:38 +0000
Subject: [PATCH 198/259] Bump github CI Ubuntu to 22.04 to get boost 1.74
 needed for TA

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 .github/workflows/cmake.yml | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index eaecd6930..bb67a827e 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -10,11 +10,11 @@ jobs:
       fail-fast: false
       matrix:
         build_type : [ Release, Debug ]
-        os : [ macos-latest, ubuntu-20.04 ]
+        os : [ macos-latest, ubuntu-22.04 ]
         include:
-          - os: ubuntu-20.04
-            cc: /usr/bin/gcc-10
-            cxx: /usr/bin/g++-10
+          - os: ubuntu-22.04
+            cc: /usr/bin/gcc-12
+            cxx: /usr/bin/g++-12
           - os: macos-latest
             cc: clang
             cxx: clang++
@@ -48,12 +48,12 @@ jobs:
       run: brew install ninja gcc@10 boost eigen open-mpi bison ccache
 
     - name: Install prerequisites Ubuntu packages
-      if: ${{ matrix.os == 'ubuntu-20.04' }}
+      if: ${{ matrix.os == 'ubuntu-22.04' }}
       run: |
         wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null
         sudo apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main"
         sudo apt-get update
-        sudo apt-get -y install ninja-build g++-10 liblapack-dev libboost-dev libboost-serialization-dev libeigen3-dev openmpi-bin libopenmpi-dev libtbb-dev ccache flex bison cmake
+        sudo apt-get -y install ninja-build g++-12 liblapack-dev libboost-dev libboost-serialization-dev libeigen3-dev openmpi-bin libopenmpi-dev libtbb-dev ccache flex bison cmake
 
     - name: Create Build Environment
       # Some projects don't allow in-source building, so create a separate build directory
@@ -63,10 +63,10 @@ jobs:
 
 
     - name: Install doxygen for Release test
-      if: ${{ matrix.os == 'ubuntu-20.04' }}
+      if: ${{ matrix.os == 'ubuntu-22.04' }}
       run: |
         if [ "${{matrix.build_type}}" = "Release" ]; then
-          sudo apt-get -y install libclang1-9 libclang-cpp9 graphviz fonts-liberation
+          sudo apt-get -y install libclang1-11 libclang-cpp11 graphviz fonts-liberation
           cd ${{github.workspace}}/build
           # If we fail getting doxygen-${DOXYGEN_VERSION}.linux.bin.tar.gz from sourceforge,
           # use EFV's gdrive mirror of 1.9.2 to work around the unreliable sourceforge
@@ -144,7 +144,7 @@ jobs:
         cmake --build test_install_userexamples/build
 
     - name: Build+Deploy Dox
-      if: ${{ matrix.os == 'ubuntu-20.04' && matrix.build_type == 'Release' && github.ref == 'refs/heads/master' }}
+      if: ${{ matrix.os == 'ubuntu-22.04' && matrix.build_type == 'Release' && github.ref == 'refs/heads/master' }}
       working-directory: ${{github.workspace}}/build
       shell: bash
       run: |

From 555ae2f5ddba7c69d1852b229f7ff7ace034c5d5 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 20 Dec 2023 17:13:34 +0000
Subject: [PATCH 199/259] Add stub implementations of ttvalue and buffer for
 madness

---
 examples/potrf/potrf.h                   |  31 ++-
 examples/potrf/testing_dpotrf.cc         |   4 +
 ttg/CMakeLists.txt                       |   2 +
 ttg/ttg/madness/buffer.h                 | 317 +++++++++++++++++++++++
 ttg/ttg/madness/fwd.h                    |   3 +-
 ttg/ttg/madness/ttg.h                    |   7 +-
 ttg/ttg/madness/ttvalue.h                |  14 +
 ttg/ttg/parsec/fwd.h                     |   2 +-
 ttg/ttg/parsec/import.h                  |   1 -
 ttg/ttg/parsec/ttg.h                     |   2 +-
 ttg/ttg/parsec/ttvalue.h                 |   4 +-
 ttg/ttg/serialization/backends/madness.h |   2 +
 ttg/ttg/ttvalue.h                        |  13 +
 13 files changed, 381 insertions(+), 21 deletions(-)
 create mode 100644 ttg/ttg/madness/buffer.h
 create mode 100644 ttg/ttg/madness/ttvalue.h
 create mode 100644 ttg/ttg/ttvalue.h

diff --git a/examples/potrf/potrf.h b/examples/potrf/potrf.h
index fb0c4f9b4..f6ba6e147 100644
--- a/examples/potrf/potrf.h
+++ b/examples/potrf/potrf.h
@@ -7,6 +7,9 @@
 #include "util.h"
 #include "../devblas_helper.h"
 
+#if (defined(TTG_ENABLE_CUDA) || defined(TTG_ENABLE_HIP))
+#define ENABLE_DEVICE_KERNEL 1
+#endif
 
 #if defined(TTG_HAVE_CUDART)
 #define ES ttg::ExecutionSpace::CUDA
@@ -29,7 +32,7 @@ namespace potrf {
   inline double FADDS_POTRF(double __n) { return (__n * (((1. / 6.) * __n) * __n - (1. / 6.))); }
   inline double FLOPS_DPOTRF(double __n) { return FMULS_POTRF(__n) + FADDS_POTRF(__n); }
 
-#if defined(TTG_HAVE_CUDART) || defined(TTG_HAVE_HIP)
+#if defined(ENABLE_DEVICE_KERNEL)
   static int device_potrf_workspace_size(MatrixTile<double> &A) {
     int Lwork;
     #if defined(TTG_HAVE_CUDA)
@@ -82,7 +85,7 @@ namespace potrf {
     hipblasDnrm2(hipblas_handle(), size, buffer, 1, norm);
   #endif
   }
-#endif // defined(TTG_HAVE_CUDART) || defined(TTG_HAVE_HIP)
+#endif // ENABLE_DEVICE_KERNEL
 
   template <typename MatrixT>
   auto make_potrf(MatrixT& A,
@@ -91,7 +94,7 @@ namespace potrf {
                   ttg::Edge<Key2, MatrixTile<typename MatrixT::element_type>>& output_trsm,
                   ttg::Edge<Key2, MatrixTile<typename MatrixT::element_type>>& output_result) {
     using T = typename MatrixT::element_type;
-#if defined(TTG_HAVE_CUDART) || defined(TTG_HAVE_HIP)
+#if defined(ENABLE_DEVICE_KERNEL)
     auto iallocator = std::make_shared<TiledArray::device_pinned_allocator<int>>();
     //std::cout << "Creating CUDA POTRF task " << std::endl;
     auto f_dev = [=, iallocator = std::move(iallocator)]
@@ -185,7 +188,7 @@ namespace potrf {
     };
     return ttg::make_tt<ES>(f_dev, ttg::edges(ttg::fuse(input, input_disp)), ttg::edges(output_result, output_trsm), "POTRF",
                         {"tile_kk/dispatcher"}, {"output_result", "output_trsm"});
-#else /* defined(TTG_HAVE_CUDART) || defined(TTG_HAVE_HIP) */
+#else /* defined(ENABLE_DEVICE_KERNEL) */
     auto f = [=](const Key1& key, MatrixTile<T>&& tile_kk,
                  std::tuple<ttg::Out<Key2, MatrixTile<T>>, ttg::Out<Key2, MatrixTile<T>>>& out) {
       const int K = key[0];
@@ -215,7 +218,7 @@ namespace potrf {
     };
     return ttg::make_tt(f, ttg::edges(ttg::fuse(input, input_disp)), ttg::edges(output_result, output_trsm), "POTRF",
                         {"tile_kk/dispatcher"}, {"output_result", "output_trsm"});
-#endif
+#endif // defined(ENABLE_DEVICE_KERNEL)
   }
 
   template <typename MatrixT>
@@ -228,7 +231,7 @@ namespace potrf {
                  ttg::Edge<Key3, MatrixTile<typename MatrixT::element_type>>& output_col,   // to GEMM
                  ttg::Edge<Key2, MatrixTile<typename MatrixT::element_type>>& output_result) {
     using T = typename MatrixT::element_type;
-#if defined(TTG_HAVE_CUDART) || defined(TTG_HAVE_HIP)
+#if defined(ENABLE_DEVICE_KERNEL)
     auto f = [=](const Key2& key, const MatrixTile<T>& tile_kk, MatrixTile<T>&& tile_mk,
                  std::tuple<ttg::Out<Key2, MatrixTile<T>>, ttg::Out<Key2, MatrixTile<T>>, ttg::Out<Key3, MatrixTile<T>>,
                             ttg::Out<Key3, MatrixTile<T>>>& out) TASKRET {
@@ -320,7 +323,7 @@ namespace potrf {
     return ttg::make_tt<ES>(f, ttg::edges(input_kk, ttg::fuse(input_mk, input_disp)),
                             ttg::edges(output_result, output_diag, output_row, output_col), "TRSM",
                             {"tile_kk", "tile_mk/dispatcher"}, {"output_result", "tile_mk", "output_row", "output_col"});
-#else
+#else // defined(ENABLE_DEVICE_KERNEL)
     auto f = [=](const Key2& key, const MatrixTile<T>& tile_kk, MatrixTile<T>&& tile_mk,
                  std::tuple<ttg::Out<Key2, MatrixTile<T>>, ttg::Out<Key2, MatrixTile<T>>, ttg::Out<Key3, MatrixTile<T>>,
                             ttg::Out<Key3, MatrixTile<T>>>& out) {
@@ -371,7 +374,7 @@ namespace potrf {
     return ttg::make_tt(f, ttg::edges(input_kk, ttg::fuse(input_mk, input_disp)),
                         ttg::edges(output_result, output_diag, output_row, output_col), "TRSM",
                         {"tile_kk", "tile_mk/dispatcher"}, {"output_result", "tile_mk", "output_row", "output_col"});
-#endif
+#endif // defined(ENABLE_DEVICE_KERNEL)
   }
 
   template <typename MatrixT>
@@ -382,7 +385,7 @@ namespace potrf {
                  ttg::Edge<Key1, MatrixTile<typename MatrixT::element_type>>& output_potrf,  // to POTRF
                  ttg::Edge<Key2, MatrixTile<typename MatrixT::element_type>>& output_syrk) {
     using T = typename MatrixT::element_type;
-#if defined(TTG_HAVE_CUDART) || defined(TTG_HAVE_HIP)
+#if defined(ENABLE_DEVICE_KERNEL)
     auto f = [=](const Key2& key, const MatrixTile<T>& tile_mk, MatrixTile<T>&& tile_kk,
                  std::tuple<ttg::Out<Key1, MatrixTile<T>>, ttg::Out<Key2, MatrixTile<T>>>& out) TASKRET {
       const int K = key[0];
@@ -455,7 +458,7 @@ namespace potrf {
     };
     return ttg::make_tt<ES>(f, ttg::edges(input_mk, ttg::fuse(input_kk, input_disp)), ttg::edges(output_potrf, output_syrk),
                             "SYRK", {"tile_mk", "tile_kk/dispatcher"}, {"output_potrf", "output_syrk"});
-#else
+#else // defined(ENABLE_DEVICE_KERNEL)
     auto f = [=](const Key2& key, const MatrixTile<T>& tile_mk, MatrixTile<T>&& tile_kk,
                  std::tuple<ttg::Out<Key1, MatrixTile<T>>, ttg::Out<Key2, MatrixTile<T>>>& out) {
       const int K = key[0];
@@ -494,7 +497,7 @@ namespace potrf {
     return ttg::make_tt(f, ttg::edges(input_mk, ttg::fuse(input_kk, input_disp)), ttg::edges(output_potrf, output_syrk),
                         "SYRK", {"tile_mk", "tile_kk/dispatcher"}, {"output_potrf", "output_syrk"});
 #endif
-  }
+  } // defined(ENABLE_DEVICE_KERNEL)
 
   template <typename MatrixT>
   auto make_gemm(MatrixT& A,
@@ -505,7 +508,7 @@ namespace potrf {
                  ttg::Edge<Key2, MatrixTile<typename MatrixT::element_type>>& output_trsm,  // to TRSM
                  ttg::Edge<Key3, MatrixTile<typename MatrixT::element_type>>& output_gemm) {
     using T = typename MatrixT::element_type;
-#if defined(TTG_HAVE_CUDART) || defined(TTG_HAVE_HIP)
+#if defined(ENABLE_DEVICE_KERNEL)
     auto f = [=](const Key3& key, const MatrixTile<T>& tile_mk, const MatrixTile<T>& tile_nk, MatrixTile<T>&& tile_mn,
                  std::tuple<ttg::Out<Key2, MatrixTile<T>>, ttg::Out<Key3, MatrixTile<T>>>& out) TASKRET {
       const int M = key[0];
@@ -585,7 +588,7 @@ namespace potrf {
     return ttg::make_tt<ES>(f, ttg::edges(input_mk, input_nk, ttg::fuse(input_disp, input_mn)),
                             ttg::edges(output_trsm, output_gemm), "GEMM", {"input_mk", "input_kn", "input_mn/dispatcher"},
                             {"output_trsm", "outout_gemm"});
-#else
+#else // defined(ENABLE_DEVICE_KERNEL)
     auto f = [=](const Key3& key, const MatrixTile<T>& tile_mk, const MatrixTile<T>& tile_nk, MatrixTile<T>&& tile_mn,
                  std::tuple<ttg::Out<Key2, MatrixTile<T>>, ttg::Out<Key3, MatrixTile<T>>>& out) {
       const int M = key[0];
@@ -624,7 +627,7 @@ namespace potrf {
     return ttg::make_tt(f, ttg::edges(input_mk, input_nk, ttg::fuse(input_disp, input_mn)),
                         ttg::edges(output_trsm, output_gemm), "GEMM", {"input_mk", "input_kn", "input_mn/dispatcher"},
                         {"output_trsm", "outout_gemm"});
-#endif
+#endif // defined(ENABLE_DEVICE_KERNEL)
   }
 
   template <typename T>
diff --git a/examples/potrf/testing_dpotrf.cc b/examples/potrf/testing_dpotrf.cc
index 4ea3b9167..5db9513e9 100644
--- a/examples/potrf/testing_dpotrf.cc
+++ b/examples/potrf/testing_dpotrf.cc
@@ -65,7 +65,9 @@ int main(int argc, char **argv)
   ttg::initialize(1, argv, nthreads);
 
   // initialize MADNESS so that TA allocators can be created
+#if defined(TTG_PARSEC_IMPORTED)
   madness::ParsecRuntime::initialize_with_existing_context(ttg::default_execution_context().impl().context());
+#endif // TTG_PARSEC_IMPORTED
   madness::initialize(argc, argv, /* nthread = */ 1, /* quiet = */ true);
 
   auto world = ttg::default_execution_context();
@@ -151,7 +153,9 @@ int main(int argc, char **argv)
         std::cout << "TTG Execution Time (milliseconds) : "
                   << elapsed / 1E3 << " : Flops " << (potrf::FLOPS_DPOTRF(N)) << " " << (potrf::FLOPS_DPOTRF(N)/1e9)/(elapsed/1e6) << " GF/s" << std::endl;
       }
+#if defined(TTG_PARSEC_IMPORTED)
       parsec_devices_reset_load(ttg::default_execution_context().impl().context());
+#endif // TTG_PARSEC_IMPORTED
     }
 
     world.dag_off();
diff --git a/ttg/CMakeLists.txt b/ttg/CMakeLists.txt
index 874e8d5d4..2954d61b1 100644
--- a/ttg/CMakeLists.txt
+++ b/ttg/CMakeLists.txt
@@ -208,9 +208,11 @@ endif(TARGET cereal::cereal)
 #########################
 if (TARGET MADworld)
   set(ttg-mad-headers
+          ${CMAKE_CURRENT_SOURCE_DIR}/ttg/madness/buffer.h
           ${CMAKE_CURRENT_SOURCE_DIR}/ttg/madness/fwd.h
           ${CMAKE_CURRENT_SOURCE_DIR}/ttg/madness/import.h
           ${CMAKE_CURRENT_SOURCE_DIR}/ttg/madness/ttg.h
+          ${CMAKE_CURRENT_SOURCE_DIR}/ttg/madness/ttvalue.h
           ${CMAKE_CURRENT_SOURCE_DIR}/ttg/madness/watch.h)
   # N.B. ttg-mad can use MADNESS serialization only
   add_ttg_library(ttg-mad "${ttg-mad-headers}" PUBLIC_HEADER "${ttg-mad-headers}" LINK_LIBRARIES "ttg;MADworld;ttg-serialization-madness" COMPILE_DEFINITIONS "WORLD_INSTANTIATE_STATIC_TEMPLATES=1")
diff --git a/ttg/ttg/madness/buffer.h b/ttg/ttg/madness/buffer.h
new file mode 100644
index 000000000..1572085f8
--- /dev/null
+++ b/ttg/ttg/madness/buffer.h
@@ -0,0 +1,317 @@
+#ifndef TTG_MADNESS_BUFFER_H
+#define TTG_MADNESS_BUFFER_H
+
+//#include "ttg/serialization/backends.h"
+
+namespace ttg_madness {
+
+template<typename T, typename Allocator>
+struct Buffer : private Allocator {
+
+  using element_type = std::decay_t<T>;
+
+  using allocator_traits = std::allocator_traits<Allocator>;
+  using allocator_type = typename  allocator_traits::allocator_type;
+
+  static_assert(std::is_trivially_copyable_v<element_type>,
+                "Only trivially copyable types are supported for devices.");
+  static_assert(std::is_default_constructible_v<element_type>,
+                "Only default constructible types are supported for devices.");
+
+private:
+  using delete_fn_t = std::function<void(element_type*)>;
+
+  using host_data_ptr   = std::add_pointer_t<element_type>;
+  host_data_ptr m_host_data = nullptr;
+  std::size_t m_count = 0;
+  bool m_owned= false;
+
+  static void delete_non_owned(element_type *ptr) {
+    // nothing to be done, we don't own the memory
+  }
+
+  allocator_type& get_allocator_reference() { return static_cast<allocator_type&>(*this); }
+
+  element_type* allocate(std::size_t n) {
+    return allocator_traits::allocate(get_allocator_reference(), n);
+  }
+
+  void deallocate() {
+    allocator_traits::deallocate(get_allocator_reference(), m_host_data, m_count);
+  }
+
+public:
+
+  Buffer() : Buffer(nullptr, 0)
+  { }
+
+  Buffer(std::size_t n)
+  : allocator_type()
+  , m_host_data(allocate(n))
+  , m_count(n)
+  , m_owned(true)
+  { }
+
+  /* Constructing a buffer using application-managed memory.
+   * The memory pointed to by ptr must be accessible during
+   * the life-time of the buffer. */
+  Buffer(element_type* ptr, std::size_t n = 1)
+  : allocator_type()
+  , m_host_data(ptr)
+  , m_count(n)
+  , m_owned(false)
+  { }
+
+  virtual ~Buffer() {
+    if (m_owned) {
+      deallocate();
+      m_owned = false;
+    }
+    unpin(); // make sure the copies are not pinned
+  }
+
+  /* allow moving device buffers */
+  Buffer(Buffer&& db)
+  : allocator_type(std::move(db))
+  , m_host_data(db.m_host_data)
+  , m_count(db.m_count)
+  , m_owned(db.m_owned)
+  {
+    db.m_host_data = nullptr;
+    db.m_count = 0;
+    db.m_owned = false;
+  }
+
+  /* explicitly disable copying of buffers
+   * TODO: should we allow this? What data to use?
+   */
+  Buffer(const Buffer& db) = delete;
+
+  /* allow moving device buffers */
+  Buffer& operator=(Buffer&& db) {
+    allocator_type::operator=(std::move(db));
+    std::swap(m_host_data, db.m_host_data);
+    std::swap(m_count, db.m_count);
+    std::swap(m_owned, db.m_owned);
+    return *this;
+  }
+
+  /* explicitly disable copying of buffers
+   * TODO: should we allow this? What data to use?
+   */
+  Buffer& operator=(const Buffer& db) = delete;
+
+  /* set the current device, useful when a device
+   * buffer was modified outside of a TTG */
+  void set_current_device(const ttg::device::Device& device) {
+    assert(is_valid());
+    if (!device.is_host()) throw std::runtime_error("MADNESS backend does not support non-host memory!");
+    /* no-op */
+  }
+
+  /* Get the owner device ID, i.e., the last updated
+   * device buffer. */
+  ttg::device::Device get_owner_device() const {
+    assert(is_valid());
+    return {}; // host only
+  }
+
+  /* Get the pointer on the currently active device. */
+  element_type* current_device_ptr() {
+    assert(is_valid());
+    return m_host_data;
+  }
+
+  /* Get the pointer on the currently active device. */
+  const element_type* current_device_ptr() const {
+    assert(is_valid());
+    return m_host_data;
+  }
+
+  /* Get the pointer on the owning device.
+   * @note: This may not be the device assigned to the currently executing task.
+   *        See \ref ttg::device::current_device for that. */
+  element_type* owner_device_ptr() {
+    assert(is_valid());
+    return m_host_data;
+  }
+
+  /* get the current device pointer */
+  const element_type* owner_device_ptr() const {
+    assert(is_valid());
+    return m_host_data;
+  }
+
+  /* get the device pointer at the given device
+   */
+  element_type* device_ptr_on(const ttg::device::Device& device) {
+    assert(is_valid());
+    if (device.is_device()) throw std::runtime_error("MADNESS missing support for non-host memory!");
+    return m_host_data;
+  }
+
+  /* get the device pointer at the given device
+   */
+  const element_type* device_ptr_on(const ttg::device::Device& device) const {
+    assert(is_valid());
+    if (device.is_device()) throw std::runtime_error("MADNESS missing support for non-host memory!");
+    return m_host_data;
+  }
+
+  element_type* host_ptr() {
+    return m_host_data;
+  }
+
+  const element_type* host_ptr() const {
+    return m_host_data;
+  }
+
+  bool is_valid_on(const ttg::device::Device& device) const {
+    assert(is_valid());
+    if (device.is_device()) throw std::runtime_error("MADNESS missing support for non-host memory!");
+    return true;
+  }
+
+  void allocate_on(const ttg::device::Device& device_id) {
+    /* TODO: need exposed PaRSEC memory allocator */
+    throw std::runtime_error("not implemented yet");
+  }
+
+  /* TODO: can we do this automatically?
+   * Pin the memory on all devices we currently track.
+   * Pinned memory won't be released by PaRSEC and can be used
+   * at any time.
+   */
+  void pin() {
+    // nothing to do
+  }
+
+  /* Unpin the memory on all devices we currently track. */
+  void unpin() {
+    // nothing to do
+  }
+
+  /* Pin the memory on a given device */
+  void pin_on(int device_id) {
+    /* TODO: how can we pin memory on a device? */
+  }
+
+  /* Pin the memory on a given device */
+  void unpin_on(int device_id) {
+    /* TODO: how can we unpin memory on a device? */
+  }
+
+  bool is_valid() const {
+    return true;
+  }
+
+  operator bool() const {
+    return true;
+  }
+
+  std::size_t size() const {
+    return m_count;
+  }
+
+  /* Reallocate the buffer with count elements */
+  void reset(std::size_t n) {
+
+    if (m_owned) {
+      deallocate();
+      m_owned = false;
+    }
+
+    if (n == 0) {
+      m_host_data = nullptr;
+      m_owned = false;
+    } else {
+      m_host_data = allocate(n);
+      m_owned = true;
+    }
+    m_count = n;
+  }
+
+  /* Reset the buffer to use the ptr to count elements */
+  void reset(T* ptr, std::size_t n = 1) {
+    /* TODO: can we resize if count is smaller than m_count? */
+    if (n == m_count) {
+      return;
+    }
+
+    if (m_owned) {
+      deallocate();
+    }
+
+    if (nullptr == ptr) {
+      m_host_data = nullptr;
+      m_count = 0;
+      m_owned = false;
+    } else {
+      m_host_data = ptr;
+      m_count = n;
+      m_owned = false;
+    }
+  }
+
+  /* serialization support */
+
+#if defined(TTG_SERIALIZATION_SUPPORTS_BOOST) && 0
+  template <typename Archive>
+  void serialize(Archive& ar, const unsigned int version) {
+    if constexpr (ttg::detail::is_output_archive_v<Archive>) {
+      std::size_t s = size();
+      ar& s;
+      /* TODO: how to serialize the array? */
+    } else {
+      std::size_t s;
+      ar & s;
+      /* initialize internal pointers and then reset */
+      reset(s);
+      /* TODO: how to deserialize the array? */
+    }
+  }
+#endif // TTG_SERIALIZATION_SUPPORTS_BOOST
+
+#if defined(TTG_SERIALIZATION_SUPPORTS_CEREAL) && 0
+  template <class Archive>
+  std::enable_if_t<std::is_base_of_v<cereal::detail::InputArchiveBase, Archive> ||
+                    std::is_base_of_v<cereal::detail::OutputArchiveBase, Archive>>
+  serialize(Archive& ar) {
+    if constexpr (ttg::detail::is_output_archive_v<Archive>)
+      std::size_t s = size();
+      ar(s);
+      /* TODO: how to serialize the array? */
+    else {
+      std::size_t s;
+      ar(s);
+      reset(s);
+      /* TODO: how to deserialize the array? */
+    }
+    ar(value);
+  }
+#endif // TTG_SERIALIZATION_SUPPORTS_CEREAL
+
+#if defined(TTG_SERIALIZATION_SUPPORTS_MADNESS) && 0
+  template <typename Archive>
+  std::enable_if_t<std::is_base_of_v<madness::archive::BufferInputArchive, Archive> ||
+                   std::is_base_of_v<madness::archive::BufferOutputArchive, Archive>>
+  serialize(Archive& ar) {
+    if constexpr (ttg::detail::is_output_archive_v<Archive>) {
+      std::size_t s = size();
+      ar& s;
+      ar << wrap(host_ptr(), s);
+    } else {
+      std::size_t s;
+      ar & s;
+      reset(s);
+      ar >> wrap(host_ptr(), s);  // MatrixTile<T>(bm.rows(), bm.cols());
+    }
+  }
+#endif // TTG_SERIALIZATION_SUPPORTS_MADNESS
+
+
+};
+
+} // namespace ttg_madness
+
+#endif // TTG_MADNESS_BUFFER_H
diff --git a/ttg/ttg/madness/fwd.h b/ttg/ttg/madness/fwd.h
index 66c6bacba..cafcf56a0 100644
--- a/ttg/ttg/madness/fwd.h
+++ b/ttg/ttg/madness/fwd.h
@@ -26,6 +26,7 @@ namespace ttg_madness {
 
   inline void ttg_finalize();
 
+  [[noreturn]]
   inline void ttg_abort();
 
   inline ttg::World ttg_default_execution_context();
@@ -49,7 +50,7 @@ namespace ttg_madness {
 
 
   /* device definitions, not currently provided by this impl */
-  template<typename T, typename Allocator>
+  template<typename T, typename Allocator = std::allocator<T>>
   struct Buffer;
 
   template<typename T>
diff --git a/ttg/ttg/madness/ttg.h b/ttg/ttg/madness/ttg.h
index 7f2ffab08..4e509108a 100644
--- a/ttg/ttg/madness/ttg.h
+++ b/ttg/ttg/madness/ttg.h
@@ -135,7 +135,10 @@ namespace ttg_madness {
     ::madness::finalize();
   }
   inline ttg::World ttg_default_execution_context() { return ttg::get_default_world(); }
-  inline void ttg_abort() { MPI_Abort(ttg_default_execution_context().impl().impl().mpi.Get_mpi_comm(), 1); }
+  inline void ttg_abort() {
+    MPI_Abort(ttg_default_execution_context().impl().impl().mpi.Get_mpi_comm(), 1);
+    assert(0); // make sure we abort
+  }
   inline void ttg_execute(ttg::World world) {
     // World executes tasks eagerly
   }
@@ -1292,5 +1295,7 @@ namespace ttg_madness {
 }  // namespace ttg_madness
 
 #include "ttg/madness/watch.h"
+#include "ttg/madness/buffer.h"
+#include "ttg/madness/ttvalue.h"
 
 #endif  // MADNESS_TTG_H_INCLUDED
diff --git a/ttg/ttg/madness/ttvalue.h b/ttg/ttg/madness/ttvalue.h
new file mode 100644
index 000000000..ad53ee5f8
--- /dev/null
+++ b/ttg/ttg/madness/ttvalue.h
@@ -0,0 +1,14 @@
+#ifndef TTG_MADNESS_TTVALUE_H
+#define TTG_MADNESS_TTVALUE_H
+
+namespace ttg_madness {
+
+  template<typename DerivedT>
+  struct TTValue
+  {
+    /* empty */
+  };
+
+} // namespace ttg_madness
+
+  #endif // TTG_MADNESS_TTVALUE_H
diff --git a/ttg/ttg/parsec/fwd.h b/ttg/ttg/parsec/fwd.h
index d6f072a9c..92411988f 100644
--- a/ttg/ttg/parsec/fwd.h
+++ b/ttg/ttg/parsec/fwd.h
@@ -55,7 +55,7 @@ namespace ttg_parsec {
   static void ttg_broadcast(ttg::World world, T &data, int source_rank);
 
   /* device definitions */
-  template<typename T, typename Allocator>
+  template<typename T, typename Allocator = std::allocator<T>>
   struct Buffer;
 
   template<typename T>
diff --git a/ttg/ttg/parsec/import.h b/ttg/ttg/parsec/import.h
index 50b720083..18ad7e89d 100644
--- a/ttg/ttg/parsec/import.h
+++ b/ttg/ttg/parsec/import.h
@@ -21,7 +21,6 @@ namespace ttg {
 
   constexpr const ttg::Runtime ttg_runtime = ttg::Runtime::PaRSEC;
 
-
 }  // namespace ttg
 
 #endif  // TTG_PARSEC_IMPORT_H
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 97acaa839..626fa8827 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -4199,7 +4199,7 @@ struct ttg::detail::value_copy_handler<ttg::Runtime::PaRSEC> {
   }
 
   template<typename Value>
-  inline std::add_lvalue_reference_t<Value> operator()(ttg::detail::persistent_value_ref<Value> vref) {
+  inline std::add_lvalue_reference_t<Value> operator()(ttg_parsec::detail::persistent_value_ref<Value> vref) {
     auto caller = ttg_parsec::detail::parsec_ttg_caller;
     if (nullptr == caller) {
       ttg::print("ERROR: ttg::send or ttg::broadcast called outside of a task!\n");
diff --git a/ttg/ttg/parsec/ttvalue.h b/ttg/ttg/parsec/ttvalue.h
index 6c3942854..b93f1687f 100644
--- a/ttg/ttg/parsec/ttvalue.h
+++ b/ttg/ttg/parsec/ttvalue.h
@@ -5,7 +5,7 @@
 
 #include "ttg/parsec/ttg_data_copy.h"
 
-namespace ttg {
+namespace ttg_parsec {
 
   /**
    * Base class for data to moved into, through, and out of
@@ -96,6 +96,6 @@ namespace ttg {
     return detail::persistent_value_ref<ValueT>{value};
   }
 
-} // namespace ttg
+} // namespace ttg_parsec
 
 #endif // TTG_PARSEC_TTVALUE_H
\ No newline at end of file
diff --git a/ttg/ttg/serialization/backends/madness.h b/ttg/ttg/serialization/backends/madness.h
index e3f64cc86..fff23b5c9 100644
--- a/ttg/ttg/serialization/backends/madness.h
+++ b/ttg/ttg/serialization/backends/madness.h
@@ -13,6 +13,8 @@
 #include <madness/world/type_traits.h>
 #endif
 
+#include "ttg/serialization/traits.h"
+
 namespace ttg::detail {
 
   /*----- if_madness_{input,output,}_archive_v -----*/
diff --git a/ttg/ttg/ttvalue.h b/ttg/ttg/ttvalue.h
new file mode 100644
index 000000000..449beafa6
--- /dev/null
+++ b/ttg/ttg/ttvalue.h
@@ -0,0 +1,13 @@
+#ifndef TTG_TTVALUE_H
+#define TTG_TTVALUE_H
+
+#include "ttg/fwd.h"
+
+namespace ttg {
+
+  template<typename T>
+  using TTValue = TTG_IMPL_NS::TTValue<T, Allocator>;
+
+} // namespace ttg
+
+#endif // TTG_TTVALUE_H
\ No newline at end of file

From 5cf2502fc01caf477779ace4c1bff78c773f71e7 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 20 Dec 2023 19:37:32 +0000
Subject: [PATCH 200/259] More fixes to ttvalue and ptr

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg.h             |  1 +
 ttg/ttg/madness/fwd.h |  3 +++
 ttg/ttg/parsec/fwd.h  |  5 ++++-
 ttg/ttg/parsec/ptr.h  | 12 ++++++------
 ttg/ttg/ptr.h         |  2 +-
 ttg/ttg/ttvalue.h     |  2 +-
 6 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/ttg/ttg.h b/ttg/ttg.h
index 1a88cea2a..85e987e85 100644
--- a/ttg/ttg.h
+++ b/ttg/ttg.h
@@ -31,6 +31,7 @@
 #include "ttg/ptr.h"
 #include "ttg/buffer.h"
 #include "ttg/devicescratch.h"
+#include "ttg/ttvalue.h"
 #include "ttg/devicescope.h"
 #include "ttg/device/device.h"
 #include "ttg/device/task.h"
diff --git a/ttg/ttg/madness/fwd.h b/ttg/ttg/madness/fwd.h
index cafcf56a0..0d340db0b 100644
--- a/ttg/ttg/madness/fwd.h
+++ b/ttg/ttg/madness/fwd.h
@@ -59,6 +59,9 @@ namespace ttg_madness {
   template<typename T>
   struct devicescratch;
 
+  template<typename T>
+  struct TTValue;
+
   template<typename T, typename... Args>
   Ptr<T> make_ptr(Args&&... args);
 
diff --git a/ttg/ttg/parsec/fwd.h b/ttg/ttg/parsec/fwd.h
index 92411988f..2ff56dc60 100644
--- a/ttg/ttg/parsec/fwd.h
+++ b/ttg/ttg/parsec/fwd.h
@@ -64,11 +64,14 @@ namespace ttg_parsec {
   template<typename T>
   struct devicescratch;
 
+  template<typename T>
+  struct TTValue;
+
   template<typename T, typename... Args>
   Ptr<T> make_ptr(Args&&... args);
 
   template<typename T>
-  auto get_ptr(T&& obj);
+  Ptr<std::decay_t<T>> get_ptr(T&& obj);
 
   template<typename... Views>
   inline bool register_device_memory(std::tuple<Views&...> &views);
diff --git a/ttg/ttg/parsec/ptr.h b/ttg/ttg/parsec/ptr.h
index c4f1200fd..8d7f62e46 100644
--- a/ttg/ttg/parsec/ptr.h
+++ b/ttg/ttg/parsec/ptr.h
@@ -123,7 +123,7 @@ namespace ttg_parsec {
 
 
     template<typename T>
-    ttg_parsec::detail::ttg_data_copy_t* get_copy(ttg_parsec::ptr<T>& p);
+    ttg_parsec::detail::ttg_data_copy_t* get_copy(ttg_parsec::Ptr<T>& p);
   } // namespace detail
 
   // fwd decl
@@ -249,8 +249,8 @@ namespace ttg_parsec {
 #endif // 0
 
   template<typename T>
-  inline ptr<std::decay_t<T>> get_ptr(T&& obj) {
-    using ptr_type = ptr<std::decay_t<T>>;
+  inline auto get_ptr(T&& obj) {
+    using ptr_type = Ptr<std::decay_t<T>>;
     if (nullptr != detail::parsec_ttg_caller) {
       for (int i = 0; i < detail::parsec_ttg_caller->data_count; ++i) {
         detail::ttg_data_copy_t *copy = detail::parsec_ttg_caller->copies[i];
@@ -267,14 +267,14 @@ namespace ttg_parsec {
   }
 
   template<typename T, typename... Args>
-  inline ptr<T> make_ptr(Args&&... args) {
+  inline Ptr<T> make_ptr(Args&&... args) {
     detail::ttg_data_copy_t *copy = detail::create_new_datacopy(T(std::forward<Args>(args)...));
-    return ptr<T>(copy);
+    return Ptr<T>(copy);
   }
 
   namespace detail {
     template<typename T>
-    detail::ttg_data_copy_t* get_copy(ttg_parsec::ptr<T>& p) {
+    detail::ttg_data_copy_t* get_copy(ttg_parsec::Ptr<T>& p) {
       return p.get_copy();
     }
   } // namespace detail
diff --git a/ttg/ttg/ptr.h b/ttg/ttg/ptr.h
index 999b6124f..272eb4409 100644
--- a/ttg/ttg/ptr.h
+++ b/ttg/ttg/ptr.h
@@ -14,7 +14,7 @@ Ptr<T> make_ptr(Args&&... args) {
 }
 
 template<typename T>
-auto get_ptr(T&& obj) {
+Ptr<std::decay_t<T>> get_ptr(T&& obj) {
   return TTG_IMPL_NS::get_ptr(std::forward<T>(obj));
 }
 
diff --git a/ttg/ttg/ttvalue.h b/ttg/ttg/ttvalue.h
index 449beafa6..4feb1376f 100644
--- a/ttg/ttg/ttvalue.h
+++ b/ttg/ttg/ttvalue.h
@@ -6,7 +6,7 @@
 namespace ttg {
 
   template<typename T>
-  using TTValue = TTG_IMPL_NS::TTValue<T, Allocator>;
+  using TTValue = TTG_IMPL_NS::TTValue<T>;
 
 } // namespace ttg
 

From 220c038e51c0171ac1f899aa44135bf0c41afee7 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 20 Dec 2023 19:38:21 +0000
Subject: [PATCH 201/259] More catch2 include fixes

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 tests/unit/device_coro.cc           | 85 +++++++++++++++--------------
 tests/unit/fibonacci-coro.cc        |  2 +-
 tests/unit/ranges.cc                |  2 +-
 tests/unit/splitmd_serialization.cc |  2 +-
 tests/unit/streams.cc               |  4 +-
 tests/unit/tt.cc                    |  2 +-
 6 files changed, 50 insertions(+), 47 deletions(-)

diff --git a/tests/unit/device_coro.cc b/tests/unit/device_coro.cc
index ae2fb96bb..124945542 100644
--- a/tests/unit/device_coro.cc
+++ b/tests/unit/device_coro.cc
@@ -1,14 +1,13 @@
-#include <catch2/catch.hpp>
+#include <catch2/catch_all.hpp>
 
 #include "ttg.h"
-#include "ttg/view.h"
 
 #include "ttg/serialization.h"
 
 #include "cuda_kernel.h"
 
 struct value_t {
-  ttg::buffer<double> db; // TODO: rename
+  ttg::Buffer<double> db; // TODO: rename
   int quark;
 
   template<typename Archive>
@@ -29,23 +28,25 @@ namespace madness::archive {
 }  // namespace madness::archive
 #endif  // TTG_SERIALIZATION_SUPPORTS_MADNESS
 
+#if defined(TTG_IMPL_DEVICE_SUPPORT)
+
 TEST_CASE("Device", "coro") {
 
   SECTION("devicebuf") {
 
     ttg::Edge<int, value_t> edge;
-    auto fn = [&](const int& key, value_t&& val) -> ttg::device_task {
+    auto fn = [&](const int& key, value_t&& val) -> ttg::device::Task {
       //ttg::print("device_task key ", key);
 
       /* wait for the view to be available on the device */
-      co_await ttg::to_device(val.db);
+      co_await ttg::device::select(val.db);
       /* once we're back here the data has been transferred */
       CHECK(val.db.current_device_ptr() != nullptr);
 
       /* NO KERNEL */
 
       /* here we suspend to wait for a kernel to complete */
-      co_await ttg::wait_kernel();
+      co_await ttg::device::wait();
 
       /* we're back, the kernel executed and we can send */
       if (key < 10) {
@@ -59,7 +60,7 @@ TEST_CASE("Device", "coro") {
 
     auto tt = ttg::make_tt<ttg::ExecutionSpace::CUDA>(fn, ttg::edges(edge), ttg::edges(edge),
                                                       "device_task", {"edge_in"}, {"edge_out"});
-    make_graph_executable(tt);
+    ttg::make_graph_executable(tt);
     if (ttg::default_execution_context().rank() == 0) tt->invoke(0, value_t{});
     std::cout << "Entering fence" << std::endl;
     ttg::ttg_fence(ttg::default_execution_context());
@@ -68,11 +69,11 @@ TEST_CASE("Device", "coro") {
   SECTION("devicebuf-inc") {
 
     ttg::Edge<int, value_t> edge;
-    auto fn = [&](const int& key, value_t&& val) -> ttg::device_task {
+    auto fn = [&](const int& key, value_t&& val) -> ttg::device::Task {
       //ttg::print("device_task key ", key);
 
       /* wait for the view to be available on the device */
-      co_await ttg::to_device(val.db);
+      co_await ttg::device::select(val.db);
       /* once we're back here the data has been transferred */
       CHECK(val.db.current_device_ptr() != nullptr);
 
@@ -84,7 +85,7 @@ TEST_CASE("Device", "coro") {
 #endif // TTG_HAVE_CUDA
 
       /* here we suspend to wait for a kernel to complete */
-      co_await ttg::wait_kernel(val.db);
+      co_await ttg::device::wait(val.db);
 
       std::cout << "KEY " << key << " VAL OUT DEV " << *val.db.current_device_ptr() << " VAL OUT HOST " << *val.db.host_ptr() << std::endl;
 
@@ -105,7 +106,7 @@ TEST_CASE("Device", "coro") {
 
     auto tt = ttg::make_tt<ttg::ExecutionSpace::CUDA>(fn, ttg::edges(edge), ttg::edges(edge),
                                                       "device_task", {"edge_in"}, {"edge_out"});
-    make_graph_executable(tt);
+    ttg::make_graph_executable(tt);
     value_t v;
     *v.db.host_ptr() = 2.0; // start from non-zero value
     if (ttg::default_execution_context().rank() == 0) tt->invoke(2, std::move(v));
@@ -116,12 +117,12 @@ TEST_CASE("Device", "coro") {
   SECTION("scratch") {
 
     ttg::Edge<int, value_t> edge;
-    auto fn = [&](const int& key, value_t&& val) -> ttg::device_task {
+    auto fn = [&](const int& key, value_t&& val) -> ttg::device::Task {
       double scratch = 0.0;
       ttg::devicescratch<double> ds = ttg::make_scratch(&scratch, ttg::scope::Allocate);
 
       /* wait for the view to be available on the device */
-      co_await ttg::to_device(ds, val.db);
+      co_await ttg::device::select(ds, val.db);
       /* once we're back here the data has been transferred */
       CHECK(ds.device_ptr()  != nullptr);
 
@@ -131,7 +132,7 @@ TEST_CASE("Device", "coro") {
 #endif // TTG_HAVE_CUDA
 
       /* here we suspend to wait for a kernel to complete */
-      co_await ttg::wait_kernel(ds);
+      co_await ttg::device::wait(ds);
 
 #ifdef TTG_HAVE_CUDA
       /* the scratch is allocated but no data is transferred in; it's incremented once */
@@ -150,7 +151,7 @@ TEST_CASE("Device", "coro") {
 
     auto tt = ttg::make_tt<ttg::ExecutionSpace::CUDA>(fn, ttg::edges(edge), ttg::edges(edge),
                                                       "device_task", {"edge_in"}, {"edge_out"});
-    make_graph_executable(tt);
+    ttg::make_graph_executable(tt);
     if (ttg::default_execution_context().rank() == 0) tt->invoke(0, value_t{});
     ttg::ttg_fence(ttg::default_execution_context());
   }
@@ -158,12 +159,12 @@ TEST_CASE("Device", "coro") {
   SECTION("scratch-syncin") {
 
     ttg::Edge<int, value_t> edge;
-    auto fn = [&](const int& key, value_t&& val) -> ttg::device_task {
+    auto fn = [&](const int& key, value_t&& val) -> ttg::device::Task {
       double scratch = key;
       ttg::devicescratch<double> ds = ttg::make_scratch(&scratch, ttg::scope::SyncIn);
 
       /* wait for the view to be available on the device */
-      co_await ttg::to_device(ds, val.db);
+      co_await ttg::device::select(ds, val.db);
       /* once we're back here the data has been transferred */
       CHECK(ds.device_ptr()  != nullptr);
 
@@ -173,7 +174,7 @@ TEST_CASE("Device", "coro") {
 #endif // TTG_HAVE_CUDA
 
       /* here we suspend to wait for a kernel to complete */
-      co_await ttg::wait_kernel(ds);
+      co_await ttg::device::wait(ds);
 
 #ifdef TTG_HAVE_CUDA
       /* scratch is increment once per task, so it should be the same as key */
@@ -192,7 +193,7 @@ TEST_CASE("Device", "coro") {
 
     auto tt = ttg::make_tt<ttg::ExecutionSpace::CUDA>(fn, ttg::edges(edge), ttg::edges(edge),
                                                       "device_task", {"edge_in"}, {"edge_out"});
-    make_graph_executable(tt);
+    ttg::make_graph_executable(tt);
     if (ttg::default_execution_context().rank() == 0) tt->invoke(0, value_t{});
     ttg::ttg_fence(ttg::default_execution_context());
   }
@@ -200,12 +201,12 @@ TEST_CASE("Device", "coro") {
   SECTION("scratch-value-out") {
 
     ttg::Edge<int, value_t> edge;
-    auto fn = [&](const int& key, value_t&& val) -> ttg::device_task {
+    auto fn = [&](const int& key, value_t&& val) -> ttg::device::Task {
       double scratch = 0.0;
       ttg::devicescratch<double> ds = ttg::make_scratch(&scratch, ttg::scope::Allocate);
 
       /* wait for the view to be available on the device */
-      co_await ttg::to_device(ds, val.db);
+      co_await ttg::device::select(ds, val.db);
       /* once we're back here the data has been transferred */
       CHECK(ds.device_ptr()  != nullptr);
 
@@ -215,7 +216,7 @@ TEST_CASE("Device", "coro") {
 #endif // TTG_HAVE_CUDA
 
       /* here we suspend to wait for a kernel to complete */
-      co_await ttg::wait_kernel(ds, val.db);
+      co_await ttg::device::wait(ds, val.db);
 
 #ifdef TTG_HAVE_CUDA
       /* buffer is increment once per task, so it should be 1 */
@@ -234,7 +235,7 @@ TEST_CASE("Device", "coro") {
 
     auto tt = ttg::make_tt<ttg::ExecutionSpace::CUDA>(fn, ttg::edges(edge), ttg::edges(edge),
                                                       "device_task", {"edge_in"}, {"edge_out"});
-    make_graph_executable(tt);
+    ttg::make_graph_executable(tt);
     if (ttg::default_execution_context().rank() == 0) tt->invoke(0, value_t{});
     ttg::ttg_fence(ttg::default_execution_context());
   }
@@ -245,12 +246,12 @@ TEST_CASE("Device", "coro") {
     ttg::Ptr<value_t> ptr;
     int last_key = 0;
     constexpr const int num_iter = 10;
-    auto fn = [&](const int& key, value_t&& val) -> ttg::device_task {
+    auto fn = [&](const int& key, value_t&& val) -> ttg::device::Task {
       double scratch = key;
       ttg::devicescratch<double> ds = ttg::make_scratch(&scratch, ttg::scope::SyncIn);
 
       /* wait for the view to be available on the device */
-      co_await ttg::to_device(ds, val.db);
+      co_await ttg::device::select(ds, val.db);
       /* once we're back here the data has been transferred */
       CHECK(ds.device_ptr()  != nullptr);
 
@@ -260,7 +261,7 @@ TEST_CASE("Device", "coro") {
 #endif // TTG_HAVE_CUDA
 
       /* here we suspend to wait for a kernel and the out-transfer to complete */
-      co_await ttg::wait_kernel(val.db, ds);
+      co_await ttg::device::wait(val.db, ds);
 
 #ifdef TTG_HAVE_CUDA
       /* buffer is increment once per task, so it should be the same as key */
@@ -284,7 +285,7 @@ TEST_CASE("Device", "coro") {
 
     auto tt = ttg::make_tt<ttg::ExecutionSpace::CUDA>(fn, ttg::edges(edge), ttg::edges(edge),
                                                       "device_task", {"edge_in"}, {"edge_out"});
-    make_graph_executable(tt);
+    ttg::make_graph_executable(tt);
     if (ttg::default_execution_context().rank() == 0) tt->invoke(0, value_t{});
     ttg::ttg_fence(ttg::default_execution_context());
     if (num_iter == last_key) {
@@ -322,10 +323,10 @@ TEST_CASE("Device", "coro") {
     auto htt = ttg::make_tt(host_fn, ttg::edges(d2h), ttg::edges(h2d),
                             "host_task", {"d2h"}, {"h2d"});
 
-    auto device_fn = [&](const int& key, value_t&& val) -> ttg::device_task {
+    auto device_fn = [&](const int& key, value_t&& val) -> ttg::device::Task {
 
       /* wait for the view to be available on the device */
-      co_await ttg::to_device(val.db);
+      co_await ttg::device::select(val.db);
 
       /* call a kernel */
 #ifdef TTG_HAVE_CUDA
@@ -333,8 +334,8 @@ TEST_CASE("Device", "coro") {
 #endif // TTG_HAVE_CUDA
 
       /* here we suspend to wait for a kernel to complete */
-      //co_await ttg::wait_kernel(val.db);
-      co_await ttg::wait_kernel();
+      //co_await ttg::device::wait(val.db);
+      co_await ttg::device::wait();
 
       /* we're back, the kernel executed and we can send */
       if (key < 10) {
@@ -346,7 +347,7 @@ TEST_CASE("Device", "coro") {
 
     auto dtt = ttg::make_tt<ttg::ExecutionSpace::CUDA>(device_fn, ttg::edges(h2d), ttg::edges(d2h),
                                                       "device_task", {"h2d"}, {"d2h"});
-    make_graph_executable(dtt);
+    ttg::make_graph_executable(dtt);
     if (ttg::default_execution_context().rank() == 0) htt->invoke(0, value_t{});
     ttg::ttg_fence(ttg::default_execution_context());
   }
@@ -354,12 +355,12 @@ TEST_CASE("Device", "coro") {
   SECTION("loop") {
 
     ttg::Edge<int, value_t> edge;
-    auto fn = [&](int key, value_t&& val) -> ttg::device_task {
+    auto fn = [&](int key, value_t&& val) -> ttg::device::Task {
       double scratch = 1.0;
       ttg::devicescratch<double> ds = ttg::make_scratch(&scratch, ttg::scope::Allocate);
 
       /* wait for the view to be available on the device */
-      co_await ttg::to_device(ds, val.db);
+      co_await ttg::device::select(ds, val.db);
       /* once we're back here the data has been transferred */
       CHECK(ds.device_ptr()  != nullptr);
 
@@ -375,7 +376,7 @@ TEST_CASE("Device", "coro") {
 #endif // TTG_HAVE_CUDA
 
         /* here we suspend to wait for a kernel and the out-transfer to complete */
-        co_await ttg::wait_kernel(val.db);
+        co_await ttg::device::wait(val.db);
 
 #ifdef TTG_HAVE_CUDA
         /* buffer is increment once per task, so it should be the same as key */
@@ -387,7 +388,7 @@ TEST_CASE("Device", "coro") {
 
     auto tt = ttg::make_tt<ttg::ExecutionSpace::CUDA>(fn, ttg::edges(edge), ttg::edges(edge),
                                                       "device_task", {"edge_in"}, {"edge_out"});
-    make_graph_executable(tt);
+    ttg::make_graph_executable(tt);
     if (ttg::default_execution_context().rank() == 0) tt->invoke(0, value_t{});
     ttg::ttg_fence(ttg::default_execution_context());
   }
@@ -395,12 +396,12 @@ TEST_CASE("Device", "coro") {
   SECTION("loop-scratchout") {
 
     ttg::Edge<int, value_t> edge;
-    auto fn = [&](int key, value_t&& val) -> ttg::device_task {
+    auto fn = [&](int key, value_t&& val) -> ttg::device::Task {
       double scratch = -10.0;
       ttg::devicescratch<double> ds = ttg::make_scratch(&scratch, ttg::scope::SyncIn);
 
       /* wait for the view to be available on the device */
-      co_await ttg::to_device(ds, val.db);
+      co_await ttg::device::select(ds, val.db);
       /* once we're back here the data has been transferred */
       CHECK(ds.device_ptr()  != nullptr);
 
@@ -416,7 +417,7 @@ TEST_CASE("Device", "coro") {
 #endif // TTG_HAVE_CUDA
 
         /* here we suspend to wait for a kernel and the out-transfer to complete */
-        co_await ttg::wait_kernel(val.db, ds);
+        co_await ttg::device::wait(val.db, ds);
 
 #ifdef TTG_HAVE_CUDA
         /* buffer is increment once per task, so it should be the same as key */
@@ -428,8 +429,10 @@ TEST_CASE("Device", "coro") {
 
     auto tt = ttg::make_tt<ttg::ExecutionSpace::CUDA>(fn, ttg::edges(edge), ttg::edges(edge),
                                                       "device_task", {"edge_in"}, {"edge_out"});
-    make_graph_executable(tt);
+    ttg::make_graph_executable(tt);
     if (ttg::default_execution_context().rank() == 0) tt->invoke(0, value_t{});
     ttg::ttg_fence(ttg::default_execution_context());
   }
-}
\ No newline at end of file
+}
+
+#endif // TTG_IMPL_DEVICE_SUPPORT
diff --git a/tests/unit/fibonacci-coro.cc b/tests/unit/fibonacci-coro.cc
index e8ed5d1ba..dd4af6aa2 100644
--- a/tests/unit/fibonacci-coro.cc
+++ b/tests/unit/fibonacci-coro.cc
@@ -1,4 +1,4 @@
-#include <catch2/catch.hpp>
+#include <catch2/catch_all.hpp>
 
 #include "ttg.h"
 
diff --git a/tests/unit/ranges.cc b/tests/unit/ranges.cc
index cd423a2bd..f532805f5 100644
--- a/tests/unit/ranges.cc
+++ b/tests/unit/ranges.cc
@@ -1,4 +1,4 @@
-#include <catch2/catch.hpp>
+#include <catch2/catch_all.hpp>
 
 #include "ttg.h"
 
diff --git a/tests/unit/splitmd_serialization.cc b/tests/unit/splitmd_serialization.cc
index 482b338ac..4323b5fda 100644
--- a/tests/unit/splitmd_serialization.cc
+++ b/tests/unit/splitmd_serialization.cc
@@ -3,7 +3,7 @@
 
 #include "ttg.h"
 
-#include <catch2/catch.hpp>
+#include <catch2/catch_all.hpp>
 
 using value_t = int;
 constexpr const int N = 10, M = 10;
diff --git a/tests/unit/streams.cc b/tests/unit/streams.cc
index 108c8610b..dfa861642 100644
--- a/tests/unit/streams.cc
+++ b/tests/unit/streams.cc
@@ -1,4 +1,4 @@
-#include <catch2/catch.hpp>
+#include <catch2/catch_all.hpp>
 #include <ctime>
 
 #include "ttg.h"
@@ -72,4 +72,4 @@ TEST_CASE("streams", "[streams][core]") {
     ttg::ttg_fence(ttg::default_execution_context());
     CHECK(reduce_ops == N);
   }
-}  // TEST_CASE("streams")
\ No newline at end of file
+}  // TEST_CASE("streams")
diff --git a/tests/unit/tt.cc b/tests/unit/tt.cc
index def14684a..e4f7524d3 100644
--- a/tests/unit/tt.cc
+++ b/tests/unit/tt.cc
@@ -1,4 +1,4 @@
-#include <catch2/catch.hpp>
+#include <catch2/catch_all.hpp>
 
 #include "ttg.h"
 

From e9b3233bf32b30a9e0dc65a6955a5ff7ecbfaff8 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 20 Dec 2023 19:55:28 +0000
Subject: [PATCH 202/259] Fix ttvalue and buffer in parsec

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/buffer.h        | 2 --
 ttg/ttg/parsec/fwd.h           | 4 ++--
 ttg/ttg/parsec/ptr.h           | 6 +++---
 ttg/ttg/parsec/ttg_data_copy.h | 7 +++++++
 ttg/ttg/ptr.h                  | 4 ++--
 5 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/ttg/ttg/parsec/buffer.h b/ttg/ttg/parsec/buffer.h
index 7325c0214..590a624f8 100644
--- a/ttg/ttg/parsec/buffer.h
+++ b/ttg/ttg/parsec/buffer.h
@@ -283,8 +283,6 @@ struct Buffer : public detail::ttg_parsec_data_wrapper_t
   /* Reallocate the buffer with count elements */
   void reset(std::size_t n) {
     /* TODO: can we resize if count is smaller than m_count? */
-    /* drop the current data and reallocate */
-    reset();
 
     if (m_owned) {
       deallocate();
diff --git a/ttg/ttg/parsec/fwd.h b/ttg/ttg/parsec/fwd.h
index 2ff56dc60..d5bc8931e 100644
--- a/ttg/ttg/parsec/fwd.h
+++ b/ttg/ttg/parsec/fwd.h
@@ -68,10 +68,10 @@ namespace ttg_parsec {
   struct TTValue;
 
   template<typename T, typename... Args>
-  Ptr<T> make_ptr(Args&&... args);
+  inline Ptr<T> make_ptr(Args&&... args);
 
   template<typename T>
-  Ptr<std::decay_t<T>> get_ptr(T&& obj);
+  inline Ptr<std::decay_t<T>> get_ptr(T&& obj);
 
   template<typename... Views>
   inline bool register_device_memory(std::tuple<Views&...> &views);
diff --git a/ttg/ttg/parsec/ptr.h b/ttg/ttg/parsec/ptr.h
index 8d7f62e46..a20c0c746 100644
--- a/ttg/ttg/parsec/ptr.h
+++ b/ttg/ttg/parsec/ptr.h
@@ -132,7 +132,7 @@ namespace ttg_parsec {
 
   // fwd decl
   template<typename T>
-  Ptr<std::decay_t<T>> get_ptr(T&& obj);
+  inline Ptr<std::decay_t<T>> get_ptr(T&& obj);
 
   template<typename T>
   struct Ptr {
@@ -249,7 +249,7 @@ namespace ttg_parsec {
 #endif // 0
 
   template<typename T>
-  inline auto get_ptr(T&& obj) {
+  inline Ptr<std::decay_t<T>> get_ptr(T&& obj) {
     using ptr_type = Ptr<std::decay_t<T>>;
     if (nullptr != detail::parsec_ttg_caller) {
       for (int i = 0; i < detail::parsec_ttg_caller->data_count; ++i) {
@@ -274,7 +274,7 @@ namespace ttg_parsec {
 
   namespace detail {
     template<typename T>
-    detail::ttg_data_copy_t* get_copy(ttg_parsec::Ptr<T>& p) {
+    inline detail::ttg_data_copy_t* get_copy(ttg_parsec::Ptr<T>& p) {
       return p.get_copy();
     }
   } // namespace detail
diff --git a/ttg/ttg/parsec/ttg_data_copy.h b/ttg/ttg/parsec/ttg_data_copy.h
index 5eb3d5b0e..a4b4575fa 100644
--- a/ttg/ttg/parsec/ttg_data_copy.h
+++ b/ttg/ttg/parsec/ttg_data_copy.h
@@ -519,6 +519,7 @@ namespace ttg_parsec {
      * definition of ttg_parsec_data_wrapper_t members that depend on ttg_data_copy_t
      */
 
+    inline
     void ttg_parsec_data_wrapper_t::remove_from_owner() {
       if (nullptr != m_ttg_copy) {
         m_ttg_copy->remove_device_data(this);
@@ -526,6 +527,7 @@ namespace ttg_parsec {
       }
     }
 
+    inline
     void ttg_parsec_data_wrapper_t::reset_parsec_data(void *ptr, size_t size) {
       if (ptr == m_data.get()) return;
 
@@ -536,6 +538,7 @@ namespace ttg_parsec {
       }
     }
 
+    inline
     ttg_parsec_data_wrapper_t::ttg_parsec_data_wrapper_t()
     : m_data(nullptr, delete_null_parsec_data)
     , m_ttg_copy(detail::ttg_data_copy_container())
@@ -544,6 +547,8 @@ namespace ttg_parsec {
         m_ttg_copy->add_device_data(this);
       }
     }
+
+    inline
     ttg_parsec_data_wrapper_t::ttg_parsec_data_wrapper_t(ttg_parsec_data_wrapper_t&& other)
     : m_data(std::move(other.m_data))
     , m_ttg_copy(detail::ttg_data_copy_container())
@@ -562,6 +567,7 @@ namespace ttg_parsec {
       //}
     }
 
+    inline
     ttg_parsec_data_wrapper_t& ttg_parsec_data_wrapper_t::operator=(ttg_parsec_data_wrapper_t&& other) {
       m_data = std::move(other.m_data);
       /* check whether the owning ttg_data_copy has already moved us */
@@ -578,6 +584,7 @@ namespace ttg_parsec {
     }
 
 
+    inline
     ttg_parsec_data_wrapper_t::~ttg_parsec_data_wrapper_t() {
       if (nullptr != m_ttg_copy) {
         m_ttg_copy->remove_device_data(this);
diff --git a/ttg/ttg/ptr.h b/ttg/ttg/ptr.h
index 272eb4409..c6c92006f 100644
--- a/ttg/ttg/ptr.h
+++ b/ttg/ttg/ptr.h
@@ -9,12 +9,12 @@ template<typename T>
 using Ptr = TTG_IMPL_NS::Ptr<T>;
 
 template<typename T, typename... Args>
-Ptr<T> make_ptr(Args&&... args) {
+inline Ptr<T> make_ptr(Args&&... args) {
   return TTG_IMPL_NS::make_ptr(std::forward<Args>(args)...);
 }
 
 template<typename T>
-Ptr<std::decay_t<T>> get_ptr(T&& obj) {
+inline Ptr<std::decay_t<T>> get_ptr(T&& obj) {
   return TTG_IMPL_NS::get_ptr(std::forward<T>(obj));
 }
 

From 65ffd4c299b4453bca071a0c72a9d7f394c9d872 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 20 Dec 2023 22:35:48 +0000
Subject: [PATCH 203/259] Add derived_has_*_op() to madness backend

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/madness/ttg.h | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/ttg/ttg/madness/ttg.h b/ttg/ttg/madness/ttg.h
index 4e509108a..254eaedd1 100644
--- a/ttg/ttg/madness/ttg.h
+++ b/ttg/ttg/madness/ttg.h
@@ -213,6 +213,26 @@ namespace ttg_madness {
    public:
     ttg::World get_world() const { return world; }
 
+    /// @return true if derivedT::have_cuda_op exists and is defined to true
+    static constexpr bool derived_has_cuda_op() {
+      return false;
+    }
+
+    /// @return true if derivedT::have_hip_op exists and is defined to true
+    static constexpr bool derived_has_hip_op() {
+      return false;
+    }
+
+    /// @return true if derivedT::have_hip_op exists and is defined to true
+    static constexpr bool derived_has_level_zero_op() {
+      return false;
+    }
+
+    /// @return true if the TT supports device execution
+    static constexpr bool derived_has_device_op() {
+      return false;
+    }
+
    protected:
     using worldobjT = ::madness::WorldObject<ttT>;
 

From e7bba1b1b5e187e1584535237f4387225b934203 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 20 Dec 2023 22:36:16 +0000
Subject: [PATCH 204/259] Allow for non-coro device tasks

---
 ttg/ttg/parsec/ttg.h | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 626fa8827..8bc55ec64 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1478,6 +1478,11 @@ namespace ttg_parsec {
 
       /* when we come back here, the flows in gpu_task are set (see register_device_memory) */
 
+      if (nullptr == task->suspended_task_address) {
+        /* short-cut in case the task returned immediately */
+        return PARSEC_HOOK_RETURN_DONE;
+      }
+
       // get the device task from the coroutine handle
       auto dev_task = ttg::device::detail::device_task_handle_type::from_address(task->suspended_task_address);
 
@@ -3542,10 +3547,6 @@ ttg::abort();  // should not happen
 
       task_t *task = (task_t*)parsec_task;
 
-      if constexpr (derived_has_device_op()) {
-        assert(task->is_dummy() || nullptr != task->suspended_task_address);
-      }
-
       /* if we still have a coroutine handle we invoke it one more time to get the sends/broadcasts */
       if (task->suspended_task_address) {
 #ifdef TTG_HAVE_DEVICE

From b653b9b4b74a716e940ac966a37f0468ed903f18 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 20 Dec 2023 22:37:09 +0000
Subject: [PATCH 205/259] Re-enbale disabled unit tests

---
 examples/spmm/spmm.cc     | 3 ++-
 tests/unit/CMakeLists.txt | 6 +++---
 tests/unit/fibonacci.cc   | 2 +-
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/examples/spmm/spmm.cc b/examples/spmm/spmm.cc
index b2ae2558a..ac198a969 100644
--- a/examples/spmm/spmm.cc
+++ b/examples/spmm/spmm.cc
@@ -92,6 +92,7 @@ namespace ttg::detail {
 }  // namespace ttg::detail
 
 #else
+using scalar_t = double;
 using blk_t = double;
 #endif
 template <typename T = blk_t>
@@ -722,7 +723,7 @@ auto norms(const btas::Tensor<T_, Range_, Store_> &t) {
 
 template <typename Blk>
 auto norms(const SpMatrix<Blk> &A) {
-  using T = decltype(std::abs(std::declval<typename Blk::value_type>()));
+  using T = scalar_t;
   T norm_2_square = 0.0;
   T norm_inf = 0.0;
   for (int i = 0; i < A.outerSize(); ++i) {
diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt
index 6b524010f..ba0db8a49 100644
--- a/tests/unit/CMakeLists.txt
+++ b/tests/unit/CMakeLists.txt
@@ -2,9 +2,9 @@ include(AddTTGExecutable)
 
 # TT unit test: core TTG ops
 set(ut_src
-        #fibonacci.cc
-        #ranges.cc
-        #tt.cc
+        fibonacci.cc
+        ranges.cc
+        tt.cc
         unit_main.cpp
 		streams.cc
     )
diff --git a/tests/unit/fibonacci.cc b/tests/unit/fibonacci.cc
index 574661b33..37d333e07 100644
--- a/tests/unit/fibonacci.cc
+++ b/tests/unit/fibonacci.cc
@@ -1,4 +1,4 @@
-#include <catch2/catch.hpp>
+#include <catch2/catch_all.hpp>
 
 #include "ttg.h"
 

From 82a7b2a8a2c8a8adbeab8f7e0aa4c899373d174f Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 21 Dec 2023 02:47:07 +0000
Subject: [PATCH 206/259] Enable madness serialization for madness backend
 buffer

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/madness/buffer.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ttg/ttg/madness/buffer.h b/ttg/ttg/madness/buffer.h
index 1572085f8..a68480be4 100644
--- a/ttg/ttg/madness/buffer.h
+++ b/ttg/ttg/madness/buffer.h
@@ -1,7 +1,7 @@
 #ifndef TTG_MADNESS_BUFFER_H
 #define TTG_MADNESS_BUFFER_H
 
-//#include "ttg/serialization/backends.h"
+#include "ttg/serialization/traits.h"
 
 namespace ttg_madness {
 
@@ -291,7 +291,7 @@ struct Buffer : private Allocator {
   }
 #endif // TTG_SERIALIZATION_SUPPORTS_CEREAL
 
-#if defined(TTG_SERIALIZATION_SUPPORTS_MADNESS) && 0
+#if defined(TTG_SERIALIZATION_SUPPORTS_MADNESS)
   template <typename Archive>
   std::enable_if_t<std::is_base_of_v<madness::archive::BufferInputArchive, Archive> ||
                    std::is_base_of_v<madness::archive::BufferOutputArchive, Archive>>

From 0d6429727104fd722d87576df2cf5e5cf1f39428 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 21 Dec 2023 02:47:41 +0000
Subject: [PATCH 207/259] Only build device_coro test if device support is
 detected

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 tests/unit/device_coro.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/unit/device_coro.cc b/tests/unit/device_coro.cc
index 124945542..60581d232 100644
--- a/tests/unit/device_coro.cc
+++ b/tests/unit/device_coro.cc
@@ -28,7 +28,7 @@ namespace madness::archive {
 }  // namespace madness::archive
 #endif  // TTG_SERIALIZATION_SUPPORTS_MADNESS
 
-#if defined(TTG_IMPL_DEVICE_SUPPORT)
+#if defined(TTG_HAVE_DEVICE) && defined(TTG_IMPL_DEVICE_SUPPORT)
 
 TEST_CASE("Device", "coro") {
 

From 5f876e3b96ccf61096e5c2e00a966426943e4bd3 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 21 Dec 2023 02:50:27 +0000
Subject: [PATCH 208/259] Fix possible use-after-free in TT::release

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/ttg.h | 16 +++-------------
 1 file changed, 3 insertions(+), 13 deletions(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 8bc55ec64..fcdf678f1 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -3773,18 +3773,6 @@ ttg::abort();  // should not happen
         self.name = nullptr;
       }
 
-      for (int i = 0; i < MAX_PARAM_COUNT; i++) {
-        parsec_flow_t *flow = (parsec_flow_t *)(self.in[i]);
-        free(flow->name);
-        delete flow;
-        *((parsec_flow_t **)&(self.in[i])) = nullptr;
-
-        flow = (parsec_flow_t *)(self.out[i]);
-        free(flow->name);
-        delete flow;
-        *((parsec_flow_t **)&(self.out[i])) = nullptr;
-      }
-
       for (std::size_t i = 0; i < numins; ++i) {
         if (inpute_reducers_taskclass[i] != nullptr) {
           std::free(inpute_reducers_taskclass[i]);
@@ -3818,14 +3806,16 @@ ttg::abort();  // should not happen
       // uintptr_t addr = (uintptr_t)self.incarnations;
       // free((void *)addr);
       free((__parsec_chore_t *)self.incarnations);
-      for (int i = 0; i < numflows; i++) {
+      for (int i = 0; i < MAX_PARAM_COUNT; i++) {
         if (NULL != self.in[i]) {
           free(self.in[i]->name);
           delete self.in[i];
+          self.in[i] = nullptr;
         }
         if (NULL != self.out[i]) {
           free(self.out[i]->name);
           delete self.out[i];
+          self.out[i] = nullptr;
         }
       }
       world.impl().deregister_op(this);

From 6f645df738251d0a697684a9e9023780c1b67805 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 21 Dec 2023 13:49:49 +0000
Subject: [PATCH 209/259] Use key directly in task_snprintf instead of locals

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/ttg.h | 23 ++++++++---------------
 1 file changed, 8 insertions(+), 15 deletions(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index fcdf678f1..44a11e88e 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -3480,24 +3480,22 @@ ttg::abort();  // should not happen
         return reinterpret_cast<parsec_key_t>(key);
     }
 
-    static char *parsec_ttg_task_snprintf(char *buffer, size_t buffer_size, const parsec_task_t *t) {
+    static char *parsec_ttg_task_snprintf(char *buffer, size_t buffer_size, const parsec_task_t *parsec_task) {
       if(buffer_size == 0)
         return buffer;
 
       if constexpr (ttg::meta::is_void_v<keyT>) {
-        snprintf(buffer, buffer_size, "%s()[]<%d>", t->task_class->name, t->priority);
+        snprintf(buffer, buffer_size, "%s()[]<%d>", parsec_task->task_class->name, parsec_task->priority);
       }  else {
-        // we use the locals array as a scratchpad to store the hash of the key and its actual address
-        // locals[0] amd locals[1] hold the hash, while locals[2] and locals[3] hold the key pointer
-        keyT *key = *(keyT**)&(t->locals[2]);
+        const task_t *task = reinterpret_cast<const task_t*>(parsec_task);
         std::stringstream ss;
-        ss << *key;
+        ss << task->key;
 
         std::string keystr = ss.str();
         std::replace(keystr.begin(), keystr.end(), '(', ':');
         std::replace(keystr.begin(), keystr.end(), ')', ':');
 
-        snprintf(buffer, buffer_size, "%s(%s)[]<%d>", t->task_class->name, keystr.c_str(), t->priority);
+        snprintf(buffer, buffer_size, "%s(%s)[]<%d>", parsec_task->task_class->name, keystr.c_str(), parsec_task->priority);
       }
       return buffer;
     }
@@ -3505,19 +3503,14 @@ ttg::abort();  // should not happen
 #if defined(PARSEC_PROF_TRACE)
     static void *parsec_ttg_task_info(void *dst, const void *data, size_t size)
     {
-      const parsec_task_t *t = reinterpret_cast<const parsec_task_t *>(data);
+      const task_t *task = reinterpret_cast<const task_t *>(data);
 
       if constexpr (ttg::meta::is_void_v<keyT>) {
         snprintf(reinterpret_cast<char*>(dst), size, "()");
       } else {
-        // we use the locals array as a scratchpad to store the hash of the key and its actual address
-        // locals[0] amd locals[1] hold the hash, while locals[2] and locals[3] hold the key pointer
-        keyT *key = *(keyT**)&(t->locals[2]);
         std::stringstream ss;
-        ss << *key;
-
-        std::string keystr = ss.str();
-        snprintf(reinterpret_cast<char*>(dst), size, "%s", keystr.c_str());
+        ss << task->key;
+        snprintf(reinterpret_cast<char*>(dst), size, "%s", keystr.str().c_str());
       }
       return dst;
     }

From ba6f7a1afe828126f4ea1bcef2575b0b23b22f86 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 21 Dec 2023 13:55:53 +0000
Subject: [PATCH 210/259] Remove legacy devicemock example

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/CMakeLists.txt             |   2 -
 examples/device_mock/device_mock.cc | 188 ----------------------------
 2 files changed, 190 deletions(-)
 delete mode 100644 examples/device_mock/device_mock.cc

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 0ae64c831..63bd9085d 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -67,8 +67,6 @@ add_ttg_executable(fw-apsp floyd-warshall/floyd_warshall.cc LINK_LIBRARIES MADwo
 add_ttg_executable(helloworld helloworld/helloworld.cpp)
 add_ttg_executable(simplegenerator simplegenerator/simplegenerator.cc RUNTIMES "mad")
 
-add_ttg_executable(device_mock device_mock/device_mock.cc LINK_LIBRARIES MADworld)
-
 if (OpenMP_CXX_FOUND AND TARGET std::execution)
         add_ttg_executable(fw-apsp-df floyd-warshall/floyd_warshall_df.cc LINK_LIBRARIES OpenMP::OpenMP_CXX std::execution MADworld)
 endif ()
diff --git a/examples/device_mock/device_mock.cc b/examples/device_mock/device_mock.cc
deleted file mode 100644
index 8722591e7..000000000
--- a/examples/device_mock/device_mock.cc
+++ /dev/null
@@ -1,188 +0,0 @@
-// clang-format off
-
-#include <ttg.h>
-#include <ttg/view.h>
-#include "../matrixtile.h"
-
-#include <parsec.h>
-#include <parsec/data_internal.h>
-#include <parsec/data_dist/matrix/matrix.h>
-#include <parsec/data_dist/matrix/sym_two_dim_rectangle_cyclic.h>
-#include <parsec/data_dist/matrix/two_dim_rectangle_cyclic.h>
-
-#include "ttg/util/meta.h"
-
-#include <ttg/util/multiindex.h>
-
-using Key2 = ttg::MultiIndex<2>;
-using Key3 = ttg::MultiIndex<3>;
-
-/* number of tiles */
-#define KT 100
-
-template<typename T>
-auto make_gemm(ttg::Edge<Key3, MatrixTile<T>>& A,
-               ttg::Edge<Key3, MatrixTile<T>>& B,
-               ttg::Edge<Key2, MatrixTile<T>>& output_result)
-{
-
-  ttg::Edge<Key3, MatrixTile<double>> C;
-  auto f_cpu = [=](const Key3& key,
-                   const MatrixTile<T>&  A,
-                   const MatrixTile<T>&  B,
-                   MatrixTile<T>&  C,
-                   std::tuple<ttg::Out<Key2, MatrixTile<T>>,
-                              ttg::Out<Key3, MatrixTile<T>>>& out)
-  {
-    int m = key[0];
-    int n = key[1];
-    int k = key[2];
-
-    /*
-    if(k == 0) {
-      dlprng(C.data(), 1789, A.mb()*B.nb());
-    }
-    dgemm(A.data(), A.mb(), A.nb(),
-          B.data(), B.mb(), B.nb(),
-          1.0,
-          C.data(), C.nb(), C.nb());
-    */
-
-    if( k == KT-1 || C.data()[0] < 1e-9 ) {
-      ttg::send<0>(Key2{m, n}, std::move(C));
-    } else {
-      ttg::send<1>(Key3{m, n, k+1}, std::move(C));
-    }
-  };
-
-  auto f_gpu_host_views = [=](const Key3& key,
-                              const MatrixTile<T>& A,
-                              const MatrixTile<T>& B,
-                              MatrixTile<T>&& C)
-  {
-    // The default ViewScope::SyncIn scope tells the runtime that the data should be copied
-    // to the device before the kernel callable is invoked.
-    ttg::View<const MatrixTile<T>, const T> dev_A = ttg::make_view( A, ttg::ViewSpan(A.data(), A.size()) );
-    ttg::View<const MatrixTile<T>, const T> dev_B = ttg::make_view( B, ttg::ViewSpan(B.data(), B.size()) );
-    ttg::View<MatrixTile<T>, T> dev_C;
-    ttg::View<T, T> dev_tmp;
-    T *host_tmp = new(T);
-    // ViewScope::SyncOut tells the runtime system that the view should be synchronized back to the
-    // host before invoking the output callable.
-    dev_tmp = ttg::make_view( *host_tmp, ttg::ViewSpan(host_tmp, 1, ttg::ViewScope::SyncOut) );
-
-    int k = key[2];
-    if(0 == k) {
-      // ViewScope::Allocate tells the runtime system that the device view needs to be allocated but doesn't need to be
-      // initialized with C.data(). However, C.data() is still associated with the device memory, so if the
-      // runtime system evicts that data from the device, it will be first copied back into C.data().
-      dev_C = ttg::make_view( C, ttg::ViewSpan(C.data(), C.size(), ttg::ViewScope::Allocate) );
-    } else {
-      dev_C = ttg::make_view( C, ttg::ViewSpan(C.data(), C.size()) );
-    }
-
-    return std::make_tuple(dev_A, dev_B, dev_C, dev_tmp);
-  };
-
-  auto f_gpu_kernel = [=](const Key3& key,
-                                ttg::View<const MatrixTile<T>, const T>& dev_A,
-                                ttg::View<const MatrixTile<T>, const T>& dev_B,
-                                ttg::View<MatrixTile<T>, T>& dev_C,
-                          ttg::View<T, T>& dev_tmp)
-  {
-    int k = key[2];
-
-    const MatrixTile<T>&  A = dev_A.get_host_object();
-    const MatrixTile<T>&  B = dev_B.get_host_object();
-    MatrixTile<T>&  C = dev_C.get_host_object();
-    T& host_tmp = dev_tmp.get_host_object();
-    auto beta = 1.0;
-    if(k == 0) {
-        //cublasDplrng(dev_C.get(0), C.mb(), C.nb());
-    }
-
-    /*
-    cublasDgemm(dev_A.get(0), A.mb(), A.nb(),
-                dev_B.get(0), B.mb(), B.nb(),
-                beta,
-                dev_C.get(0), C.mb(), C.nb());
-
-    cudaMemcpyAsync(&dev_C.get(0)[0], host_tmp, sizeof(T), cudaDeviceToHost);
-    */
-  };
-
-  auto f_gpu_output_flows = [=](const Key3& key,
-                                const MatrixTile<T>&  A,
-                                const MatrixTile<T>&  B,
-                                MatrixTile<T>&  C,
-                                T& host_tmp)
-  {
-    int m = key[0];
-    int n = key[1];
-    int k = key[2];
-
-    if( k == KT-1 || host_tmp < 1e-9 ) {
-        ttg::send<0>(Key2{m, n}, std::move(C));
-    } else {
-      ttg::send<1>(Key3{m, n, k+1}, std::move(C));
-    }
-    delete &host_tmp;
-  };
-
-  //ttg::meta::type_printer<decltype(ttg::edges(output_result, C))> x;
-
-  /* If we only have GPU */
-  auto gemm_tt = ttg::make_device_tt<Key3>(f_gpu_host_views, f_gpu_kernel, f_gpu_output_flows, ttg::ExecutionSpace::CUDA,
-                                     ttg::edges(A, B, C), ttg::edges(output_result, C),
-                                     "GEMM", {"A", "B", "C"}, {"output_result", "C"});
-
-#if 0
-  /* Alternative: to get both type of tasklets: */
-  auto gemm_tt = ttg::make_device_tt(f_cpu, f_gpu_host_views, f_gpu_kernel, f_gpu_output_flows, ttg::ExecutionSpace::CUDA,
-                                     ttg::edges(A, B), ttg::edges(output_result, C),
-                                     "GEMM", {"A", "B"}, {"output_result", "C"});
-#endif
-  return gemm_tt;
-}
-
-int main(int argc, char **argv)
-{
-
-  std::chrono::time_point<std::chrono::high_resolution_clock> beg, end;
-  int N = 1024;
-  int M = N;
-  int NB = 128;
-  int check = 0;
-  int nthreads = -1;
-  const char* prof_filename = nullptr;
-
-  if (argc > 1) {
-    N = M = atoi(argv[1]);
-  }
-
-  if (argc > 2) {
-    NB = atoi(argv[2]);
-  }
-
-  if (argc > 3) {
-    check = atoi(argv[3]);
-  }
-
-  if (argc > 4) {
-    nthreads = atoi(argv[4]);
-  }
-
-  ttg::initialize(argc, argv, nthreads);
-
-  auto world = ttg::default_execution_context();
-
-  ttg::Edge<Key3, MatrixTile<double>> edge_a, edge_b;
-  ttg::Edge<Key2, MatrixTile<double>> edge_out;
-
-  auto gemm_tt = make_gemm(edge_a, edge_b, edge_out);
-
-  ttg::fence();
-
-  ttg::finalize();
-  return 0;
-}

From c899f074d71ffb92609daf65c33d5974c15714e9 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 21 Dec 2023 14:51:12 +0000
Subject: [PATCH 211/259] Fix device chain benchmark

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/CMakeLists.txt                   |  8 ++++++-
 examples/task-benchmarks/chain-ttg-dev.cc | 27 +++++++++--------------
 2 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 63bd9085d..87aa0b127 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -47,7 +47,13 @@ if (TARGET tiledarray)
                           RUNTIMES "parsec")
     endif()
 
-    add_ttg_executable(chain-ttg-dev task-benchmarks/chain-ttg-dev.cc LINK_LIBRARIES tiledarray RUNTIMES "parsec")
+    if (TTG_HAVE_CUDA)
+      add_ttg_executable(chain-ttg-cuda task-benchmarks/chain-ttg-dev.cc LINK_LIBRARIES tiledarray RUNTIMES "parsec")
+    endif(TTG_HAVE_CUDA)
+
+    if (TTG_HAVE_HIP)
+      add_ttg_executable(chain-ttg-hip task-benchmarks/chain-ttg-dev.cc LINK_LIBRARIES tiledarray RUNTIMES "parsec")
+    endif(TTG_HAVE_HIP)
 endif()
 
 if (TARGET MADworld)
diff --git a/examples/task-benchmarks/chain-ttg-dev.cc b/examples/task-benchmarks/chain-ttg-dev.cc
index ac96e16b3..80f14bff4 100644
--- a/examples/task-benchmarks/chain-ttg-dev.cc
+++ b/examples/task-benchmarks/chain-ttg-dev.cc
@@ -20,7 +20,7 @@ std::atomic<int> task_counter = 0;
 struct A : public ttg::TTValue<A> {
   // TODO: allocate pinned memory
   int v = 0;
-  ttg::buffer<int> b;
+  ttg::Buffer<int> b;
   A() : b(&v, 1) { }
 
   A(A&& a) = default;
@@ -53,10 +53,9 @@ auto make_ttg<1>(bool do_move) {
       send<0>(0, A{});
     }, edges(), edges(I2N));
 
-  auto next = make_tt<ES, int>([=](const int &key, auto&& value) -> ttg::device_task {
+  auto next = make_tt<ES, int>([=](const int &key, auto&& value) -> ttg::device::Task {
     //++task_counter;
-    co_await ttg::to_device(value.b);
-    //co_await ttg::wait_kernel(); // empty kernel
+    co_await ttg::device::select(value.b);
     if (key < NUM_TASKS) {
       if (do_move) {
         co_await ttg::device::forward(ttg::device::send<0>(key+1, std::move(value)));
@@ -81,9 +80,8 @@ auto make_ttg<2>(bool do_move) {
     send<1>(0, A{});
   }, edges(), edges(I2N1, I2N2));
 
-  auto next = make_tt<ES, int>([=](const int &key, A&& v1, A&& v2) -> ttg::device_task {
-    co_await ttg::to_device(v1.b, v2.b);
-    //co_await ttg::wait_kernel(); // empty kernel
+  auto next = make_tt<ES, int>([=](const int &key, A&& v1, A&& v2) -> ttg::device::Task {
+    co_await ttg::device::select(v1.b, v2.b);
     if (key < NUM_TASKS) {
       if (do_move) {
         co_await ttg::device::forward(ttg::device::send<0>(key+1, std::move(v1)),
@@ -112,9 +110,8 @@ auto make_ttg<4>(bool do_move) {
       send<3>(0, A{});
     }, edges(), edges(I2N1, I2N2, I2N3, I2N4));
 
-  auto next = make_tt<ES, int>([=](const int &key, A&& v1, A&& v2, A&& v3, A&& v4) -> ttg::device_task {
-    co_await ttg::to_device(v1.b, v2.b, v3.b, v4.b);
-    //co_await ttg::wait_kernel(); // empty kernel
+  auto next = make_tt<ES, int>([=](const int &key, A&& v1, A&& v2, A&& v3, A&& v4) -> ttg::device::Task {
+    co_await ttg::device::select(v1.b, v2.b, v3.b, v4.b);
     if (key < NUM_TASKS) {
       if (do_move) {
         co_await ttg::device::forward(ttg::device::send<0>(key+1, std::move(v1)),
@@ -153,9 +150,8 @@ auto make_ttg<8>(bool do_move) {
       send<7>(0, A{});
     }, edges(), edges(I2N1, I2N2, I2N3, I2N4, I2N5, I2N6, I2N7, I2N8));
 
-  auto next = make_tt<ES, int>([=](const int &key, auto&& v1, auto&& v2, auto&& v3, auto&& v4, auto&& v5, auto&& v6, auto&& v7, auto&& v8) -> ttg::device_task {
-    co_await ttg::to_device(v1.b, v2.b, v3.b, v4.b, v5.b, v6.b, v7.b, v8.b);
-    //co_await ttg::wait_kernel(); // empty kernel
+  auto next = make_tt<ES, int>([=](const int &key, auto&& v1, auto&& v2, auto&& v3, auto&& v4, auto&& v5, auto&& v6, auto&& v7, auto&& v8) -> ttg::device::Task {
+    co_await ttg::device::select(v1.b, v2.b, v3.b, v4.b, v5.b, v6.b, v7.b, v8.b);
     if (key < NUM_TASKS) {
       if (do_move) {
         co_await ttg::device::forward(ttg::device::send<0>(key+1, std::move(v1)),
@@ -191,9 +187,8 @@ auto make_ttg<0>(bool do_move) {
 
   auto init = make_tt<void>([](std::tuple<Out<int, void>> &outs) { sendk<0>(0, outs); }, edges(), edges(I2N));
 
-  auto next = make_tt<ES>([](const int& key) -> ttg::device_task {
-    co_await ttg::to_device();
-    //co_await ttg::wait_kernel();
+  auto next = make_tt<ES>([](const int& key) -> ttg::device::Task {
+    co_await ttg::device::select();
     if (key < NUM_TASKS) {
       co_await ttg::device::forward(ttg::device::sendk<0>(key+1));
     }

From bd454dd2625706cb32595ce6a25f77f9dd7a5919 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 21 Dec 2023 14:51:47 +0000
Subject: [PATCH 212/259] Comment preliminary mra device code pieces

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/madness/mrattg.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/examples/madness/mrattg.cc b/examples/madness/mrattg.cc
index 2b2d6c113..2341ba864 100644
--- a/examples/madness/mrattg.cc
+++ b/examples/madness/mrattg.cc
@@ -124,7 +124,8 @@ auto make_project(functorT& f,
   return ttg::make_tt(F, edges(fuse(refine, ctl)), edges(refine, result), name, {"control"}, {"refine", "result"});
 }
 
-
+/* below is a preliminary attempt at a device version, needs revisiting */
+#if 0
 /// Returns an std::unique_ptr to the object
 template <typename functorT, typename T, size_t K, Dimension NDIM>
 auto make_project_device(functorT& f,
@@ -161,7 +162,7 @@ auto make_project_device(functorT& f,
   ctlEdge<NDIM> refine("refine");
   return ttg::make_tt(F, edges(fuse(refine, ctl)), edges(refine, result), name, {"control"}, {"refine", "result"});
 }
-
+#endif // 0
 
 
 namespace detail {

From ec67c6b3cecab025d1b8ac68e8727eb123e80324 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 21 Dec 2023 14:52:42 +0000
Subject: [PATCH 213/259] Gracefully handle missing TA allocators

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/matrixtile.h            | 27 ++++++++++++++++++++-
 examples/potrf/testing_dlauum.cc |  4 ++++
 examples/potrf/testing_dpoinv.cc | 40 ++++++++++++++++++--------------
 examples/potrf/testing_dpotrf.cc |  9 +++----
 examples/potrf/testing_dtrtri.cc |  6 ++++-
 5 files changed, 60 insertions(+), 26 deletions(-)

diff --git a/examples/matrixtile.h b/examples/matrixtile.h
index a6e3a02c2..a2080f3ed 100644
--- a/examples/matrixtile.h
+++ b/examples/matrixtile.h
@@ -5,11 +5,36 @@
 #include <memory>
 #include <optional>
 
+#include <ttg.h>
+
 #include <ttg/serialization/splitmd_data_descriptor.h>
 
+
 #include <TiledArray/device/allocators.h>
+#if defined(TILEDARRAY_HAS_DEVICE)
+#define ALLOCATOR TiledArray::device_pinned_allocator<T>
+
+inline void allocator_init() {
+  // initialize MADNESS so that TA allocators can be created
+#if defined(TTG_PARSEC_IMPORTED)
+  madness::ParsecRuntime::initialize_with_existing_context(ttg::default_execution_context().impl().context());
+#endif // TTG_PARSEC_IMPORTED
+  madness::initialize(argc, argv, /* nthread = */ 1, /* quiet = */ true);
+}
+
+inline void allocator_fini() {
+  madness::finalize();
+}
+#else  // TILEDARRAY_HAS_DEVICE
+#define ALLOCATOR std::allocator<T>
+
+inline void allocator_init() { }
+
+inline void allocator_fini() { }
+
+#endif // TILEDARRAY_HAS_DEVICE
 
-template <typename T, class Allocator = TiledArray::device_pinned_allocator<double>>
+template <typename T, class Allocator = ALLOCATOR>
 class MatrixTile : public ttg::TTValue<MatrixTile<T, Allocator>> {
  public:
   using metadata_t = typename std::tuple<std::size_t, std::size_t, std::size_t>;
diff --git a/examples/potrf/testing_dlauum.cc b/examples/potrf/testing_dlauum.cc
index 717600094..df87c63a6 100644
--- a/examples/potrf/testing_dlauum.cc
+++ b/examples/potrf/testing_dlauum.cc
@@ -58,6 +58,9 @@ int main(int argc, char **argv)
 
   ttg::initialize(argc, argv, nthreads);
 
+  /* set up TA to get the allocator */
+  allocator_init();
+
   auto world = ttg::default_execution_context();
 
   if( prof_filename != nullptr ) {
@@ -140,6 +143,7 @@ int main(int argc, char **argv)
   world.dag_off();
   world.profile_off();
 
+  allocator_fini();
   ttg::finalize();
   return ret;
 }
diff --git a/examples/potrf/testing_dpoinv.cc b/examples/potrf/testing_dpoinv.cc
index a1e419cc8..bd10b24b7 100644
--- a/examples/potrf/testing_dpoinv.cc
+++ b/examples/potrf/testing_dpoinv.cc
@@ -94,10 +94,13 @@ int main(int argc, char **argv)
   ttg::initialize(ttg_argc, ttg_argv, nthreads);
   delete[] ttg_argv;
 
+  /* set up TA to get the allocator */
+  allocator_init();
+
   ttg::trace_on();
 
   auto world = ttg::default_execution_context();
-  
+
   if(nullptr != prof_filename) {
     world.profile_on();
     world.dag_on(prof_filename);
@@ -118,12 +121,12 @@ int main(int argc, char **argv)
 
   parsec_matrix_sym_block_cyclic_t dcA;
   parsec_matrix_sym_block_cyclic_init(&dcA, parsec_matrix_type_t::PARSEC_MATRIX_DOUBLE,
-                                world.rank(), 
-                                NB, NB, 
+                                world.rank(),
+                                NB, NB,
+                                N, M,
+                                0, 0,
                                 N, M,
-                                0, 0, 
-                                N, M, 
-                                P, Q, 
+                                P, Q,
                                 PARSEC_MATRIX_LOWER);
   dcA.mat = parsec_data_allocate((size_t)dcA.super.nb_local_tiles *
                                  (size_t)dcA.super.bsiz *
@@ -248,15 +251,15 @@ int main(int argc, char **argv)
           end = endpotri;
 
           auto elapsed = (std::chrono::duration_cast<std::chrono::microseconds>(endpotrf - begpotrf).count());
-          std::cout << "POINV (POTRF+POTRI) (" << (defer_cow_hint ? "with" : "without") << " defer writer) -- N= " << N << " NB= " << NB <<  " P= " << P << " Q= " << Q << " nthreads= " << nthreads 
+          std::cout << "POINV (POTRF+POTRI) (" << (defer_cow_hint ? "with" : "without") << " defer writer) -- N= " << N << " NB= " << NB <<  " P= " << P << " Q= " << Q << " nthreads= " << nthreads
                     << " POTRF TTG Execution Time (milliseconds) : " << elapsed / 1E3 << " : Flops " << (potrf::FLOPS_DPOTRF(N)) << " " << (potrf::FLOPS_DPOTRF(N)/1e9)/(elapsed/1e6) << " GF/s" << std::endl;
-          
+
           elapsed = (std::chrono::duration_cast<std::chrono::microseconds>(endpotri - begpotri).count());
-          std::cout << "POINV (POTRF+POTRI) (" << (defer_cow_hint ? "with" : "without") << " defer writer) -- N= " << N << " NB= " << NB <<  " P= " << P << " Q= " << Q << " nthreads= " << nthreads 
+          std::cout << "POINV (POTRF+POTRI) (" << (defer_cow_hint ? "with" : "without") << " defer writer) -- N= " << N << " NB= " << NB <<  " P= " << P << " Q= " << Q << " nthreads= " << nthreads
                     << " POTRI TTG Execution Time (milliseconds) : " << elapsed / 1E3 << " : Flops " << (potri::FLOPS_DPOTRI(N)) << " " << (potri::FLOPS_DPOTRI(N)/1e9)/(elapsed/1e6) << " GF/s" << std::endl;
 
           elapsed = (std::chrono::duration_cast<std::chrono::microseconds>(end - beg).count());
-          std::cout << "POINV (POTRF+POTRI) (" << (defer_cow_hint ? "with" : "without") << " defer writer) N= " << N << " NB= " << NB <<  " P= " << P << " Q= " << Q << " nthreads= " << nthreads 
+          std::cout << "POINV (POTRF+POTRI) (" << (defer_cow_hint ? "with" : "without") << " defer writer) N= " << N << " NB= " << NB <<  " P= " << P << " Q= " << Q << " nthreads= " << nthreads
                     << " TTG Execution Time (milliseconds) : " << elapsed / 1E3 << " : Flops " << (potrf::FLOPS_DPOTRF(N) + potri::FLOPS_DPOTRI(N)) << " " << ((potrf::FLOPS_DPOTRF(N)+potri::FLOPS_DPOTRI(N))/1e9)/(elapsed/1e6) << " GF/s" << std::endl;
         }
       } else {
@@ -328,20 +331,20 @@ int main(int argc, char **argv)
           end = endlauum;
 
           auto elapsed = (std::chrono::duration_cast<std::chrono::microseconds>(endpotrf - begpotrf).count());
-          std::cout << "POINV (POTRF+TRTRI+LAUUM) (" << (defer_cow_hint ? "with" : "without") << " defer writer) -- N= " << N << " NB= " << NB <<  " P= " << P << " Q= " << Q << " nthreads= " << nthreads 
+          std::cout << "POINV (POTRF+TRTRI+LAUUM) (" << (defer_cow_hint ? "with" : "without") << " defer writer) -- N= " << N << " NB= " << NB <<  " P= " << P << " Q= " << Q << " nthreads= " << nthreads
                     << " POTRF TTG Execution Time (milliseconds) : " << elapsed / 1E3 << " : Flops " << (potrf::FLOPS_DPOTRF(N)) << " " << (potrf::FLOPS_DPOTRF(N)/1e9)/(elapsed/1e6) << " GF/s" << std::endl;
-          
+
           elapsed = (std::chrono::duration_cast<std::chrono::microseconds>(endtrtri - begtrtri).count());
-          std::cout << "POINV (POTRF+TRTRI+LAUUM) (" << (defer_cow_hint ? "with" : "without") << " defer writer) -- N= " << N << " NB= " << NB <<  " P= " << P << " Q= " << Q << " nthreads= " << nthreads 
+          std::cout << "POINV (POTRF+TRTRI+LAUUM) (" << (defer_cow_hint ? "with" : "without") << " defer writer) -- N= " << N << " NB= " << NB <<  " P= " << P << " Q= " << Q << " nthreads= " << nthreads
                     << " TRTRI TTG Execution Time (milliseconds) : " << elapsed / 1E3 << " : Flops " << (trtri_LOWER::FLOPS_DTRTRI(N)) << " " << (trtri_LOWER::FLOPS_DTRTRI(N)/1e9)/(elapsed/1e6) << " GF/s" << std::endl;
 
           elapsed = (std::chrono::duration_cast<std::chrono::microseconds>(endlauum - beglauum).count());
-          std::cout << "POINV (POTRF+TRTRI+LAUUM) (" << (defer_cow_hint ? "with" : "without") << " defer writer) -- N= " << N << " NB= " << NB <<  " P= " << P << " Q= " << Q << " nthreads= " << nthreads 
+          std::cout << "POINV (POTRF+TRTRI+LAUUM) (" << (defer_cow_hint ? "with" : "without") << " defer writer) -- N= " << N << " NB= " << NB <<  " P= " << P << " Q= " << Q << " nthreads= " << nthreads
                     << " LAUUM TTG Execution Time (milliseconds) : " << elapsed / 1E3 << " : Flops " << (lauum::FLOPS_DLAUUM(N)) << " " << (lauum::FLOPS_DLAUUM(N)/1e9)/(elapsed/1e6) << " GF/s" << std::endl;
 
           elapsed = (std::chrono::duration_cast<std::chrono::microseconds>(end - beg).count());
-          std::cout << "POINV (POTRF+TRTRI+LAUUM) (" << (defer_cow_hint ? "with" : "without") << " defer writer) N= " << N << " NB= " << NB <<  " P= " << P << " Q= " << Q << " nthreads= " << nthreads 
-                    << " TTG Execution Time (milliseconds) : " << elapsed / 1E3 
+          std::cout << "POINV (POTRF+TRTRI+LAUUM) (" << (defer_cow_hint ? "with" : "without") << " defer writer) N= " << N << " NB= " << NB <<  " P= " << P << " Q= " << Q << " nthreads= " << nthreads
+                    << " TTG Execution Time (milliseconds) : " << elapsed / 1E3
                     << " : Flops " << (potrf::FLOPS_DPOTRF(N) + potri::FLOPS_DPOTRI(N)) << " " << ((potrf::FLOPS_DPOTRF(N)+potri::FLOPS_DPOTRI(N))/1e9)/(elapsed/1e6) << " GF/s" << std::endl;
         }
       }
@@ -379,8 +382,8 @@ int main(int argc, char **argv)
         end = std::chrono::high_resolution_clock::now();
         auto elapsed = (std::chrono::duration_cast<std::chrono::microseconds>(end - beg).count());
         end = std::chrono::high_resolution_clock::now();
-        std::cout << "POINV (POINV) (" << (defer_cow_hint ? "with" : "without") << " defer writer) N= " << N << " NB= " << NB <<  " P= " << P << " Q= " << Q << " nthreads= " << nthreads 
-                  << " TTG Execution Time (milliseconds) : " << elapsed / 1E3 
+        std::cout << "POINV (POINV) (" << (defer_cow_hint ? "with" : "without") << " defer writer) N= " << N << " NB= " << NB <<  " P= " << P << " Q= " << Q << " nthreads= " << nthreads
+                  << " TTG Execution Time (milliseconds) : " << elapsed / 1E3
                   << " : Flops " << (potrf::FLOPS_DPOTRF(N) + potri::FLOPS_DPOTRI(N)) << " " << ((potrf::FLOPS_DPOTRF(N)+potri::FLOPS_DPOTRI(N))/1e9)/(elapsed/1e6) << " GF/s" << std::endl;
       }
     }
@@ -395,6 +398,7 @@ int main(int argc, char **argv)
   parsec_data_free(dcA.mat); dcA.mat = NULL;
   parsec_tiled_matrix_destroy( (parsec_tiled_matrix_t*)&dcA);
 
+  allocator_fini();
   ttg::finalize();
   return ret;
 }
diff --git a/examples/potrf/testing_dpotrf.cc b/examples/potrf/testing_dpotrf.cc
index 5db9513e9..1658dbed3 100644
--- a/examples/potrf/testing_dpotrf.cc
+++ b/examples/potrf/testing_dpotrf.cc
@@ -64,11 +64,8 @@ int main(int argc, char **argv)
   // TODO: need to filter out our arguments to make parsec happy
   ttg::initialize(1, argv, nthreads);
 
-  // initialize MADNESS so that TA allocators can be created
-#if defined(TTG_PARSEC_IMPORTED)
-  madness::ParsecRuntime::initialize_with_existing_context(ttg::default_execution_context().impl().context());
-#endif // TTG_PARSEC_IMPORTED
-  madness::initialize(argc, argv, /* nthread = */ 1, /* quiet = */ true);
+  /* set up TA to get the allocator */
+  allocator_init();
 
   auto world = ttg::default_execution_context();
   if(nullptr != prof_filename) {
@@ -230,7 +227,7 @@ int main(int argc, char **argv)
 
   world.profile_off();
 
-  madness::finalize();
+  allocator_fini();
   ttg::finalize();
   return ret;
 }
diff --git a/examples/potrf/testing_dtrtri.cc b/examples/potrf/testing_dtrtri.cc
index 107fbf8c2..bebe8cccf 100644
--- a/examples/potrf/testing_dtrtri.cc
+++ b/examples/potrf/testing_dtrtri.cc
@@ -74,6 +74,9 @@ int main(int argc, char **argv)
 
   ttg::initialize(argc, argv, nthreads);
 
+  /* set up TA to get the allocator */
+  allocator_init();
+
   auto world = ttg::default_execution_context();
 
   if(nullptr != prof_filename) {
@@ -228,6 +231,7 @@ int main(int argc, char **argv)
   world.dag_off();
   world.profile_off();
 
+  allocator_fini();
   ttg::finalize();
   return ret;
 }
@@ -309,7 +313,7 @@ int check_dtrtri( lapack::Diag diag, lapack::Uplo uplo, double *A, double *Ainv,
 
     std::cout << "============" << std::endl;
     std::cout << "Checking TRTRI " << std::endl;
-    std::cout <<  "-- ||A||_one = " << Anorm << " ||A^(-1)||_one = " << Ainvnorm << " ||I - A * A^(-1)||_one = " 
+    std::cout <<  "-- ||A||_one = " << Anorm << " ||A^(-1)||_one = " << Ainvnorm << " ||I - A * A^(-1)||_one = "
               << Rnorm << ", cond = " << Rcond << ", result = " << result << std::endl;
 
     if ( std::isinf(Ainvnorm) || std::isnan(result) || std::isinf(result) || (result > 10.0) ) {

From 8ccab449e7430abe6c7254a66b6941f2da6d9c6c Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 21 Dec 2023 16:36:51 +0000
Subject: [PATCH 214/259] Use stock doxygen for relase test

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 .github/workflows/cmake.yml | 20 +-------------------
 1 file changed, 1 insertion(+), 19 deletions(-)

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index bb67a827e..8a22b24dd 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -53,7 +53,7 @@ jobs:
         wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null
         sudo apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main"
         sudo apt-get update
-        sudo apt-get -y install ninja-build g++-12 liblapack-dev libboost-dev libboost-serialization-dev libeigen3-dev openmpi-bin libopenmpi-dev libtbb-dev ccache flex bison cmake
+        sudo apt-get -y install ninja-build g++-12 liblapack-dev libboost-dev libboost-serialization-dev libeigen3-dev openmpi-bin libopenmpi-dev libtbb-dev ccache flex bison cmake doxygen
 
     - name: Create Build Environment
       # Some projects don't allow in-source building, so create a separate build directory
@@ -61,24 +61,6 @@ jobs:
       run: |
         cmake -E make_directory ${{github.workspace}}/build
 
-
-    - name: Install doxygen for Release test
-      if: ${{ matrix.os == 'ubuntu-22.04' }}
-      run: |
-        if [ "${{matrix.build_type}}" = "Release" ]; then
-          sudo apt-get -y install libclang1-11 libclang-cpp11 graphviz fonts-liberation
-          cd ${{github.workspace}}/build
-          # If we fail getting doxygen-${DOXYGEN_VERSION}.linux.bin.tar.gz from sourceforge,
-          # use EFV's gdrive mirror of 1.9.2 to work around the unreliable sourceforge
-          # the sharing link: https://drive.google.com/file/d/16GXpH4YOEUxGXQrXOKdAIibhdfzATY0d/view?usp=sharing
-          wget https://downloads.sourceforge.net/project/doxygen/rel-${DOXYGEN_VERSION}/doxygen-${DOXYGEN_VERSION}.linux.bin.tar.gz || wget -4 --no-check-certificate -O doxygen-${DOXYGEN_VERSION}.linux.bin.tar.gz "https://drive.google.com/uc?export=download&id=16GXpH4YOEUxGXQrXOKdAIibhdfzATY0d"
-          tar xzf ./doxygen-${DOXYGEN_VERSION}.linux.bin.tar.gz
-          export DOXYGEN_DIR=${{github.workspace}}/build/doxygen-${DOXYGEN_VERSION}
-          ${DOXYGEN_DIR}/bin/doxygen --version
-          # doxygen should be in PATH in subsequent steps
-          echo "${DOXYGEN_DIR}/bin" >> $GITHUB_PATH
-        fi
-
     - name: Prepare ccache timestamp
       id: ccache_cache_timestamp
       shell: cmake -P {0}

From beed36a78e5e3a9f25c7523293646b61cac29537 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 21 Dec 2023 23:27:07 +0000
Subject: [PATCH 215/259] Fix const handling in reducer tasks

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/ttg.h | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 44a11e88e..b9e58077c 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -846,6 +846,7 @@ namespace ttg_parsec {
           parsec_task_t *next_task = copy->get_next_task();
           copy->set_next_task(nullptr);
           parsec_ttg_task_base_t *deferred_op = (parsec_ttg_task_base_t *)next_task;
+          copy->mark_mutable();
           deferred_op->release_task();
         } else if ((1 == copy->num_ref()) || (1 == copy->drop_ref())) {
           /* we are the last reference, delete the copy */
@@ -1743,6 +1744,8 @@ ttg::abort();  // should not happen
 
       auto& reducer = std::get<i>(baseobj->input_reducers);
 
+      //std::cout << "static_reducer_op " << parent_task->key << std::endl;
+
       if (obj->tracing()) {
         if constexpr (!ttg::meta::is_void_v<keyT>)
           ttg::trace(obj->get_world().rank(), ":", obj->get_name(), " : ", parent_task->key, ": reducer executing");
@@ -2347,12 +2350,12 @@ ttg::abort();  // should not happen
 
       auto get_copy_fn = [&](detail::parsec_ttg_task_base_t *task, auto&& value, bool is_const){
         detail::ttg_data_copy_t *copy = copy_in;
-        if (nullptr != detail::parsec_ttg_caller) {
+        if (nullptr == copy && nullptr != detail::parsec_ttg_caller) {
           copy = detail::find_copy_in_task(detail::parsec_ttg_caller, &value);
         }
         if (nullptr != copy) {
           /* retain the data copy */
-          copy = detail::register_data_copy<valueT>(copy, task, true);
+          copy = detail::register_data_copy<valueT>(copy, task, is_const);
         } else {
           /* create a new copy */
           copy = detail::create_new_datacopy(std::forward<Value>(value));
@@ -2364,6 +2367,7 @@ ttg::abort();  // should not happen
         auto submit_reducer_task = [&](auto *parent_task){
           /* check if we need to create a task */
           std::size_t c = parent_task->streams[i].reduce_count.fetch_add(1, std::memory_order_release);
+          //std::cout << "submit_reducer_task " << key << " c " << c << std::endl;
           if (0 == c) {
             /* we are responsible for creating the reduction task */
             detail::reducer_task_t *reduce_task;
@@ -2379,6 +2383,7 @@ ttg::abort();  // should not happen
             using decay_valueT = std::decay_t<valueT>;
 
             /* first input value, create a task and bind it to the copy */
+            //std::cout << "Creating new reducer task for " << key << std::endl;
             detail::reducer_task_t *reduce_task;
             reduce_task = create_new_reducer_task<i>(task, true);
 
@@ -2392,6 +2397,9 @@ ttg::abort();  // should not happen
             task->streams[i].size = 1;
             task->streams[i].reduce_count.store(1, std::memory_order_relaxed);
 
+            /* release the task if we're not deferred
+             * TODO: can we delay that until we get the second value?
+             */
             if (copy->get_next_task() != &reduce_task->parsec_task) {
               reduce_task->release_task(reduce_task);
             }
@@ -3785,6 +3793,10 @@ ttg::abort();  // should not happen
       }
     }
 
+    void print_incomplete_tasks() {
+      parsec_hash_table_for_all(&tasks_table, ht_iter_cb, this);
+    }
+
     virtual void release() override { do_release(); }
 
     void do_release() {
@@ -3793,7 +3805,7 @@ ttg::abort();  // should not happen
       }
       alive = false;
       /* print all outstanding tasks */
-      parsec_hash_table_for_all(&tasks_table, ht_iter_cb, this);
+      print_incomplete_tasks();
       parsec_hash_table_fini(&tasks_table);
       parsec_mempool_destruct(&mempools);
       // uintptr_t addr = (uintptr_t)self.incarnations;

From f30f90198156a79e00ee16e30ade920f801a8aec Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Thu, 21 Dec 2023 23:27:42 +0000
Subject: [PATCH 216/259] Only enable cuda op if cuda was detected

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/test/test.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/examples/test/test.cc b/examples/test/test.cc
index 61fbecbd7..f739881b4 100644
--- a/examples/test/test.cc
+++ b/examples/test/test.cc
@@ -22,7 +22,9 @@ class A : public TT<keyT, std::tuple<Out<void, int>, Out<keyT, int>>, A, ttg::ty
     const std::string &name)
       : baseT(inedges, outedges, name, {"inputA"}, {"resultA", "iterateA"}) {}
 
+#if defined(TTG_HAVE_CUDA)
   static constexpr const bool have_cuda_op = true;
+#endif // TTG_HAVE_CUDA
 
   void op(const keyT &key, const baseT::input_refs_tuple_type &t, baseT::output_terminals_type &out) {
     // int& value = baseT::get<0>(t);  // !! ERROR, trying to get int& from const int

From ece8cc3ce28e545595c76597345e6aa448368b1d Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 22 Dec 2023 03:40:15 +0000
Subject: [PATCH 217/259] Fix streams unit test

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 tests/unit/streams.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/unit/streams.cc b/tests/unit/streams.cc
index dfa861642..1c4d099c1 100644
--- a/tests/unit/streams.cc
+++ b/tests/unit/streams.cc
@@ -15,8 +15,8 @@ TEST_CASE("streams", "[streams][core]") {
     ttg::Edge<int, int> O2S;
     const auto nranks = ttg::default_execution_context().size();
 
-    constexpr std::size_t N = 10000;
-    constexpr std::size_t SLICE = 500;
+    constexpr std::size_t N = 12000;
+    constexpr std::size_t SLICE = 600;
     constexpr const timespec ts = { .tv_sec = 0, .tv_nsec = 10000 };
     constexpr int VALUE = 1;
     std::atomic<std::size_t> reduce_ops = 0;
@@ -62,7 +62,7 @@ TEST_CASE("streams", "[streams][core]") {
     }, SLICE);
 
     make_graph_executable(op);
-    ttg::ttg_fence(ttg::default_execution_context());
+    ttg::execute(ttg::default_execution_context());
     if (ttg::default_execution_context().rank() == 0) {
       for (std::size_t i = 0; i < N; ++i) {
         op->invoke(i, VALUE);
@@ -70,6 +70,6 @@ TEST_CASE("streams", "[streams][core]") {
     }
 
     ttg::ttg_fence(ttg::default_execution_context());
-    CHECK(reduce_ops == N);
+    CHECK(reduce_ops == N/nranks);
   }
 }  // TEST_CASE("streams")

From db4f736525166cbbf741cf2e32d3f515f262cc6a Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 22 Dec 2023 04:23:41 +0000
Subject: [PATCH 218/259] Make sure new reducer target copies are marked
 mutable

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/t9/t9_streaming.cc |   2 +-
 tests/unit/streams.cc       | 135 ++++++++++++++++++++++++++++++++++++
 ttg/ttg/parsec/ttg.h        |  13 ++--
 3 files changed, 145 insertions(+), 5 deletions(-)

diff --git a/examples/t9/t9_streaming.cc b/examples/t9/t9_streaming.cc
index f79b80321..8511e1bd2 100644
--- a/examples/t9/t9_streaming.cc
+++ b/examples/t9/t9_streaming.cc
@@ -281,7 +281,7 @@ auto make_reconstruct(const nodeEdge& in, nodeEdge& out, const std::string& name
 }
 
 // cannot easily replace this with wrapper due to persistent state
-class Norm2 : public TT<Key, std::tuple<>, Norm2, ttg::typelist<Node>> {
+class Norm2 : public TT<Key, std::tuple<>, Norm2, ttg::typelist<const Node>> {
   using baseT = typename Norm2::ttT;
   double sumsq;
   std::mutex charon;
diff --git a/tests/unit/streams.cc b/tests/unit/streams.cc
index 1c4d099c1..b31506e5a 100644
--- a/tests/unit/streams.cc
+++ b/tests/unit/streams.cc
@@ -9,6 +9,7 @@
 
 
 TEST_CASE("streams", "[streams][core]") {
+
   // in distributed memory we must count how many messages the reducer will receive
   SECTION("concurrent-stream-size") {
     ttg::Edge<int, int> I2O;
@@ -72,4 +73,138 @@ TEST_CASE("streams", "[streams][core]") {
     ttg::ttg_fence(ttg::default_execution_context());
     CHECK(reduce_ops == N/nranks);
   }
+
+  SECTION("streams-readonly-input") {
+    ttg::Edge<int, int> I2O;
+    ttg::Edge<int, int> O2S;
+    ttg::Edge<int, int> O2D;
+    const auto nranks = ttg::default_execution_context().size();
+
+    constexpr std::size_t N = 12000;
+    constexpr std::size_t SLICE = 600;
+    constexpr const timespec ts = { .tv_sec = 0, .tv_nsec = 10000 };
+    constexpr int VALUE = 1;
+    std::atomic<std::size_t> reduce_ops = 0;
+
+    auto dummy = ttg::make_tt([&](const int &n, const int &i){
+      CHECK(i == VALUE);
+    }, ttg::edges(O2D));
+
+    auto op = ttg::make_tt(
+        [&](const int &n, const int& i,
+           std::tuple<ttg::Out<int, int>, ttg::Out<int, int>> &outs) {
+          int key = n/SLICE;
+          nanosleep(&ts, nullptr);
+          ttg::send<1>(n, i, outs); // send to a dummy to check ref-counting
+          if (n < N-1) {
+            ttg::send<0>(key, i, outs);
+            //ttg::print("sent to sink ", key);
+          } else {
+            // set the size of the last reducer
+            if (N%SLICE > 0) {
+              ttg::set_size<0>(key, N%SLICE, outs);
+              std::cout << "set_size key " << key << " size " << N%SLICE << std::endl;
+            }
+            // forward the value
+            ttg::send<0>(key, i, outs);
+            //ttg::print("finalized last sink ", key);
+          }
+        },
+        ttg::edges(I2O), ttg::edges(O2S, O2D));
+
+    auto sink_op = ttg::make_tt(
+        [&](const int key, const int &value) {
+          std::cout << "sink " << key << std::endl;
+          if (!(value == SLICE || key == (N/SLICE))) {
+            std::cout << "SINK ERROR: key " << key << " value " << value << " SLICE " << SLICE << " N " << N << std::endl;
+          }
+          CHECK((value == SLICE || key == (N/SLICE)));
+          reduce_ops++;
+        },
+        ttg::edges(O2S), ttg::edges());
+
+    op->set_keymap([=](const auto &key) { return nranks - 1; });
+    op->set_trace_instance(true);
+    sink_op->set_input_reducer<0>([&](int &a, const int &b) {
+      a += 1; // we count invocations
+      CHECK(b == VALUE);
+      reduce_ops++;
+    }, SLICE);
+
+    make_graph_executable(op);
+    ttg::execute(ttg::default_execution_context());
+    if (ttg::default_execution_context().rank() == 0) {
+      for (std::size_t i = 0; i < N; ++i) {
+        op->invoke(i, VALUE);
+      }
+    }
+
+    ttg::ttg_fence(ttg::default_execution_context());
+    CHECK(reduce_ops == N/nranks);
+  }
+
+  SECTION("streams-temporary-input") {
+    ttg::Edge<int, int> I2O;
+    ttg::Edge<int, int> O2S;
+    const auto nranks = ttg::default_execution_context().size();
+
+    constexpr std::size_t N = 12000;
+    constexpr std::size_t SLICE = 600;
+    constexpr const timespec ts = { .tv_sec = 0, .tv_nsec = 10000 };
+    constexpr int VALUE = 1;
+    std::atomic<std::size_t> reduce_ops = 0;
+
+    auto op = ttg::make_tt(
+        [&](const int &n, const int& i,
+           std::tuple<ttg::Out<int, int>> &outs) {
+          int key = n/SLICE;
+          nanosleep(&ts, nullptr);
+          int tmp = i; // temporary data, not tracked
+          if (n < N-1) {
+            std::get<0>(outs).send(key, int{i});
+            //ttg::send<0>(key, int{i}, outs);
+            //ttg::print("sent to sink ", key);
+          } else {
+            // set the size of the last reducer
+            if (N%SLICE > 0) {
+              ttg::set_size<0>(key, N%SLICE, outs);
+              std::cout << "set_size key " << key << " size " << N%SLICE << std::endl;
+            }
+            // forward the value
+            ttg::send<0>(key, int{i}, outs);
+            //ttg::print("finalized last sink ", key);
+          }
+        },
+        ttg::edges(I2O), ttg::edges(O2S));
+
+    auto sink_op = ttg::make_tt(
+        [&](const int key, const int &value) {
+          std::cout << "sink " << key << std::endl;
+          if (!(value == SLICE || key == (N/SLICE))) {
+            std::cout << "SINK ERROR: key " << key << " value " << value << " SLICE " << SLICE << " N " << N << std::endl;
+          }
+          CHECK((value == SLICE || key == (N/SLICE)));
+          reduce_ops++;
+        },
+        ttg::edges(O2S), ttg::edges());
+
+    op->set_keymap([=](const auto &key) { return nranks - 1; });
+    op->set_trace_instance(true);
+    sink_op->set_input_reducer<0>([&](int &a, const int &b) {
+      a += 1; // we count invocations
+      CHECK(b == VALUE);
+      reduce_ops++;
+    }, SLICE);
+
+    make_graph_executable(op);
+    ttg::execute(ttg::default_execution_context());
+    if (ttg::default_execution_context().rank() == 0) {
+      for (std::size_t i = 0; i < N; ++i) {
+        op->invoke(i, VALUE);
+      }
+    }
+
+    ttg::ttg_fence(ttg::default_execution_context());
+    CHECK(reduce_ops == N/nranks);
+  }
 }  // TEST_CASE("streams")
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index b9e58077c..c0dcaf3b7 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1789,6 +1789,8 @@ ttg::abort();  // should not happen
             break;
           }
           source_copy = ((detail::ttg_data_copy_self_t *)(item))->self;
+          assert(target_copy->num_readers() == target_copy->mutable_tag);
+          assert(source_copy->num_readers() > 0);
           reducer(*reinterpret_cast<std::decay_t<value_t> *>(target_copy->get_ptr()),
                   *reinterpret_cast<std::decay_t<value_t> *>(source_copy->get_ptr()));
           detail::release_data_copy(source_copy);
@@ -2359,6 +2361,9 @@ ttg::abort();  // should not happen
         } else {
           /* create a new copy */
           copy = detail::create_new_datacopy(std::forward<Value>(value));
+          if (!is_const) {
+            copy->mark_mutable();
+          }
         }
         return copy;
       };
@@ -2387,16 +2392,16 @@ ttg::abort();  // should not happen
             detail::reducer_task_t *reduce_task;
             reduce_task = create_new_reducer_task<i>(task, true);
 
+            /* protected by the bucket lock */
+            task->streams[i].size = 1;
+            task->streams[i].reduce_count.store(1, std::memory_order_relaxed);
+
             /* get the copy to use as input for this task */
             detail::ttg_data_copy_t *copy = get_copy_fn(reduce_task, std::forward<Value>(value), false);
 
             /* put the copy into the task */
             task->copies[i] = copy;
 
-            /* protected by the bucket lock */
-            task->streams[i].size = 1;
-            task->streams[i].reduce_count.store(1, std::memory_order_relaxed);
-
             /* release the task if we're not deferred
              * TODO: can we delay that until we get the second value?
              */

From 4547d8cc805af3193e1a3fe174345f15241ab7b2 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 22 Dec 2023 13:14:35 +0000
Subject: [PATCH 219/259] Add ttg/ttvalue.h to cmake

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ttg/CMakeLists.txt b/ttg/CMakeLists.txt
index 2954d61b1..9a08adc6a 100644
--- a/ttg/CMakeLists.txt
+++ b/ttg/CMakeLists.txt
@@ -60,6 +60,7 @@ set(ttg-impl-headers
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/serialization.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/terminal.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/traverse.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/ttg/ttvalue.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/world.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/make_tt.h
         ${CMAKE_CURRENT_BINARY_DIR}/ttg/config.h

From f9900fe84a780331ac0e715913ab2627e451d73d Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 22 Dec 2023 14:08:08 +0000
Subject: [PATCH 220/259] Reset readers on reduced copy

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/ttg.h | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index c0dcaf3b7..3157a02b5 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1737,6 +1737,7 @@ ttg::abort();  // should not happen
       using rtask_t = detail::reducer_task_t;
       using value_t = std::tuple_element_t<i, actual_input_tuple_type>;
       constexpr const bool val_is_void = ttg::meta::is_void_v<value_t>;
+      constexpr const bool input_is_const = std::is_const_v<value_t>;
       rtask_t *rtask = (rtask_t*)parsec_task;
       task_t *parent_task = static_cast<task_t*>(rtask->parent_task);
       ttT *baseobj = parent_task->tt;
@@ -1810,7 +1811,11 @@ ttg::abort();  // should not happen
       //          << " of " << parent_task->streams[i].goal << " complete " << complete
       //          << " c " << c << std::endl;
       if (complete && c == 0) {
-        /* task is still in the hash table, have release_task remove it */
+        if constexpr(input_is_const) {
+          /* make the consumer task a reader if its input is const */
+          target_copy->reset_readers();
+        }
+        /* task may not be runnable yet because other inputs are missing, have release_task decide */
         parent_task->remove_from_hash = true;
         parent_task->release_task(parent_task);
       }

From d42cb9217f954b13d95ae06c424e4deeeaaec120 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 19 Jan 2024 10:08:53 -0500
Subject: [PATCH 221/259] Fix keystr string name

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/ttg.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 3157a02b5..1f6c6ae18 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -3528,7 +3528,7 @@ ttg::abort();  // should not happen
       } else {
         std::stringstream ss;
         ss << task->key;
-        snprintf(reinterpret_cast<char*>(dst), size, "%s", keystr.str().c_str());
+        snprintf(reinterpret_cast<char*>(dst), size, "%s", ss.str().c_str());
       }
       return dst;
     }

From 7b9c248a7df5ed68c7484667b14ffea91565ef53 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 29 Jan 2024 17:50:24 -0500
Subject: [PATCH 222/259] PaRSEC task: copy task locals to comply with
 profiling reqs

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/task.h | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/ttg/ttg/parsec/task.h b/ttg/ttg/parsec/task.h
index 99f683278..44148e1db 100644
--- a/ttg/ttg/parsec/task.h
+++ b/ttg/ttg/parsec/task.h
@@ -304,7 +304,12 @@ namespace ttg_parsec {
                                true /* deferred until other readers have completed */)
       , parent_task(task)
       , is_first(is_first)
-      { }
+      {
+        /* store the first 4 integers from the parent task (needed for profiling) */
+        for (int i = 0; i < 4; ++i) {
+          parsec_task.locals[i] = task->parsec_task.locals[i];
+        }
+      }
 
       static void release_task(parsec_ttg_task_base_t* task_base) {
         /* reducer tasks have one mutable input so the task can be submitted on the first release */

From 641c81ff2b7bcf2ac39430b19f6951458e13b6e8 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 29 Jan 2024 17:59:17 -0500
Subject: [PATCH 223/259] SPMM: Make writing C back into Eigen optional

The Eigen format is not particularly suitable for thread-parallel
writes so for benchmarks we drop the last part of pushing tiles back
into the Eigen matrix. Tiles are still communicated and reduced properly.

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/spmm/spmm.cc | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/examples/spmm/spmm.cc b/examples/spmm/spmm.cc
index ac198a969..ec0c0512a 100644
--- a/examples/spmm/spmm.cc
+++ b/examples/spmm/spmm.cc
@@ -192,16 +192,21 @@ class Write_SpMatrix : public TT<Key<2>, std::tuple<>, Write_SpMatrix<Blk>, ttg:
   using baseT = typename Write_SpMatrix::ttT;
 
   template <typename Keymap2>
-  Write_SpMatrix(SpMatrix<Blk> &matrix, Edge<Key<2>, Blk> &in, Keymap2 &&ij_keymap)
-      : baseT(edges(in), edges(), "write_spmatrix", {"Cij"}, {}, ij_keymap), matrix_(matrix) {}
+  Write_SpMatrix(SpMatrix<Blk> &matrix, Edge<Key<2>, Blk> &in, Keymap2 &&ij_keymap, bool write_back = true)
+      : baseT(edges(in), edges(), "write_spmatrix", {"Cij"}, {}, ij_keymap)
+      , matrix_(matrix)
+      , write_back_(write_back)
+  { }
 
   void op(const Key<2> &key, typename baseT::input_values_tuple_type &&elem, std::tuple<> &) {
-    std::lock_guard<std::mutex> lock(mtx_);
-    ttg::trace("rank =", default_execution_context().rank(),
-               "/ thread_id =", reinterpret_cast<std::uintptr_t>(pthread_self()), "spmm.cc Write_SpMatrix wrote {",
-               key[0], ",", key[1], "} = ", baseT::template get<0>(elem), " in ", static_cast<void *>(&matrix_),
-               " with mutex @", static_cast<void *>(&mtx_), " for object @", static_cast<void *>(this));
-    values_.emplace_back(key[0], key[1], baseT::template get<0>(elem));
+    if (write_back_) {
+      std::lock_guard<std::mutex> lock(mtx_);
+      ttg::trace("rank =", default_execution_context().rank(),
+                "/ thread_id =", reinterpret_cast<std::uintptr_t>(pthread_self()), "spmm.cc Write_SpMatrix wrote {",
+                key[0], ",", key[1], "} = ", baseT::template get<0>(elem), " in ", static_cast<void *>(&matrix_),
+                " with mutex @", static_cast<void *>(&mtx_), " for object @", static_cast<void *>(this));
+      values_.emplace_back(key[0], key[1], baseT::template get<0>(elem));
+    }
   }
 
   /// grab completion status as a future<void>
@@ -225,6 +230,7 @@ class Write_SpMatrix : public TT<Key<2>, std::tuple<>, Write_SpMatrix<Blk>, ttg:
   SpMatrix<Blk> &matrix_;
   std::vector<SpMatrixTriplet<Blk>> values_;
   mutable std::shared_ptr<std::shared_future<void>> completion_status_;
+  bool write_back_;
 };
 
 /// sparse mm via 2.5D SUMMA
@@ -1254,6 +1260,7 @@ static void timed_measurement(SpMatrix<> &A, SpMatrix<> &B, const std::function<
   Read_SpMatrix a("A", A, ctl, eA, ij_keymap);
   Read_SpMatrix b("B", B, ctl, eB, ij_keymap);
   Write_SpMatrix<> c(C, eC, ij_keymap);
+  Write_SpMatrix<> c(C, eC, ij_keymap, false);
   auto &c_status = c.status();
   assert(!has_value(c_status));
   //  SpMM25D a_times_b(world, eA, eB, eC, A, B);

From d584e061368b30563887a72ca3b5668088f2dd41 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 29 Jan 2024 18:13:02 -0500
Subject: [PATCH 224/259] SPMM: distribute A/B over all procs and throttle
 broadcasts

We distribute A and B over all three dimensions instead of just the first
two. The 2.5D approach does not seem suitable for our approach
since matrices will likely be distributed across all processes.

Also, add the ability to throttle the number of parallel bcasts to
avoid overloading the network. This should be revisited, since it seems
to the an issue esp with UCX as a backend in OMPI.

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/spmm/spmm.cc | 503 ++++++++++++++++++++++++++----------------
 1 file changed, 316 insertions(+), 187 deletions(-)

diff --git a/examples/spmm/spmm.cc b/examples/spmm/spmm.cc
index ec0c0512a..5ed195eea 100644
--- a/examples/spmm/spmm.cc
+++ b/examples/spmm/spmm.cc
@@ -142,17 +142,21 @@ template <std::size_t Rank>
 using Key = MultiIndex<Rank>;
 
 /// maps {i,j} to rank within first (R=0) layer of the 3-d process grid
-inline int ij2rank(int i, int j, int P, int Q) {
-  std::vector<int> vec;
+inline int ij2rank(int i, int j, int P, int Q, int R) {
   int p = (i % P);
   int q = (j % Q);
-  int rank = (q * P) + p;
+  //int rank = (q * P) + p;
+  //int pq = (q * P) + p;
+  int l = (i*j) % R;
+  int rank = (l * P * Q) + (q * P) + p;
+//  size_t hash = Key<2>{i, j}.hash();
+//  int rank = hash%(P*Q*R);
+  //std::cout << "ij2rank " << Key<2>{i, j} << " rank " << rank << std::endl;
   return rank;
 }
 
 /// maps {i,j,k} to rank within a 3-d process grid
 inline int ijk2rank(int i, int j, int k, int P, int Q, int R) {
-  std::vector<int> vec;
   int p = (i % P);
   int q = (j % Q);
   int l = (k % R);
@@ -161,28 +165,36 @@ inline int ijk2rank(int i, int j, int k, int P, int Q, int R) {
 }
 
 // flow data from an existing SpMatrix on rank 0
-template <typename Blk = blk_t, typename Keymap = std::function<int(const Key<2> &)>>
-class Read_SpMatrix : public TT<Key<2>, std::tuple<Out<Key<2>, Blk>>, Read_SpMatrix<Blk, Keymap>, ttg::typelist<void>> {
+template <typename Blk = blk_t,
+          typename Keymap = std::function<int(const Key<3> &)>,
+          typename OutKeymap = std::function<int(const Key<2> &)>>
+class Read_SpMatrix : public TT<Key<3>, std::tuple<Out<Key<2>, Blk>>, Read_SpMatrix<Blk, Keymap>, ttg::typelist<void>> {
  public:
   using baseT = typename Read_SpMatrix::ttT;
-  Read_SpMatrix(const char *label, const SpMatrix<Blk> &matrix, Edge<Key<2>> &ctl, Edge<Key<2>, Blk> &out,
-                Keymap &ij_keymap)
+  Read_SpMatrix(const char *label, const SpMatrix<Blk> &matrix, Edge<Key<3>> &ctl, Edge<Key<2>, Blk> &out,
+                Keymap &ijk_keymap, std::function<int(const Key<2> &)> ij_keymap)
       : baseT(edges(ctl), edges(out), std::string("read_spmatrix(") + label + ")", {"ctl"}, {std::string(label) + "ij"},
-              ij_keymap)
-      , matrix_(matrix) {}
+              ijk_keymap)
+      , matrix_(matrix)
+      , keymap_(ij_keymap) {}
 
-  void op(const Key<2> &, std::tuple<Out<Key<2>, Blk>> &out) {
+  void op(const Key<3> &, std::tuple<Out<Key<2>, Blk>> &out) {
     auto rank = ttg::default_execution_context().rank();
     for (int k = 0; k < matrix_.outerSize(); ++k) {
-      for (typename SpMatrix<Blk>::InnerIterator it(matrix_, k); it; ++it) {
-        if (rank == this->get_keymap()(Key<2>(std::initializer_list<long>({it.row(), it.col()}))))
-          ::send<0>(Key<2>(std::initializer_list<long>({it.row(), it.col()})), it.value(), out);
+      for (typename SpMatrix<Blk>::InnerIterator it(matrix_, k);
+           it;
+           ++it) {
+        auto key = Key<2>(std::initializer_list<long>({it.row(), it.col()}));
+        if (rank == keymap_(key)) {
+          ::send<0>(key, it.value(), out);
+        }
       }
     }
   }
 
  private:
   const SpMatrix<Blk> &matrix_;
+  std::function<int(const Key<2> &)> keymap_;
 };
 
 // flow (move?) data into an existing SpMatrix on rank 0
@@ -245,35 +257,44 @@ class SpMM25D {
   /// @param ijk_keymap maps {i,j,k} to process, controls distribution of tasks performing C[i][j] += A[i][k]*B[k][j]
   /// @param R the number of "layers" in the 3-D process grid
   SpMM25D(Edge<Key<2>, Blk> &a, Edge<Key<2>, Blk> &b, Edge<Key<2>, Blk> &c, const SpMatrix<Blk> &a_mat,
-          const SpMatrix<Blk> &b_mat, const std::vector<std::vector<long>> &a_rowidx_to_colidx,
-          const std::vector<std::vector<long>> &a_colidx_to_rowidx,
-          const std::vector<std::vector<long>> &b_rowidx_to_colidx,
-          const std::vector<std::vector<long>> &b_colidx_to_rowidx, const std::vector<int> &mTiles,
-          const std::vector<int> &nTiles, const std::vector<int> &kTiles, Keymap2 ij_keymap, Keymap3 ijk_keymap, long R)
-      : a_rowidx_to_colidx_(a_rowidx_to_colidx)
-      , b_colidx_to_rowidx_(b_colidx_to_rowidx)
-      , a_colidx_to_rowidx_(a_colidx_to_rowidx)
-      , b_rowidx_to_colidx_(b_rowidx_to_colidx)
+          const SpMatrix<Blk> &b_mat, const std::vector<std::vector<long>> &a_cols_of_row,
+          const std::vector<std::vector<long>> &a_rows_of_col,
+          const std::vector<std::vector<long>> &b_cols_of_row,
+          const std::vector<std::vector<long>> &b_rows_of_col, const std::vector<int> &mTiles,
+          const std::vector<int> &nTiles, const std::vector<int> &kTiles, Keymap2 ij_keymap, Keymap3 ijk_keymap,
+          long R, long parallel_bcasts = 1)
+      : a_cols_of_row_(a_cols_of_row)
+      , b_rows_of_col_(b_rows_of_col)
+      , a_rows_of_col_(a_rows_of_col)
+      , b_cols_of_row_(b_cols_of_row)
       , ij_keymap_(std::move(ij_keymap))
-      , ijk_keymap_(std::move(ijk_keymap)) {
-    bcast_a_ = std::make_unique<BcastA>(a, local_a_ijk_, b_rowidx_to_colidx_, ij_keymap_, ijk_keymap_);
-    local_bcast_a_ = std::make_unique<LocalBcastA>(local_a_ijk_, a_ijk_, b_rowidx_to_colidx_, ijk_keymap_);
-    bcast_b_ = std::make_unique<BcastB>(b, local_b_ijk_, a_colidx_to_rowidx_, ij_keymap_, ijk_keymap_);
-    local_bcast_b_ = std::make_unique<LocalBcastB>(local_b_ijk_, b_ijk_, a_colidx_to_rowidx_, ijk_keymap_);
-    multiplyadd_ = std::make_unique<MultiplyAdd>(a_ijk_, b_ijk_, c_ijk_, c_ij_p_, a_rowidx_to_colidx_,
-                                                 b_colidx_to_rowidx_, mTiles, nTiles, ijk_keymap_);
+      , ijk_keymap_(std::move(ijk_keymap))
+      , parallel_bcasts_(parallel_bcasts) {
+    Edge<Key<2>, void> a_ctl, b_ctl;
+    bcast_a_ = std::make_unique<BcastA>(a, a_ctl, local_a_ijk_, a_cols_of_row_, b_cols_of_row_,
+                                        ij_keymap_, ijk_keymap_, parallel_bcasts_);
+    local_bcast_a_ = std::make_unique<LocalBcastA>(local_a_ijk_, a_ijk_, b_cols_of_row_, ijk_keymap_);
+    bcast_b_ = std::make_unique<BcastB>(b, b_ctl, local_b_ijk_, a_rows_of_col_, b_cols_of_row_,
+                                        ij_keymap_, ijk_keymap_, parallel_bcasts_);
+    local_bcast_b_ = std::make_unique<LocalBcastB>(local_b_ijk_, b_ijk_, a_rows_of_col_, ijk_keymap_);
+    multiplyadd_ = std::make_unique<MultiplyAdd>(a_ijk_, b_ijk_, c_ijk_, c_ij_p_, a_cols_of_row_,
+                                                 b_rows_of_col_, mTiles, nTiles, ijk_keymap_);
     reduce_c_ = std::make_unique<ReduceC>(c_ij_p_, c, ij_keymap_);
-    reduce_c_->template set_input_reducer<0>([](Blk &c_ij, const Blk &c_ij_p) { c_ij = c_ij + c_ij_p; });
+    reduce_c_->template set_input_reducer<0>(
+      [&](Blk &c_ij, const Blk &c_ij_p) {
+        reduce_count++;
+        c_ij = c_ij + c_ij_p;
+      });
     // compute how many contributions each C[i][j] should expect ... MultiplyAdd already does this, but need a way to
     // send message from each process p to the process owning C[i][j] to expect a contribution from it for now replicate
     // this logic ...
     // TODO: do this in MultiplyAdd (need to allreduce this info so that everyone has it)
     // N.B. only need to set stream size on the rank that will accumulate the C[i][j] contribution
     const auto my_rank = ttg::default_execution_context().rank();
-    for (auto i = 0ul; i != a_rowidx_to_colidx_.size(); ++i) {
-      if (a_rowidx_to_colidx_[i].empty()) continue;
-      for (auto j = 0ul; j != b_colidx_to_rowidx_.size(); ++j) {
-        if (b_colidx_to_rowidx_[j].empty()) continue;
+    for (auto i = 0ul; i != a_cols_of_row_.size(); ++i) {
+      if (a_cols_of_row_[i].empty()) continue;
+      for (auto j = 0ul; j != b_rows_of_col_.size(); ++j) {
+        if (b_rows_of_col_[j].empty()) continue;
 
         if (ij_keymap_(Key<2>{i, j}) == my_rank) {
           decltype(i) k;
@@ -299,6 +320,35 @@ class SpMM25D {
       }
     }
 
+    auto world = ttg::default_execution_context();
+      /* initial ctl input for bcasts for A */
+      int to_start = parallel_bcasts;
+      for (int i = 0;
+           0 < to_start && i < a_cols_of_row_.size();
+           ++i) {
+        for (auto k : a_cols_of_row_[i]) {
+          auto key = Key<2>(i, k);
+          if (world.rank() == ij_keymap_(key)) {
+            bcast_a_->template in<1>()->sendk(key);
+            if (0 == --to_start) break;
+          }
+        }
+      }
+
+      /* initial ctl input for bcasts for B */
+      to_start = parallel_bcasts;
+      for (int k = 0;
+           0 < to_start && k < b_cols_of_row_.size();
+           ++k) {
+        for (auto j : b_cols_of_row_[k]) {
+          auto key = Key<2>(k, j);
+          if (world.rank() == ij_keymap_(key)) {
+            bcast_b_->template in<1>()->sendk(key);
+            if (0 == --to_start) break;
+          }
+        }
+      }
+
     TTGUNUSED(bcast_a_);
     TTGUNUSED(bcast_b_);
     TTGUNUSED(multiplyadd_);
@@ -312,10 +362,10 @@ class SpMM25D {
     using baseT = typename LocalBcastA::ttT;
 
     LocalBcastA(Edge<Key<3>, Blk> &a, Edge<Key<3>, Blk> &a_ijk,
-                const std::vector<std::vector<long>> &b_rowidx_to_colidx, const Keymap3 &ijk_keymap)
+                const std::vector<std::vector<long>> &b_cols_of_row, const Keymap3 &ijk_keymap)
         : baseT(edges(a), edges(a_ijk), "SpMM25D::local_bcast_a", {"a_ikp"}, {"a_ijk"},
                 [](const Key<3> &ikp) { return ikp[2]; })
-        , b_rowidx_to_colidx_(b_rowidx_to_colidx)
+        , b_cols_of_row_(b_cols_of_row)
         , ijk_keymap_(ijk_keymap) {}
 
     void op(const Key<3> &ikp, typename baseT::input_values_tuple_type &&a_ik, std::tuple<Out<Key<3>, Blk>> &a_ijk) {
@@ -326,10 +376,10 @@ class SpMM25D {
       auto world = default_execution_context();
       assert(p == world.rank());
       ttg::trace("LocalBcastA(", i, ", ", k, ", ", p, ")");
-      if (k >= b_rowidx_to_colidx_.size()) return;
+      if (k >= b_cols_of_row_.size()) return;
       // local broadcast a_ik to all {i,j,k} such that b_kj exists
       std::vector<Key<3>> ijk_keys;
-      for (auto &j : b_rowidx_to_colidx_[k]) {
+      for (auto &j : b_cols_of_row_[k]) {
         if (ijk_keymap_(Key<3>({i, j, k})) == world.rank()) {
           ttg::trace("Broadcasting A[", i, "][", k, "] on proc ", p, " to j=", j);
           ijk_keys.emplace_back(Key<3>({i, j, k}));
@@ -339,32 +389,39 @@ class SpMM25D {
     }
 
    private:
-    const std::vector<std::vector<long>> &b_rowidx_to_colidx_;
+    const std::vector<std::vector<long>> &b_cols_of_row_;
     const Keymap3 &ijk_keymap_;
   };  // class LocalBcastA
 
   /// broadcast `A[i][k]` to all processors which will contain at least one `C[i][j]` such that `B[k][j]` exists
-  class BcastA : public TT<Key<2>, std::tuple<Out<Key<3>, Blk>>, BcastA, ttg::typelist<Blk>> {
+  class BcastA : public TT<Key<2>, std::tuple<Out<Key<3>, Blk>, Out<Key<2>, void>>, BcastA, ttg::typelist<Blk, void>> {
    public:
     using baseT = typename BcastA::ttT;
 
-    BcastA(Edge<Key<2>, Blk> &a_ik, Edge<Key<3>, Blk> &a_ikp, const std::vector<std::vector<long>> &b_rowidx_to_colidx,
-           const Keymap2 &ij_keymap, const Keymap3 &ijk_keymap)
-        : baseT(edges(a_ik), edges(a_ikp), "SpMM25D::bcast_a", {"a_ik"}, {"a_ikp"}, ij_keymap)
-        , b_rowidx_to_colidx_(b_rowidx_to_colidx)
-        , ijk_keymap_(ijk_keymap) {}
-
-    void op(const Key<2> &ik, typename baseT::input_values_tuple_type &&a_ik, std::tuple<Out<Key<3>, Blk>> &a_ikp) {
-      const auto i = ik[0];
-      const auto k = ik[1];
+    BcastA(Edge<Key<2>, Blk> &a_ik, Edge<Key<2>, void> &ctl, Edge<Key<3>, Blk> &a_ikp,
+           const std::vector<std::vector<long>> &a_cols_of_row,
+           const std::vector<std::vector<long>> &b_cols_of_row,
+           const Keymap2 &ij_keymap, const Keymap3 &ijk_keymap,
+           const int parallel_bcasts)
+        : baseT(edges(a_ik, ctl), edges(a_ikp, ctl), "SpMM25D::bcast_a", {"a_ik", "ctl"}, {"a_ikp", "ctl"}, ij_keymap)
+        , a_cols_of_row_(a_cols_of_row)
+        , b_cols_of_row_(b_cols_of_row)
+        , ijk_keymap_(ijk_keymap)
+        , ij_keymap_(ij_keymap)
+        , parallel_bcasts_(parallel_bcasts) {}
+
+    void op(const Key<2> &ik, typename baseT::input_values_tuple_type &&a_ik,
+            std::tuple<Out<Key<3>, Blk>, Out<Key<2>, void>> &outs) {
+      auto i = ik[0]; // row
+      auto k = ik[1]; // col
       ttg::trace("BcastA(", i, ", ", k, ")");
       std::vector<Key<3>> ikp_keys;
 
-      if (k >= b_rowidx_to_colidx_.size()) return;
+      if (k >= b_cols_of_row_.size()) return;
       auto world = default_execution_context();
       std::vector<bool> procmap(world.size());
-      for (auto &j : b_rowidx_to_colidx_[k]) {
-        const long p = ijk_keymap_(Key<3>(
+      for (auto &j : b_cols_of_row_[k]) {
+        const int p = ijk_keymap_(Key<3>(
             {i, j, k}));  // N.B. in 2.5D SUMMA different k contributions to C[i][j] are computed on different nodes
         if (!procmap[p]) {
           ttg::trace("Broadcasting A[", i, "][", k, "] to proc ", p);
@@ -372,12 +429,39 @@ class SpMM25D {
           procmap[p] = true;
         }
       }
-      ::broadcast<0>(ikp_keys, std::move(baseT::template get<0>(a_ik)), a_ikp);
+      ::broadcast<0>(ikp_keys, std::move(baseT::template get<0>(a_ik)), outs);
+
+      /* enable next broadcast through a control message
+       * we don't check whether this tile is in B here, this is
+       * done inside the next task (see above) */
+      std::size_t num_rows = a_cols_of_row_.size();
+      auto it = std::find(a_cols_of_row_[i].begin(), a_cols_of_row_[i].end(), k);
+      ++it; // skip the current tile
+      long to_skip = parallel_bcasts_;
+      do {
+        for (; it != a_cols_of_row_[i].end(); ++it) {
+          Key<2> key = {i, *it};
+          if (world.rank() == this->get_keymap()(key)) {
+            if (0 == --to_skip) {
+              ::sendk<1>(key, outs);
+              return;
+            }
+          }
+        }
+        if ((i+1) < num_rows) {
+          it = a_cols_of_row_[++i].begin();
+        } else {
+          break;
+        }
+      } while (1);
     }
 
    private:
-    const std::vector<std::vector<long>> &b_rowidx_to_colidx_;
+    const std::vector<std::vector<long>> &a_cols_of_row_;
+    const std::vector<std::vector<long>> &b_cols_of_row_;
     const Keymap3 &ijk_keymap_;
+    const Keymap2 &ij_keymap_;
+    const int parallel_bcasts_;
   };  // class BcastA
 
   /// Locally broadcast `B[k][j]` assigned to this processor `p` to matmul tasks `{i,j,k}` for all `k` such that
@@ -387,10 +471,10 @@ class SpMM25D {
     using baseT = typename LocalBcastB::ttT;
 
     LocalBcastB(Edge<Key<3>, Blk> &b_kjp, Edge<Key<3>, Blk> &b_ijk,
-                const std::vector<std::vector<long>> &a_colidx_to_rowidx, const Keymap3 &ijk_keymap)
+                const std::vector<std::vector<long>> &a_rows_of_col, const Keymap3 &ijk_keymap)
         : baseT(edges(b_kjp), edges(b_ijk), "SpMM25D::local_bcast_b", {"b_kjp"}, {"b_ijk"},
                 [](const Key<3> &kjp) { return kjp[2]; })
-        , a_colidx_to_rowidx_(a_colidx_to_rowidx)
+        , a_rows_of_col_(a_rows_of_col)
         , ijk_keymap_(ijk_keymap) {}
 
     void op(const Key<3> &kjp, typename baseT::input_values_tuple_type &&b_kj, std::tuple<Out<Key<3>, Blk>> &b_ijk) {
@@ -400,10 +484,10 @@ class SpMM25D {
       auto world = default_execution_context();
       assert(p == world.rank());
       ttg::trace("BcastB(", k, ", ", j, ", ", p, ")");
-      if (k >= a_colidx_to_rowidx_.size()) return;
+      if (k >= a_rows_of_col_.size()) return;
       // broadcast b_kj to all ijk for which c_ij is on this processor and a_ik exists
       std::vector<Key<3>> ijk_keys;
-      for (auto &i : a_colidx_to_rowidx_[k]) {
+      for (auto &i : a_rows_of_col_[k]) {
         if (ijk_keymap_(Key<3>({i, j, k})) == world.rank()) {
           ttg::trace("Broadcasting B[", k, "][", j, "] on proc ", p, " to i=", i);
           ijk_keys.emplace_back(Key<3>({i, j, k}));
@@ -413,31 +497,37 @@ class SpMM25D {
     }
 
    private:
-    const std::vector<std::vector<long>> &a_colidx_to_rowidx_;
+    const std::vector<std::vector<long>> &a_rows_of_col_;
     const Keymap3 &ijk_keymap_;
   };  // class LocalBcastB
 
   /// broadcast `B[k][j]` to all processors which will contain at least one `C[i][j]` such that `A[i][k]` exists
-  class BcastB : public TT<Key<2>, std::tuple<Out<Key<3>, Blk>>, BcastB, ttg::typelist<Blk>> {
+  class BcastB : public TT<Key<2>, std::tuple<Out<Key<3>, Blk>, Out<Key<2>, void>>, BcastB, ttg::typelist<Blk, void>> {
    public:
     using baseT = typename BcastB::ttT;
 
-    BcastB(Edge<Key<2>, Blk> &b_kj, Edge<Key<3>, Blk> &b_kjp, const std::vector<std::vector<long>> &a_colidx_to_rowidx,
-           const Keymap2 &ij_keymap, const Keymap3 &ijk_keymap)
-        : baseT(edges(b_kj), edges(b_kjp), "SpMM25D::bcast_b", {"b_kj"}, {"b_kjp"}, ij_keymap)
-        , a_colidx_to_rowidx_(a_colidx_to_rowidx)
-        , ijk_keymap_(ijk_keymap) {}
-
-    void op(const Key<2> &kj, typename baseT::input_values_tuple_type &&b_kj, std::tuple<Out<Key<3>, Blk>> &b_kjp) {
-      const auto k = kj[0];
-      const auto j = kj[1];
+    BcastB(Edge<Key<2>, Blk> &b_kj, Edge<Key<2>, void> ctl, Edge<Key<3>, Blk> &b_kjp,
+           const std::vector<std::vector<long>> &a_rows_of_col,
+           const std::vector<std::vector<long>> &b_cols_of_row,
+           const Keymap2 &ij_keymap, const Keymap3 &ijk_keymap,
+           const int parallel_bcasts)
+        : baseT(edges(b_kj, ctl), edges(b_kjp, ctl), "SpMM25D::bcast_b", {"b_kj", "ctl"}, {"b_kjp", "ctl"}, ij_keymap)
+        , a_rows_of_col_(a_rows_of_col)
+        , b_cols_of_row_(b_cols_of_row)
+        , ijk_keymap_(ijk_keymap)
+        , parallel_bcasts_(parallel_bcasts) {}
+
+    void op(const Key<2> &kj, typename baseT::input_values_tuple_type &&b_kj,
+            std::tuple<Out<Key<3>, Blk>, Out<Key<2>, void>> &outs) {
+      auto k = kj[0]; // row
+      auto j = kj[1]; // col
       // broadcast b_kj to all processors which will contain at least one c_ij such that a_ik exists
       std::vector<Key<3>> kjp_keys;
       ttg::trace("BcastB(", k, ", ", j, ")");
-      if (k >= a_colidx_to_rowidx_.size()) return;
+      if (k >= a_rows_of_col_.size()) return;
       auto world = default_execution_context();
       std::vector<bool> procmap(world.size());
-      for (auto &i : a_colidx_to_rowidx_[k]) {
+      for (auto &i : a_rows_of_col_[k]) {
         long p = ijk_keymap_(Key<3>({i, j, k}));
         if (!procmap[p]) {
           ttg::trace("Broadcasting B[", k, "][", j, "] to proc ", p);
@@ -445,12 +535,38 @@ class SpMM25D {
           procmap[p] = true;
         }
       }
-      ::broadcast<0>(kjp_keys, std::move(baseT::template get<0>(b_kj)), b_kjp);
+      ::broadcast<0>(kjp_keys, std::move(baseT::template get<0>(b_kj)), outs);
+
+      /* enable next broadcast through a control message
+       * we don't check whether this tile is in A here, this is
+       * done inside the next task (see above) */
+      std::size_t num_rows = b_cols_of_row_.size();
+      auto it = std::find(b_cols_of_row_[k].begin(), b_cols_of_row_[k].end(), j);
+      ++it; // skip the current tile
+      long to_skip = parallel_bcasts_;
+      do {
+        for (; it != b_cols_of_row_[k].end(); ++it) {
+          Key<2> key = {k, *it};
+          if (world.rank() == this->get_keymap()(key)) {
+            if (0 == --to_skip) {
+              ::sendk<1>(key, outs);
+              return;
+            }
+          }
+        }
+        if ((k+1) < num_rows) {
+          it = b_cols_of_row_[++k].begin();
+        } else {
+          break;
+        }
+      } while (1);
     }
 
    private:
-    const std::vector<std::vector<long>> &a_colidx_to_rowidx_;
+    const std::vector<std::vector<long>> &a_rows_of_col_;
+    const std::vector<std::vector<long>> &b_cols_of_row_;
     const Keymap3 &ijk_keymap_;
+    const int parallel_bcasts_;
   };  // class BcastB
 
   /// multiply task has 3 input flows: a_ijk, b_ijk, and c_ijk, c_ijk contains the running total for this kayer of the
@@ -461,21 +577,21 @@ class SpMM25D {
     using baseT = typename MultiplyAdd::ttT;
 
     MultiplyAdd(Edge<Key<3>, Blk> &a_ijk, Edge<Key<3>, Blk> &b_ijk, Edge<Key<3>, Blk> &c_ijk, Edge<Key<2>, Blk> &c,
-                const std::vector<std::vector<long>> &a_rowidx_to_colidx,
-                const std::vector<std::vector<long>> &b_colidx_to_rowidx, const std::vector<int> &mTiles,
+                const std::vector<std::vector<long>> &a_cols_of_row,
+                const std::vector<std::vector<long>> &b_rows_of_col, const std::vector<int> &mTiles,
                 const std::vector<int> &nTiles, const Keymap3 &ijk_keymap)
         : baseT(edges(a_ijk, b_ijk, c_ijk), edges(c, c_ijk), "SpMM25D::MultiplyAdd", {"a_ijk", "b_ijk", "c_ijk"},
                 {"c_ij", "c_ijk"}, ijk_keymap)
-        , a_rowidx_to_colidx_(a_rowidx_to_colidx)
-        , b_colidx_to_rowidx_(b_colidx_to_rowidx) {
+        , a_cols_of_row_(a_cols_of_row)
+        , b_rows_of_col_(b_rows_of_col) {
       this->set_priomap([=,this](const Key<3> &ijk) { return this->prio(ijk); });  // map a key to an integral priority value
 
       // for each {i,j} determine first k that contributes AND belongs to this node,
       // initialize input {i,j,first_k} flow to 0
-      for (auto i = 0ul; i != a_rowidx_to_colidx_.size(); ++i) {
-        if (a_rowidx_to_colidx_[i].empty()) continue;
-        for (auto j = 0ul; j != b_colidx_to_rowidx_.size(); ++j) {
-          if (b_colidx_to_rowidx_[j].empty()) continue;
+      for (auto i = 0ul; i != a_cols_of_row_.size(); ++i) {
+        if (a_cols_of_row_[i].empty()) continue;
+        for (auto j = 0ul; j != b_rows_of_col_.size(); ++j) {
+          if (b_rows_of_col_[j].empty()) continue;
 
           const auto p = ttg::default_execution_context().rank();
           decltype(i) k;
@@ -490,7 +606,7 @@ class SpMM25D {
 #endif
             this->template in<2>()->send(Key<3>({i, j, k}), zero);
           } else {
-            if (tracing() && a_rowidx_to_colidx_.size() * b_colidx_to_rowidx_.size() < 400)
+            if (tracing() && a_cols_of_row_.size() * b_rows_of_col_.size() < 400)
               ttg::print("C[", i, "][", j, "] is empty");
           }
         }
@@ -528,8 +644,8 @@ class SpMM25D {
     }
 
    private:
-    const std::vector<std::vector<long>> &a_rowidx_to_colidx_;
-    const std::vector<std::vector<long>> &b_colidx_to_rowidx_;
+    const std::vector<std::vector<long>> &a_cols_of_row_;
+    const std::vector<std::vector<long>> &b_rows_of_col_;
 
     /* Compute the length of the remaining sequence on that tile */
     int32_t prio(const Key<3> &key) const {
@@ -549,11 +665,11 @@ class SpMM25D {
    public:  // to be able to reuse this logic in SpMM25D
     // given {i,j} return first k such that A[i][k] and B[k][j] exist
     std::tuple<long, bool> compute_first_k(long i, long j) const {
-      const auto &a_k_range = a_rowidx_to_colidx_.at(i);
+      const auto &a_k_range = a_cols_of_row_.at(i);
       auto a_iter = a_k_range.begin();
       auto a_iter_fence = a_k_range.end();
       if (a_iter == a_iter_fence) return std::make_tuple(-1, false);
-      const auto &b_k_range = b_colidx_to_rowidx_.at(j);
+      const auto &b_k_range = b_rows_of_col_.at(j);
       auto b_iter = b_k_range.begin();
       auto b_iter_fence = b_k_range.end();
       if (b_iter == b_iter_fence) return std::make_tuple(-1, false);
@@ -581,11 +697,11 @@ class SpMM25D {
     // given {i,j,k} such that A[i][k] and B[k][j] exist
     // return next k such that this condition holds
     std::tuple<long, bool> compute_next_k(long i, long j, long k) const {
-      const auto &a_k_range = a_rowidx_to_colidx_.at(i);
+      const auto &a_k_range = a_cols_of_row_.at(i);
       auto a_iter_fence = a_k_range.end();
       auto a_iter = std::find(a_k_range.begin(), a_iter_fence, k);
       assert(a_iter != a_iter_fence);
-      const auto &b_k_range = b_colidx_to_rowidx_.at(j);
+      const auto &b_k_range = b_rows_of_col_.at(j);
       auto b_iter_fence = b_k_range.end();
       auto b_iter = std::find(b_k_range.begin(), b_iter_fence, k);
       assert(b_iter != b_iter_fence);
@@ -664,10 +780,11 @@ class SpMM25D {
   Edge<Key<3>, Blk> local_b_ijk_;
   Edge<Key<3>, Blk> c_ijk_;
   Edge<Key<2>, Blk> c_ij_p_;
-  const std::vector<std::vector<long>> &a_rowidx_to_colidx_;
-  const std::vector<std::vector<long>> &b_colidx_to_rowidx_;
-  const std::vector<std::vector<long>> &a_colidx_to_rowidx_;
-  const std::vector<std::vector<long>> &b_rowidx_to_colidx_;
+  Edge<Key<2>, void> a_bcast_ctl_, b_bcast_ctl_;
+  const std::vector<std::vector<long>> &a_cols_of_row_;
+  const std::vector<std::vector<long>> &b_rows_of_col_;
+  const std::vector<std::vector<long>> &a_rows_of_col_;
+  const std::vector<std::vector<long>> &b_cols_of_row_;
   std::unique_ptr<BcastA> bcast_a_;
   std::unique_ptr<LocalBcastA> local_bcast_a_;
   std::unique_ptr<BcastB> bcast_b_;
@@ -676,28 +793,33 @@ class SpMM25D {
   std::unique_ptr<ReduceC> reduce_c_;
   Keymap2 ij_keymap_;
   Keymap3 ijk_keymap_;
+  long parallel_bcasts_;
 };
 
-class Control : public TT<void, std::tuple<Out<Key<2>>>, Control> {
+class Control : public TT<void, std::tuple<Out<Key<3>>>, Control> {
   using baseT = typename Control::ttT;
-  int P;
-  int Q;
+  int P = 0;
+  int Q = 0;
+  int R = 0;
 
  public:
-  explicit Control(Edge<Key<2>> &ctl) : baseT(edges(), edges(ctl), "Control", {}, {"ctl"}), P(0), Q(0) {}
+  explicit Control(Edge<Key<3>> &ctl) : baseT(edges(), edges(ctl), "Control", {}, {"ctl"}) {}
 
-  void op(std::tuple<Out<Key<2>>> &out) const {
+  void op(std::tuple<Out<Key<3>>> &out) const {
     for (int p = 0; p < P; p++) {
       for (int q = 0; q < Q; q++) {
-        ttg::trace("Control: start computing on process {", p, ", ", q, "}");
-        ::sendk<0>(Key<2>{p, q}, out);
+        for (int r = 0; r < R; r++) {
+          ttg::trace("Control: start computing on process {", p, ", ", q, ", ", r, "}");
+          ::sendk<0>(Key<3>{p, q, r}, out);
+        }
       }
     }
   }
 
-  void start(const int _p, const int _q) {
+  void start(const int _p, const int _q, const int _r) {
     P = _p;
     Q = _q;
+    R = _r;
     invoke();
   }
 };
@@ -928,10 +1050,10 @@ static void initSpHardCoded(const std::function<int(const Key<2> &)> &keymap, Sp
 static void initBlSpHardCoded(const std::function<int(const Key<2> &)> &keymap, SpMatrix<> &A, SpMatrix<> &B,
                               SpMatrix<> &C, SpMatrix<> &Aref, SpMatrix<> &Bref, bool buildRefs,
                               std::vector<int> &mTiles, std::vector<int> &nTiles, std::vector<int> &kTiles,
-                              std::vector<std::vector<long>> &a_rowidx_to_colidx,
-                              std::vector<std::vector<long>> &a_colidx_to_rowidx,
-                              std::vector<std::vector<long>> &b_rowidx_to_colidx,
-                              std::vector<std::vector<long>> &b_colidx_to_rowidx, int &m, int &n, int &k) {
+                              std::vector<std::vector<long>> &a_cols_of_row,
+                              std::vector<std::vector<long>> &a_rows_of_col,
+                              std::vector<std::vector<long>> &b_cols_of_row,
+                              std::vector<std::vector<long>> &b_rows_of_col, int &m, int &n, int &k) {
   m = 2;
   n = 3;
   k = 4;
@@ -1001,19 +1123,19 @@ static void initBlSpHardCoded(const std::function<int(const Key<2> &)> &keymap,
     Aref_elements.emplace_back(1, 2, .2);
   }
 #endif
-  a_rowidx_to_colidx.resize(2);
-  a_rowidx_to_colidx[0].emplace_back(1);  // A[0][1]
-  a_rowidx_to_colidx[0].emplace_back(2);  // A[0][2]
-  a_rowidx_to_colidx[0].emplace_back(3);  // A[0][3]
-  a_rowidx_to_colidx[1].emplace_back(0);  // A[1][0]
-  a_rowidx_to_colidx[1].emplace_back(2);  // A[1][2]
-
-  a_colidx_to_rowidx.resize(4);
-  a_colidx_to_rowidx[0].emplace_back(1);  // A[1][0]
-  a_colidx_to_rowidx[1].emplace_back(0);  // A[0][1]
-  a_colidx_to_rowidx[2].emplace_back(0);  // A[0][2]
-  a_colidx_to_rowidx[2].emplace_back(1);  // A[1][2]
-  a_colidx_to_rowidx[3].emplace_back(0);  // A[0][3]
+  a_cols_of_row.resize(2);
+  a_cols_of_row[0].emplace_back(1);  // A[0][1]
+  a_cols_of_row[0].emplace_back(2);  // A[0][2]
+  a_cols_of_row[0].emplace_back(3);  // A[0][3]
+  a_cols_of_row[1].emplace_back(0);  // A[1][0]
+  a_cols_of_row[1].emplace_back(2);  // A[1][2]
+
+  a_rows_of_col.resize(4);
+  a_rows_of_col[0].emplace_back(1);  // A[1][0]
+  a_rows_of_col[1].emplace_back(0);  // A[0][1]
+  a_rows_of_col[2].emplace_back(0);  // A[0][2]
+  a_rows_of_col[2].emplace_back(1);  // A[1][2]
+  a_rows_of_col[3].emplace_back(0);  // A[0][3]
 
   A.setFromTriplets(A_elements.begin(), A_elements.end());
   std::cout << "A_elements.begin()" << A_elements.begin() << "A_elements.end()" << A_elements.end() << "\n";
@@ -1078,23 +1200,23 @@ static void initBlSpHardCoded(const std::function<int(const Key<2> &)> &keymap,
     B_elements.emplace_back(3, 2, 0.2);
   }
 #endif
-  b_rowidx_to_colidx.resize(4);
-  b_rowidx_to_colidx[0].emplace_back(0);  // B[0][0]
-  b_rowidx_to_colidx[1].emplace_back(0);  // B[1][0]
-  b_rowidx_to_colidx[1].emplace_back(1);  // B[1][1]
-  b_rowidx_to_colidx[1].emplace_back(2);  // B[1][2]
-  b_rowidx_to_colidx[2].emplace_back(2);  // B[2][2]
-  b_rowidx_to_colidx[3].emplace_back(0);  // B[3][0]
-  b_rowidx_to_colidx[3].emplace_back(2);  // B[3][2]
-
-  b_colidx_to_rowidx.resize(3);
-  b_colidx_to_rowidx[0].emplace_back(0);  // B[0][0]
-  b_colidx_to_rowidx[0].emplace_back(1);  // B[1][0]
-  b_colidx_to_rowidx[0].emplace_back(3);  // B[3][0]
-  b_colidx_to_rowidx[1].emplace_back(1);  // B[1][1]
-  b_colidx_to_rowidx[2].emplace_back(1);  // B[1][2]
-  b_colidx_to_rowidx[2].emplace_back(2);  // B[2][2]
-  b_colidx_to_rowidx[2].emplace_back(3);  // A[3][2]
+  b_cols_of_row.resize(4);
+  b_cols_of_row[0].emplace_back(0);  // B[0][0]
+  b_cols_of_row[1].emplace_back(0);  // B[1][0]
+  b_cols_of_row[1].emplace_back(1);  // B[1][1]
+  b_cols_of_row[1].emplace_back(2);  // B[1][2]
+  b_cols_of_row[2].emplace_back(2);  // B[2][2]
+  b_cols_of_row[3].emplace_back(0);  // B[3][0]
+  b_cols_of_row[3].emplace_back(2);  // B[3][2]
+
+  b_rows_of_col.resize(3);
+  b_rows_of_col[0].emplace_back(0);  // B[0][0]
+  b_rows_of_col[0].emplace_back(1);  // B[1][0]
+  b_rows_of_col[0].emplace_back(3);  // B[3][0]
+  b_rows_of_col[1].emplace_back(1);  // B[1][1]
+  b_rows_of_col[2].emplace_back(1);  // B[1][2]
+  b_rows_of_col[2].emplace_back(2);  // B[2][2]
+  b_rows_of_col[2].emplace_back(3);  // A[3][2]
 
   B.setFromTriplets(B_elements.begin(), B_elements.end());
   if (buildRefs && 0 == rank) {
@@ -1106,10 +1228,10 @@ static void initBlSpHardCoded(const std::function<int(const Key<2> &)> &keymap,
 static void initBlSpRandom(const std::function<int(const Key<2> &)> &keymap, size_t M, size_t N, size_t K, int minTs,
                            int maxTs, double avgDensity, SpMatrix<> &A, SpMatrix<> &B, SpMatrix<> &Aref,
                            SpMatrix<> &Bref, bool buildRefs, std::vector<int> &mTiles, std::vector<int> &nTiles,
-                           std::vector<int> &kTiles, std::vector<std::vector<long>> &a_rowidx_to_colidx,
-                           std::vector<std::vector<long>> &a_colidx_to_rowidx,
-                           std::vector<std::vector<long>> &b_rowidx_to_colidx,
-                           std::vector<std::vector<long>> &b_colidx_to_rowidx, double &average_tile_size,
+                           std::vector<int> &kTiles, std::vector<std::vector<long>> &a_cols_of_row,
+                           std::vector<std::vector<long>> &a_rows_of_col,
+                           std::vector<std::vector<long>> &b_cols_of_row,
+                           std::vector<std::vector<long>> &b_rows_of_col, double &average_tile_size,
                            double &Adensity, double &Bdensity, unsigned int seed) {
   int rank = ttg::default_execution_context().rank();
 
@@ -1172,10 +1294,10 @@ static void initBlSpRandom(const std::function<int(const Key<2> &)> &keymap, siz
     if (fills.find({mt, kt}) != fills.end()) continue;
     fills.insert({mt, kt});
 
-    if (mt >= a_rowidx_to_colidx.size()) a_rowidx_to_colidx.resize(mt + 1);
-    a_rowidx_to_colidx[mt].emplace_back(kt);
-    if (kt >= a_colidx_to_rowidx.size()) a_colidx_to_rowidx.resize(kt + 1);
-    a_colidx_to_rowidx[kt].emplace_back(mt);
+    if (mt >= a_cols_of_row.size()) a_cols_of_row.resize(mt + 1);
+    a_cols_of_row[mt].emplace_back(kt);
+    if (kt >= a_rows_of_col.size()) a_rows_of_col.resize(kt + 1);
+    a_rows_of_col[kt].emplace_back(mt);
 
     filling += mTiles[mt] * kTiles[kt];
     avg_nb += mTiles[mt] * kTiles[kt];
@@ -1185,10 +1307,10 @@ static void initBlSpRandom(const std::function<int(const Key<2> &)> &keymap, siz
     if (rank != keymap({mt, kt})) continue;
     A_elements.emplace_back(mt, kt, blk_t(btas::Range(mTiles[mt], kTiles[kt]), value));
   }
-  for (auto &row : a_rowidx_to_colidx) {
+  for (auto &row : a_cols_of_row) {
     std::sort(row.begin(), row.end());
   }
-  for (auto &col : a_colidx_to_rowidx) {
+  for (auto &col : a_rows_of_col) {
     std::sort(col.begin(), col.end());
   }
   A.setFromTriplets(A_elements.begin(), A_elements.end());
@@ -1204,10 +1326,10 @@ static void initBlSpRandom(const std::function<int(const Key<2> &)> &keymap, siz
     if (fills.find({kt, nt}) != fills.end()) continue;
     fills.insert({kt, nt});
 
-    if (kt >= b_rowidx_to_colidx.size()) b_rowidx_to_colidx.resize(kt + 1);
-    b_rowidx_to_colidx[kt].emplace_back(nt);
-    if (nt >= b_colidx_to_rowidx.size()) b_colidx_to_rowidx.resize(nt + 1);
-    b_colidx_to_rowidx[nt].emplace_back(kt);
+    if (kt >= b_cols_of_row.size()) b_cols_of_row.resize(kt + 1);
+    b_cols_of_row[kt].emplace_back(nt);
+    if (nt >= b_rows_of_col.size()) b_rows_of_col.resize(nt + 1);
+    b_rows_of_col[nt].emplace_back(kt);
 
     filling += kTiles[kt] * nTiles[nt];
     avg_nb += kTiles[kt] * nTiles[nt];
@@ -1217,10 +1339,10 @@ static void initBlSpRandom(const std::function<int(const Key<2> &)> &keymap, siz
     if (rank != keymap({kt, nt})) continue;
     B_elements.emplace_back(kt, nt, blk_t(btas::Range(kTiles[kt], nTiles[nt]), value));
   }
-  for (auto &row : b_rowidx_to_colidx) {
+  for (auto &row : b_cols_of_row) {
     std::sort(row.begin(), row.end());
   }
-  for (auto &col : b_colidx_to_rowidx) {
+  for (auto &col : b_rows_of_col) {
     std::sort(col.begin(), col.end());
   }
   B.setFromTriplets(B_elements.begin(), B_elements.end());
@@ -1237,12 +1359,12 @@ static void initBlSpRandom(const std::function<int(const Key<2> &)> &keymap, siz
 static void timed_measurement(SpMatrix<> &A, SpMatrix<> &B, const std::function<int(const Key<2> &)> &ij_keymap,
                               const std::function<int(const Key<3> &)> &ijk_keymap, const std::string &tiling_type,
                               double gflops, double avg_nb, double Adensity, double Bdensity,
-                              const std::vector<std::vector<long>> &a_rowidx_to_colidx,
-                              const std::vector<std::vector<long>> &a_colidx_to_rowidx,
-                              const std::vector<std::vector<long>> &b_rowidx_to_colidx,
-                              const std::vector<std::vector<long>> &b_colidx_to_rowidx, std::vector<int> &mTiles,
+                              const std::vector<std::vector<long>> &a_cols_of_row,
+                              const std::vector<std::vector<long>> &a_rows_of_col,
+                              const std::vector<std::vector<long>> &b_cols_of_row,
+                              const std::vector<std::vector<long>> &b_rows_of_col, std::vector<int> &mTiles,
                               std::vector<int> &nTiles, std::vector<int> &kTiles, int M, int N, int K, int minTs,
-                              int maxTs, int P, int Q, int R) {
+                              int maxTs, int P, int Q, int R, int parallel_bcasts) {
   int MT = (int)A.rows();
   int NT = (int)B.cols();
   int KT = (int)A.cols();
@@ -1251,21 +1373,22 @@ static void timed_measurement(SpMatrix<> &A, SpMatrix<> &B, const std::function<
   SpMatrix<> C;
   C.resize(MT, NT);
 
+  reduce_count = 0;
+
   // flow graph needs to exist on every node
-  Edge<Key<2>> ctl("control");
+  Edge<Key<3>> ctl("control");
   Control control(ctl);
   Edge<Key<2>, blk_t> eA, eB;
   Edge<Key<2>, blk_t> eC;
 
-  Read_SpMatrix a("A", A, ctl, eA, ij_keymap);
-  Read_SpMatrix b("B", B, ctl, eB, ij_keymap);
-  Write_SpMatrix<> c(C, eC, ij_keymap);
+  Read_SpMatrix a("A", A, ctl, eA, ijk_keymap, ij_keymap);
+  Read_SpMatrix b("B", B, ctl, eB, ijk_keymap, ij_keymap);
   Write_SpMatrix<> c(C, eC, ij_keymap, false);
   auto &c_status = c.status();
   assert(!has_value(c_status));
   //  SpMM25D a_times_b(world, eA, eB, eC, A, B);
-  SpMM25D<> a_times_b(eA, eB, eC, A, B, a_rowidx_to_colidx, a_colidx_to_rowidx, b_rowidx_to_colidx, b_colidx_to_rowidx,
-                      mTiles, nTiles, kTiles, ij_keymap, ijk_keymap, R);
+  SpMM25D<> a_times_b(eA, eB, eC, A, B, a_cols_of_row, a_rows_of_col, b_cols_of_row, b_rows_of_col,
+                      mTiles, nTiles, kTiles, ij_keymap, ijk_keymap, R, parallel_bcasts);
   TTGUNUSED(a);
   TTGUNUSED(b);
   TTGUNUSED(a_times_b);
@@ -1279,7 +1402,7 @@ static void timed_measurement(SpMatrix<> &A, SpMatrix<> &B, const std::function<
   }, end{0}, diff{0};
   gettimeofday(&start, nullptr);
   // ready, go! need only 1 kick, so must be done by 1 thread only
-  if (ttg::default_execution_context().rank() == 0) control.start(P, Q);
+  if (ttg::default_execution_context().rank() == 0) control.start(P, Q, R);
   fence();
   gettimeofday(&end, nullptr);
   timersub(&end, &start, &diff);
@@ -1297,10 +1420,11 @@ static void timed_measurement(SpMatrix<> &A, SpMatrix<> &B, const std::function<
               << " A_density= " << Adensity << " B_density= " << Bdensity << " gflops= " << gflops << " seconds= " << tc
               << " gflops/s= " << gflops / tc << std::endl;
   }
+  std::cout << "num reductions " << reduce_count.load() << " tiles " << MT*KT << std::endl;
 }
 
 #if !defined(BLOCK_SPARSE_GEMM)
-static void make_rowidx_to_colidx_from_eigen(const SpMatrix<> &mat, std::vector<std::vector<long>> &r2c) {
+static void make_cols_of_row_from_eigen(const SpMatrix<> &mat, std::vector<std::vector<long>> &r2c) {
   for (int k = 0; k < mat.outerSize(); ++k) {  // cols, if col-major, rows otherwise
     for (typename SpMatrix<blk_t>::InnerIterator it(mat, k); it; ++it) {
       const long row = it.row();
@@ -1315,7 +1439,7 @@ static void make_rowidx_to_colidx_from_eigen(const SpMatrix<> &mat, std::vector<
   }
 }
 
-static void make_colidx_to_rowidx_from_eigen(const SpMatrix<> &mat, std::vector<std::vector<long>> &c2r) {
+static void make_rows_of_col_from_eigen(const SpMatrix<> &mat, std::vector<std::vector<long>> &c2r) {
   for (int k = 0; k < mat.outerSize(); ++k) {  // cols, if col-major, rows otherwise
     for (typename SpMatrix<blk_t>::InnerIterator it(mat, k); it; ++it) {
       const long row = it.row();
@@ -1418,10 +1542,15 @@ int main(int argc, char **argv) {
       std::string tiling_type;
       int M = 0, N = 0, K = 0;
       int minTs = 0, maxTs = 0;
+      int parallel_bcasts = 1;
 
       double avg_nb = nan("undefined");
       double Adensity = nan("undefined");
       double Bdensity = nan("undefined");
+      if (cmdOptionExists(argv, argv + argc, "-b")) {
+        std::string pStr = getCmdOption(argv, argv + argc, "-b");
+        parallel_bcasts = std::stol(pStr);
+      }
 
       std::string PStr(getCmdOption(argv, argv + argc, "-P"));
       P = parseOption(PStr, P);
@@ -1446,10 +1575,10 @@ int main(int argc, char **argv) {
         }
       }
 
-      auto ij_keymap = [P, Q](const Key<2> &ij) {
+      auto ij_keymap = [P, Q, R](const Key<2> &ij) {
         int i = (int)ij[0];
         int j = (int)ij[1];
-        int r = ij2rank(i, j, P, Q);
+        int r = ij2rank(i, j, P, Q, R);
         return r;
       };
 
@@ -1473,10 +1602,10 @@ int main(int argc, char **argv) {
       std::vector<int> mTiles;
       std::vector<int> nTiles;
       std::vector<int> kTiles;
-      std::vector<std::vector<long>> a_rowidx_to_colidx;
-      std::vector<std::vector<long>> a_colidx_to_rowidx;
-      std::vector<std::vector<long>> b_rowidx_to_colidx;
-      std::vector<std::vector<long>> b_colidx_to_rowidx;
+      std::vector<std::vector<long>> a_cols_of_row;
+      std::vector<std::vector<long>> a_rows_of_col;
+      std::vector<std::vector<long>> b_cols_of_row;
+      std::vector<std::vector<long>> b_rows_of_col;
 
       std::string checkStr(getCmdOption(argv, argv + argc, "-x"));
       int check = parseOption(checkStr, !(argc >= 2));
@@ -1504,10 +1633,10 @@ int main(int argc, char **argv) {
       }
 
       // We still need to build the metadata from the  matrices.
-      make_rowidx_to_colidx_from_eigen(A, a_rowidx_to_colidx);
-      make_colidx_to_rowidx_from_eigen(A, a_colidx_to_rowidx);
-      make_rowidx_to_colidx_from_eigen(B, b_rowidx_to_colidx);
-      make_colidx_to_rowidx_from_eigen(B, b_colidx_to_rowidx);
+      make_cols_of_row_from_eigen(A, a_cols_of_row);
+      make_rows_of_col_from_eigen(A, a_rows_of_col);
+      make_cols_of_row_from_eigen(B, b_cols_of_row);
+      make_rows_of_col_from_eigen(B, b_rows_of_col);
       // This is only needed to compute the flops
       for (int mt = 0; mt < M; mt++) mTiles.emplace_back(1);
       for (int nt = 0; nt < N; nt++) nTiles.emplace_back(1);
@@ -1529,18 +1658,18 @@ int main(int argc, char **argv) {
         timing = (check == 0);
         tiling_type = "RandomIrregularTiling";
         initBlSpRandom(ij_keymap, M, N, K, minTs, maxTs, avg, A, B, Aref, Bref, check, mTiles, nTiles, kTiles,
-                       a_rowidx_to_colidx, a_colidx_to_rowidx, b_rowidx_to_colidx, b_colidx_to_rowidx, avg_nb, Adensity,
+                       a_cols_of_row, a_rows_of_col, b_cols_of_row, b_rows_of_col, avg_nb, Adensity,
                        Bdensity, seed);
 
         C.resize(mTiles.size(), nTiles.size());
       } else {
         tiling_type = "HardCodedBlockSparseMatrix";
-        initBlSpHardCoded(ij_keymap, A, B, C, Aref, Bref, true, mTiles, nTiles, kTiles, a_rowidx_to_colidx,
-                          a_colidx_to_rowidx, b_rowidx_to_colidx, b_colidx_to_rowidx, M, N, K);
+        initBlSpHardCoded(ij_keymap, A, B, C, Aref, Bref, true, mTiles, nTiles, kTiles, a_cols_of_row,
+                          a_rows_of_col, b_cols_of_row, b_rows_of_col, M, N, K);
       }
 #endif  // !defined(BLOCK_SPARSE_GEMM)
 
-      gflops = compute_gflops(a_rowidx_to_colidx, b_rowidx_to_colidx, mTiles, nTiles, kTiles);
+      gflops = compute_gflops(a_cols_of_row, b_cols_of_row, mTiles, nTiles, kTiles);
 
       std::string nbrunStr(getCmdOption(argv, argv + argc, "-n"));
       int nb_runs = parseOption(nbrunStr, 1);
@@ -1550,24 +1679,24 @@ int main(int argc, char **argv) {
         execute();
         for (int nrun = 0; nrun < nb_runs; nrun++) {
           timed_measurement(A, B, ij_keymap, ijk_keymap, tiling_type, gflops, avg_nb, Adensity, Bdensity,
-                            a_rowidx_to_colidx, a_colidx_to_rowidx, b_rowidx_to_colidx, b_colidx_to_rowidx, mTiles,
-                            nTiles, kTiles, M, N, K, minTs, maxTs, P, Q, R);
+                            a_cols_of_row, a_rows_of_col, b_cols_of_row, b_rows_of_col, mTiles,
+                            nTiles, kTiles, M, N, K, minTs, maxTs, P, Q, R, parallel_bcasts);
         }
       } else {
         // flow graph needs to exist on every node
         // N.B. to validate C we need it on node 0!
         auto keymap_write = [](const Key<2> &key) { return 0; };
-        Edge<Key<2>> ctl("control");
+        Edge<Key<3>> ctl("control");
         Control control(ctl);
         Edge<Key<2>, blk_t> eA, eB, eC;
-        Read_SpMatrix a("A", A, ctl, eA, ij_keymap);
-        Read_SpMatrix b("B", B, ctl, eB, ij_keymap);
+        Read_SpMatrix a("A", A, ctl, eA, ijk_keymap, ij_keymap);
+        Read_SpMatrix b("B", B, ctl, eB, ijk_keymap, ij_keymap);
         Write_SpMatrix<> c(C, eC, keymap_write);
         auto &c_status = c.status();
         assert(!has_value(c_status));
         //  SpMM25D a_times_b(world, eA, eB, eC, A, B);
-        SpMM25D<> a_times_b(eA, eB, eC, A, B, a_rowidx_to_colidx, a_colidx_to_rowidx, b_rowidx_to_colidx,
-                            b_colidx_to_rowidx, mTiles, nTiles, kTiles, ij_keymap, ijk_keymap, R);
+        SpMM25D<> a_times_b(eA, eB, eC, A, B, a_cols_of_row, a_rows_of_col, b_cols_of_row,
+                            b_rows_of_col, mTiles, nTiles, kTiles, ij_keymap, ijk_keymap, R);
         TTGUNUSED(a_times_b);
         // calling the Dot constructor with 'true' argument disables the type
         if (default_execution_context().rank() == 0) std::cout << Dot{/*disable_type=*/true}(&control) << std::endl;
@@ -1578,7 +1707,7 @@ int main(int argc, char **argv) {
         TTGUNUSED(connected);
 
         // ready, go! need only 1 kick, so must be done by 1 thread only
-        if (ttg::default_execution_context().rank() == 0) control.start(P, Q);
+        if (ttg::default_execution_context().rank() == 0) control.start(P, Q, R);
 
         execute();
         fence();

From a85c2c4634cd7277098f9e28f9b70404d82e0450 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Tue, 30 Jan 2024 16:53:23 -0500
Subject: [PATCH 225/259] btas::Tensor: do not fill with zeroes when
 constructing (e.g., in zero-copy serialization), unless necessary

---
 examples/spmm/spmm.cc      | 13 ++++++++-----
 examples/spmm/spmm_cuda.cc | 13 ++++++++-----
 2 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/examples/spmm/spmm.cc b/examples/spmm/spmm.cc
index ac198a969..94edf672b 100644
--- a/examples/spmm/spmm.cc
+++ b/examples/spmm/spmm.cc
@@ -70,8 +70,8 @@ namespace ttg {
         return boost::container::small_vector<iovec, 1>{};
     }
     static auto create_from_metadata(const std::pair<int, int> &meta) {
-      if (meta != std::pair{0, 0})
-        return blk_t(btas::Range(std::get<0>(meta), std::get<1>(meta)), 0.0);
+      if (meta != std::pair{0, 0}) // N.B. allocate only, do not fill with zeroes
+        return blk_t(btas::Range(std::get<0>(meta), std::get<1>(meta)));
       else
         return blk_t{};
     }
@@ -121,10 +121,13 @@ namespace btas {
   btas::Tensor<T_, Range_, Store_> gemm(btas::Tensor<T_, Range_, Store_> &&C, const btas::Tensor<T_, Range_, Store_> &A,
                                         const btas::Tensor<T_, Range_, Store_> &B) {
     using array = btas::DEFAULT::index<int>;
-    if (C.empty()) {
-      C = btas::Tensor<T_, Range_, Store_>(btas::Range(A.range().extent(0), B.range().extent(1)), 0.0);
+    if (C.empty()) {  // first contribution to C = allocate it and gemm with beta=0
+      C = btas::Tensor<T_, Range_, Store_>(btas::Range(A.range().extent(0), B.range().extent(1)));
+      btas::contract_222(1.0, A, array{1, 2}, B, array{2, 3}, 0.0, C, array{1, 3}, false, false);
+    }
+    else {   // subsequent contributions to C = gemm with beta=1
+      btas::contract_222(1.0, A, array{1, 2}, B, array{2, 3}, 1.0, C, array{1, 3}, false, false);
     }
-    btas::contract_222(1.0, A, array{1, 2}, B, array{2, 3}, 1.0, C, array{1, 3}, false, false);
     return std::move(C);
   }
 }  // namespace btas
diff --git a/examples/spmm/spmm_cuda.cc b/examples/spmm/spmm_cuda.cc
index 6235554b2..90cca96f8 100644
--- a/examples/spmm/spmm_cuda.cc
+++ b/examples/spmm/spmm_cuda.cc
@@ -372,8 +372,8 @@ namespace ttg {
         return boost::container::small_vector<iovec, 1>{};
     }
     static auto create_from_metadata(const std::pair<int, int> &meta) {
-      if (meta != std::pair{0, 0})
-        return blk_t(btas::Range(std::get<0>(meta), std::get<1>(meta)), 0.0);
+      if (meta != std::pair{0, 0}) // N.B. allocate only, do not fill with zeroes
+        return blk_t(btas::Range(std::get<0>(meta), std::get<1>(meta)));
       else
         return blk_t{};
     }
@@ -422,10 +422,13 @@ namespace btas {
   btas::Tensor<T_, Range_, Store_> gemm(btas::Tensor<T_, Range_, Store_> &&C, const btas::Tensor<T_, Range_, Store_> &A,
                                         const btas::Tensor<T_, Range_, Store_> &B) {
     using array = btas::DEFAULT::index<int>;
-    if (C.empty()) {
-      C = btas::Tensor<T_, Range_, Store_>(btas::Range(A.range().extent(0), B.range().extent(1)), 0.0);
+    if (C.empty()) {  // first contribution to C = allocate it and gemm with beta=0
+      C = btas::Tensor<T_, Range_, Store_>(btas::Range(A.range().extent(0), B.range().extent(1)));
+      btas::contract_222(1.0, A, array{1, 2}, B, array{2, 3}, 0.0, C, array{1, 3}, false, false);
+    }
+    else {   // subsequent contributions to C = gemm with beta=1
+      btas::contract_222(1.0, A, array{1, 2}, B, array{2, 3}, 1.0, C, array{1, 3}, false, false);
     }
-    btas::contract_222(1.0, A, array{1, 2}, B, array{2, 3}, 1.0, C, array{1, 3}, false, false);
     return std::move(C);
   }
 }  // namespace btas

From 1f2a4fbd3af8342e64627cf02024c003cb531f14 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Wed, 31 Jan 2024 16:03:28 -0500
Subject: [PATCH 226/259] [cmake] bump MADNESS, BTAS, and TA tags to sync with
 latest; MADNESS points to main repo now

---
 CMakeLists.txt                                |  2 +-
 .../ExternalDependenciesVersions.cmake        |  7 +-
 cmake/modules/FindOrFetchBoost.cmake          | 97 ++++++++++---------
 cmake/modules/FindOrFetchMADNESS.cmake        |  2 +-
 examples/CMakeLists.txt                       |  8 +-
 ttg/CMakeLists.txt                            | 17 +++-
 6 files changed, 74 insertions(+), 59 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 684d3a58c..1607cfcb9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -91,7 +91,7 @@ endif (BUILD_TESTING)
 #### optional prerequisites
 ###########################
 # Boost
-include(FindOrFetchBoost)
+include("${PROJECT_SOURCE_DIR}/cmake/modules/FindOrFetchBoost.cmake")
 # Cereal
 #include(FindOrFetchCereal)
 # C++ coroutines
diff --git a/cmake/modules/ExternalDependenciesVersions.cmake b/cmake/modules/ExternalDependenciesVersions.cmake
index 8d763f5db..b3d7c8e55 100644
--- a/cmake/modules/ExternalDependenciesVersions.cmake
+++ b/cmake/modules/ExternalDependenciesVersions.cmake
@@ -3,9 +3,10 @@
 
 # need Boost.CallableTraits (header only, part of Boost 1.66 released in Dec 2017) for wrap.h to work
 set(TTG_TRACKED_BOOST_VERSION 1.66)
+set(TTG_TRACKED_VG_CMAKE_KIT_TAG d5c0a6f9ff6dc97cbb5132912733e1eb1cf73f1e)  # used to provide "real" FindOrFetchBoost
 set(TTG_TRACKED_CATCH2_VERSION 3.5.0)
 set(TTG_TRACKED_CEREAL_VERSION 1.3.0)
-set(TTG_TRACKED_MADNESS_TAG cb195817d7807c4aead10ba200cd20649036cbae)
+set(TTG_TRACKED_MADNESS_TAG 8788aea9758bfe6479cc23d39e6c77b7528009db)
 set(TTG_TRACKED_PARSEC_TAG 25d1931e863b6741e453112d2117d85ad32e7fba)
-set(TTG_TRACKED_BTAS_TAG a02be0d29fb4a788ecef43de711dcd6d6f1cb6b8)
-set(TTG_TRACKED_TILEDARRAY_TAG f0115e9e4a3f988224afbfb3c241e92171e916b8)
+set(TTG_TRACKED_BTAS_TAG 4e8f5233aa7881dccdfcc37ce07128833926d3c2)
+set(TTG_TRACKED_TILEDARRAY_TAG 493c109379a1b64ddd5ef59f7e33b95633b68d73)
diff --git a/cmake/modules/FindOrFetchBoost.cmake b/cmake/modules/FindOrFetchBoost.cmake
index e1733e925..ed1fc9b2a 100644
--- a/cmake/modules/FindOrFetchBoost.cmake
+++ b/cmake/modules/FindOrFetchBoost.cmake
@@ -1,50 +1,55 @@
-if (NOT TARGET Boost::boost)
-  find_package(Boost ${TTG_TRACKED_BOOST_VERSION} QUIET CONFIG OPTIONAL_COMPONENTS serialization)
-endif(NOT TARGET Boost::boost)
+# Boost can be discovered by every (sub)package but only the top package can *build* it ...
+# in either case must declare the components used by TA
+set(required_components
+        headers
+        callable_traits
+)
+if (TTG_PARSEC_USE_BOOST_SERIALIZATION)
+    list(APPEND required_components
+            serialization
+            iostreams
+    )
+endif()
+if (DEFINED Boost_REQUIRED_COMPONENTS)
+    list(APPEND Boost_REQUIRED_COMPONENTS
+            ${required_components})
+    list(REMOVE_DUPLICATES Boost_REQUIRED_COMPONENTS)
+else()
+    set(Boost_REQUIRED_COMPONENTS "${required_components}" CACHE STRING "Components of Boost to discovered or built")
+endif()
+set(optional_components
+)
+if (DEFINED Boost_OPTIONAL_COMPONENTS)
+    list(APPEND Boost_OPTIONAL_COMPONENTS
+            ${optional_components}
+    )
+    list(REMOVE_DUPLICATES Boost_OPTIONAL_COMPONENTS)
+else()
+    set(Boost_OPTIONAL_COMPONENTS "${optional_components}" CACHE STRING "Optional components of Boost to discovered or built")
+endif()
 
-if (TARGET Boost::boost)
-  set(_msg "Found Boost at ${Boost_CONFIG}")
-  if (TARGET Boost::serialization)
-    list(APPEND _msg " includes Boost::serialization")
-  endif(TARGET Boost::serialization)
-  message(STATUS "${_msg}")
+if (NOT DEFINED Boost_FETCH_IF_MISSING AND TTG_FETCH_BOOST)
+    set(Boost_FETCH_IF_MISSING 1)
+endif()
 
-  # Boost::* targets by default are not GLOBAL, so to allow users of TTG to safely use them we need to make them global
-  # more discussion here: https://gitlab.kitware.com/cmake/cmake/-/issues/17256
-  foreach(tgt boost;headers;${Boost_BTAS_DEPS_LIBRARIES})
-    if (TARGET Boost::${tgt})
-      get_target_property(_boost_tgt_${tgt}_is_imported_global Boost::${tgt} IMPORTED_GLOBAL)
-      if (NOT _boost_tgt_${tgt}_is_imported_global)
-        set_target_properties(Boost::${tgt} PROPERTIES IMPORTED_GLOBAL TRUE)
-      endif()
-      unset(_boost_tgt_${tgt}_is_imported_global)
+# Bring ValeevGroup cmake toolkit, if not yet available
+if (NOT DEFINED vg_cmake_kit_SOURCE_DIR)
+    include(FetchContent)
+    if (DEFINED PROJECT_BINARY_DIR)
+        set(VG_CMAKE_KIT_PREFIX_DIR PROJECT_BINARY_DIR)
+    else ()
+        set(VG_CMAKE_KIT_PREFIX_DIR CMAKE_CURRENT_BINARY_DIR)
     endif()
-  endforeach()
-
-elseif (TTG_FETCH_BOOST)
-
-  FetchContent_Declare(
-          CMAKEBOOST
-          GIT_REPOSITORY      https://github.com/Orphis/boost-cmake
-  )
-  FetchContent_MakeAvailable(CMAKEBOOST)
-  FetchContent_GetProperties(CMAKEBOOST
-          SOURCE_DIR CMAKEBOOST_SOURCE_DIR
-          BINARY_DIR CMAKEBOOST_BINARY_DIR
-          )
-
-  # current boost-cmake/master does not install boost correctly, so warn that installed TTG will not be usable
-  # boost-cmake/install_rules https://github.com/Orphis/boost-cmake/pull/45 is supposed to fix it but is inactive
-  message(WARNING "Building Boost from source makes TTG unusable from the install location! Install Boost using package manager or manually and reconfigure/reinstall TTG to fix this")
-
-  if (TARGET Boost::serialization AND TARGET Boost_serialization)
-    install(TARGETS Boost_serialization EXPORT boost)
-    export(EXPORT boost
-           FILE "${PROJECT_BINARY_DIR}/boost-targets.cmake")
-    install(EXPORT boost
-            FILE "boost-targets.cmake"
-            DESTINATION "${CMAKE_INSTALL_CMAKEDIR}"
-            COMPONENT boost-libs)
-  endif()
-
+    FetchContent_Declare(
+            vg_cmake_kit
+            QUIET
+            GIT_REPOSITORY      https://github.com/ValeevGroup/kit-cmake.git
+            GIT_TAG             ${TTG_TRACKED_VG_CMAKE_KIT_TAG}
+            SOURCE_DIR ${${VG_CMAKE_KIT_PREFIX_DIR}}/cmake/vg
+            BINARY_DIR ${${VG_CMAKE_KIT_PREFIX_DIR}}/cmake/vg-build
+            SUBBUILD_DIR ${${VG_CMAKE_KIT_PREFIX_DIR}}/cmake/vg-subbuild
+    )
+    FetchContent_MakeAvailable(vg_cmake_kit)
 endif()
+include(${vg_cmake_kit_SOURCE_DIR}/modules/FindOrFetchBoost.cmake)
+
diff --git a/cmake/modules/FindOrFetchMADNESS.cmake b/cmake/modules/FindOrFetchMADNESS.cmake
index f112e4ff7..24f0bc798 100644
--- a/cmake/modules/FindOrFetchMADNESS.cmake
+++ b/cmake/modules/FindOrFetchMADNESS.cmake
@@ -10,7 +10,7 @@ if (NOT TARGET MADworld)
   set(MADNESS_TASK_BACKEND PaRSEC CACHE STRING "The task backend to use for MADNESS tasks")
   FetchContent_Declare(
           MADNESS
-          GIT_REPOSITORY https://github.com/therault/madness.git
+          GIT_REPOSITORY https://github.com/m-a-d-n-e-s-s/madness.git
           GIT_TAG ${TTG_TRACKED_MADNESS_TAG}
   )
   FetchContent_MakeAvailable(MADNESS)
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 87aa0b127..10b808540 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -10,7 +10,7 @@ if (TARGET tiledarray)
     add_ttg_executable(spmm spmm/spmm.cc LINK_LIBRARIES TiledArray_Eigen)
     # block-sparse needs BTAS ... it's always provided by TA
     # since only need to use matrices, limit BTAS_TARGET_MAX_INDEX_RANK to 2
-    add_ttg_executable(bspmm spmm/spmm.cc LINK_LIBRARIES tiledarray TiledArray_Eigen BTAS Boost::boost COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2)
+    add_ttg_executable(bspmm spmm/spmm.cc LINK_LIBRARIES tiledarray TiledArray_Eigen BTAS COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2)
 
     add_ttg_executable(testing_dpotrf potrf/testing_dpotrf.cc LINK_LIBRARIES tiledarray lapackpp)
     add_ttg_executable(testing_dtrtri potrf/testing_dtrtri.cc LINK_LIBRARIES tiledarray lapackpp)
@@ -19,7 +19,7 @@ if (TARGET tiledarray)
 
     if (TARGET CUDA::cublas)
       add_ttg_executable(bspmm-cuda spmm/spmm_cuda.cc
-                         LINK_LIBRARIES tiledarray TiledArray_Eigen BTAS Boost::boost CUDA::cublas
+                         LINK_LIBRARIES tiledarray TiledArray_Eigen BTAS CUDA::cublas
                          COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2
                          RUNTIMES "parsec")
 
@@ -31,7 +31,7 @@ if (TARGET tiledarray)
       endif(TARGET CUDA::cusolver)
     elseif (TARGET roc::hipblas)
       add_ttg_executable(bspmm-hip spmm/spmm_cuda.cc
-                          LINK_LIBRARIES tiledarray TiledArray_Eigen BTAS Boost::boost roc::hipblas
+                          LINK_LIBRARIES tiledarray TiledArray_Eigen roc::hipblas
                           COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2
                           RUNTIMES "parsec")
       if (TARGET roc::hipsolver)
@@ -42,7 +42,7 @@ if (TARGET tiledarray)
       endif(TARGET roc::hipsolver)
     elseif (TARGET MKL::MKL_DPCPP)
       add_ttg_executable(bspmm-lz spmm/spmm_cuda.cc
-                          LINK_LIBRARIES tiledarray TiledArray_Eigen BTAS Boost::boost MKL::MKL_DPCPP level_zero::ze_loader m
+                          LINK_LIBRARIES tiledarray TiledArray_Eigen BTAS MKL::MKL_DPCPP level_zero::ze_loader m
                           COMPILE_DEFINITIONS BLOCK_SPARSE_GEMM=1;BTAS_TARGET_MAX_INDEX_RANK=2
                           RUNTIMES "parsec")
     endif()
diff --git a/ttg/CMakeLists.txt b/ttg/CMakeLists.txt
index 9a08adc6a..5d40c5b20 100644
--- a/ttg/CMakeLists.txt
+++ b/ttg/CMakeLists.txt
@@ -94,9 +94,13 @@ set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/version.cc
         COMPILE_DEFINITIONS "TTG_GIT_REVISION=\"${TTG_GIT_REVISION}\";TTG_GIT_DESCRIPTION=\"${TTG_GIT_DESCRIPTION}\"")
 
 # optional dependencies
-if (TARGET Boost::boost)
-  list(APPEND ttg-deps Boost::boost)
-else () # if Boost::boost is missing must use bundled Boost.CallableTraits
+if (TARGET Boost::headers)
+  if (TARGET Boost::callable_traits)  # using modularized Boost?
+      list(APPEND ttg-deps Boost::callable_traits)
+  else()
+      list(APPEND ttg-deps Boost::headers)
+  endif()
+else () # if Boost::headers is missing must use bundled Boost.CallableTraits
   list(APPEND ttg-defs "$<BUILD_INTERFACE:TTG_USE_BUNDLED_BOOST_CALLABLE_TRAITS=1>")
   list(APPEND ttg-incs "$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/ttg/external>")
 endif ()
@@ -167,6 +171,11 @@ if (TARGET MADworld)
 endif(TARGET MADworld)
 if (TARGET Boost::serialization)
   list(APPEND ttg-serialization-deps Boost::serialization)
+  list(APPEND ttg-serialization-boost-deps Boost::serialization)
+  if (TARGET Boost::iostreams)  # using modularized Boost?
+      list(APPEND ttg-serialization-deps Boost::iostreams)
+      list(APPEND ttg-serialization-boost-deps Boost::iostreams)
+  endif()
   list(APPEND ttg-serialization-compile-definitions TTG_SERIALIZATION_SUPPORTS_BOOST=1)
 endif (TARGET Boost::serialization)
 if (TARGET cereal::cereal)
@@ -192,7 +201,7 @@ if (TARGET Boost::serialization)
   add_ttg_library(ttg-serialization-boost
           "${ttg-serialization-sources}"
           PUBLIC_HEADER "${ttg-serialization-headers}"
-          LINK_LIBRARIES "Boost::serialization"
+          LINK_LIBRARIES "${ttg-serialization-boost-deps}"
           COMPILE_DEFINITIONS "TTG_SERIALIZATION_SUPPORTS_BOOST=1")
 endif(TARGET Boost::serialization)
 # make cereal-only serialization target

From 1125f96568bd987f75bb37ab546c721a1e930f41 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Wed, 31 Jan 2024 17:14:03 -0500
Subject: [PATCH 227/259] fixup

---
 ttg/ttg/parsec/ptr.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ttg/ttg/parsec/ptr.h b/ttg/ttg/parsec/ptr.h
index a20c0c746..8184d2657 100644
--- a/ttg/ttg/parsec/ptr.h
+++ b/ttg/ttg/parsec/ptr.h
@@ -38,7 +38,7 @@ namespace ttg_parsec {
 
       void register_self() {
         /* insert ourselves from the list of ptr */
-        std::lock_guard {m_ptr_map_mtx};
+        std::lock_guard _{m_ptr_map_mtx};
         m_ptr_map.insert(std::pair{this, true});
       }
 

From 549ba94193e296f6248b1bf95ef24bf469fc0706 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Wed, 31 Jan 2024 17:14:42 -0500
Subject: [PATCH 228/259] introduce initialize_fpe to turn on/off
 floating-point exceptions

---
 ttg/ttg/util/bug.cpp | 98 ++++++++++++++++++++++++++++++++++++++++++++
 ttg/ttg/util/bug.h   |  9 ++++
 2 files changed, 107 insertions(+)

diff --git a/ttg/ttg/util/bug.cpp b/ttg/ttg/util/bug.cpp
index 1e91e8fd2..27b743096 100644
--- a/ttg/ttg/util/bug.cpp
+++ b/ttg/ttg/util/bug.cpp
@@ -28,6 +28,7 @@
 #include "bug.h"
 
 #include <unistd.h>
+#include <cfenv>
 #include <csignal>
 #include <cstdio>
 #include <cstdlib>
@@ -46,6 +47,103 @@
 using namespace std;
 using namespace ttg;
 
+namespace ttg {
+  void initialize_fpe() {
+#if defined(__APPLE__) && defined(__MACH__)
+
+    // Public domain polyfill for feenableexcept on OS X
+    // http://www-personal.umich.edu/~williams/archive/computation/fe-handling-example.c
+
+#ifndef HAVE_FEENABLEEXCEPT
+    auto feenableexcept = [](int excepts) -> int {
+      static fenv_t fenv;
+      const auto new_excepts = excepts & FE_ALL_EXCEPT;
+
+      if (fegetenv(&fenv)) {
+        return -1;
+      }
+#if defined(__x86_64__)
+      // previous masks
+      const unsigned int old_excepts = fenv.__control & FE_ALL_EXCEPT;
+
+      // unmask
+      fenv.__control &= ~new_excepts;
+      fenv.__mxcsr &= ~(new_excepts << 7);
+#elif defined(__arm64__)
+      if (new_excepts & FE_INVALID) fenv.__fpcr |= __fpcr_trap_invalid;
+      if (new_excepts & FE_DIVBYZERO) fenv.__fpcr |= __fpcr_trap_divbyzero;
+      if (new_excepts & FE_OVERFLOW) fenv.__fpcr |= __fpcr_trap_overflow;
+      if (new_excepts & FE_UNDERFLOW) fenv.__fpcr |= __fpcr_trap_underflow;
+      if (new_excepts & FE_INEXACT) fenv.__fpcr |= __fpcr_trap_inexact;
+#else
+#error "MacOS on unknown architecture"
+#endif
+      return fesetenv(&fenv);
+    };
+#define HAVE_FEENABLEEXCEPT 1
+#endif  // not defined HAVE_FEENABLEEXCEPT
+
+#ifndef HAVE_FEDISABLEEXCEPT
+    auto fedisableexcept = [](int excepts) -> int {
+      static fenv_t fenv;
+      const auto new_excepts = excepts & FE_ALL_EXCEPT;
+      // all previous masks
+
+      if (fegetenv(&fenv)) {
+        return -1;
+      }
+#if defined(__x86_64__)
+      const unsigned int old_excepts = fenv.__control & FE_ALL_EXCEPT;
+
+      // mask
+      fenv.__control |= new_excepts;
+      fenv.__mxcsr |= new_excepts << 7;
+#elif defined(__arm64__)
+      if (new_excepts & FE_INVALID) fenv.__fpcr &= ~__fpcr_trap_invalid;
+      if (new_excepts & FE_DIVBYZERO) fenv.__fpcr &= ~__fpcr_trap_divbyzero;
+      if (new_excepts & FE_OVERFLOW) fenv.__fpcr &= ~__fpcr_trap_overflow;
+      if (new_excepts & FE_UNDERFLOW) fenv.__fpcr &= ~__fpcr_trap_underflow;
+      if (new_excepts & FE_INEXACT) fenv.__fpcr &= ~__fpcr_trap_inexact;
+#else
+#error "MacOS on unknown architecture"
+#endif
+
+      return fesetenv(&fenv);
+    };
+
+#define HAVE_FEDISABLEEXCEPT 1
+#endif  // not defined HAVE_FEDISABLEEXCEPT
+#endif  // mac
+
+#ifdef HAVE_FEENABLEEXCEPT
+    // this uses a glibc extension to trap on individual exceptions
+    int enable_excepts = 0;
+#ifdef FE_DIVBYZERO
+    enable_excepts |= FE_DIVBYZERO;
+#endif
+#ifdef FE_INVALID
+    enable_excepts |= FE_INVALID;
+#endif
+#ifdef FE_OVERFLOW
+    enable_excepts |= FE_OVERFLOW;
+#endif
+    feenableexcept(enable_excepts);
+#endif
+
+#ifdef HAVE_FEDISABLEEXCEPT
+    // this uses a glibc extension to not trap on individual exceptions
+    int disable_excepts = 0;
+#ifdef FE_UNDERFLOW
+    disable_excepts |= FE_UNDERFLOW;
+#endif
+#ifdef FE_INEXACT
+    disable_excepts |= FE_INEXACT;
+#endif
+    fedisableexcept(disable_excepts);
+#endif
+  }
+}
+
 //////////////////////////////////////////////////////////////////////
 // static variables
 
diff --git a/ttg/ttg/util/bug.h b/ttg/ttg/util/bug.h
index 27212a809..df2791908 100644
--- a/ttg/ttg/util/bug.h
+++ b/ttg/ttg/util/bug.h
@@ -250,6 +250,15 @@ namespace ttg {
 
   }  // namespace detail
 
+  /// @brief Initializes the floating point exceptions.
+  ///
+  /// Enables (if available) FE_DIVBYZERO, FE_INVALID, and FE_OVERFLOW;
+  /// FE_UNDERFLOW and FE_INEXACT are disabled (if available).
+  /// @warning This should be called from the main thread *before* any threads
+  /// have been created (i.e. before madness::initialize()),
+  ///          so that all threads inherit the same floating point environment.
+  void initialize_fpe();
+
   /**
    * The Debugger class describes what should be done when a catastrophic
    * error causes unexpected program termination.  It can try things such as

From f05530fae941072f1a69517ceda3baf69b2d12bf Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Thu, 1 Feb 2024 00:24:59 -0500
Subject: [PATCH 229/259] can use bundled Boost.CallableTraits only if don't
 have real Boost and TTG_FETCH_BOOST is not set

---
 CMakeLists.txt                                |  2 +-
 .../ExternalDependenciesVersions.cmake        |  2 +-
 cmake/modules/FindOrFetchBoost.cmake          | 29 ++++++++++++++-----
 cmake/ttg-config.cmake.in                     | 22 ++++----------
 ttg/CMakeLists.txt                            |  6 ++--
 5 files changed, 32 insertions(+), 29 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1607cfcb9..1502787ec 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -58,7 +58,7 @@ option(TTG_EXAMPLES "Whether to build examples" OFF)
 option(TTG_ENABLE_ASAN "Whether to enable address sanitizer" OFF)
 
 option(TTG_FETCH_BOOST "Whether to fetch+build Boost, if missing" OFF)
-option(TTG_IGNORE_BUNDLED_EXTERNALS "Whether to skip installation and use of bundled external depenedencies (Boost.CallableTraits)" OFF)
+option(TTG_IGNORE_BUNDLED_EXTERNALS "Whether to skip installation and use of bundled external dependencies (Boost.CallableTraits)" OFF)
 option(TTG_ENABLE_TRACE "Whether to enable ttg::trace() output" OFF)
 # See https://medium.com/@alasher/colored-c-compiler-output-with-ninja-clang-gcc-10bfe7f2b949
 option (FORCE_COLORED_OUTPUT "Always produce ANSI-colored output (GNU/Clang only)." TRUE)
diff --git a/cmake/modules/ExternalDependenciesVersions.cmake b/cmake/modules/ExternalDependenciesVersions.cmake
index b3d7c8e55..d4bc3bea5 100644
--- a/cmake/modules/ExternalDependenciesVersions.cmake
+++ b/cmake/modules/ExternalDependenciesVersions.cmake
@@ -3,7 +3,7 @@
 
 # need Boost.CallableTraits (header only, part of Boost 1.66 released in Dec 2017) for wrap.h to work
 set(TTG_TRACKED_BOOST_VERSION 1.66)
-set(TTG_TRACKED_VG_CMAKE_KIT_TAG d5c0a6f9ff6dc97cbb5132912733e1eb1cf73f1e)  # used to provide "real" FindOrFetchBoost
+set(TTG_TRACKED_VG_CMAKE_KIT_TAG 7ea2d4d3f8854b9e417f297fd74d6fc49aa13fd5)  # used to provide "real" FindOrFetchBoost
 set(TTG_TRACKED_CATCH2_VERSION 3.5.0)
 set(TTG_TRACKED_CEREAL_VERSION 1.3.0)
 set(TTG_TRACKED_MADNESS_TAG 8788aea9758bfe6479cc23d39e6c77b7528009db)
diff --git a/cmake/modules/FindOrFetchBoost.cmake b/cmake/modules/FindOrFetchBoost.cmake
index ed1fc9b2a..dc1a0a654 100644
--- a/cmake/modules/FindOrFetchBoost.cmake
+++ b/cmake/modules/FindOrFetchBoost.cmake
@@ -1,15 +1,31 @@
 # Boost can be discovered by every (sub)package but only the top package can *build* it ...
-# in either case must declare the components used by TA
+# in either case must declare the components used by TTG
 set(required_components
         headers
         callable_traits
 )
+set(optional_components
+)
 if (TTG_PARSEC_USE_BOOST_SERIALIZATION)
-    list(APPEND required_components
+    list(APPEND optional_components
             serialization
             iostreams
     )
 endif()
+
+# if not allowed to fetch Boost make all Boost optional
+if (NOT DEFINED Boost_FETCH_IF_MISSING AND TTG_FETCH_BOOST)
+    set(Boost_FETCH_IF_MISSING 1)
+endif()
+if (NOT Boost_FETCH_IF_MISSING)
+    foreach(__component IN LISTS required_components)
+    list(APPEND optional_components
+            ${__component}
+    )
+    endforeach()
+    set(required_components )
+endif()
+
 if (DEFINED Boost_REQUIRED_COMPONENTS)
     list(APPEND Boost_REQUIRED_COMPONENTS
             ${required_components})
@@ -17,8 +33,6 @@ if (DEFINED Boost_REQUIRED_COMPONENTS)
 else()
     set(Boost_REQUIRED_COMPONENTS "${required_components}" CACHE STRING "Components of Boost to discovered or built")
 endif()
-set(optional_components
-)
 if (DEFINED Boost_OPTIONAL_COMPONENTS)
     list(APPEND Boost_OPTIONAL_COMPONENTS
             ${optional_components}
@@ -28,10 +42,6 @@ else()
     set(Boost_OPTIONAL_COMPONENTS "${optional_components}" CACHE STRING "Optional components of Boost to discovered or built")
 endif()
 
-if (NOT DEFINED Boost_FETCH_IF_MISSING AND TTG_FETCH_BOOST)
-    set(Boost_FETCH_IF_MISSING 1)
-endif()
-
 # Bring ValeevGroup cmake toolkit, if not yet available
 if (NOT DEFINED vg_cmake_kit_SOURCE_DIR)
     include(FetchContent)
@@ -53,3 +63,6 @@ if (NOT DEFINED vg_cmake_kit_SOURCE_DIR)
 endif()
 include(${vg_cmake_kit_SOURCE_DIR}/modules/FindOrFetchBoost.cmake)
 
+if (TARGET Boost::headers)
+    set(TTG_HAS_BOOST 1)
+endif()
diff --git a/cmake/ttg-config.cmake.in b/cmake/ttg-config.cmake.in
index 41663c806..82f7c2ae6 100644
--- a/cmake/ttg-config.cmake.in
+++ b/cmake/ttg-config.cmake.in
@@ -6,11 +6,10 @@ set(TTG_EXT_VERSION "@TTG_EXT_VERSION@")
 
 set(PaRSEC_CONFIG "@PaRSEC_CONFIG@")
 set(MADNESS_CONFIG "@MADNESS_CONFIG@")
-set(Boost_CONFIG "@Boost_CONFIG@")
 set(CXX_COROUTINE_COMPONENT "@CXX_COROUTINE_COMPONENT@")
 
 set(TTG_TRACKED_BOOST_VERSION "@TTG_TRACKED_BOOST_VERSION@")
-
+set(TTG_HAS_BOOST @TTG_HAS_BOOST@)
 set(TTG_IGNORE_BUNDLED_EXTERNALS @TTG_IGNORE_BUNDLED_EXTERNALS@)
 
 # make TTG CMake modules discoverable + load AddTTGExecutable by default
@@ -19,6 +18,10 @@ include(AddTTGExecutable)
 
 @PACKAGE_INIT@
 
+if (TTG_HAS_BOOST)
+@Boost_CONFIG_FILE_CONTENTS@
+endif(TTG_HAS_BOOST)
+
 if (NOT TARGET MADworld AND MADNESS_CONFIG)
   get_filename_component(MADNESS_CONFIG_DIR "${MADNESS_CONFIG}" DIRECTORY)
   find_package(MADNESS 0.10.1 CONFIG QUIET REQUIRED COMPONENTS world PATHS "${MADNESS_CONFIG_DIR}" NO_DEFAULT_PATH)
@@ -29,12 +32,6 @@ if (NOT TARGET PaRSEC::parsec AND PaRSEC_CONFIG)
   find_package(PaRSEC CONFIG QUIET REQUIRED COMPONENTS parsec PATHS "${PaRSEC_CONFIG_DIR}" NO_DEFAULT_PATH)
 endif()
 
-# if Boost was discovered and used at TTG configure time discover Boost at the same path
-if (NOT TARGET Boost::boost AND Boost_CONFIG)
-  get_filename_component(Boost_CONFIG_DIR "${Boost_CONFIG}" DIRECTORY)
-  find_package(Boost ${TTG_TRACKED_BOOST_VERSION} CONFIG QUIET REQUIRED OPTIONAL_COMPONENTS serialization PATHS "${Boost_CONFIG_DIR}" NO_DEFAULT_PATH)
-endif()
-
 # if C++ coroutines were used discover same version of them
 if (NOT TARGET std::coroutine AND CXX_COROUTINE_COMPONENT)
   find_package(CXXStdCoroutine MODULE QUIET REQUIRED COMPONENTS "${CXX_COROUTINE_COMPONENT}")
@@ -48,13 +45,4 @@ if(NOT TARGET ttg)
   endif()
 endif()
 
-# if don't have Boost, use bundled Boost.CallableTraits
-if (NOT TARGET Boost::boost)
-  if (TTG_IGNORE_BUNDLED_EXTERNALS)
-    find_package(Boost ${TTG_TRACKED_BOOST_VERSION} QUIET REQUIRED)
-  else()
-    target_compile_definitions(ttg INTERFACE TTG_USE_BUNDLED_BOOST_CALLABLE_TRAITS=1)
-  endif()
-endif()
-
 set(TTG_FOUND TRUE)
diff --git a/ttg/CMakeLists.txt b/ttg/CMakeLists.txt
index 5d40c5b20..307135ca0 100644
--- a/ttg/CMakeLists.txt
+++ b/ttg/CMakeLists.txt
@@ -101,8 +101,10 @@ if (TARGET Boost::headers)
       list(APPEND ttg-deps Boost::headers)
   endif()
 else () # if Boost::headers is missing must use bundled Boost.CallableTraits
-  list(APPEND ttg-defs "$<BUILD_INTERFACE:TTG_USE_BUNDLED_BOOST_CALLABLE_TRAITS=1>")
-  list(APPEND ttg-incs "$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/ttg/external>")
+  list(APPEND ttg-defs "TTG_USE_BUNDLED_BOOST_CALLABLE_TRAITS=1")
+  list(APPEND ttg-incs
+          "$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/ttg/external>"
+          "$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}/ttg/external>")
 endif ()
 if (TARGET TTG_Libunwind)
   list(APPEND ttg-deps TTG_Libunwind)

From 9765f756657a38870334343603ca66ce4f46f5c7 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Thu, 1 Feb 2024 03:21:55 -0500
Subject: [PATCH 230/259] bump MADNESS tag to fix up CI issues

---
 cmake/modules/ExternalDependenciesVersions.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/modules/ExternalDependenciesVersions.cmake b/cmake/modules/ExternalDependenciesVersions.cmake
index d4bc3bea5..1e55b870e 100644
--- a/cmake/modules/ExternalDependenciesVersions.cmake
+++ b/cmake/modules/ExternalDependenciesVersions.cmake
@@ -6,7 +6,7 @@ set(TTG_TRACKED_BOOST_VERSION 1.66)
 set(TTG_TRACKED_VG_CMAKE_KIT_TAG 7ea2d4d3f8854b9e417f297fd74d6fc49aa13fd5)  # used to provide "real" FindOrFetchBoost
 set(TTG_TRACKED_CATCH2_VERSION 3.5.0)
 set(TTG_TRACKED_CEREAL_VERSION 1.3.0)
-set(TTG_TRACKED_MADNESS_TAG 8788aea9758bfe6479cc23d39e6c77b7528009db)
+set(TTG_TRACKED_MADNESS_TAG 2eb3bcf0138127ee2dbc651f1aabd3e9b0def4e3)
 set(TTG_TRACKED_PARSEC_TAG 25d1931e863b6741e453112d2117d85ad32e7fba)
 set(TTG_TRACKED_BTAS_TAG 4e8f5233aa7881dccdfcc37ce07128833926d3c2)
 set(TTG_TRACKED_TILEDARRAY_TAG 493c109379a1b64ddd5ef59f7e33b95633b68d73)

From b70444ca12ec180e107c44e44011a5743e7c5454 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Thu, 1 Feb 2024 03:25:45 -0500
Subject: [PATCH 231/259] [ci] install libboost-random-dev to help TA build
 itself

---
 .github/workflows/cmake.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index 8a22b24dd..68933c61c 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -53,7 +53,7 @@ jobs:
         wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null
         sudo apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main"
         sudo apt-get update
-        sudo apt-get -y install ninja-build g++-12 liblapack-dev libboost-dev libboost-serialization-dev libeigen3-dev openmpi-bin libopenmpi-dev libtbb-dev ccache flex bison cmake doxygen
+        sudo apt-get -y install ninja-build g++-12 liblapack-dev libboost-dev libboost-serialization-dev libboost-random-dev libeigen3-dev openmpi-bin libopenmpi-dev libtbb-dev ccache flex bison cmake doxygen
 
     - name: Create Build Environment
       # Some projects don't allow in-source building, so create a separate build directory

From 1ffaf6aa00b96a39bd74456f1effda222ae1ed20 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Fri, 2 Feb 2024 09:36:24 -0500
Subject: [PATCH 232/259] implemented Read{A,B} that read A and B more
 intelligently

---
 examples/spmm/spmm.cc | 147 ++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 140 insertions(+), 7 deletions(-)

diff --git a/examples/spmm/spmm.cc b/examples/spmm/spmm.cc
index 94edf672b..f3ae517b6 100644
--- a/examples/spmm/spmm.cc
+++ b/examples/spmm/spmm.cc
@@ -163,7 +163,13 @@ inline int ijk2rank(int i, int j, int k, int P, int Q, int R) {
   return rank;
 }
 
-// flow data from an existing SpMatrix on rank 0
+/// Pushes out data from an existing SpMatrix whose data is distributed on a 2-d grid.
+
+/// Data is pushed in the order of the appearance of the data in the container, without any tailoring to
+/// the order in which the data is consumed; thus this is likely to generate tasks in a suboptimal order.
+/// \note Reading should in general occur in the same order as the data will be consumed.
+/// If all consuming tasks can execute concurrently this should be OK, albeit the runtime will likely throttle
+/// sends, thus task dependencies further "down" the DAG may result in some reading orders being better than others
 template <typename Blk = blk_t, typename Keymap = std::function<int(const Key<2> &)>>
 class Read_SpMatrix : public TT<Key<2>, std::tuple<Out<Key<2>, Blk>>, Read_SpMatrix<Blk, Keymap>, ttg::typelist<void>> {
  public:
@@ -174,18 +180,142 @@ class Read_SpMatrix : public TT<Key<2>, std::tuple<Out<Key<2>, Blk>>, Read_SpMat
               ij_keymap)
       , matrix_(matrix) {}
 
-  void op(const Key<2> &, std::tuple<Out<Key<2>, Blk>> &out) {
+  // key is this process' coordinate in the 2-d grid of processes (managed by ij_keymap) ...
+  // but it's not used at all since all this TT does is generate consuming tasks that use local tiles ...
+  // the consumers better use same keymap (ij_keymap) as this TT to avoid for the data flow from this to be local
+  void op(const Key<2> & /* pq */, std::tuple<Out<Key<2>, Blk>> &out) {
+    auto rank = ttg::default_execution_context().rank();
+    // this code assumes col-major layout
+    static_assert(SpMatrix<Blk>::IsRowMajor == false, "SpMatrix must be col-major");
+    for (int j = 0; j < matrix_.outerSize(); ++j) {
+      for (typename SpMatrix<Blk>::InnerIterator it(matrix_, j); it; ++it) {
+        assert(j == it.col());
+        const auto i = it.row();
+        // IF the receiver uses the same keymap, these sends are local
+        if (rank == this->get_keymap()(Key<2>(std::initializer_list<long>({i, j})))) {
+          ::send<0>(Key<2>(std::initializer_list<long>({i, j})), it.value(), out);
+        }
+      }
+    }
+  }
+
+ private:
+  const SpMatrix<Blk> &matrix_;
+};
+
+enum class ReadSchedule {
+  SingleK,   // appropriate for 2D (see ReadA)
+  MultipleK  // appropriate for 3D (see ReadA)
+};
+// change this to control the schedule of sends (2-D vs 3-D)
+constexpr auto DefaultReadSchedule = ReadSchedule::SingleK;
+
+/// flow data from A distributed on a 2-d grid of processes in the order they are likely to be consumed
+
+/// The order of sends needs to tailored as follows:
+/// - for 2-D SUMMA (R=1): read A[i][k] for all i and k=0 first, then k=1, etc. Clearly, Read_SpMatrix is going to
+/// generate reads in the wrong (transposed) order (send all k for i=0, then for i=1, etc.).
+/// - for 2.5/3-D SUMMA (R>1): same order of sends as for 2-D SUMMA will be suboptimal since all k=0 will only
+/// generate work on the r=0 process plane. Instead we *may* want to one tile needed on each plane, then one more for
+/// each plane. Hence the need for ReadSchedule.
+template <ReadSchedule KSchedule = DefaultReadSchedule, typename Blk = blk_t, typename Keymap2 = std::function<int(const Key<2> &)>>
+class ReadA : public TT<Key<2>, std::tuple<Out<Key<2>, Blk>>, ReadA<KSchedule, Blk, Keymap2>, ttg::typelist<void>> {
+ public:
+  using baseT = typename ReadA::ttT;
+  ReadA(const SpMatrix<Blk> &matrix, Edge<Key<2>> &ctl, Edge<Key<2>, Blk> &out, const Keymap2 &ij_keymap, long R)
+      : baseT(edges(ctl), edges(out), std::string("SpMM25D::read_a"), {"ctl"}, {"a_ik"}, ij_keymap)
+      , matrix_(matrix)
+      , R_(R) {}
+
+  // key is this process' coordinate in the 2-d grid of processes (managed by ij_keymap) ...
+  // but it's not used at all since all this TT does is generate consuming tasks that use local tiles ...
+  // the consumers better use same keymap (ij_keymap) as this TT to avoid for the data flow from this to be local
+  void op(const Key<2> & /* pq */, std::tuple<Out<Key<2>, Blk>> &out) {
+    auto rank = ttg::default_execution_context().rank();
+    const int I = matrix_.rows();
+    const int K = matrix_.cols();
+
+    // this assumes col-major layout of SpMatrix
+    static_assert(SpMatrix<Blk>::IsRowMajor == false, "SpMatrix must be col-major");
+
+    // MultipleK schedule is not yet correctly implemented
+    static_assert(KSchedule == ReadSchedule::SingleK, "MultipleK schedule not yet implemented");
+
+    // loop over blocks of k at a time, block size controlled by KSchedule
+    const int k_blk_size = (KSchedule == ReadSchedule::SingleK) ? 1 : R_;
+    for (std::pair<int, int> k_blk = {0, std::min(k_blk_size, K)}; k_blk.first < K;
+         k_blk = {k_blk.first + k_blk_size, std::min(k_blk.first + k_blk_size + k_blk_size, K)}) {
+
+      // N.B. : due to the CSC layout of A iterating over (blocks of) columns is efficient
+      for (int k = k_blk.first; k < k_blk.second; ++k) {
+        for (typename SpMatrix<Blk>::InnerIterator it(matrix_, k); it; ++it) {
+          assert(k == it.col());
+          const auto i = it.row();
+          // IF the receiver uses the same keymap, these sends are local
+          if (rank == this->get_keymap()(Key<2>(std::initializer_list<long>({i, k})))) {
+            ::send<0>(Key<2>(std::initializer_list<long>({i, k})), it.value(), out);
+          }
+        }
+      }
+
+    }
+
+  }
+
+ private:
+  const SpMatrix<Blk> &matrix_;
+  long R_;
+};
+
+// flow data from an existing SpMatrix on rank 0
+template <ReadSchedule KSchedule = DefaultReadSchedule, typename Blk = blk_t, typename Keymap2 = std::function<int(const Key<2> &)>>
+class ReadB : public TT<Key<2>, std::tuple<Out<Key<2>, Blk>>, ReadB<KSchedule, Blk, Keymap2>, ttg::typelist<void>> {
+ public:
+  using baseT = typename ReadB::ttT;
+  ReadB(const SpMatrix<Blk> &matrix, Edge<Key<2>> &ctl, Edge<Key<2>, Blk> &out, const Keymap2 &ij_keymap, long R)
+      : baseT(edges(ctl), edges(out), std::string("read_b"), {"ctl"}, {"b_kj"}, ij_keymap), matrix_(matrix), R_(R) {}
+
+  // key is this process' coordinate in the 2-d grid of processes (managed by ij_keymap) ...
+  // but it's not used at all since all this TT does is generate consuming tasks that use local tiles ...
+  // the consumers better use same keymap (ij_keymap) as this TT to avoid for the data flow from this to be local
+  void op(const Key<2> & /* pq */, std::tuple<Out<Key<2>, Blk>> &out) {
     auto rank = ttg::default_execution_context().rank();
-    for (int k = 0; k < matrix_.outerSize(); ++k) {
-      for (typename SpMatrix<Blk>::InnerIterator it(matrix_, k); it; ++it) {
-        if (rank == this->get_keymap()(Key<2>(std::initializer_list<long>({it.row(), it.col()}))))
-          ::send<0>(Key<2>(std::initializer_list<long>({it.row(), it.col()})), it.value(), out);
+    const int J = matrix_.cols();
+    const int K = matrix_.rows();
+
+    // this assumes col-major layout of SpMatrix
+    static_assert(SpMatrix<Blk>::IsRowMajor == false, "SpMatrix must be col-major");
+
+    // MultipleK schedule is not yet correctly implemented
+    static_assert(KSchedule == ReadSchedule::SingleK, "MultipleK schedule not yet implemented");
+
+    // loop over blocks of k at a time, block size controlled by KSchedule
+    const int k_blk_size = (KSchedule == ReadSchedule::SingleK) ? 1 : R_;
+    for (std::pair<int, int> k_blk = {0, std::min(k_blk_size, K)}; k_blk.first < K;
+         k_blk = {k_blk.first + k_blk_size, std::min(k_blk.first + k_blk_size + k_blk_size, K)}) {
+
+      // WARNING : due to the CSC layout of B iterating over (blocks of) columns is inefficient
+      for (int j = 0; j < matrix_.outerSize(); ++j) {
+        for (typename SpMatrix<Blk>::InnerIterator it(matrix_, j); it; ++it) {
+          assert(j == it.col());
+          const auto k = it.row();
+          // if k past the k block, we are done with this i
+          if (k >= k_blk.second) break;
+          // continue iterating until k has not reached this k block
+          if (k < k_blk.first) continue;
+          // IF the receiver uses the same keymap, these sends are local
+          if (rank == this->get_keymap()(Key<2>(std::initializer_list<long>({k, j})))) {
+            ::send<0>(Key<2>(std::initializer_list<long>({k, j})), it.value(), out);
+          }
+        }
+
       }
     }
   }
 
  private:
   const SpMatrix<Blk> &matrix_;
+  long R_;
 };
 
 // flow (move?) data into an existing SpMatrix on rank 0
@@ -1153,7 +1283,7 @@ static void initBlSpRandom(const std::function<int(const Key<2> &)> &keymap, siz
   size_t avg_nb = 0;
   int avg_nb_nb = 0;
 
-  struct tuple_hash : public std::unary_function<std::tuple<int, int>, std::size_t> {
+  struct tuple_hash {
     std::size_t operator()(const std::tuple<int, int> &k) const {
       return static_cast<size_t>(std::get<0>(k)) | (static_cast<size_t>(std::get<1>(k)) << 32);
     }
@@ -1256,6 +1386,9 @@ static void timed_measurement(SpMatrix<> &A, SpMatrix<> &B, const std::function<
 
   Read_SpMatrix a("A", A, ctl, eA, ij_keymap);
   Read_SpMatrix b("B", B, ctl, eB, ij_keymap);
+  // uncomment this to use more intelligent schedule of reads
+//  ReadA<> a(A, ctl, eA, ij_keymap, R);
+//  ReadB<> b(B, ctl, eB, ij_keymap, R);
   Write_SpMatrix<> c(C, eC, ij_keymap);
   auto &c_status = c.status();
   assert(!has_value(c_status));

From 41cc81430ae883ce443bff1c1481f6aaa6e99ade Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 2 Feb 2024 09:44:23 -0500
Subject: [PATCH 233/259] Bump PaRSEC tag

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 CMakeLists.txt                                   | 2 +-
 cmake/modules/ExternalDependenciesVersions.cmake | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 684d3a58c..c1813ee31 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -183,7 +183,7 @@ set(TTG_HAVE_MPI MPI_FOUND)
 if (MPI_FOUND)
   include(CheckIncludeFiles)
   set(CMAKE_REQUIRED_INCLUDES ${MPI_C_INCLUDE_DIRS})
-  check_include_file("mpi-ext.h" TTG_HAVE_MPIEXT)
+  check_include_files("mpi-ext.h" TTG_HAVE_MPIEXT)
 else(MPI_FOUND)
   set(TTG_HAVE_MPIEXT $<BOOL:false>)
 endif(MPI_FOUND)
diff --git a/cmake/modules/ExternalDependenciesVersions.cmake b/cmake/modules/ExternalDependenciesVersions.cmake
index 8d763f5db..f967539d3 100644
--- a/cmake/modules/ExternalDependenciesVersions.cmake
+++ b/cmake/modules/ExternalDependenciesVersions.cmake
@@ -6,6 +6,7 @@ set(TTG_TRACKED_BOOST_VERSION 1.66)
 set(TTG_TRACKED_CATCH2_VERSION 3.5.0)
 set(TTG_TRACKED_CEREAL_VERSION 1.3.0)
 set(TTG_TRACKED_MADNESS_TAG cb195817d7807c4aead10ba200cd20649036cbae)
-set(TTG_TRACKED_PARSEC_TAG 25d1931e863b6741e453112d2117d85ad32e7fba)
+#set(TTG_TRACKED_PARSEC_TAG 25d1931e863b6741e453112d2117d85ad32e7fba)
+set(TTG_TRACKED_PARSEC_TAG 8ed7cdf02ad607051c130a6deeb8f12c84f2e4d3)
 set(TTG_TRACKED_BTAS_TAG a02be0d29fb4a788ecef43de711dcd6d6f1cb6b8)
 set(TTG_TRACKED_TILEDARRAY_TAG f0115e9e4a3f988224afbfb3c241e92171e916b8)

From 8993e49ef20d3d2e4b860cb9dc773963dffe6bf1 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 2 Feb 2024 09:46:15 -0500
Subject: [PATCH 234/259] Implement support for inlining of objects with
 SplitMetadata

For small objects it is beneficial of packing the data into the
activation message instead of going through an extra GET.


Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/ttg.h | 332 +++++++++++++++++++++++++++----------------
 1 file changed, 211 insertions(+), 121 deletions(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 1f6c6ae18..b9edccad4 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -145,6 +145,7 @@ namespace ttg_parsec {
     std::size_t key_offset = 0;
     fn_id_t fn_id = MSG_INVALID;
     std::int8_t num_iovecs = 0;
+    bool inline_data = false;
     int32_t param_id = -1;
     int num_keys = 0;
     int sender = -1;
@@ -165,6 +166,26 @@ namespace ttg_parsec {
 
   namespace detail {
 
+    constexpr const int PARSEC_TTG_MAX_AM_SIZE = 1 * 1024*1024;
+
+    struct msg_t {
+      msg_header_t tt_id;
+      static constexpr std::size_t max_payload_size = PARSEC_TTG_MAX_AM_SIZE - sizeof(msg_header_t);
+      unsigned char bytes[max_payload_size];
+
+      msg_t() = default;
+      msg_t(uint64_t tt_id,
+            uint32_t taskpool_id,
+            msg_header_t::fn_id_t fn_id,
+            int32_t param_id,
+            int sender,
+            int num_keys = 1)
+      : tt_id(fn_id, taskpool_id, tt_id, param_id, sender, num_keys)
+      {}
+    };
+
+    inline std::size_t max_inline_size = msg_t::max_payload_size;
+
     static int static_unpack_msg(parsec_comm_engine_t *ce, uint64_t tag, void *data, long unsigned int size,
                                  int src_rank, void *obj) {
       static_set_arg_fct_type static_set_arg_fct;
@@ -225,7 +246,7 @@ namespace ttg_parsec {
 
     static void ttg_parsec_ce_up(parsec_comm_engine_t *comm_engine, void *user_data)
     {
-      parsec_ce.tag_register(WorldImpl::parsec_ttg_tag(), &detail::static_unpack_msg, user_data, PARSEC_TTG_MAX_AM_SIZE);
+      parsec_ce.tag_register(WorldImpl::parsec_ttg_tag(), &detail::static_unpack_msg, user_data, detail::PARSEC_TTG_MAX_AM_SIZE);
       parsec_ce.tag_register(WorldImpl::parsec_ttg_rma_tag(), &detail::get_remote_complete_cb, user_data, 128);
     }
 
@@ -242,7 +263,6 @@ namespace ttg_parsec {
     int parsec_ttg_profile_backend_allocate_datacopy, parsec_ttg_profile_backend_free_datacopy;
 #endif
 
-    static constexpr const int PARSEC_TTG_MAX_AM_SIZE = 1 * 1024*1024;
     WorldImpl(int *argc, char **argv[], int ncores, parsec_context_t *c = nullptr)
         : WorldImplBase(query_comm_size(), query_comm_rank())
         , ctx(c)
@@ -293,7 +313,7 @@ namespace ttg_parsec {
 #endif
 
       if( NULL != parsec_ce.tag_register) {
-        parsec_ce.tag_register(WorldImpl::parsec_ttg_tag(), &detail::static_unpack_msg, this, PARSEC_TTG_MAX_AM_SIZE);
+        parsec_ce.tag_register(WorldImpl::parsec_ttg_tag(), &detail::static_unpack_msg, this, detail::PARSEC_TTG_MAX_AM_SIZE);
         parsec_ce.tag_register(WorldImpl::parsec_ttg_rma_tag(), &detail::get_remote_complete_cb, this, 128);
       }
 
@@ -997,6 +1017,15 @@ namespace ttg_parsec {
         throw std::runtime_error("PaRSEC: Found non-GPU device in GPU ID range!");
       }
     }
+
+    /* parse the maximum inline size */
+    const char* ttg_max_inline_cstr = std::getenv("TTG_MAX_INLINE");
+    if (nullptr != ttg_max_inline_cstr) {
+      std::size_t inline_size = std::atol(ttg_max_inline_cstr);
+      if (inline_size < detail::max_inline_size) {
+        detail::max_inline_size = inline_size;
+      }
+    }
   }
   inline void ttg_finalize() {
     // We need to notify the current taskpool of termination if we are in user termination detection mode
@@ -1075,20 +1104,6 @@ namespace ttg_parsec {
       parsec_task_class_t self;
     };
 
-    struct msg_t {
-      msg_header_t tt_id;
-      unsigned char bytes[WorldImpl::PARSEC_TTG_MAX_AM_SIZE - sizeof(msg_header_t)];
-
-      msg_t() = default;
-      msg_t(uint64_t tt_id,
-            uint32_t taskpool_id,
-            msg_header_t::fn_id_t fn_id,
-            int32_t param_id,
-            int sender,
-            int num_keys = 1)
-      : tt_id(fn_id, taskpool_id, tt_id, param_id, sender, num_keys)
-      {}
-    };
   }  // namespace detail
 
   template <typename keyT, typename output_terminalsT, typename derivedT, typename input_valueTs>
@@ -1995,6 +2010,7 @@ ttg::abort();  // should not happen
         if constexpr (!ttg::meta::is_void_v<valueT>) {
           using decvalueT = std::decay_t<valueT>;
           int32_t num_iovecs = msg->tt_id.num_iovecs;
+          //bool inline_data = msg->inline_data;
           detail::ttg_data_copy_t *copy;
           if constexpr (ttg::has_split_metadata<decvalueT>::value) {
             ttg::SplitMetadataDescriptor<decvalueT> descr;
@@ -2048,26 +2064,37 @@ ttg::abort();  // should not happen
             int remote = msg->tt_id.sender;
             assert(remote < world.size());
 
-            /* nothing else to do if the object is empty */
-            /* extract the callback tag */
-            parsec_ce_tag_t cbtag;
-            std::memcpy(&cbtag, msg->bytes + pos, sizeof(cbtag));
-            pos += sizeof(cbtag);
-
-            /* create the value from the metadata */
-            auto activation = new detail::rma_delayed_activate(
-                std::move(keylist), copy, num_iovecs, [this](std::vector<keyT> &&keylist, detail::ttg_data_copy_t *copy) {
-                  set_arg_from_msg_keylist<i, decvalueT>(keylist, copy);
-                  this->world.impl().decrement_inflight_msg();
-                });
             auto &val = *static_cast<decvalueT *>(copy->get_ptr());
 
-            using ActivationT = std::decay_t<decltype(*activation)>;
+            bool inline_data = msg->tt_id.inline_data;
 
             int nv = 0;
             /* start the RMA transfers */
             auto handle_iovecs_fn =
               [&](auto&& iovecs) {
+
+                if (inline_data) {
+                  /* unpack the data from the message */
+                  for (auto &&iov : iovecs) {
+                    ++nv;
+                    std::memcpy(iov.data, msg->bytes + pos, iov.num_bytes);
+                    pos += iov.num_bytes;
+                  }
+                } else {
+                  /* extract the callback tag */
+                  parsec_ce_tag_t cbtag;
+                  std::memcpy(&cbtag, msg->bytes + pos, sizeof(cbtag));
+                  pos += sizeof(cbtag);
+
+                  /* create the value from the metadata */
+                  auto activation = new detail::rma_delayed_activate(
+                      std::move(keylist), copy, num_iovecs, [this](std::vector<keyT> &&keylist, detail::ttg_data_copy_t *copy) {
+                        set_arg_from_msg_keylist<i, decvalueT>(keylist, copy);
+                        this->world.impl().decrement_inflight_msg();
+                      });
+
+                  using ActivationT = std::decay_t<decltype(*activation)>;
+
                   for (auto &&iov : iovecs) {
                     ++nv;
                     parsec_ce_mem_reg_handle_t rreg;
@@ -2095,6 +2122,7 @@ ttg::abort();  // should not happen
                                   /*world.impl().parsec_ttg_rma_tag()*/
                                   cbtag, &fn_ptr, sizeof(std::intptr_t));
                   }
+                }
             };
             if constexpr (ttg::has_split_metadata<decvalueT>::value) {
               ttg::SplitMetadataDescriptor<decvalueT> descr;
@@ -2106,6 +2134,10 @@ ttg::abort();  // should not happen
 
             assert(num_iovecs == nv);
             assert(size == (key_end_pos + sizeof(msg_header_t)));
+
+            if (inline_data) {
+              set_arg_from_msg_keylist<i, decvalueT>(ttg::span<keyT>(&keylist[0], num_keys), copy);
+            }
           }
           // case 2 and 3
         } else if constexpr (!ttg::meta::is_void_v<keyT> && std::is_void_v<valueT>) {
@@ -2546,6 +2578,35 @@ ttg::abort();  // should not happen
       set_arg_impl<i>(key, ttg::Void{});
     }
 
+    template<typename Value, typename Key>
+    bool can_inline_data(Value* value_ptr, detail::ttg_data_copy_t *copy, const Key& key, std::size_t num_keys) {
+      using decvalueT = std::decay_t<Value>;
+      bool inline_data = false;
+      /* check whether to send data in inline */
+      std::size_t iov_size = 0;
+      std::size_t metadata_size = 0;
+      if constexpr (ttg::has_split_metadata<std::decay_t<Value>>::value) {
+        ttg::SplitMetadataDescriptor<decvalueT> descr;
+        auto iovs = descr.get_data(*const_cast<decvalueT *>(value_ptr));
+        iov_size = std::accumulate(iovs.begin(), iovs.end(), 0,
+                                    [](std::size_t s, auto& iov){ return s + iov.num_bytes; });
+        auto metadata = descr.get_metadata(*const_cast<decvalueT *>(value_ptr));
+        metadata_size = ttg::default_data_descriptor<decltype(metadata)>::payload_size(&metadata);
+      } else {
+        /* TODO: how can we query the iovecs of the buffers here without actually packing the data? */
+        metadata_size = ttg::default_data_descriptor<ttg::meta::remove_cvr_t<Value>>::payload_size(value_ptr);
+        iov_size = std::accumulate(copy->iovec_begin(), copy->iovec_end(), 0,
+                                    [](std::size_t s, auto& iov){ return s + iov.num_bytes; });
+      }
+      /* key is packed at the end */
+      std::size_t key_pack_size = ttg::default_data_descriptor<Key>::payload_size(&key);
+      std::size_t pack_size = key_pack_size + metadata_size + iov_size;
+      if (pack_size < detail::max_inline_size) {
+        inline_data = true;
+      }
+      return inline_data;
+    }
+
     // Used to set the i'th argument
     template <std::size_t i, typename Key, typename Value>
     void set_arg_impl(const Key &key, Value &&value, detail::ttg_data_copy_t *copy_in = nullptr) {
@@ -2600,46 +2661,58 @@ ttg::abort();  // should not happen
           }
         }
 
+        bool inline_data = can_inline_data(value_ptr, copy, key, 1);
+        msg->tt_id.inline_data = inline_data;
+
         auto handle_iovec_fn = [&](auto&& iovecs){
 
-          /* TODO: at the moment, the tag argument to parsec_ce.get() is treated as a
-           * raw function pointer instead of a preregistered AM tag, so play that game.
-           * Once this is fixed in PaRSEC we need to use parsec_ttg_rma_tag instead! */
-          parsec_ce_tag_t cbtag = reinterpret_cast<parsec_ce_tag_t>(&detail::get_remote_complete_cb);
-          std::memcpy(msg->bytes + pos, &cbtag, sizeof(cbtag));
-          pos += sizeof(cbtag);
+          if (inline_data) {
+            /* inline data is packed right after the tt_id in the message */
+            for (auto &&iov : iovecs) {
+              std::memcpy(msg->bytes + pos, iov.data, iov.num_bytes);
+              pos += iov.num_bytes;
+            }
+          } else {
 
-          /**
-           * register the generic iovecs and pack the registration handles
-           * memory layout: [<lreg_size, lreg, release_cb_ptr>, ...]
-           */
-          for (auto &&iov : iovecs) {
-            copy = detail::register_data_copy<decvalueT>(copy, nullptr, true);
-            parsec_ce_mem_reg_handle_t lreg;
-            size_t lreg_size;
-            /* TODO: only register once when we can broadcast the data! */
-            parsec_ce.mem_register(iov.data, PARSEC_MEM_TYPE_NONCONTIGUOUS, iov.num_bytes, parsec_datatype_int8_t,
-                                   iov.num_bytes, &lreg, &lreg_size);
-            auto lreg_ptr = std::shared_ptr<void>{lreg, [](void *ptr) {
-                                                    parsec_ce_mem_reg_handle_t memreg = (parsec_ce_mem_reg_handle_t)ptr;
-                                                    parsec_ce.mem_unregister(&memreg);
-                                                  }};
-            int32_t lreg_size_i = lreg_size;
-            std::memcpy(msg->bytes + pos, &lreg_size_i, sizeof(lreg_size_i));
-            pos += sizeof(lreg_size_i);
-            std::memcpy(msg->bytes + pos, lreg, lreg_size);
-            pos += lreg_size;
-            //std::cout << "set_arg_impl lreg " << lreg << std::endl;
-            /* TODO: can we avoid the extra indirection of going through std::function? */
-            std::function<void(void)> *fn = new std::function<void(void)>([=]() mutable {
-              /* shared_ptr of value and registration captured by value so resetting
-               * them here will eventually release the memory/registration */
-              detail::release_data_copy(copy);
-              lreg_ptr.reset();
-            });
-            std::intptr_t fn_ptr{reinterpret_cast<std::intptr_t>(fn)};
-            std::memcpy(msg->bytes + pos, &fn_ptr, sizeof(fn_ptr));
-            pos += sizeof(fn_ptr);
+            /* TODO: at the moment, the tag argument to parsec_ce.get() is treated as a
+            * raw function pointer instead of a preregistered AM tag, so play that game.
+            * Once this is fixed in PaRSEC we need to use parsec_ttg_rma_tag instead! */
+            parsec_ce_tag_t cbtag = reinterpret_cast<parsec_ce_tag_t>(&detail::get_remote_complete_cb);
+            std::memcpy(msg->bytes + pos, &cbtag, sizeof(cbtag));
+            pos += sizeof(cbtag);
+
+            /**
+             * register the generic iovecs and pack the registration handles
+             * memory layout: [<lreg_size, lreg, release_cb_ptr>, ...]
+             */
+            for (auto &&iov : iovecs) {
+              copy = detail::register_data_copy<decvalueT>(copy, nullptr, true);
+              parsec_ce_mem_reg_handle_t lreg;
+              size_t lreg_size;
+              /* TODO: only register once when we can broadcast the data! */
+              parsec_ce.mem_register(iov.data, PARSEC_MEM_TYPE_NONCONTIGUOUS, iov.num_bytes, parsec_datatype_int8_t,
+                                    iov.num_bytes, &lreg, &lreg_size);
+              auto lreg_ptr = std::shared_ptr<void>{lreg, [](void *ptr) {
+                                                      parsec_ce_mem_reg_handle_t memreg = (parsec_ce_mem_reg_handle_t)ptr;
+                                                      parsec_ce.mem_unregister(&memreg);
+                                                    }};
+              int32_t lreg_size_i = lreg_size;
+              std::memcpy(msg->bytes + pos, &lreg_size_i, sizeof(lreg_size_i));
+              pos += sizeof(lreg_size_i);
+              std::memcpy(msg->bytes + pos, lreg, lreg_size);
+              pos += lreg_size;
+              //std::cout << "set_arg_impl lreg " << lreg << std::endl;
+              /* TODO: can we avoid the extra indirection of going through std::function? */
+              std::function<void(void)> *fn = new std::function<void(void)>([=]() mutable {
+                /* shared_ptr of value and registration captured by value so resetting
+                * them here will eventually release the memory/registration */
+                detail::release_data_copy(copy);
+                lreg_ptr.reset();
+              });
+              std::intptr_t fn_ptr{reinterpret_cast<std::intptr_t>(fn)};
+              std::memcpy(msg->bytes + pos, &fn_ptr, sizeof(fn_ptr));
+              pos += sizeof(fn_ptr);
+            }
           }
         };
 
@@ -2769,57 +2842,72 @@ ttg::abort();  // should not happen
         copy = detail::find_copy_in_task(detail::parsec_ttg_caller, &value);
         assert(nullptr != copy);
 
-        std::vector<std::pair<int32_t, std::shared_ptr<void>>> memregs;
-        auto register_iovs_fn = [&memregs](auto&& iovs){
-          for (auto &&iov : iovs) {
-            parsec_ce_mem_reg_handle_t lreg;
-            size_t lreg_size;
-            parsec_ce.mem_register(iov.data, PARSEC_MEM_TYPE_NONCONTIGUOUS, iov.num_bytes, parsec_datatype_int8_t,
-                                  iov.num_bytes, &lreg, &lreg_size);
-            /* TODO: use a static function for deregistration here? */
-            memregs.push_back(std::make_pair(static_cast<int32_t>(lreg_size),
-                                            /* TODO: this assumes that parsec_ce_mem_reg_handle_t is void* */
-                                            std::shared_ptr<void>{lreg, [](void *ptr) {
-                                                                    parsec_ce_mem_reg_handle_t memreg =
-                                                                        (parsec_ce_mem_reg_handle_t)ptr;
-                                                                    //std::cout << "broadcast_arg memunreg lreg " << memreg << std::endl;
-                                                                    parsec_ce.mem_unregister(&memreg);
-                                                                  }}));
-            //std::cout << "broadcast_arg memreg lreg " << lreg << std::endl;
-          }
-        };
-
         using msg_t = detail::msg_t;
         auto &world_impl = world.impl();
         std::unique_ptr<msg_t> msg = std::make_unique<msg_t>(get_instance_id(), world_impl.taskpool()->taskpool_id,
                                                              msg_header_t::MSG_SET_ARG, i, world_impl.rank());
 
+        /* check if we inline the data */
+        /* TODO: this assumes the worst case: that all keys are packed at once (i.e., go to the same remote). Can we do better?*/
+        bool inline_data = can_inline_data(&value, copy, keylist_sorted[0], keylist_sorted.size());
+        msg->tt_id.inline_data = inline_data;
+
+        std::vector<std::pair<int32_t, std::shared_ptr<void>>> memregs;
+        auto handle_iovs_fn = [&](auto&& iovs){
+
+          if (inline_data) {
+            /* inline data is packed right after the tt_id in the message */
+            for (auto &&iov : iovs) {
+              std::memcpy(msg->bytes + pos, iov.data, iov.num_bytes);
+              pos += iov.num_bytes;
+            }
+          } else {
+
+            /* TODO: at the moment, the tag argument to parsec_ce.get() is treated as a
+              * raw function pointer instead of a preregistered AM tag, so play that game.
+              * Once this is fixed in PaRSEC we need to use parsec_ttg_rma_tag instead! */
+            parsec_ce_tag_t cbtag = reinterpret_cast<parsec_ce_tag_t>(&detail::get_remote_complete_cb);
+            std::memcpy(msg->bytes + pos, &cbtag, sizeof(cbtag));
+            pos += sizeof(cbtag);
+
+            for (auto &&iov : iovs) {
+              parsec_ce_mem_reg_handle_t lreg;
+              size_t lreg_size;
+              parsec_ce.mem_register(iov.data, PARSEC_MEM_TYPE_NONCONTIGUOUS, iov.num_bytes, parsec_datatype_int8_t,
+                                    iov.num_bytes, &lreg, &lreg_size);
+              /* TODO: use a static function for deregistration here? */
+              memregs.push_back(std::make_pair(static_cast<int32_t>(lreg_size),
+                                              /* TODO: this assumes that parsec_ce_mem_reg_handle_t is void* */
+                                              std::shared_ptr<void>{lreg, [](void *ptr) {
+                                                                      parsec_ce_mem_reg_handle_t memreg =
+                                                                          (parsec_ce_mem_reg_handle_t)ptr;
+                                                                      //std::cout << "broadcast_arg memunreg lreg " << memreg << std::endl;
+                                                                      parsec_ce.mem_unregister(&memreg);
+                                                                    }}));
+              //std::cout << "broadcast_arg memreg lreg " << lreg << std::endl;
+            }
+          }
+        };
+
         if constexpr (ttg::has_split_metadata<std::decay_t<Value>>::value) {
           ttg::SplitMetadataDescriptor<decvalueT> descr;
-          auto iovs = descr.get_data(*const_cast<decvalueT *>(&value));
-          num_iovs = std::distance(std::begin(iovs), std::end(iovs));
-          memregs.reserve(num_iovs);
-          register_iovs_fn(iovs);
           /* pack the metadata */
           auto metadata = descr.get_metadata(value);
           size_t metadata_size = sizeof(metadata);
           pos = pack(metadata, msg->bytes, pos);
+          auto iovs = descr.get_data(*const_cast<decvalueT *>(&value));
+          num_iovs = std::distance(std::begin(iovs), std::end(iovs));
+          memregs.reserve(num_iovs);
+          handle_iovs_fn(iovs);
           //std::cout << "broadcast_arg splitmd num_iovecs " << num_iovs << std::endl;
         } else if constexpr (!ttg::has_split_metadata<std::decay_t<Value>>::value) {
           /* serialize the object once */
           pos = pack(value, msg->bytes, pos, copy);
           num_iovs = std::distance(copy->iovec_begin(), copy->iovec_end());
-          register_iovs_fn(copy->iovec_span());
+          handle_iovs_fn(copy->iovec_span());
           copy->iovec_reset();
         }
 
-        /* TODO: at the moment, the tag argument to parsec_ce.get() is treated as a
-          * raw function pointer instead of a preregistered AM tag, so play that game.
-          * Once this is fixed in PaRSEC we need to use parsec_ttg_rma_tag instead! */
-        parsec_ce_tag_t cbtag = reinterpret_cast<parsec_ce_tag_t>(&detail::get_remote_complete_cb);
-        std::memcpy(msg->bytes + pos, &cbtag, sizeof(cbtag));
-        pos += sizeof(cbtag);
-
         msg->tt_id.num_iovecs = num_iovs;
 
         std::size_t save_pos = pos;
@@ -2844,28 +2932,30 @@ ttg::abort();  // should not happen
            * memory layout: [<lreg_size, lreg, lreg_fn>, ...]
            * NOTE: we need to pack these for every receiver to ensure correct ref-counting of the registration
            */
-          for (int idx = 0; idx < num_iovs; ++idx) {
-            // auto [lreg_size, lreg_ptr] = memregs[idx];
-            int32_t lreg_size;
-            std::shared_ptr<void> lreg_ptr;
-            std::tie(lreg_size, lreg_ptr) = memregs[idx];
-            std::memcpy(msg->bytes + pos, &lreg_size, sizeof(lreg_size));
-            pos += sizeof(lreg_size);
-            std::memcpy(msg->bytes + pos, lreg_ptr.get(), lreg_size);
-            pos += lreg_size;
-            //std::cout << "broadcast_arg lreg_ptr " << lreg_ptr.get() << std::endl;
-            /* mark another reader on the copy */
-            copy = detail::register_data_copy<valueT>(copy, nullptr, true);
-            /* create a function that will be invoked upon RMA completion at the target */
-            std::function<void(void)> *fn = new std::function<void(void)>([=]() mutable {
-              /* shared_ptr of value and registration captured by value so resetting
-                * them here will eventually release the memory/registration */
-              detail::release_data_copy(copy);
-              lreg_ptr.reset();
-            });
-            std::intptr_t fn_ptr{reinterpret_cast<std::intptr_t>(fn)};
-            std::memcpy(msg->bytes + pos, &fn_ptr, sizeof(fn_ptr));
-            pos += sizeof(fn_ptr);
+          if (!inline_data) {
+            for (int idx = 0; idx < num_iovs; ++idx) {
+              // auto [lreg_size, lreg_ptr] = memregs[idx];
+              int32_t lreg_size;
+              std::shared_ptr<void> lreg_ptr;
+              std::tie(lreg_size, lreg_ptr) = memregs[idx];
+              std::memcpy(msg->bytes + pos, &lreg_size, sizeof(lreg_size));
+              pos += sizeof(lreg_size);
+              std::memcpy(msg->bytes + pos, lreg_ptr.get(), lreg_size);
+              pos += lreg_size;
+              //std::cout << "broadcast_arg lreg_ptr " << lreg_ptr.get() << std::endl;
+              /* mark another reader on the copy */
+              copy = detail::register_data_copy<valueT>(copy, nullptr, true);
+              /* create a function that will be invoked upon RMA completion at the target */
+              std::function<void(void)> *fn = new std::function<void(void)>([=]() mutable {
+                /* shared_ptr of value and registration captured by value so resetting
+                  * them here will eventually release the memory/registration */
+                detail::release_data_copy(copy);
+                lreg_ptr.reset();
+              });
+              std::intptr_t fn_ptr{reinterpret_cast<std::intptr_t>(fn)};
+              std::memcpy(msg->bytes + pos, &fn_ptr, sizeof(fn_ptr));
+              pos += sizeof(fn_ptr);
+            }
           }
 
           /* mark the beginning of the keys */

From becdbf4e4aff2826e884dc1e5464117a558f44c1 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 2 Feb 2024 09:53:57 -0500
Subject: [PATCH 235/259] SPMM: Allow changing the work distribution

Work may be performed by the owner of A, B, or C, or the default
honoring the R parameter.

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/spmm/spmm.cc | 74 +++++++++++++++++++++++++++++++++++--------
 1 file changed, 60 insertions(+), 14 deletions(-)

diff --git a/examples/spmm/spmm.cc b/examples/spmm/spmm.cc
index 6c5090dd8..f13d8aad2 100644
--- a/examples/spmm/spmm.cc
+++ b/examples/spmm/spmm.cc
@@ -1384,8 +1384,13 @@ static void timed_measurement(SpMatrix<> &A, SpMatrix<> &B, const std::function<
   Edge<Key<2>, blk_t> eA, eB;
   Edge<Key<2>, blk_t> eC;
 
-  Read_SpMatrix a("A", A, ctl, eA, ijk_keymap, ij_keymap);
-  Read_SpMatrix b("B", B, ctl, eB, ijk_keymap, ij_keymap);
+  /* the Read_SpMatrix tasks get process coordinates, not tile coordinates  */
+  auto read_keymap = [&](const Key<3>& key){
+      return ijk2rank(key[0], key[1], key[2], P, Q, R);
+    };
+
+  Read_SpMatrix a("A", A, ctl, eA, read_keymap, ij_keymap);
+  Read_SpMatrix b("B", B, ctl, eB, read_keymap, ij_keymap);
   Write_SpMatrix<> c(C, eC, ij_keymap, false);
   auto &c_status = c.status();
   assert(!has_value(c_status));
@@ -1423,7 +1428,7 @@ static void timed_measurement(SpMatrix<> &A, SpMatrix<> &B, const std::function<
               << " A_density= " << Adensity << " B_density= " << Bdensity << " gflops= " << gflops << " seconds= " << tc
               << " gflops/s= " << gflops / tc << std::endl;
   }
-  std::cout << "num reductions " << reduce_count.load() << " tiles " << MT*KT << std::endl;
+  //std::cout << "num reductions " << reduce_count.load() << " tiles " << MT*KT << std::endl;
 }
 
 #if !defined(BLOCK_SPARSE_GEMM)
@@ -1459,6 +1464,13 @@ static void make_rows_of_col_from_eigen(const SpMatrix<> &mat, std::vector<std::
 }
 #endif
 
+/* where to distribute the work to */
+enum class WORKDIST {
+  A = 0, // distribute work based on A's distribution
+  B = 1, // distribute work based on B's distribution
+  C = 2, // distribute work based on C's distribution
+};
+
 static double compute_gflops(const std::vector<std::vector<long>> &a_r2c, const std::vector<std::vector<long>> &b_r2c,
                              const std::vector<int> &mTiles, const std::vector<int> &nTiles,
                              const std::vector<int> &kTiles) {
@@ -1545,7 +1557,7 @@ int main(int argc, char **argv) {
       std::string tiling_type;
       int M = 0, N = 0, K = 0;
       int minTs = 0, maxTs = 0;
-      int parallel_bcasts = 1;
+      int parallel_bcasts = std::numeric_limits<int>::max();
 
       double avg_nb = nan("undefined");
       double Adensity = nan("undefined");
@@ -1578,6 +1590,18 @@ int main(int argc, char **argv) {
         }
       }
 
+      WORKDIST dist = WORKDIST::C;
+      if (cmdOptionExists(argv, argv + argc, "-D")) {
+        std::string DStr(getCmdOption(argv, argv+argc, "-D"));
+        if (DStr == "a") {
+          dist = WORKDIST::A;
+        } else if (DStr == "b") {
+          dist = WORKDIST::B;
+        } else if (DStr == "c") {
+          dist = WORKDIST::C;
+        }
+      }
+
       auto ij_keymap = [P, Q, R](const Key<2> &ij) {
         int i = (int)ij[0];
         int j = (int)ij[1];
@@ -1585,16 +1609,33 @@ int main(int argc, char **argv) {
         return r;
       };
 
-      auto ijk_keymap = [P, Q, R](const Key<3> &ijk) {
-        int i = (int)ijk[0];
-        int j = (int)ijk[1];
-        int k = (int)ijk[2];
-        int r = ijk2rank(i, j, k, P, Q, R);
-        return r;
-      };
+      std::function<int(const Key<3> &ijk)> ijk_keymap;
+
+      if (dist == WORKDIST::A) {
+        ijk_keymap = [&](const Key<3> &ijk) {
+            int i = ijk[0], j = ijk[1], k = ijk[2];
+            return ij2rank(i, k, P, Q, R);
+          };
+      } else if (dist == WORKDIST::B) {
+        ijk_keymap = [&](const Key<3> &ijk) {
+            int i = ijk[0], j = ijk[1], k = ijk[2];
+            return ij2rank(k, j, P, Q, R);
+          };
+      } else if (dist == WORKDIST::C) {
+        ijk_keymap = [&](const Key<3> &ijk) {
+            int i = ijk[0], j = ijk[1], k = ijk[2];
+            return ij2rank(i, j, P, Q, R);
+          };
+      } else {
+        ijk_keymap = [&](const Key<3> &ijk) {
+            int i = ijk[0], j = ijk[1], k = ijk[2];
+            int r = ijk2rank(i, j, k, P, Q, R);
+            return r;
+          };
+      }
 
       std::string seedStr(getCmdOption(argv, argv + argc, "-s"));
-      unsigned int seed = parseOption(seedStr, 0);
+      unsigned long seed = parseOption(seedStr, 0L);
       if (seed == 0) {
         std::random_device rd;
         seed = rd();
@@ -1689,11 +1730,16 @@ int main(int argc, char **argv) {
         // flow graph needs to exist on every node
         // N.B. to validate C we need it on node 0!
         auto keymap_write = [](const Key<2> &key) { return 0; };
+
+        /* the Read_SpMatrix tasks get process coordinates, not tile coordinates  */
+        auto read_keymap = [&](const Key<3>& key){
+            return ijk2rank(key[0], key[1], key[2], P, Q, R);
+          };
         Edge<Key<3>> ctl("control");
         Control control(ctl);
         Edge<Key<2>, blk_t> eA, eB, eC;
-        Read_SpMatrix a("A", A, ctl, eA, ijk_keymap, ij_keymap);
-        Read_SpMatrix b("B", B, ctl, eB, ijk_keymap, ij_keymap);
+        Read_SpMatrix a("A", A, ctl, eA, read_keymap, ij_keymap);
+        Read_SpMatrix b("B", B, ctl, eB, read_keymap, ij_keymap);
         Write_SpMatrix<> c(C, eC, keymap_write);
         auto &c_status = c.status();
         assert(!has_value(c_status));

From 3dad2c1d5fd394d17f40b506efe82d02542ec58f Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Fri, 2 Feb 2024 11:17:10 -0500
Subject: [PATCH 236/259] Read{A,B} can use MultipleK schedule

---
 examples/spmm/spmm.cc | 45 +++++++++++++++++++++++++++++--------------
 1 file changed, 31 insertions(+), 14 deletions(-)

diff --git a/examples/spmm/spmm.cc b/examples/spmm/spmm.cc
index f3ae517b6..ff3250862 100644
--- a/examples/spmm/spmm.cc
+++ b/examples/spmm/spmm.cc
@@ -238,22 +238,39 @@ class ReadA : public TT<Key<2>, std::tuple<Out<Key<2>, Blk>>, ReadA<KSchedule, B
     // this assumes col-major layout of SpMatrix
     static_assert(SpMatrix<Blk>::IsRowMajor == false, "SpMatrix must be col-major");
 
-    // MultipleK schedule is not yet correctly implemented
-    static_assert(KSchedule == ReadSchedule::SingleK, "MultipleK schedule not yet implemented");
-
     // loop over blocks of k at a time, block size controlled by KSchedule
     const int k_blk_size = (KSchedule == ReadSchedule::SingleK) ? 1 : R_;
+
+    // this keeps iterators over each k-column in this block
+    // there is only 1 task per process, so no need to synchronize access to this
+    static std::vector<typename SpMatrix<Blk>::InnerIterator> column_iterators;
+
     for (std::pair<int, int> k_blk = {0, std::min(k_blk_size, K)}; k_blk.first < K;
          k_blk = {k_blk.first + k_blk_size, std::min(k_blk.first + k_blk_size + k_blk_size, K)}) {
 
+      // for each k in the block send one A[i][k], then next i, etc. this means keep track of iterators over k-column
+      // for each k in the block
       // N.B. : due to the CSC layout of A iterating over (blocks of) columns is efficient
+      column_iterators.resize(0);
       for (int k = k_blk.first; k < k_blk.second; ++k) {
-        for (typename SpMatrix<Blk>::InnerIterator it(matrix_, k); it; ++it) {
-          assert(k == it.col());
-          const auto i = it.row();
-          // IF the receiver uses the same keymap, these sends are local
-          if (rank == this->get_keymap()(Key<2>(std::initializer_list<long>({i, k})))) {
-            ::send<0>(Key<2>(std::initializer_list<long>({i, k})), it.value(), out);
+        column_iterators.emplace_back(matrix_, k);
+      }
+
+      int k_remaining = k_blk.second - k_blk.first;
+      while(k_remaining != 0) {
+        for (int k = k_blk.first, k_in_blk = 0; k < k_blk.second; ++k, ++k_in_blk) {
+          if (auto& it = column_iterators[k_in_blk]) {
+            assert(k == it.col());
+            const auto i = it.row();
+            // IF the receiver uses the same keymap, these sends are local
+            if (rank == this->get_keymap()(Key<2>(std::initializer_list<long>({i, k})))) {
+              ::send<0>(Key<2>(std::initializer_list<long>({i, k})), it.value(), out);
+            }
+            ++it;
+          }
+          else { // this k is done
+            assert(k_remaining != 0);
+            --k_remaining;
           }
         }
       }
@@ -286,13 +303,10 @@ class ReadB : public TT<Key<2>, std::tuple<Out<Key<2>, Blk>>, ReadB<KSchedule, B
     // this assumes col-major layout of SpMatrix
     static_assert(SpMatrix<Blk>::IsRowMajor == false, "SpMatrix must be col-major");
 
-    // MultipleK schedule is not yet correctly implemented
-    static_assert(KSchedule == ReadSchedule::SingleK, "MultipleK schedule not yet implemented");
-
     // loop over blocks of k at a time, block size controlled by KSchedule
     const int k_blk_size = (KSchedule == ReadSchedule::SingleK) ? 1 : R_;
     for (std::pair<int, int> k_blk = {0, std::min(k_blk_size, K)}; k_blk.first < K;
-         k_blk = {k_blk.first + k_blk_size, std::min(k_blk.first + k_blk_size + k_blk_size, K)}) {
+      k_blk = {k_blk.first + k_blk_size, std::min(k_blk.first + k_blk_size + k_blk_size, K)}) {
 
       // WARNING : due to the CSC layout of B iterating over (blocks of) columns is inefficient
       for (int j = 0; j < matrix_.outerSize(); ++j) {
@@ -580,7 +594,7 @@ class SpMM25D {
     const Keymap3 &ijk_keymap_;
   };  // class BcastB
 
-  /// multiply task has 3 input flows: a_ijk, b_ijk, and c_ijk, c_ijk contains the running total for this kayer of the
+  /// multiply task has 3 input flows: a_ijk, b_ijk, and c_ijk, c_ijk contains the running total for this layer of the
   /// 3-D process grid only
   class MultiplyAdd : public TT<Key<3>, std::tuple<Out<Key<2>, Blk>, Out<Key<3>, Blk>>, MultiplyAdd,
                                 ttg::typelist<const Blk, const Blk, Blk>> {
@@ -1691,6 +1705,9 @@ int main(int argc, char **argv) {
         Edge<Key<2>, blk_t> eA, eB, eC;
         Read_SpMatrix a("A", A, ctl, eA, ij_keymap);
         Read_SpMatrix b("B", B, ctl, eB, ij_keymap);
+        // uncomment this to use more intelligent schedule of reads
+//        ReadA<> a(A, ctl, eA, ij_keymap, R);
+//        ReadB<> b(B, ctl, eB, ij_keymap, R);
         Write_SpMatrix<> c(C, eC, keymap_write);
         auto &c_status = c.status();
         assert(!has_value(c_status));

From 4416f3bf41fe5f4a01b34eb457001fe90e1103bb Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 2 Feb 2024 17:40:48 -0500
Subject: [PATCH 237/259] Enforce strict ordering of broadcasts in column/row

For matrix A, we make sure that tiles are sent out in column-order (left to right).
For matrix B, we want to make sure that tiles are sent out in row-order (top to bottom).

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 examples/spmm/spmm.cc | 246 ++++++++++++++++++++++++++++++++----------
 1 file changed, 188 insertions(+), 58 deletions(-)

diff --git a/examples/spmm/spmm.cc b/examples/spmm/spmm.cc
index f13d8aad2..5806f4548 100644
--- a/examples/spmm/spmm.cc
+++ b/examples/spmm/spmm.cc
@@ -43,6 +43,9 @@ using namespace ttg;
 using scalar_t = double;
 using blk_t = btas::Tensor<scalar_t, btas::DEFAULT::range, btas::mohndle<btas::varray<scalar_t>, btas::Handle::shared_ptr>>;
 
+//#include <atomic>
+//static std::atomic<uint64_t> reduce_count = 0;
+
 #if defined(TTG_USE_PARSEC)
 namespace ttg {
   template <>
@@ -274,10 +277,11 @@ class SpMM25D {
       , ijk_keymap_(std::move(ijk_keymap))
       , parallel_bcasts_(parallel_bcasts) {
     Edge<Key<2>, void> a_ctl, b_ctl;
-    bcast_a_ = std::make_unique<BcastA>(a, a_ctl, local_a_ijk_, a_cols_of_row_, b_cols_of_row_,
+    Edge<Key<2>, int> a_rowctl, b_colctl; // TODO: can we have multiple control inputs per TT?
+    bcast_a_ = std::make_unique<BcastA>(a, a_ctl, a_rowctl, local_a_ijk_, a_rows_of_col_, a_cols_of_row_, b_cols_of_row_,
                                         ij_keymap_, ijk_keymap_, parallel_bcasts_);
     local_bcast_a_ = std::make_unique<LocalBcastA>(local_a_ijk_, a_ijk_, b_cols_of_row_, ijk_keymap_);
-    bcast_b_ = std::make_unique<BcastB>(b, b_ctl, local_b_ijk_, a_rows_of_col_, b_cols_of_row_,
+    bcast_b_ = std::make_unique<BcastB>(b, b_ctl, b_colctl, local_b_ijk_, a_rows_of_col_, b_cols_of_row_, b_rows_of_col_,
                                         ij_keymap_, ijk_keymap_, parallel_bcasts_);
     local_bcast_b_ = std::make_unique<LocalBcastB>(local_b_ijk_, b_ijk_, a_rows_of_col_, ijk_keymap_);
     multiplyadd_ = std::make_unique<MultiplyAdd>(a_ijk_, b_ijk_, c_ijk_, c_ij_p_, a_cols_of_row_,
@@ -285,7 +289,7 @@ class SpMM25D {
     reduce_c_ = std::make_unique<ReduceC>(c_ij_p_, c, ij_keymap_);
     reduce_c_->template set_input_reducer<0>(
       [&](Blk &c_ij, const Blk &c_ij_p) {
-        reduce_count++;
+        //reduce_count++;
         c_ij = c_ij + c_ij_p;
       });
     // compute how many contributions each C[i][j] should expect ... MultiplyAdd already does this, but need a way to
@@ -293,7 +297,9 @@ class SpMM25D {
     // this logic ...
     // TODO: do this in MultiplyAdd (need to allreduce this info so that everyone has it)
     // N.B. only need to set stream size on the rank that will accumulate the C[i][j] contribution
-    const auto my_rank = ttg::default_execution_context().rank();
+    auto world = ttg::default_execution_context();
+    const auto my_rank = world.rank();
+    std::vector<bool> c_ij_procmask(world.size(), false);
     for (auto i = 0ul; i != a_cols_of_row_.size(); ++i) {
       if (a_cols_of_row_[i].empty()) continue;
       for (auto j = 0ul; j != b_rows_of_col_.size(); ++j) {
@@ -303,54 +309,76 @@ class SpMM25D {
           decltype(i) k;
           bool have_k;
           std::tie(k, have_k) = multiplyadd_->compute_first_k(i, j);
-          std::vector<bool> c_ij_procmask(R, false);
-          if (have_k) {
-            const auto pR = k % R;  // k values are distributed round-robin among the layers of the 3-D grid
+          while (have_k) {
+            const auto pR = ijk_keymap_(Key<3>{i, j, k});
             assert(pR < c_ij_procmask.size());
             c_ij_procmask[pR] = true;
-            while (have_k) {
-              std::tie(k, have_k) = multiplyadd_->compute_next_k(i, j, k);
-              if (have_k) {
-                const auto pR = k % R;
-                assert(pR < c_ij_procmask.size());
-                c_ij_procmask[pR] = true;
-              }
-            }
+            /* get next k */
+            std::tie(k, have_k) = multiplyadd_->compute_next_k(i, j, k);
           }
           const auto c_ij_nprocs = std::count_if(c_ij_procmask.begin(), c_ij_procmask.end(), [](bool b) { return b; });
           if (c_ij_nprocs > 0) reduce_c_->template set_argstream_size<0>(Key<2>{i, j}, c_ij_nprocs);
+          /* reset the map */
+          std::fill(c_ij_procmask.begin(), c_ij_procmask.end(), false);
         }
       }
     }
 
-    auto world = ttg::default_execution_context();
-      /* initial ctl input for bcasts for A */
-      int to_start = parallel_bcasts;
-      for (int i = 0;
-           0 < to_start && i < a_cols_of_row_.size();
-           ++i) {
-        for (auto k : a_cols_of_row_[i]) {
-          auto key = Key<2>(i, k);
-          if (world.rank() == ij_keymap_(key)) {
-            bcast_a_->template in<1>()->sendk(key);
-            if (0 == --to_start) break;
-          }
+    /* kick off the first broadcast in each row of A
+     * this is used to enforce strict ordering within a row of A */
+    for (int i = 0; i < a_cols_of_row_.size(); ++i) {
+      for (int k : a_cols_of_row_[i]) {
+        auto key = Key<2>(i, k);
+        if (world.rank() == ij_keymap_(key)) {
+          bcast_a_->template in<1>()->send(key, 0);
+          break;
         }
       }
+    }
 
-      /* initial ctl input for bcasts for B */
-      to_start = parallel_bcasts;
-      for (int k = 0;
-           0 < to_start && k < b_cols_of_row_.size();
-           ++k) {
-        for (auto j : b_cols_of_row_[k]) {
-          auto key = Key<2>(k, j);
-          if (world.rank() == ij_keymap_(key)) {
-            bcast_b_->template in<1>()->sendk(key);
-            if (0 == --to_start) break;
-          }
+    /* initial ctl input for a number of bcasts for A
+     * this is used to limit the number of concurrent bcasts */
+    int to_start = parallel_bcasts;
+    for (int k = 0;
+          0 < to_start && k < a_rows_of_col_.size();
+          ++k) {
+      for (auto i : a_rows_of_col_[k]) {
+        auto key = Key<2>(i, k);
+        if (world.rank() == ij_keymap_(key)) {
+          //std::cout << "SPMM kick off BcastA " << key << std::endl;
+          bcast_a_->template in<2>()->sendk(key);
+          if (0 == --to_start) break;
+        }
+      }
+    }
+
+    /* kick off the first broadcast in each column of B
+     * this is used to enforce strict ordering within a column of B */
+    for (int j = 0; j < b_rows_of_col_.size(); ++j) {
+      for (int k : b_rows_of_col_[j]) {
+        auto key = Key<2>(k, j);
+        if (world.rank() == ij_keymap_(key)) {
+          //std::cout << "BcastB kick off " << key << std::endl;
+          bcast_b_->template in<1>()->send(key, 0);
+          break;
+        }
+      }
+    }
+
+    /* initial ctl input for bcasts for B */
+    to_start = parallel_bcasts;
+    for (int k = 0;
+          0 < to_start && k < b_cols_of_row_.size();
+          ++k) {
+      for (auto j : b_cols_of_row_[k]) {
+        auto key = Key<2>(k, j);
+        if (world.rank() == ij_keymap_(key)) {
+          //std::cout << "SPMM kick off BcastB " << key << std::endl;
+          bcast_b_->template in<2>()->sendk(key);
+          if (0 == --to_start) break;
         }
       }
+    }
 
     TTGUNUSED(bcast_a_);
     TTGUNUSED(bcast_b_);
@@ -397,26 +425,34 @@ class SpMM25D {
   };  // class LocalBcastA
 
   /// broadcast `A[i][k]` to all processors which will contain at least one `C[i][j]` such that `B[k][j]` exists
-  class BcastA : public TT<Key<2>, std::tuple<Out<Key<3>, Blk>, Out<Key<2>, void>>, BcastA, ttg::typelist<Blk, void>> {
+  class BcastA : public TT<Key<2>, std::tuple<Out<Key<3>, Blk>, Out<Key<2>, int>, Out<Key<2>, void>>, BcastA, ttg::typelist<Blk, int, void>> {
    public:
     using baseT = typename BcastA::ttT;
 
-    BcastA(Edge<Key<2>, Blk> &a_ik, Edge<Key<2>, void> &ctl, Edge<Key<3>, Blk> &a_ikp,
+    BcastA(Edge<Key<2>, Blk> &a_ik, Edge<Key<2>, void> &ctl,
+           Edge<Key<2>, int> &rowctl, Edge<Key<3>, Blk> &a_ikp,
+           const std::vector<std::vector<long>> &a_rows_of_col,
            const std::vector<std::vector<long>> &a_cols_of_row,
            const std::vector<std::vector<long>> &b_cols_of_row,
            const Keymap2 &ij_keymap, const Keymap3 &ijk_keymap,
            const int parallel_bcasts)
-        : baseT(edges(a_ik, ctl), edges(a_ikp, ctl), "SpMM25D::bcast_a", {"a_ik", "ctl"}, {"a_ikp", "ctl"}, ij_keymap)
+        : baseT(edges(a_ik, rowctl, ctl), edges(a_ikp, rowctl, ctl), "SpMM25D::bcast_a", {"a_ik", "rowctl", "ctl"}, {"a_ikp", "rowctl", "ctl"}, ij_keymap)
+        , a_rows_of_col_(a_rows_of_col)
         , a_cols_of_row_(a_cols_of_row)
         , b_cols_of_row_(b_cols_of_row)
         , ijk_keymap_(ijk_keymap)
         , ij_keymap_(ij_keymap)
-        , parallel_bcasts_(parallel_bcasts) {}
+        , parallel_bcasts_(parallel_bcasts) {
+
+      this->set_priomap([](const Key<2>& key){
+        return std::numeric_limits<int>::max() - key[0];
+      });
+    }
 
     void op(const Key<2> &ik, typename baseT::input_values_tuple_type &&a_ik,
-            std::tuple<Out<Key<3>, Blk>, Out<Key<2>, void>> &outs) {
-      auto i = ik[0]; // row
-      auto k = ik[1]; // col
+            std::tuple<Out<Key<3>, Blk>, Out<Key<2>, int>, Out<Key<2>, void>> &outs) {
+      const auto i = ik[0]; // row
+      const auto k = ik[1]; // col
       ttg::trace("BcastA(", i, ", ", k, ")");
       std::vector<Key<3>> ikp_keys;
 
@@ -428,19 +464,58 @@ class SpMM25D {
             {i, j, k}));  // N.B. in 2.5D SUMMA different k contributions to C[i][j] are computed on different nodes
         if (!procmap[p]) {
           ttg::trace("Broadcasting A[", i, "][", k, "] to proc ", p);
+          //std::cout << "[" << world.rank() << "] BcastA key " << ik << " op " << Key<3>({i, j, k}) << " to proc " << p << std::endl;
           ikp_keys.emplace_back(Key<3>({i, k, p}));
           procmap[p] = true;
         }
+        // TODO: debug
+        //if (p != world.rank() && ij_keymap_(Key<2>{k, j}) != p) {
+        //  std::cout << "[" << world.rank() << "] BCAST A " << ik << " for C update " << Key<3>({i, k, p}) << " on " << p << " has B from " << ij_keymap_(Key<2>{k, j}) << std::endl;
+        //}
       }
       ::broadcast<0>(ikp_keys, std::move(baseT::template get<0>(a_ik)), outs);
 
+      /* enable the next broadcast on this row */
+      int row = i;
+      int col = k;
+      auto rowit = std::find(a_cols_of_row_[row].begin(), a_cols_of_row_[row].end(), col);
+      for (++rowit; rowit != a_cols_of_row_[row].end(); ++rowit) {
+        Key<2> key = {row, *rowit};
+        if (world.rank() == this->get_keymap()(key)) {
+          ::send<1>(key, std::move(baseT::template get<1>(a_ik)), outs);
+          break;
+        }
+      }
+
+
       /* enable next broadcast through a control message
        * we don't check whether this tile is in B here, this is
-       * done inside the next task (see above) */
-      std::size_t num_rows = a_cols_of_row_.size();
-      auto it = std::find(a_cols_of_row_[i].begin(), a_cols_of_row_[i].end(), k);
-      ++it; // skip the current tile
+       * done inside the next task (see above)
+       * we walk the matrix A column-major in an attempt to send from top to bottom, left to right */
       long to_skip = parallel_bcasts_;
+
+      auto colit = std::find(a_rows_of_col_[col].begin(), a_rows_of_col_[col].end(), row);
+      ++colit; // skip to next row
+      do {
+        for (; colit != a_rows_of_col_[col].end(); ++colit) {
+          Key<2> key = {*colit, col};
+          if (world.rank() == this->get_keymap()(key)) {
+            if (0 == --to_skip) {
+              //std::cout << "BcastA sending to " << key << " from " << ik << std::endl;
+              ::sendk<2>(key, outs);
+              return;
+            }
+          }
+        }
+        /* nothing for us in this column, move on to the next column */
+        if (++col < a_rows_of_col_.size()) {
+          colit = a_rows_of_col_[col].begin();
+        } else {
+          break;
+        }
+      } while (1);
+
+#if 0
       do {
         for (; it != a_cols_of_row_[i].end(); ++it) {
           Key<2> key = {i, *it};
@@ -457,9 +532,12 @@ class SpMM25D {
           break;
         }
       } while (1);
+#endif // 0
     }
 
    private:
+    //const std::vector<std::vector<long>> &a_cols_of_row_;
+    const std::vector<std::vector<long>> &a_rows_of_col_;
     const std::vector<std::vector<long>> &a_cols_of_row_;
     const std::vector<std::vector<long>> &b_cols_of_row_;
     const Keymap3 &ijk_keymap_;
@@ -505,25 +583,32 @@ class SpMM25D {
   };  // class LocalBcastB
 
   /// broadcast `B[k][j]` to all processors which will contain at least one `C[i][j]` such that `A[i][k]` exists
-  class BcastB : public TT<Key<2>, std::tuple<Out<Key<3>, Blk>, Out<Key<2>, void>>, BcastB, ttg::typelist<Blk, void>> {
+  class BcastB : public TT<Key<2>, std::tuple<Out<Key<3>, Blk>, Out<Key<2>, int>, Out<Key<2>, void>>, BcastB, ttg::typelist<Blk, int, void>> {
    public:
     using baseT = typename BcastB::ttT;
 
-    BcastB(Edge<Key<2>, Blk> &b_kj, Edge<Key<2>, void> ctl, Edge<Key<3>, Blk> &b_kjp,
+    BcastB(Edge<Key<2>, Blk> &b_kj, Edge<Key<2>, void> ctl, Edge<Key<2>, int> colctl, Edge<Key<3>, Blk> &b_kjp,
            const std::vector<std::vector<long>> &a_rows_of_col,
            const std::vector<std::vector<long>> &b_cols_of_row,
+           const std::vector<std::vector<long>> &b_rows_of_col,
            const Keymap2 &ij_keymap, const Keymap3 &ijk_keymap,
            const int parallel_bcasts)
-        : baseT(edges(b_kj, ctl), edges(b_kjp, ctl), "SpMM25D::bcast_b", {"b_kj", "ctl"}, {"b_kjp", "ctl"}, ij_keymap)
+        : baseT(edges(b_kj, colctl, ctl), edges(b_kjp, colctl, ctl), "SpMM25D::bcast_b", {"b_kj", "colctl", "ctl"}, {"b_kjp", "colctl", "ctl"}, ij_keymap)
         , a_rows_of_col_(a_rows_of_col)
         , b_cols_of_row_(b_cols_of_row)
+        , b_rows_of_col_(b_rows_of_col)
         , ijk_keymap_(ijk_keymap)
-        , parallel_bcasts_(parallel_bcasts) {}
+        , parallel_bcasts_(parallel_bcasts)
+    {
+      this->set_priomap([](const Key<2>& key){
+        return std::numeric_limits<int>::max() - key[1];
+      });
+    }
 
     void op(const Key<2> &kj, typename baseT::input_values_tuple_type &&b_kj,
-            std::tuple<Out<Key<3>, Blk>, Out<Key<2>, void>> &outs) {
-      auto k = kj[0]; // row
-      auto j = kj[1]; // col
+            std::tuple<Out<Key<3>, Blk>, Out<Key<2>, int>, Out<Key<2>, void>> &outs) {
+      const auto k = kj[0]; // row
+      const auto j = kj[1]; // col
       // broadcast b_kj to all processors which will contain at least one c_ij such that a_ik exists
       std::vector<Key<3>> kjp_keys;
       ttg::trace("BcastB(", k, ", ", j, ")");
@@ -534,15 +619,58 @@ class SpMM25D {
         long p = ijk_keymap_(Key<3>({i, j, k}));
         if (!procmap[p]) {
           ttg::trace("Broadcasting B[", k, "][", j, "] to proc ", p);
+          //std::cout << "[" << world.rank() << "] BcastB key " << kj << " op " << Key<3>({i, j, k}) << " to proc " << p << std::endl;
           kjp_keys.emplace_back(Key<3>({k, j, p}));
           procmap[p] = true;
         }
       }
       ::broadcast<0>(kjp_keys, std::move(baseT::template get<0>(b_kj)), outs);
 
+      /* enable the next broadcast on this row */
+      int row = k;
+      int col = j;
+      auto colit = std::find(b_rows_of_col_[col].begin(), b_rows_of_col_[col].end(), row);
+      for (++colit; colit != b_rows_of_col_[col].end(); ++colit) {
+        Key<2> key = {*colit, col};
+        if (world.rank() == this->get_keymap()(key)) {
+          //std::cout << "BcastB kick off " << key << std::endl;
+          ::send<1>(key, std::move(baseT::template get<1>(b_kj)), outs);
+          break;
+        }
+      }
+
       /* enable next broadcast through a control message
        * we don't check whether this tile is in A here, this is
-       * done inside the next task (see above) */
+       * done inside the next task (see above)
+       * we run across a row to enable broadcasts */
+      long to_skip = parallel_bcasts_;
+
+      // iterator over the current row
+      auto rowit = std::find(b_cols_of_row_[row].begin(), b_cols_of_row_[row].end(), col);
+      ++rowit; // skip to next col
+      do {
+        for (; rowit != b_cols_of_row_[row].end(); ++rowit) {
+          Key<2> key = {row, *rowit};
+          if (world.rank() == this->get_keymap()(key)) {
+            if (0 == --to_skip) {
+              //std::cout << "BcastB sending to " << key << " from " << kj << " pb " << parallel_bcasts_ << std::endl;
+              ::sendk<2>(key, outs);
+              return;
+            } else {
+              //std::cout << "BcastB skipping " << key << " from " << kj << " pb " << parallel_bcasts_ << std::endl;
+            }
+          }
+        }
+        /* nothing for us in this row, move on to the next row */
+        if (++row != b_cols_of_row_.size()) {
+          rowit = b_cols_of_row_[row].begin();
+        } else {
+          break;
+        }
+      } while (1);
+
+
+#if 0
       std::size_t num_rows = b_cols_of_row_.size();
       auto it = std::find(b_cols_of_row_[k].begin(), b_cols_of_row_[k].end(), j);
       ++it; // skip the current tile
@@ -563,11 +691,13 @@ class SpMM25D {
           break;
         }
       } while (1);
+#endif // 0
     }
 
    private:
     const std::vector<std::vector<long>> &a_rows_of_col_;
     const std::vector<std::vector<long>> &b_cols_of_row_;
+    const std::vector<std::vector<long>> &b_rows_of_col_;
     const Keymap3 &ijk_keymap_;
     const int parallel_bcasts_;
   };  // class BcastB
@@ -1376,7 +1506,7 @@ static void timed_measurement(SpMatrix<> &A, SpMatrix<> &B, const std::function<
   SpMatrix<> C;
   C.resize(MT, NT);
 
-  reduce_count = 0;
+  //reduce_count = 0;
 
   // flow graph needs to exist on every node
   Edge<Key<3>> ctl("control");

From 59779c002f9e06cc3bea6ca1115df4ceef36a136 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 2 Feb 2024 17:52:30 -0500
Subject: [PATCH 238/259] Bump PaRSEC sha

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 cmake/modules/ExternalDependenciesVersions.cmake | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/cmake/modules/ExternalDependenciesVersions.cmake b/cmake/modules/ExternalDependenciesVersions.cmake
index 8ea99c19c..e27f33347 100644
--- a/cmake/modules/ExternalDependenciesVersions.cmake
+++ b/cmake/modules/ExternalDependenciesVersions.cmake
@@ -7,7 +7,6 @@ set(TTG_TRACKED_VG_CMAKE_KIT_TAG 7ea2d4d3f8854b9e417f297fd74d6fc49aa13fd5)  # us
 set(TTG_TRACKED_CATCH2_VERSION 3.5.0)
 set(TTG_TRACKED_CEREAL_VERSION 1.3.0)
 set(TTG_TRACKED_MADNESS_TAG 2eb3bcf0138127ee2dbc651f1aabd3e9b0def4e3)
-#set(TTG_TRACKED_PARSEC_TAG 25d1931e863b6741e453112d2117d85ad32e7fba)
-set(TTG_TRACKED_PARSEC_TAG 8ed7cdf02ad607051c130a6deeb8f12c84f2e4d3)
+set(TTG_TRACKED_PARSEC_TAG 7cf50058cabffcb0453c243a1b5dc4383ec0105d)
 set(TTG_TRACKED_BTAS_TAG 4e8f5233aa7881dccdfcc37ce07128833926d3c2)
 set(TTG_TRACKED_TILEDARRAY_TAG 493c109379a1b64ddd5ef59f7e33b95633b68d73)

From 94b2b7554e8bb2cb4b17566eb25d1def043c8697 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Wed, 14 Feb 2024 09:35:51 -0500
Subject: [PATCH 239/259] Bump PaRSEC version for MPI backend instrumentation

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 cmake/modules/ExternalDependenciesVersions.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/modules/ExternalDependenciesVersions.cmake b/cmake/modules/ExternalDependenciesVersions.cmake
index e27f33347..8fbeda386 100644
--- a/cmake/modules/ExternalDependenciesVersions.cmake
+++ b/cmake/modules/ExternalDependenciesVersions.cmake
@@ -7,6 +7,6 @@ set(TTG_TRACKED_VG_CMAKE_KIT_TAG 7ea2d4d3f8854b9e417f297fd74d6fc49aa13fd5)  # us
 set(TTG_TRACKED_CATCH2_VERSION 3.5.0)
 set(TTG_TRACKED_CEREAL_VERSION 1.3.0)
 set(TTG_TRACKED_MADNESS_TAG 2eb3bcf0138127ee2dbc651f1aabd3e9b0def4e3)
-set(TTG_TRACKED_PARSEC_TAG 7cf50058cabffcb0453c243a1b5dc4383ec0105d)
+set(TTG_TRACKED_PARSEC_TAG 0b3140f58ad9dc78a3d64da9fd73ecc7f443ece7)
 set(TTG_TRACKED_BTAS_TAG 4e8f5233aa7881dccdfcc37ce07128833926d3c2)
 set(TTG_TRACKED_TILEDARRAY_TAG 493c109379a1b64ddd5ef59f7e33b95633b68d73)

From c06654f95b5bd2ff61e25e77d7a5539e783152d5 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 23 Feb 2024 13:57:31 -0500
Subject: [PATCH 240/259] Add missing <numeric> header for CI

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/ttg.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index b9edccad4..1dcbd6878 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -59,6 +59,7 @@
 #include <map>
 #include <memory>
 #include <mutex>
+#include <numeric>
 #include <sstream>
 #include <string>
 #include <tuple>

From b8c2cc8aa5dd458f97ecdb084a6b887345e453c7 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Fri, 23 Feb 2024 13:58:43 -0500
Subject: [PATCH 241/259] Order broadcast keys so that we send to myrank+1
 first

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/ttg.h | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 1dcbd6878..2ba474305 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -2817,6 +2817,7 @@ ttg::abort();  // should not happen
     broadcast_arg(const ttg::span<const Key> &keylist, const Value &value) {
       using valueT = std::tuple_element_t<i, input_values_full_tuple_type>;
       auto world = ttg_default_execution_context();
+      auto np = world.size();
       int rank = world.rank();
       uint64_t pos = 0;
       bool have_remote = keylist.end() != std::find_if(keylist.begin(), keylist.end(),
@@ -2830,7 +2831,10 @@ ttg::abort();  // should not happen
         std::sort(keylist_sorted.begin(), keylist_sorted.end(), [&](const Key &a, const Key &b) mutable {
           int rank_a = keymap(a);
           int rank_b = keymap(b);
-          return rank_a < rank_b;
+          // sort so that the keys for my rank are first, rank+1 next, ..., wrapping around to 0
+          int pos_a = (rank_a + np - rank) % np;
+          int pos_b = (rank_b + np - rank) % np;
+          return pos_a < pos_b;
         });
 
         /* Assuming there are no local keys, will be updated while iterating over the keys */

From 67b0fb3af22658147d8bcf63b78945eba73f77ac Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Fri, 23 Feb 2024 14:20:49 -0500
Subject: [PATCH 242/259] Reverted ReadA/B additions

 Partially reverts 1ffaf6aa00b96a39bd74456f1effda222ae1ed20 and 3dad2c1d5fd394d17f40b506efe82d02542ec58f
---
 examples/spmm/spmm.cc | 291 ++----------------------------------------
 1 file changed, 14 insertions(+), 277 deletions(-)

diff --git a/examples/spmm/spmm.cc b/examples/spmm/spmm.cc
index f5b9812ee..3e9d1d3f8 100644
--- a/examples/spmm/spmm.cc
+++ b/examples/spmm/spmm.cc
@@ -180,20 +180,23 @@ inline int ijk2rank(int i, int j, int k, int P, int Q, int R) {
 template <typename Blk = blk_t,
           typename Keymap = std::function<int(const Key<3> &)>,
           typename OutKeymap = std::function<int(const Key<2> &)>>
-class Read_SpMatrix : public TT<Key<3>, std::tuple<Out<Key<2>, Blk>>, Read_SpMatrix<Blk, Keymap>, ttg::typelist<void>> {
+class Read_SpMatrix : public TT<Key<3>,
+                                std::tuple<Out<Key<2>, Blk>>,
+                                Read_SpMatrix<Blk, Keymap, OutKeymap>,
+                                ttg::typelist<void>> {
  public:
   using baseT = typename Read_SpMatrix::ttT;
   Read_SpMatrix(const char *label, const SpMatrix<Blk> &matrix, Edge<Key<3>> &ctl, Edge<Key<2>, Blk> &out,
-                Keymap &ijk_keymap, std::function<int(const Key<2> &)> ij_keymap)
+                Keymap &pqr_keymap, std::function<int(const Key<2> &)> ij_keymap)
       : baseT(edges(ctl), edges(out), std::string("read_spmatrix(") + label + ")", {"ctl"}, {std::string(label) + "ij"},
-              ijk_keymap)
+              pqr_keymap)
       , matrix_(matrix)
-      , keymap_(ij_keymap) {}
+      , ij_keymap_(ij_keymap) {}
 
   // key is this process' coordinate in the 2-d grid of processes (managed by ij_keymap) ...
   // but it's not used at all since all this TT does is generate consuming tasks that use local tiles ...
   // the consumers better use same keymap (ij_keymap) as this TT to avoid for the data flow from this to be local
-  void op(const Key<3> & /* pq */, std::tuple<Out<Key<2>, Blk>> &out) {
+  void op(const Key<3> & /* pqr */, std::tuple<Out<Key<2>, Blk>> &out) {
     auto rank = ttg::default_execution_context().rank();
     // this code assumes col-major layout
     static_assert(SpMatrix<Blk>::IsRowMajor == false, "SpMatrix must be col-major");
@@ -202,7 +205,7 @@ class Read_SpMatrix : public TT<Key<3>, std::tuple<Out<Key<2>, Blk>>, Read_SpMat
         assert(j == it.col());
         const auto i = it.row();
         // IF the receiver uses the same keymap, these sends are local
-        if (rank == this->get_keymap()(Key<2>(std::initializer_list<long>({i, j})))) {
+        if (rank == this->ij_keymap_(Key<2>(std::initializer_list<long>({i, j})))) {
           ::send<0>(Key<2>(std::initializer_list<long>({i, j})), it.value(), out);
         }
       }
@@ -211,265 +214,7 @@ class Read_SpMatrix : public TT<Key<3>, std::tuple<Out<Key<2>, Blk>>, Read_SpMat
 
  private:
   const SpMatrix<Blk> &matrix_;
-  std::function<int(const Key<2> &)> keymap_;
-};
-
-enum class ReadSchedule {
-  SingleK,   // appropriate for 2D (see ReadA)
-  MultipleK  // appropriate for 3D (see ReadA)
-};
-// change this to control the schedule of sends (2-D vs 3-D)
-constexpr auto DefaultReadSchedule = ReadSchedule::SingleK;
-
-/// flow data from A distributed on a 2-d grid of processes in the order they are likely to be consumed
-
-/// The order of sends needs to tailored as follows:
-/// - for 2-D SUMMA (R=1): read A[i][k] for all i and k=0 first, then k=1, etc. Clearly, Read_SpMatrix is going to
-/// generate reads in the wrong (transposed) order (send all k for i=0, then for i=1, etc.).
-/// - for 2.5/3-D SUMMA (R>1): same order of sends as for 2-D SUMMA will be suboptimal since all k=0 will only
-/// generate work on the r=0 process plane. Instead we *may* want to one tile needed on each plane, then one more for
-/// each plane. Hence the need for ReadSchedule.
-template <ReadSchedule KSchedule = DefaultReadSchedule, typename Blk = blk_t, typename Keymap2 = std::function<int(const Key<2> &)>>
-class ReadA : public TT<Key<2>, std::tuple<Out<Key<2>, Blk>>, ReadA<KSchedule, Blk, Keymap2>, ttg::typelist<void>> {
- public:
-  using baseT = typename ReadA::ttT;
-  ReadA(const SpMatrix<Blk> &matrix, Edge<Key<2>> &ctl, Edge<Key<2>, Blk> &out, const Keymap2 &ij_keymap, long R)
-      : baseT(edges(ctl), edges(out), std::string("SpMM25D::read_a"), {"ctl"}, {"a_ik"}, ij_keymap)
-      , matrix_(matrix)
-      , R_(R) {}
-
-  // key is this process' coordinate in the 2-d grid of processes (managed by ij_keymap) ...
-  // but it's not used at all since all this TT does is generate consuming tasks that use local tiles ...
-  // the consumers better use same keymap (ij_keymap) as this TT to avoid for the data flow from this to be local
-  void op(const Key<2> & /* pq */, std::tuple<Out<Key<2>, Blk>> &out) {
-    auto rank = ttg::default_execution_context().rank();
-    const int I = matrix_.rows();
-    const int K = matrix_.cols();
-
-    // this assumes col-major layout of SpMatrix
-    static_assert(SpMatrix<Blk>::IsRowMajor == false, "SpMatrix must be col-major");
-
-    // loop over blocks of k at a time, block size controlled by KSchedule
-    const int k_blk_size = (KSchedule == ReadSchedule::SingleK) ? 1 : R_;
-
-    // this keeps iterators over each k-column in this block
-    // there is only 1 task per process, so no need to synchronize access to this
-    static std::vector<typename SpMatrix<Blk>::InnerIterator> column_iterators;
-
-    for (std::pair<int, int> k_blk = {0, std::min(k_blk_size, K)}; k_blk.first < K;
-         k_blk = {k_blk.first + k_blk_size, std::min(k_blk.first + k_blk_size + k_blk_size, K)}) {
-
-      // for each k in the block send one A[i][k], then next i, etc. this means keep track of iterators over k-column
-      // for each k in the block
-      // N.B. : due to the CSC layout of A iterating over (blocks of) columns is efficient
-      column_iterators.resize(0);
-      for (int k = k_blk.first; k < k_blk.second; ++k) {
-        column_iterators.emplace_back(matrix_, k);
-      }
-
-      int k_remaining = k_blk.second - k_blk.first;
-      while(k_remaining != 0) {
-        for (int k = k_blk.first, k_in_blk = 0; k < k_blk.second; ++k, ++k_in_blk) {
-          if (auto& it = column_iterators[k_in_blk]) {
-            assert(k == it.col());
-            const auto i = it.row();
-            // IF the receiver uses the same keymap, these sends are local
-            if (rank == this->get_keymap()(Key<2>(std::initializer_list<long>({i, k})))) {
-              ::send<0>(Key<2>(std::initializer_list<long>({i, k})), it.value(), out);
-            }
-            ++it;
-          }
-          else { // this k is done
-            assert(k_remaining != 0);
-            --k_remaining;
-          }
-        }
-      }
-
-    }
-
-  }
-
- private:
-  const SpMatrix<Blk> &matrix_;
-  long R_;
-};
-
-// flow data from an existing SpMatrix on rank 0
-template <ReadSchedule KSchedule = DefaultReadSchedule, typename Blk = blk_t, typename Keymap2 = std::function<int(const Key<2> &)>>
-class ReadB : public TT<Key<2>, std::tuple<Out<Key<2>, Blk>>, ReadB<KSchedule, Blk, Keymap2>, ttg::typelist<void>> {
- public:
-  using baseT = typename ReadB::ttT;
-  ReadB(const SpMatrix<Blk> &matrix, Edge<Key<2>> &ctl, Edge<Key<2>, Blk> &out, const Keymap2 &ij_keymap, long R)
-      : baseT(edges(ctl), edges(out), std::string("read_b"), {"ctl"}, {"b_kj"}, ij_keymap), matrix_(matrix), R_(R) {}
-
-  // key is this process' coordinate in the 2-d grid of processes (managed by ij_keymap) ...
-  // but it's not used at all since all this TT does is generate consuming tasks that use local tiles ...
-  // the consumers better use same keymap (ij_keymap) as this TT to avoid for the data flow from this to be local
-  void op(const Key<2> & /* pq */, std::tuple<Out<Key<2>, Blk>> &out) {
-    auto rank = ttg::default_execution_context().rank();
-    const int J = matrix_.cols();
-    const int K = matrix_.rows();
-
-    // this assumes col-major layout of SpMatrix
-    static_assert(SpMatrix<Blk>::IsRowMajor == false, "SpMatrix must be col-major");
-
-    // loop over blocks of k at a time, block size controlled by KSchedule
-    const int k_blk_size = (KSchedule == ReadSchedule::SingleK) ? 1 : R_;
-    for (std::pair<int, int> k_blk = {0, std::min(k_blk_size, K)}; k_blk.first < K;
-      k_blk = {k_blk.first + k_blk_size, std::min(k_blk.first + k_blk_size + k_blk_size, K)}) {
-
-      // WARNING : due to the CSC layout of B iterating over (blocks of) columns is inefficient
-      for (int j = 0; j < matrix_.outerSize(); ++j) {
-        for (typename SpMatrix<Blk>::InnerIterator it(matrix_, j); it; ++it) {
-          assert(j == it.col());
-          const auto k = it.row();
-          // if k past the k block, we are done with this i
-          if (k >= k_blk.second) break;
-          // continue iterating until k has not reached this k block
-          if (k < k_blk.first) continue;
-          // IF the receiver uses the same keymap, these sends are local
-          if (rank == this->get_keymap()(Key<2>(std::initializer_list<long>({k, j})))) {
-            ::send<0>(Key<2>(std::initializer_list<long>({k, j})), it.value(), out);
-          }
-        }
-
-      }
-    }
-  }
-
- private:
-  const SpMatrix<Blk> &matrix_;
-  long R_;
-};
-
-enum class ReadSchedule {
-  SingleK,   // appropriate for 2D (see ReadA)
-  MultipleK  // appropriate for 3D (see ReadA)
-};
-// change this to control the schedule of sends (2-D vs 3-D)
-constexpr auto DefaultReadSchedule = ReadSchedule::SingleK;
-
-/// flow data from A distributed on a 2-d grid of processes in the order they are likely to be consumed
-
-/// The order of sends needs to tailored as follows:
-/// - for 2-D SUMMA (R=1): read A[i][k] for all i and k=0 first, then k=1, etc. Clearly, Read_SpMatrix is going to
-/// generate reads in the wrong (transposed) order (send all k for i=0, then for i=1, etc.).
-/// - for 2.5/3-D SUMMA (R>1): same order of sends as for 2-D SUMMA will be suboptimal since all k=0 will only
-/// generate work on the r=0 process plane. Instead we *may* want to one tile needed on each plane, then one more for
-/// each plane. Hence the need for ReadSchedule.
-template <ReadSchedule KSchedule = DefaultReadSchedule, typename Blk = blk_t, typename Keymap2 = std::function<int(const Key<2> &)>>
-class ReadA : public TT<Key<2>, std::tuple<Out<Key<2>, Blk>>, ReadA<KSchedule, Blk, Keymap2>, ttg::typelist<void>> {
- public:
-  using baseT = typename ReadA::ttT;
-  ReadA(const SpMatrix<Blk> &matrix, Edge<Key<2>> &ctl, Edge<Key<2>, Blk> &out, const Keymap2 &ij_keymap, long R)
-      : baseT(edges(ctl), edges(out), std::string("SpMM25D::read_a"), {"ctl"}, {"a_ik"}, ij_keymap)
-      , matrix_(matrix)
-      , R_(R) {}
-
-  // key is this process' coordinate in the 2-d grid of processes (managed by ij_keymap) ...
-  // but it's not used at all since all this TT does is generate consuming tasks that use local tiles ...
-  // the consumers better use same keymap (ij_keymap) as this TT to avoid for the data flow from this to be local
-  void op(const Key<2> & /* pq */, std::tuple<Out<Key<2>, Blk>> &out) {
-    auto rank = ttg::default_execution_context().rank();
-    const int I = matrix_.rows();
-    const int K = matrix_.cols();
-
-    // this assumes col-major layout of SpMatrix
-    static_assert(SpMatrix<Blk>::IsRowMajor == false, "SpMatrix must be col-major");
-
-    // loop over blocks of k at a time, block size controlled by KSchedule
-    const int k_blk_size = (KSchedule == ReadSchedule::SingleK) ? 1 : R_;
-
-    // this keeps iterators over each k-column in this block
-    // there is only 1 task per process, so no need to synchronize access to this
-    static std::vector<typename SpMatrix<Blk>::InnerIterator> column_iterators;
-
-    for (std::pair<int, int> k_blk = {0, std::min(k_blk_size, K)}; k_blk.first < K;
-         k_blk = {k_blk.first + k_blk_size, std::min(k_blk.first + k_blk_size + k_blk_size, K)}) {
-
-      // for each k in the block send one A[i][k], then next i, etc. this means keep track of iterators over k-column
-      // for each k in the block
-      // N.B. : due to the CSC layout of A iterating over (blocks of) columns is efficient
-      column_iterators.resize(0);
-      for (int k = k_blk.first; k < k_blk.second; ++k) {
-        column_iterators.emplace_back(matrix_, k);
-      }
-
-      int k_remaining = k_blk.second - k_blk.first;
-      while(k_remaining != 0) {
-        for (int k = k_blk.first, k_in_blk = 0; k < k_blk.second; ++k, ++k_in_blk) {
-          if (auto& it = column_iterators[k_in_blk]) {
-            assert(k == it.col());
-            const auto i = it.row();
-            // IF the receiver uses the same keymap, these sends are local
-            if (rank == this->get_keymap()(Key<2>(std::initializer_list<long>({i, k})))) {
-              ::send<0>(Key<2>(std::initializer_list<long>({i, k})), it.value(), out);
-            }
-            ++it;
-          }
-          else { // this k is done
-            assert(k_remaining != 0);
-            --k_remaining;
-          }
-        }
-      }
-
-    }
-
-  }
-
- private:
-  const SpMatrix<Blk> &matrix_;
-  long R_;
-};
-
-// flow data from an existing SpMatrix on rank 0
-template <ReadSchedule KSchedule = DefaultReadSchedule, typename Blk = blk_t, typename Keymap2 = std::function<int(const Key<2> &)>>
-class ReadB : public TT<Key<2>, std::tuple<Out<Key<2>, Blk>>, ReadB<KSchedule, Blk, Keymap2>, ttg::typelist<void>> {
- public:
-  using baseT = typename ReadB::ttT;
-  ReadB(const SpMatrix<Blk> &matrix, Edge<Key<2>> &ctl, Edge<Key<2>, Blk> &out, const Keymap2 &ij_keymap, long R)
-      : baseT(edges(ctl), edges(out), std::string("read_b"), {"ctl"}, {"b_kj"}, ij_keymap), matrix_(matrix), R_(R) {}
-
-  // key is this process' coordinate in the 2-d grid of processes (managed by ij_keymap) ...
-  // but it's not used at all since all this TT does is generate consuming tasks that use local tiles ...
-  // the consumers better use same keymap (ij_keymap) as this TT to avoid for the data flow from this to be local
-  void op(const Key<2> & /* pq */, std::tuple<Out<Key<2>, Blk>> &out) {
-    auto rank = ttg::default_execution_context().rank();
-    const int J = matrix_.cols();
-    const int K = matrix_.rows();
-
-    // this assumes col-major layout of SpMatrix
-    static_assert(SpMatrix<Blk>::IsRowMajor == false, "SpMatrix must be col-major");
-
-    // loop over blocks of k at a time, block size controlled by KSchedule
-    const int k_blk_size = (KSchedule == ReadSchedule::SingleK) ? 1 : R_;
-    for (std::pair<int, int> k_blk = {0, std::min(k_blk_size, K)}; k_blk.first < K;
-      k_blk = {k_blk.first + k_blk_size, std::min(k_blk.first + k_blk_size + k_blk_size, K)}) {
-
-      // WARNING : due to the CSC layout of B iterating over (blocks of) columns is inefficient
-      for (int j = 0; j < matrix_.outerSize(); ++j) {
-        for (typename SpMatrix<Blk>::InnerIterator it(matrix_, j); it; ++it) {
-          assert(j == it.col());
-          const auto k = it.row();
-          // if k past the k block, we are done with this i
-          if (k >= k_blk.second) break;
-          // continue iterating until k has not reached this k block
-          if (k < k_blk.first) continue;
-          // IF the receiver uses the same keymap, these sends are local
-          if (rank == this->get_keymap()(Key<2>(std::initializer_list<long>({k, j})))) {
-            ::send<0>(Key<2>(std::initializer_list<long>({k, j})), it.value(), out);
-          }
-        }
-
-      }
-    }
-  }
-
- private:
-  const SpMatrix<Blk> &matrix_;
-  long R_;
+  std::function<int(const Key<2> &)> ij_keymap_;
 };
 
 // flow (move?) data into an existing SpMatrix on rank 0
@@ -1775,7 +1520,10 @@ static void timed_measurement(SpMatrix<> &A, SpMatrix<> &B, const std::function<
   SpMatrix<> C;
   C.resize(MT, NT);
 
-  //reduce_count = 0;
+  /* the Read_SpMatrix tasks get process coordinates, not tile coordinates  */
+  auto read_keymap = [&](const Key<3>& key){
+    return ijk2rank(key[0], key[1], key[2], P, Q, R);
+  };
 
   // flow graph needs to exist on every node
   Edge<Key<3>> ctl("control");
@@ -1783,16 +1531,8 @@ static void timed_measurement(SpMatrix<> &A, SpMatrix<> &B, const std::function<
   Edge<Key<2>, blk_t> eA, eB;
   Edge<Key<2>, blk_t> eC;
 
-  /* the Read_SpMatrix tasks get process coordinates, not tile coordinates  */
-  auto read_keymap = [&](const Key<3>& key){
-      return ijk2rank(key[0], key[1], key[2], P, Q, R);
-    };
-
   Read_SpMatrix a("A", A, ctl, eA, read_keymap, ij_keymap);
   Read_SpMatrix b("B", B, ctl, eB, read_keymap, ij_keymap);
-  // uncomment this to use more intelligent schedule of reads
-//  ReadA<> a(A, ctl, eA, ij_keymap, R);
-//  ReadB<> b(B, ctl, eB, ij_keymap, R);
   Write_SpMatrix<> c(C, eC, ij_keymap, false);
   auto &c_status = c.status();
   assert(!has_value(c_status));
@@ -2142,9 +1882,6 @@ int main(int argc, char **argv) {
         Edge<Key<2>, blk_t> eA, eB, eC;
         Read_SpMatrix a("A", A, ctl, eA, read_keymap, ij_keymap);
         Read_SpMatrix b("B", B, ctl, eB, read_keymap, ij_keymap);
-        // uncomment this to use more intelligent schedule of reads
-//        ReadA<> a(A, ctl, eA, ij_keymap, R);
-//        ReadB<> b(B, ctl, eB, ij_keymap, R);
         Write_SpMatrix<> c(C, eC, keymap_write);
         auto &c_status = c.status();
         assert(!has_value(c_status));

From b741e786890a93fed490310ec06f1594f1352b36 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Fri, 23 Feb 2024 14:21:45 -0500
Subject: [PATCH 243/259] [cleanup] Design-Device.md

---
 doc/dox/contrib/Design-Device.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/dox/contrib/Design-Device.md b/doc/dox/contrib/Design-Device.md
index 2c40f64fd..2c3c576f3 100644
--- a/doc/dox/contrib/Design-Device.md
+++ b/doc/dox/contrib/Design-Device.md
@@ -75,7 +75,7 @@ Additional memory-related concerns common to both models:
 How should we map the following host task onto the device?
 ```cpp
 make_tt([](auto& key, auto& data1, auto& data2) -> void {
-    double data3 = blap::dot(data1.data(), data2.data());
+    double data3 = blas::dot(data1.data(), data2.data());
     if (data3 >= 0.)
         send<0>(data1);
     else
@@ -86,7 +86,7 @@ make_tt([](auto& key, auto& data1, auto& data2) -> void {
 Ideally the task will receive `data1` and `data2` already transferred to the memory space(s) accessible from the device execution space:
 ```cpp
 make_device_tt([](auto& key, auto& data1, auto& data2) -> void {
-    double data3 = blap::device_dot(data1.data(), data2.data());
+    double data3 = blas::device_dot(data1.data(), data2.data());
     if (data3 >= 0.)
         send<0>(data1);
     else

From ed270da8170a018bf3d9c020fb1c06e20e9519a1 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Fri, 23 Feb 2024 15:00:27 -0500
Subject: [PATCH 244/259] [dox] new vars config.in.h [skip ci]

---
 ttg/ttg/config.in.h | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/ttg/ttg/config.in.h b/ttg/ttg/config.in.h
index f04d089de..51e58b4a2 100644
--- a/ttg/ttg/config.in.h
+++ b/ttg/ttg/config.in.h
@@ -11,19 +11,28 @@
 /** the C++ namespace containing the coroutine API */
 #define TTG_CXX_COROUTINE_NAMESPACE @CXX_COROUTINE_NAMESPACE@
 
+/** whether TTG has CUDA language support */
 #cmakedefine TTG_HAVE_CUDA
 
+/** whether TTG has CUDA runtime support */
 #cmakedefine TTG_HAVE_CUDART
 
+/** whether TTG has HIP support */
 #cmakedefine TTG_HAVE_HIP
 
-#cmakedefine TTG_HAVE_LEVEL_ZERO
-
+/** whether TTG has HIP BLAS library */
 #cmakedefine TTG_HAVE_HIPBLAS
 
+/** whether TTG has Intel Level Zero support */
+#cmakedefine TTG_HAVE_LEVEL_ZERO
+
+/** whether TTG has any device programming model (CUDA/HIP/LEVEL_ZERO) support */
 #cmakedefine TTG_HAVE_DEVICE
 
+/** whether TTG has MPI library */
 #cmakedefine TTG_HAVE_MPI
+
+/** whether TTG has the mpi-ext.h header */
 #cmakedefine TTG_HAVE_MPIEXT
 
 #endif  // TTG_CONFIG_IN_H

From 12d1c1fdd0111aab50497ed23ba85ee1cd72c6e0 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Fri, 23 Feb 2024 16:02:26 -0500
Subject: [PATCH 245/259] [dox] stub for device::Device

---
 ttg/ttg/device/device.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ttg/ttg/device/device.h b/ttg/ttg/device/device.h
index fbe036612..6690982f6 100644
--- a/ttg/ttg/device/device.h
+++ b/ttg/ttg/device/device.h
@@ -16,6 +16,8 @@ namespace ttg::device {
 #else
   constexpr ttg::ExecutionSpace available_execution_space = ttg::ExecutionSpace::Invalid;
 #endif
+
+  /// Represents a device in a specific execution space
   class Device {
     int m_id = 0;
     ttg::ExecutionSpace m_space = ttg::ExecutionSpace::Host;

From 27c03f78ab4577401260848c35de27cbd506bc5d Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Fri, 23 Feb 2024 16:06:30 -0500
Subject: [PATCH 246/259] [dox] stub for ttg_madness::Buffer

---
 ttg/ttg/madness/buffer.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ttg/ttg/madness/buffer.h b/ttg/ttg/madness/buffer.h
index a68480be4..aac36c24b 100644
--- a/ttg/ttg/madness/buffer.h
+++ b/ttg/ttg/madness/buffer.h
@@ -5,6 +5,7 @@
 
 namespace ttg_madness {
 
+/// A runtime-managed buffer mirrored between host and device memory
 template<typename T, typename Allocator>
 struct Buffer : private Allocator {
 

From fcbd2e8ec330c2175f9735ed73e8e1c66eff93d8 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Fri, 23 Feb 2024 16:59:48 -0500
Subject: [PATCH 247/259] [cleanup] device/task.h

---
 ttg/ttg/device/task.h | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/ttg/ttg/device/task.h b/ttg/ttg/device/task.h
index ea7c6e9a0..55cf5f0d5 100644
--- a/ttg/ttg/device/task.h
+++ b/ttg/ttg/device/task.h
@@ -88,7 +88,7 @@ namespace ttg::device {
 
     using send_coro_handle_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<send_coro_promise_type>;
 
-    /// task that can be resumed after some events occur
+    /// a coroutine for sending data from the device
     struct send_coro_state : public send_coro_handle_type {
       using base_type = send_coro_handle_type;
 
@@ -112,6 +112,7 @@ namespace ttg::device {
       inline bool completed();
     };
 
+    /// the promise type for the send coroutine
     struct send_coro_promise_type {
 
       /* do not suspend the coroutine on first invocation, we want to run
@@ -145,8 +146,8 @@ namespace ttg::device {
     };
 
     template<typename Key, typename Value, ttg::Runtime Runtime = ttg::ttg_runtime>
-    inline send_coro_state send_coro (const Key& key, Value&& value, ttg::Out<Key, std::decay_t<Value>> &t,
-                                      ttg::detail::value_copy_handler<Runtime>& ch) {
+    inline send_coro_state send_coro(const Key& key, Value&& value, ttg::Out<Key, std::decay_t<Value>> &t,
+                                     ttg::detail::value_copy_handler<Runtime>& ch) {
       ttg::detail::value_copy_handler<Runtime> copy_handler = std::move(ch); // destroyed at the end of the coro
       Key k = key;
       t.prepare_send(k, std::forward<Value>(value));
@@ -155,8 +156,8 @@ namespace ttg::device {
     };
 
     template<typename Value, ttg::Runtime Runtime = ttg::ttg_runtime>
-    inline send_coro_state sendv_coro (Value&& value, ttg::Out<void, std::decay_t<Value>> &t,
-                                        ttg::detail::value_copy_handler<Runtime>& ch) {
+    inline send_coro_state sendv_coro(Value&& value, ttg::Out<void, std::decay_t<Value>> &t,
+                                      ttg::detail::value_copy_handler<Runtime>& ch) {
       ttg::detail::value_copy_handler<Runtime> copy_handler = std::move(ch); // destroyed at the end of the coro
       t.prepare_send(std::forward<Value>(value));
       co_await ttg::Void{}; // we'll come back once the task is done
@@ -164,7 +165,7 @@ namespace ttg::device {
     };
 
     template<typename Key, ttg::Runtime Runtime = ttg::ttg_runtime>
-    inline send_coro_state sendk_coro (const Key& key, ttg::Out<Key, void> &t) {
+    inline send_coro_state sendk_coro(const Key& key, ttg::Out<Key, void> &t) {
       // no need to prepare the send but we have to suspend once
       Key k = key;
       co_await ttg::Void{}; // we'll come back once the task is done
@@ -172,7 +173,7 @@ namespace ttg::device {
     };
 
     template<ttg::Runtime Runtime = ttg::ttg_runtime>
-    inline send_coro_state send_coro (ttg::Out<void, void> &t) {
+    inline send_coro_state send_coro(ttg::Out<void, void> &t) {
       // no need to prepare the send but we have to suspend once
       co_await ttg::Void{}; // we'll come back once the task is done
       t.send();
@@ -415,7 +416,14 @@ namespace ttg::device {
     using device_task_handle_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<device_task_promise_type>;
   } // namespace detail
 
-  /// task that can be resumed after some events occur
+  /// A device::Task is a coroutine (a callable that can be suspended and resumed).
+
+  /// \note Since task execution in TTG is not preempable, tasks should not block.
+  /// The purpose of suspending a task is to yield control back to the runtime until some events occur;
+  /// in the meantime its executor (e.g., a user-space thread) can perform other work.
+  /// Once the task function reaches a point where further progress is pending completion of one or more asynchronous
+  /// actions the function needs to be suspended via a coroutine await (`co_await`).
+  /// Resumption will be handled by the runtime.
   struct Task : public detail::device_task_handle_type {
     using base_type = detail::device_task_handle_type;
 

From 9cd39b8b4704063b5a4fd9591ae46978edb8d877 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Fri, 23 Feb 2024 20:31:09 -0500
Subject: [PATCH 248/259] unit test informational logging interferes with
 catch_discover_tests, only produce if Catch's `verbosity` != `quiet`

---
 tests/unit/unit_main.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/unit/unit_main.cpp b/tests/unit/unit_main.cpp
index e4a923cd3..9384ebac9 100644
--- a/tests/unit/unit_main.cpp
+++ b/tests/unit/unit_main.cpp
@@ -30,7 +30,8 @@ int main(int argc, char** argv) {
   ttg::diagnose_off();  // turn off diagnostics
 
   const auto nranks = ttg::default_execution_context().size();
-  std::cout << "ready to run TTG unit tests with " << nranks << " rank" << (nranks > 1 ? "s" : "") << std::endl;
+  if (session.config().verbosity() != Catch::Verbosity::Quiet)
+    std::cout << "ready to run TTG unit tests with " << nranks << " rank" << (nranks > 1 ? "s" : "") << std::endl;
 
   ttg::execute();
 #endif

From 8104b49b46d6a7e5382e014a85d162ad2aba3c69 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Sat, 24 Feb 2024 00:34:02 -0500
Subject: [PATCH 249/259] pass required Boost version to FindOrFetchBoost

---
 cmake/modules/ExternalDependenciesVersions.cmake |  5 +++--
 cmake/modules/FindOrFetchBoost.cmake             | 13 +++++++++++++
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/cmake/modules/ExternalDependenciesVersions.cmake b/cmake/modules/ExternalDependenciesVersions.cmake
index 8fbeda386..88cf52b11 100644
--- a/cmake/modules/ExternalDependenciesVersions.cmake
+++ b/cmake/modules/ExternalDependenciesVersions.cmake
@@ -1,8 +1,6 @@
 # for each dependency track both current and previous id (the variable for the latter must contain PREVIOUS)
 # to be able to auto-update them
 
-# need Boost.CallableTraits (header only, part of Boost 1.66 released in Dec 2017) for wrap.h to work
-set(TTG_TRACKED_BOOST_VERSION 1.66)
 set(TTG_TRACKED_VG_CMAKE_KIT_TAG 7ea2d4d3f8854b9e417f297fd74d6fc49aa13fd5)  # used to provide "real" FindOrFetchBoost
 set(TTG_TRACKED_CATCH2_VERSION 3.5.0)
 set(TTG_TRACKED_CEREAL_VERSION 1.3.0)
@@ -10,3 +8,6 @@ set(TTG_TRACKED_MADNESS_TAG 2eb3bcf0138127ee2dbc651f1aabd3e9b0def4e3)
 set(TTG_TRACKED_PARSEC_TAG 0b3140f58ad9dc78a3d64da9fd73ecc7f443ece7)
 set(TTG_TRACKED_BTAS_TAG 4e8f5233aa7881dccdfcc37ce07128833926d3c2)
 set(TTG_TRACKED_TILEDARRAY_TAG 493c109379a1b64ddd5ef59f7e33b95633b68d73)
+
+# need Boost.CallableTraits (header only, part of Boost 1.66 released in Dec 2017) for wrap.h to work
+set(TTG_OLDEST_BOOST_VERSION 1.66)
diff --git a/cmake/modules/FindOrFetchBoost.cmake b/cmake/modules/FindOrFetchBoost.cmake
index dc1a0a654..1e133eee0 100644
--- a/cmake/modules/FindOrFetchBoost.cmake
+++ b/cmake/modules/FindOrFetchBoost.cmake
@@ -1,3 +1,16 @@
+# update the Boost version that we can tolerate
+if (NOT DEFINED Boost_OLDEST_BOOST_VERSION)
+    set(Boost_OLDEST_BOOST_VERSION ${TTG_OLDEST_BOOST_VERSION})
+else()
+    if (${Boost_OLDEST_BOOST_VERSION} VERSION_LESS ${TTG_OLDEST_BOOST_VERSION})
+        if (DEFINED CACHE{Boost_OLDEST_BOOST_VERSION})
+            set(Boost_OLDEST_BOOST_VERSION "${TTG_OLDEST_BOOST_VERSION}" CACHE STRING "Oldest Boost version to use" FORCE)
+        else()
+            set(Boost_OLDEST_BOOST_VERSION ${TTG_OLDEST_BOOST_VERSION})
+        endif()
+    endif()
+endif()
+
 # Boost can be discovered by every (sub)package but only the top package can *build* it ...
 # in either case must declare the components used by TTG
 set(required_components

From 1826821f5dc9e33812473e2da079d452393dd036 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Sat, 24 Feb 2024 00:34:33 -0500
Subject: [PATCH 250/259] HIP LANGUAGE needs CMake 3.21 or later

---
 CMakeLists.txt | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 78574acbc..59f2d583d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -32,6 +32,8 @@ project(ttg
 
 if (NOT DEFINED CMAKE_CXX_STANDARD)
     set(CMAKE_CXX_STANDARD "20" CACHE STRING "The C++ standard")
+elseif (${CMAKE_CXX_STANDARD} LESS 20)
+    message(FATAL_ERROR "TTG requires C++ compiler with C++20, but CMAKE_CXX_STANDARD is set to ${CMAKE_CXX_STANDARD}; bump up CMAKE_CXX_STANDARD to 20 or above")
 endif()
 if (NOT CMAKE_CXX_EXTENSIONS)
   set(CMAKE_CXX_EXTENSIONS OFF CACHE BOOL "The C++ standard extensions allowed?")
@@ -53,7 +55,7 @@ set(CMAKE_INSTALL_CMAKEDIR "lib/cmake/ttg"
 option(TTG_PARSEC_USE_BOOST_SERIALIZATION "Whether to select Boost serialization methods in PaRSEC backend" ON)
 option(TTG_ENABLE_CUDA "Whether to TTG will look for CUDA" OFF)
 option(TTG_ENABLE_HIP "Whether to TTG will look for HIP" OFF)
-option(TTG_ENABLE_LEVEL_ZERO "Whether to TTG will look for LEVEL_ZERO" OFF)
+option(TTG_ENABLE_LEVEL_ZERO "Whether to TTG will look for Intel oneAPI Level Zero" OFF)
 option(TTG_EXAMPLES "Whether to build examples" OFF)
 option(TTG_ENABLE_ASAN "Whether to enable address sanitizer" OFF)
 
@@ -116,6 +118,8 @@ if (TTG_ENABLE_CUDA)
 endif(TTG_ENABLE_CUDA)
 
 if (TTG_ENABLE_HIP)
+  # HIP LANGUAGE introduced in 3.21
+  cmake_minimum_required(VERSION 3.21)
   include(CheckLanguage)
   check_language(HIP)
   if(CMAKE_HIP_COMPILER)

From 56ed9425dabd15cccfc5fcd99f24afb3f7735c51 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Sat, 24 Feb 2024 00:34:44 -0500
Subject: [PATCH 251/259] update INSTALL.md

---
 INSTALL.md | 45 ++++++++++++++++++++++++++-------------------
 1 file changed, 26 insertions(+), 19 deletions(-)

diff --git a/INSTALL.md b/INSTALL.md
index 7a2470f7b..bb459f618 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -13,12 +13,12 @@ $ cmake --build ttg/build --target install
 TTG is usable only on POSIX systems.
 
 ## mandatory prerequisites
-- [CMake](https://cmake.org/), version 3.14 or higher
-- C++ compiler with support for the [C++17 standard](http://www.iso.org/standard/68564.html), or a more recent standard. This includes the following compilers:
-  - [GNU C++](https://gcc.gnu.org/), version 7.0 or higher
-  - [Clang](https://clang.llvm.org/), version 5 or higher
-  - [Apple Clang](https://en.wikipedia.org/wiki/Xcode), version 9.3 or higher
-  - [Intel C++ compiler](https://software.intel.com/en-us/c-compilers), version 19 or higher
+- [CMake](https://cmake.org/), version 3.14 or higher; version 3.21 or higher is required to support execution on HIP/ROCm-capable devices.
+- C++ compiler with support for the [C++20 standard](http://www.iso.org/standard/68564.html), or a more recent standard. This includes the following compilers:
+  - [GNU C++](https://gcc.gnu.org/), version 10.0 or higher; GCC is the only compiler that can be used for accelerator programming.
+  - [Clang](https://clang.llvm.org/), version 10 or higher
+  - [Apple Clang](https://en.wikipedia.org/wiki/Xcode), version 10.0 or higher
+  - [Intel C++ compiler](https://software.intel.com/en-us/c-compilers), version 2021.1 or higher
 - one or more of the following runtimes:
   - [PaRSEC](https://bitbucket.org/icldistcomp/parsec): this distributed-memory runtime is the primary runtime intended for high-performance implementation of TTG
   - [MADNESS](https://github.org/m-a-d-n-e-s-s/madness): this distributed-memory runtime is to be used primarily for developmental purposes
@@ -27,12 +27,15 @@ While the list of prerequisites is short, note that the runtimes have many more
 Also: it is _strongly_ recommended that the runtimes are built as parts of the TTG build process (this requires some of the optional prerequisites, listed below). This will make sure that the correct versions of the runtimes are used.
 
 ## optional prerequisites
-- [Git](https://git-scm.com) 1.8 or later: needed to obtain the source code for PaRSEC or MADNESS runtimes
-- [Boost](https://boost.org/) version 1.66 or later: needed to use TTG with classes serializable by the [Boost.Serialization](https://www.boost.org/doc/libs/master/libs/serialization/doc/index.html) library.
-  - The [Boost.Serialization](https://www.boost.org/doc/libs/master/libs/serialization/doc/index.html) library is not header-only, i.e., it must be compiled.
-  - If the Boost package is not detected TTG can download and build Boost as part of its build process; to do that configure TTG with the CMake cache variable `TTG_FETCH_BOOST` set to `ON` (e.g., by adding `-DTTG_FETCH_BOOST=ON` to the CMake executable command line)
-  - *Note to package maintainers*: TTG also requires Boost.CallableTraits; if Boost is not found or built, TTG installs and uses a bundled copy of Boost.CallableTraits. To avoid the installation and use of the bundled Boost.CallableTraits configure TTG with the CMake cache variable `TTG_IGNORE_BUNDLED_EXTERNALS` set to `ON`.
-- ([Doxygen](http://www.doxygen.nl/), version 1.8.12 or later: needed for building documentation
+- [Git](https://git-scm.com): needed to obtain the source code for any prerequisite built from source code as part of TTG, such as PaRSEC or MADNESS runtimes
+- [Boost](https://boost.org/) version 1.81 or later. If the Boost package is not detected TTG can download and build Boost as part of its build process, but this is NOT recommended, you should obtain Boost via the system or third-party package manager. Experts may try to build Boost from source as part of TTG by configuring it with the CMake cache variable `TTG_FETCH_BOOST` set to `ON` (e.g., by adding `-DTTG_FETCH_BOOST=ON` to the CMake executable command line). The following primary Boost libraries/modules (and their transitive dependents) are used:
+  - (required) [Boost.CallableTraits](): used to introspect generic callables given to `make_tt`. P.S. TTG has a bundled copy of `Boost.CallableTraits` which is used and installed if Boost is not found or built from source. To avoid the installation and use of the bundled Boost.CallableTraits configure TTG with the CMake cache variable `TTG_IGNORE_BUNDLED_EXTERNALS` set to `ON`.
+  - (optional) [Boost.Serialization](https://www.boost.org/doc/libs/master/libs/serialization/doc/index.html): needed to use TTG with classes serializable by the [Boost.Serialization](https://www.boost.org/doc/libs/master/libs/serialization/doc/index.html) library. Note that `Boost.Serialization` is not header-only, i.e., it must be compiled. This is only required if TTG is configured with CMake cache variable `TTG_PARSEC_USE_BOOST_SERIALIZATION` set to `ON`.
+- ([Doxygen](http://www.doxygen.nl/), version 1.8.12 or later: needed for building documentation.
+- for execution on GPGPUs and other accelerators, the following are required:
+  - [CUDA compiler and runtime](https://developer.nvidia.com/cuda-zone) -- for execution on NVIDIA's CUDA-enabled accelerators. CUDA 11 or later is required.
+  - [HIP/ROCm compiler and runtime](https://developer.nvidia.com/cuda-zone) -- for execution on AMD's ROCm-enabled accelerators.
+  - [oneAPI DPC++/SYCL/LevelZero compiler and runtime](https://developer.nvidia.com/cuda-zone) -- for execution on Intel accelerators.
 
 ## transitive prerequisites
 
@@ -60,10 +63,14 @@ TTG includes several examples that may require additional prerequisites. These a
 
 ## useful cmake cache variables:
 
-| Variable                       |Default             | Description                                                                                                                                                                                           |
-|--------------------------------|--------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `BUILD_TESTING`                | `ON`               | whether target `check-ttg` and its relatives will actually build and run unit tests                                                                                                                   |
-| `TTG_EXAMPLES`                 | `OFF`              | whether target `check-ttg` and its relatives will actually build and run examples; setting this to `ON` will cause detection of several optional prerequisites, and (if missing) building from source |
-| `TTG_ENABLE_TRACE`             | `OFF`              | setting this to `ON` will enable the ability to instrument TTG code for tracing (see `ttg::trace()`, etc.); if this is set to `OFF`, `ttg::trace()` is a no-op                                        |
-| `TTG_FETCH_BOOST`              | `OFF`              | whether to download and build Boost automatically, if missing                                                                                                                                         |
-| `TTG_IGNORE_BUNDLED_EXTERNALS` | `OFF`              | whether to install and use bundled external dependencies (currently, only Boost.CallableTraits)                                                                                                       |
+| Variable                             |Default             | Description                                                                                                                                                                                                                                  |
+|--------------------------------------|--------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `TTG_ENABLE_CUDA`                    | `OFF`              | whether to enable CUDA device support                                                                                                                                                                                                        |
+| `TTG_ENABLE_HIP`                     | `OFF`              | whether to enable HIP/ROCm device support                                                                                                                                                                                                    |
+| `TTG_ENABLE_LEVEL_ZERO`              | `OFF`              | whether to enable Intel oneAPI Level Zero device support                                                                                                                                                                                     |
+| `BUILD_TESTING`                      | `ON`               | whether target `check-ttg` and its relatives will actually build and run unit tests                                                                                                                                                          |
+| `TTG_EXAMPLES`                       | `OFF`              | whether target `check-ttg` and its relatives will actually build and run examples; setting this to `ON` will cause detection of several optional prerequisites, and (if missing) building from source                                        |
+| `TTG_ENABLE_TRACE`                   | `OFF`              | setting this to `ON` will enable the ability to instrument TTG code for tracing (see `ttg::trace()`, etc.); if this is set to `OFF`, `ttg::trace()` is a no-op                                                                               |
+| `TTG_PARSEC_USE_BOOST_SERIALIZATION` | `OFF`       | whether to use Boost.Serialization for serialization for the PaRSEC backend; if this is set to `OFF`, PaRSEC backend will only be able to use trivially-copyable data types or, if MADNESS backend is available, MADNESS-serializable types. |
+| `TTG_FETCH_BOOST`                    | `OFF`              | whether to download and build Boost automatically, if missing                                                                                                                                                                                |
+| `TTG_IGNORE_BUNDLED_EXTERNALS`       | `OFF`              | whether to install and use bundled external dependencies (currently, only Boost.CallableTraits)                                                                                                                                              |

From 0c65e11d09000bae3541e3806f571513325b2661 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Sun, 25 Feb 2024 23:52:28 -0500
Subject: [PATCH 252/259] dox cleanup

---
 doc/dox/config/Doxyfile.in     |  2 +-
 ttg/ttg/base/tt.h              |  1 -
 ttg/ttg/func.h                 | 37 ++++++++++++++++++----------------
 ttg/ttg/madness/ttg.h          |  4 ++--
 ttg/ttg/parsec/ttg.h           |  4 ++--
 ttg/ttg/serialization/traits.h |  2 +-
 ttg/ttg/terminal.h             |  1 +
 ttg/ttg/util/bug.h             |  3 +--
 ttg/ttg/util/iovec.h           |  5 ++---
 9 files changed, 30 insertions(+), 29 deletions(-)

diff --git a/doc/dox/config/Doxyfile.in b/doc/dox/config/Doxyfile.in
index a1154935d..cdf1a56f1 100644
--- a/doc/dox/config/Doxyfile.in
+++ b/doc/dox/config/Doxyfile.in
@@ -399,7 +399,7 @@ DISTRIBUTE_GROUP_DOC   = YES
 # is disabled and one has to add nested compounds explicitly via \ingroup.
 # The default value is: NO.
 
-GROUP_NESTED_COMPOUNDS = NO
+GROUP_NESTED_COMPOUNDS = YES
 
 # Set the SUBGROUPING tag to YES to allow class member groups of the same type
 # (for instance a group of public functions) to be put as a subgroup of that
diff --git a/ttg/ttg/base/tt.h b/ttg/ttg/base/tt.h
index c686bee2a..d2fbd467a 100644
--- a/ttg/ttg/base/tt.h
+++ b/ttg/ttg/base/tt.h
@@ -264,7 +264,6 @@ namespace ttg {
     virtual void release() {}
 
     /// Marks this executable
-    /// @return nothing
     virtual void make_executable() = 0;
 
     /// Queries if this ready to execute
diff --git a/ttg/ttg/func.h b/ttg/ttg/func.h
index e1394be89..28b75ace9 100644
--- a/ttg/ttg/func.h
+++ b/ttg/ttg/func.h
@@ -98,11 +98,14 @@ namespace ttg {
   inline void connect(ttg::TerminalBase *out, ttg::TerminalBase *in) { out->connect(in); }
 
   /// \brief Connect producer output terminal outindex to consumer input terminal inindex (via unique or otherwise
-  /// wrapped pointers to TTs) \tparam outindex The index of the output terminal on the producer. \tparam inindex  The
-  /// index of the input terminal on the consumer. \param p The producer TT \param c The consumer TT
+  /// wrapped pointers to TTs)
+  /// \tparam outindex The index of the output terminal on the producer.
+  /// \tparam inindex  The index of the input terminal on the consumer.
+  /// \param p The producer TT
+  /// \param c The consumer TT
   template <std::size_t outindex, std::size_t inindex, typename producer_tt_ptr, typename successor_tt_ptr>
-  inline void connect(producer_tt_ptr &p, successor_tt_ptr &s) {
-    connect(p->template out<outindex>(), s->template in<inindex>());
+  inline void connect(producer_tt_ptr &p, successor_tt_ptr &c) {
+    connect(p->template out<outindex>(), c->template in<inindex>());
   }
 
   /// \brief Connect producer output terminal outindex to consumer input terminal inindex (via bare pointers to TTs)
@@ -111,13 +114,13 @@ namespace ttg {
   /// \param p The producer TT
   /// \param c The consumer TT
   template <std::size_t outindex, std::size_t inindex, typename producer_tt_ptr, typename successor_tt_ptr>
-  inline void connect(producer_tt_ptr *p, successor_tt_ptr *s) {
-    connect(p->template out<outindex>(), s->template in<inindex>());
+  inline void connect(producer_tt_ptr *p, successor_tt_ptr *c) {
+    connect(p->template out<outindex>(), c->template in<inindex>());
   }
 
   /// \brief Connect producer output terminal outindex to consumer input terminal inindex (via TTBase pointers)
-  /// \tparam outindex The index of the output terminal on the producer.
-  /// \tparam inindex  The index of the input terminal on the consumer.
+  /// \param outindex The index of the output terminal on the producer.
+  /// \param inindex  The index of the input terminal on the consumer.
   /// \param producer The producer TT
   /// \param consumer The consumer TT
   inline void connect(size_t outindex, size_t inindex, TTBase *producer, TTBase *consumer) {
@@ -149,7 +152,7 @@ namespace ttg {
   /// \brief Sends a task id and a value to the given output terminal
   /// \param[in] key: the id of the task(s) receiving the value
   /// \param[in] value: the value to send to the receiving task(s)
-  /// \param[in] out: the output terminal
+  /// \param[in] t: the output terminal
   // clang-format on
   template <typename keyT, typename valueT, typename output_terminalT, ttg::Runtime Runtime = ttg::ttg_runtime>
   inline void send(const keyT &key, valueT &&value, ttg::Out<keyT, valueT> &t) {
@@ -160,7 +163,7 @@ namespace ttg {
   // clang-format off
   /// \brief Sends a task id (without an accompanying value) to the given output terminal
   /// \param[in] key: the id of the task(s) receiving the value
-  /// \param[in] out: the output terminal
+  /// \param[in] t: the output terminal
   // clang-format on
   template <typename keyT>
   inline void sendk(const keyT &key, ttg::Out<keyT, void> &t) {
@@ -170,7 +173,7 @@ namespace ttg {
   // clang-format off
   /// \brief Sends a value (without an accompanying task id) to the given output terminal
   /// \param[in] value: the value to send to the receiving task(s)
-  /// \param[in] out: the output terminal
+  /// \param[in] t: the output terminal
   // clang-format on
   template <typename valueT, ttg::Runtime Runtime = ttg::ttg_runtime>
   inline void sendv(valueT &&value, ttg::Out<void, valueT> &t) {
@@ -180,7 +183,7 @@ namespace ttg {
 
   // clang-format off
   /// \brief Sends a control message (message without an accompanying task id or a value) to the given output terminal
-  /// \param[in] out: the output terminal
+  /// \param[in] t: the output terminal
   // clang-format on
   inline void send(ttg::Out<void, void> &t) { t.send(); }
 
@@ -189,7 +192,7 @@ namespace ttg {
   /// \tparam <i> Identifies which output terminal in \p t to select for sending
   /// \param[in] key: the id of the task(s) receiving the value
   /// \param[in] value: the value to send to the receiving task(s)
-  /// \param[in] out: a tuple of output terminals (typically, this is the output terminal of the template task where this is invoked)
+  /// \param[in] t: a tuple of output terminals (typically, this is the output terminal of the template task where this is invoked)
   // clang-format on
   template <size_t i, typename keyT, typename valueT, typename... out_keysT, typename... out_valuesT,
             ttg::Runtime Runtime = ttg::ttg_runtime>
@@ -230,7 +233,7 @@ namespace ttg {
   /// \brief Sends a task id (without an accompanying value) to the template tasks attached to the output terminal selected in the explicitly given terminal tuple \p t
   /// \tparam <i> Identifies which output terminal in \p t to select for sending
   /// \param[in] key: the id of the task(s) receiving the value
-  /// \param[in] out: a tuple of output terminals (typically, this is the output terminal of the template task where this is invoked)
+  /// \param[in] t: a tuple of output terminals (typically, this is the output terminal of the template task where this is invoked)
   // clang-format on
   template <size_t i, typename keyT, typename... out_keysT, typename... out_valuesT>
   inline std::enable_if_t<!meta::is_void_v<keyT>, void> sendk(const keyT &key,
@@ -264,7 +267,7 @@ namespace ttg {
   /// \brief Sends a value (without an accompanying task id) to the template tasks attached to the output terminal selected in the explicitly given terminal tuple \p t
   /// \tparam <i> Identifies which output terminal in \p t to select for sending
   /// \param[in] value: the value to send to the receiving task(s)
-  /// \param[in] out: a tuple of output terminals (typically, this is the output terminal of the template task where this is invoked)
+  /// \param[in] t: a tuple of output terminals (typically, this is the output terminal of the template task where this is invoked)
   // clang-format on
   template <size_t i, typename valueT, typename... out_keysT, typename... out_valuesT,
             ttg::Runtime Runtime = ttg::ttg_runtime>
@@ -276,7 +279,7 @@ namespace ttg {
 
   // clang-format off
   /// \brief Sends a value (without an accompanying task id) to the template tasks attached to the output terminal of this template task
-  /// \param[in] i Identifies which output terminal of this template task to select for sending
+  /// \param[in] <i> Identifies which output terminal of this template task to select for sending
   /// \param[in] value: the value to send to the receiving task(s)
   // clang-format on
   template <typename valueT, ttg::Runtime Runtime = ttg::ttg_runtime>
@@ -300,7 +303,7 @@ namespace ttg {
   // clang-format off
   /// \brief Sends a control message (message without an accompanying task id or a value) to the template tasks attached to the output terminal selected in the explicitly given terminal tuple \p t
   /// \tparam <i> Identifies which output terminal in \p t to select for sending
-  /// \param[in] out: a tuple of output terminals (typically, this is the output terminal of the template task where this is invoked)
+  /// \param[in] t: a tuple of output terminals (typically, this is the output terminal of the template task where this is invoked)
   // clang-format on
   template <size_t i, typename... out_keysT, typename... out_valuesT>
   inline void send(std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
diff --git a/ttg/ttg/madness/ttg.h b/ttg/ttg/madness/ttg.h
index 8dfc46971..2a0f46169 100644
--- a/ttg/ttg/madness/ttg.h
+++ b/ttg/ttg/madness/ttg.h
@@ -1164,7 +1164,7 @@ namespace ttg_madness {
     /// define the reducer function to be called when additional inputs are
     /// received on a streaming terminal
     ///   @tparam <i> the index of the input terminal that is used as a streaming terminal
-    ///   @param[in] reducer: a function of prototype (input_type<i> &a, const input_type<i> &b)
+    ///   @param[in] reducer: a function of prototype `void(input_type<i> &a, const input_type<i> &b)`
     ///                       that function should aggregate b into a
     template <std::size_t i, typename Reducer>
     void set_input_reducer(Reducer &&reducer) {
@@ -1175,7 +1175,7 @@ namespace ttg_madness {
     /// define the reducer function to be called when additional inputs are
     /// received on a streaming terminal
     ///   @tparam <i> the index of the input terminal that is used as a streaming terminal
-    ///   @param[in] reducer: a function of prototype (input_type<i> &a, const input_type<i> &b)
+    ///   @param[in] reducer: a function of prototype `void(input_type<i> &a, const input_type<i> &b)`
     ///                       that function should aggregate b into a
     ///   @param[in] size: the default number of inputs that are received in this streaming terminal,
     ///                    for each task
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 50072b2f7..2f10d75d3 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -3936,7 +3936,7 @@ ttg::abort();  // should not happen
     /// define the reducer function to be called when additional inputs are
     /// received on a streaming terminal
     ///   @tparam <i> the index of the input terminal that is used as a streaming terminal
-    ///   @param[in] reducer: a function of prototype (input_type<i> &a, const input_type<i> &b)
+    ///   @param[in] reducer: a function of prototype `void(input_type<i> &a, const input_type<i> &b)`
     ///                       that function should aggregate b into a
     template <std::size_t i, typename Reducer>
     void set_input_reducer(Reducer &&reducer) {
@@ -4015,7 +4015,7 @@ ttg::abort();  // should not happen
     /// define the reducer function to be called when additional inputs are
     /// received on a streaming terminal
     ///   @tparam <i> the index of the input terminal that is used as a streaming terminal
-    ///   @param[in] reducer: a function of prototype (input_type<i> &a, const input_type<i> &b)
+    ///   @param[in] reducer: a function of prototype `void(input_type<i> &a, const input_type<i> &b)`
     ///                       that function should aggregate b into a
     ///   @param[in] size: the default number of inputs that are received in this streaming terminal,
     ///                    for each task
diff --git a/ttg/ttg/serialization/traits.h b/ttg/ttg/serialization/traits.h
index bd6d8f825..bf70298d6 100644
--- a/ttg/ttg/serialization/traits.h
+++ b/ttg/ttg/serialization/traits.h
@@ -23,7 +23,7 @@ namespace ttg::detail {
 
   /// helps to detect that `T` has a member serialization method that
   /// accepts single argument of type `Archive`
-  /// @note use in combination with `ttg::meta`::is_detected_v
+  /// @note use in combination with ttg::meta::is_detected_v
   template <typename T, typename Archive>
   using has_member_serialize_t = decltype(std::declval<T&>().serialize(std::declval<Archive&>()));
 
diff --git a/ttg/ttg/terminal.h b/ttg/ttg/terminal.h
index 8711b3b06..dc9f4b08b 100644
--- a/ttg/ttg/terminal.h
+++ b/ttg/ttg/terminal.h
@@ -188,6 +188,7 @@ namespace ttg {
     ///     will continue adding data onto this terminal
     /// \param[in] setsize_callback: if the terminal is a reduce terminal, announces how many items will be set
     ///     unto this terminal for reduction
+    /// \param[in] prepare_send_callback: for resumable/device tasks this is called before actual send
     void set_callback(const send_callback_type &send_callback, const move_callback_type &move_callback,
                       const broadcast_callback_type &bcast_callback = broadcast_callback_type{},
                       const setsize_callback_type &setsize_callback = setsize_callback_type{},
diff --git a/ttg/ttg/util/bug.h b/ttg/ttg/util/bug.h
index 25f831191..1404fa01a 100644
--- a/ttg/ttg/util/bug.h
+++ b/ttg/ttg/util/bug.h
@@ -286,12 +286,11 @@ namespace ttg {
 
     static std::shared_ptr<Debugger> default_debugger_;
 
-    /** prints out a backtrace
+    /** prints out a backtrace to `std::cout`
      *
      * @param prefix this string will be prepended at the beginning of each line
      * of Backtrace
      * @param reason optional string specifying the reason for traceback
-     * @return backtrace
      */
     static void __traceback(const std::string &prefix, const char *reason = nullptr);
 
diff --git a/ttg/ttg/util/iovec.h b/ttg/ttg/util/iovec.h
index 2a8d672b3..ad20d066d 100644
--- a/ttg/ttg/util/iovec.h
+++ b/ttg/ttg/util/iovec.h
@@ -7,12 +7,11 @@ namespace ttg {
 
   /**
    * Used to describe transfer payload in types using the \sa SplitMetadataDescriptor.
-   * @member data Pointer to the data to be read from / written to.
-   * @member num_bytes The number of bytes to read from / write to the memory location
-   *                   \sa data.
    */
   struct iovec {
+    /// The number of bytes to read from / write to the memory location given by `data`.
     std::size_t num_bytes;
+    /// Pointer to the data to be read from / written to.
     void* data;
   };
 

From a8aeec07832aa01883aa653d9a6445f8eca6f7a2 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Sun, 25 Feb 2024 23:53:07 -0500
Subject: [PATCH 253/259] localize all uses of TTG_CXX_COROUTINE_NAMESPACE into
 util/coroutine.h

---
 ttg/ttg/device/task.h    | 33 ++++++++++-----------
 ttg/ttg/madness/ttg.h    |  8 +++---
 ttg/ttg/make_tt.h        |  4 +--
 ttg/ttg/parsec/ttg.h     |  2 +-
 ttg/ttg/util/coroutine.h | 62 +++++++++++++++++++++++++---------------
 5 files changed, 63 insertions(+), 46 deletions(-)

diff --git a/ttg/ttg/device/task.h b/ttg/ttg/device/task.h
index 55cf5f0d5..1375e61ab 100644
--- a/ttg/ttg/device/task.h
+++ b/ttg/ttg/device/task.h
@@ -51,7 +51,8 @@ namespace ttg::device {
       constexpr bool await_ready() const noexcept { return false; }
 
       /* always suspend */
-      constexpr void await_suspend( TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<> ) const noexcept {}
+      template <typename Promise>
+      constexpr void await_suspend( ttg::coroutine_handle<Promise> ) const noexcept {}
 
       void await_resume() noexcept {
         if constexpr (sizeof...(Ts) > 0) {
@@ -86,7 +87,7 @@ namespace ttg::device {
   namespace detail {
     struct send_coro_promise_type;
 
-    using send_coro_handle_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<send_coro_promise_type>;
+    using send_coro_handle_type = ttg::coroutine_handle<send_coro_promise_type>;
 
     /// a coroutine for sending data from the device
     struct send_coro_state : public send_coro_handle_type {
@@ -118,21 +119,21 @@ namespace ttg::device {
       /* do not suspend the coroutine on first invocation, we want to run
       * the coroutine immediately and suspend only once.
       */
-      TTG_CXX_COROUTINE_NAMESPACE::suspend_never initial_suspend() {
+      ttg::suspend_never initial_suspend() {
         return {};
       }
 
       /* we don't suspend the coroutine at the end.
       * it can be destroyed once the send/broadcast is done
       */
-      TTG_CXX_COROUTINE_NAMESPACE::suspend_never final_suspend() noexcept {
+      ttg::suspend_never final_suspend() noexcept {
         return {};
       }
 
       send_coro_state get_return_object() { return send_coro_state{send_coro_handle_type::from_promise(*this)}; }
 
       /* the send coros only have an empty co_await */
-      TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(ttg::Void) {
+      ttg::suspend_always await_transform(ttg::Void) {
         return {};
       }
 
@@ -413,7 +414,7 @@ namespace ttg::device {
     // fwd-decl
     struct device_task_promise_type;
     // base type for ttg::device::Task
-    using device_task_handle_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<device_task_promise_type>;
+    using device_task_handle_type = ttg::coroutine_handle<device_task_promise_type>;
   } // namespace detail
 
   /// A device::Task is a coroutine (a callable that can be suspended and resumed).
@@ -458,7 +459,7 @@ namespace ttg::device {
       /* do not suspend the coroutine on first invocation, we want to run
       * the coroutine immediately and suspend when we get the device transfers.
       */
-      TTG_CXX_COROUTINE_NAMESPACE::suspend_never initial_suspend() {
+      ttg::suspend_never initial_suspend() {
         m_state = ttg::device::detail::TTG_DEVICE_CORO_INIT;
         return {};
       }
@@ -467,19 +468,19 @@ namespace ttg::device {
       * so we can access the promise.
       * TODO: necessary? maybe we can save one suspend here
       */
-      TTG_CXX_COROUTINE_NAMESPACE::suspend_always final_suspend() noexcept {
+      ttg::suspend_always final_suspend() noexcept {
         m_state = ttg::device::detail::TTG_DEVICE_CORO_COMPLETE;
         return {};
       }
 
       /* Allow co_await on a tuple */
       template<typename... Views>
-      TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(std::tuple<Views&...> &views) {
+      ttg::suspend_always await_transform(std::tuple<Views&...> &views) {
         return yield_value(views);
       }
 
       template<typename... Ts>
-      TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(detail::to_device_t<Ts...>&& a) {
+      ttg::suspend_always await_transform(detail::to_device_t<Ts...>&& a) {
         bool need_transfer = !(TTG_IMPL_NS::register_device_memory(a.ties));
         /* TODO: are we allowed to not suspend here and launch the kernel directly? */
         m_state = ttg::device::detail::TTG_DEVICE_CORO_WAIT_TRANSFER;
@@ -496,13 +497,13 @@ namespace ttg::device {
         return a;
       }
 
-      TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(std::vector<device::detail::send_t>&& v) {
+      ttg::suspend_always await_transform(std::vector<device::detail::send_t>&& v) {
         m_sends = std::forward<std::vector<device::detail::send_t>>(v);
         m_state = ttg::device::detail::TTG_DEVICE_CORO_SENDOUT;
         return {};
       }
 
-      TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(device::detail::send_t&& v) {
+      ttg::suspend_always await_transform(device::detail::send_t&& v) {
         m_sends.clear();
         m_sends.push_back(std::forward<device::detail::send_t>(v));
         m_state = ttg::device::detail::TTG_DEVICE_CORO_SENDOUT;
@@ -560,7 +561,7 @@ namespace ttg::device {
 
   struct device_reducer_promise_type;
 
-  using device_reducer_handle_type = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<device_reducer_promise_type>;
+  using device_reducer_handle_type = ttg::coroutine_handle<device_reducer_promise_type>;
 
   /// task that can be resumed after some events occur
   struct device_reducer : public device_reducer_handle_type {
@@ -596,7 +597,7 @@ namespace ttg::device {
     /* do not suspend the coroutine on first invocation, we want to run
     * the coroutine immediately and suspend when we get the device transfers.
     */
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_never initial_suspend() {
+    ttg::suspend_never initial_suspend() {
       m_state = ttg::device::detail::TTG_DEVICE_CORO_INIT;
       return {};
     }
@@ -605,13 +606,13 @@ namespace ttg::device {
     * so we can access the promise.
     * TODO: necessary? maybe we can save one suspend here
     */
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_always final_suspend() noexcept {
+    ttg::suspend_always final_suspend() noexcept {
       m_state = ttg::device::detail::TTG_DEVICE_CORO_COMPLETE;
       return {};
     }
 
     template<typename... Ts>
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_always await_transform(detail::to_device_t<Ts...>&& a) {
+    ttg::suspend_always await_transform(detail::to_device_t<Ts...>&& a) {
       bool need_transfer = !(TTG_IMPL_NS::register_device_memory(a.ties));
       /* TODO: are we allowed to not suspend here and launch the kernel directly? */
       m_state = ttg::device::detail::TTG_DEVICE_CORO_WAIT_TRANSFER;
diff --git a/ttg/ttg/madness/ttg.h b/ttg/ttg/madness/ttg.h
index 2a0f46169..86d05a15f 100644
--- a/ttg/ttg/madness/ttg.h
+++ b/ttg/ttg/madness/ttg.h
@@ -359,7 +359,7 @@ namespace ttg_madness {
             ttg::abort();
         } else {  // resume suspended coroutine
 #ifdef TTG_HAS_COROUTINE
-          auto ret = static_cast<ttg::resumable_task>(ttg::coroutine_handle<>::from_address(suspended_task_address));
+          auto ret = static_cast<ttg::resumable_task>(ttg::coroutine_handle<ttg::resumable_task_state>::from_address(suspended_task_address));
           assert(ret.ready());
           ret.resume();
           if (ret.completed()) {
@@ -388,14 +388,14 @@ namespace ttg_madness {
           // so mark the events finished manually, parsec will rerun this task again and it should complete the second
           // time
           auto events =
-              static_cast<ttg::resumable_task>(ttg::coroutine_handle<>::from_address(suspended_task_address)).events();
+              static_cast<ttg::resumable_task>(ttg::coroutine_handle<ttg::resumable_task_state>::from_address(suspended_task_address)).events();
           for (auto &event_ptr : events) {
             event_ptr->finish();
           }
-          assert(ttg::coroutine_handle<>::from_address(suspended_task_address).promise().ready());
+          assert(ttg::coroutine_handle<ttg::resumable_task_state>::from_address(suspended_task_address).promise().ready());
 
           // resume the coroutine
-          auto ret = static_cast<ttg::resumable_task>(ttg::coroutine_handle<>::from_address(suspended_task_address));
+          auto ret = static_cast<ttg::resumable_task>(ttg::coroutine_handle<ttg::resumable_task_state>::from_address(suspended_task_address));
           assert(ret.ready());
           ret.resume();
           if (ret.completed()) {
diff --git a/ttg/ttg/make_tt.h b/ttg/ttg/make_tt.h
index 1e03b3ad5..1711d8556 100644
--- a/ttg/ttg/make_tt.h
+++ b/ttg/ttg/make_tt.h
@@ -151,7 +151,7 @@ class CallableWrapTTArgs
   using op_return_type =
 #ifdef TTG_HAS_COROUTINE
       std::conditional_t<std::is_same_v<returnT, ttg::resumable_task>,
-                         ttg::coroutine_handle<>,
+                         ttg::coroutine_handle<ttg::resumable_task_state>,
 #ifdef TTG_HAVE_DEVICE
                          std::conditional_t<std::is_same_v<returnT, ttg::device::Task>,
                                             ttg::device::Task::base_type,
@@ -178,7 +178,7 @@ class CallableWrapTTArgs
     if constexpr (!std::is_void_v<returnT>) {  // protect from compiling for void returnT
 #ifdef TTG_HAS_COROUTINE
       if constexpr (std::is_same_v<returnT, ttg::resumable_task>) {
-        ttg::coroutine_handle<> coro_handle;
+        ttg::coroutine_handle<ttg::resumable_task_state> coro_handle;
         // if task completed destroy it
         if (ret.completed()) {
           ret.destroy();
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 2f10d75d3..3f8771764 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1723,7 +1723,7 @@ ttg::abort();  // should not happen
       }
       else {
 #ifdef TTG_HAS_COROUTINE
-        auto ret = static_cast<ttg::resumable_task>(ttg::coroutine_handle<>::from_address(suspended_task_address));
+        auto ret = static_cast<ttg::resumable_task>(ttg::coroutine_handle<ttg::resumable_task_state>::from_address(suspended_task_address));
         assert(ret.ready());
         ret.resume();
         if (ret.completed()) {
diff --git a/ttg/ttg/util/coroutine.h b/ttg/ttg/util/coroutine.h
index 3577032f4..9de101e46 100644
--- a/ttg/ttg/util/coroutine.h
+++ b/ttg/ttg/util/coroutine.h
@@ -13,16 +13,29 @@
 
 namespace ttg {
 
-  struct resumable_task_state;
+  // import std coroutine API into ttg namespace
 
-  // import coroutine_handle, with default promise redefined to resumable_task_state
-  template <typename Promise = resumable_task_state>
+  using suspend_always = TTG_CXX_COROUTINE_NAMESPACE::suspend_always;
+  using suspend_never = TTG_CXX_COROUTINE_NAMESPACE::suspend_never;
+  template <typename Promise>
   using coroutine_handle = TTG_CXX_COROUTINE_NAMESPACE::coroutine_handle<Promise>;
 
+  /// @defgroup resumable_task resumable_task coroutine
+
+  /// resumable_task is the original prototype TTG coroutine that awaits on generic events.
+  /// There is no proper support for it by TTG runtimes, but it can be useful for understanding how
+  /// coroutines work with TTG and potentially in the future as a model for universal resumable tasks
+
+  /// @{
+
+  // fwd-declares
+
+  struct resumable_task_state;
+
   template <std::size_t N>
   struct resumable_task_events;
 
-  /// represents a one-time event
+  /// represents a generic one-time event
   struct event {
     void finish() { finished_ = true; }
 
@@ -34,15 +47,15 @@ namespace ttg {
   };
 
   /// task that can be resumed after some events occur
-  struct resumable_task : public ttg::coroutine_handle<> {
-    using base_type = ttg::coroutine_handle<>;
+  struct resumable_task : public ttg::coroutine_handle<resumable_task_state> {
+    using base_type = ttg::coroutine_handle<resumable_task_state>;
 
-    /// these are members mandated by the promise_type concept
-    ///@{
+    /// @name members mandated by the promise_type concept
+    /// @{
 
     using promise_type = struct resumable_task_state;
 
-    ///@}
+    /// @}
 
     resumable_task(base_type base) : base_type(std::move(base)) {}
 
@@ -69,28 +82,28 @@ namespace ttg {
     resumable_task_state& operator=(resumable_task_state&&) = delete;
 
     constexpr static inline std::size_t MaxNumEvents = 20;
-    using handle_type = coroutine_handle<>;
+    using handle_type = coroutine_handle<resumable_task_state>;
 
-    /// these are members mandated by the promise_type concept
-    ///@{
+    /// @name members mandated by the promise_type concept
+    /// @{
 
     resumable_task get_return_object() { return resumable_task{handle_type::from_promise(*this)}; }
 
     /// @note start task eagerly
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_never initial_suspend() noexcept { return {}; }
+    suspend_never initial_suspend() noexcept { return {}; }
 
     /// @note suspend task before destroying it so the runtime can know that the task is completed
-    TTG_CXX_COROUTINE_NAMESPACE::suspend_always final_suspend() noexcept {
+    suspend_always final_suspend() noexcept {
       completed_ = true;
       return {};
     }
     void return_void() {}
     void unhandled_exception() {}
 
-    ///@}
+    /// @}
 
-    /// these are optional members of the promise_type concept
-    ///@{
+    /// @name optional members of the promise_type concept
+    /// @{
 
     // these can be used to use optional storage provided by the runtime (e.g. part of the runtime's task data struct)
     // N.B. the existing buffer must be passed to operator new via TLS
@@ -105,7 +118,7 @@ namespace ttg {
     //      ::operator delete(ptr, size);
     //    }
 
-    ///@}
+    /// @}
 
     /// @return true if ready to resume
     constexpr bool ready() const {
@@ -160,12 +173,12 @@ namespace ttg {
     template <typename... Events>
     constexpr resumable_task_events(Events&&... events) : events_{(&events)...} {}
 
-    /// these are members mandated by the Awaiter concept
-    ///@{
+    /// @name members mandated by the Awaiter concept
+    /// @{
 
     constexpr bool await_ready() const { return await_ready(std::make_index_sequence<N>{}); }
 
-    void await_suspend(coroutine_handle<> pending_task) {
+    void await_suspend(coroutine_handle<resumable_task_state> pending_task) {
       pending_task_ = pending_task;
       pending_task_.promise().set_events(events_);
     }
@@ -177,11 +190,11 @@ namespace ttg {
       }
     }
 
-    ///@}
+    /// @}
 
    private:
     std::array<event*, N> events_;
-    coroutine_handle<> pending_task_;
+    coroutine_handle<resumable_task_state> pending_task_;
   };  // resumable_task_events
 
   // deduce the number of events properly
@@ -189,6 +202,9 @@ namespace ttg {
   resumable_task_events(Events&&...) -> resumable_task_events<sizeof...(Events)>;
 
   static_assert(resumable_task_events<0>{}.await_ready() == true);
+
+  /// @}
+
 }  // namespace ttg
 
 #endif  // TTG_UTIL_COROUTINE_H

From 0444a03dc8e92c3cdde8adbd82588660363d5d06 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Mon, 26 Feb 2024 00:05:00 -0500
Subject: [PATCH 254/259] use proper INCLUDE_PATH in Doxygen.in

---
 doc/dox/config/Doxyfile.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/dox/config/Doxyfile.in b/doc/dox/config/Doxyfile.in
index cdf1a56f1..581310a05 100644
--- a/doc/dox/config/Doxyfile.in
+++ b/doc/dox/config/Doxyfile.in
@@ -2145,7 +2145,7 @@ SEARCH_INCLUDES        = YES
 # preprocessor.
 # This tag requires that the tag SEARCH_INCLUDES is set to YES.
 
-INCLUDE_PATH           =
+INCLUDE_PATH           = @PROJECT_SOURCE_DIR@/ttg @PROJECT_BINARY_DIR@/ttg
 
 # You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard
 # patterns (like *.h and *.hpp) to filter out the header-files in the

From ab38b01bb9d0b6468d4b96a016965eaa17125154 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Mon, 26 Feb 2024 10:02:51 -0500
Subject: [PATCH 255/259] reenable support for resumable_task in PaRSEC to and
 reenable fibonacci-coro.cc

---
 tests/unit/CMakeLists.txt      |  18 +++--
 tests/unit/fibonacci-coro.cc   |   2 -
 ttg/CMakeLists.txt             |   2 +-
 ttg/ttg.h                      |   2 +-
 ttg/ttg/{util => }/coroutine.h |  26 +++++-
 ttg/ttg/device/task.h          | 100 +++++++++++------------
 ttg/ttg/madness/ttg.h          |  16 ++--
 ttg/ttg/parsec/task.h          |   2 +
 ttg/ttg/parsec/ttg.h           | 140 ++++++++++++++++-----------------
 ttg/ttg/tt.h                   |  37 ++++++---
 10 files changed, 183 insertions(+), 162 deletions(-)
 rename ttg/ttg/{util => }/coroutine.h (89%)

diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt
index ba0db8a49..69378cbf3 100644
--- a/tests/unit/CMakeLists.txt
+++ b/tests/unit/CMakeLists.txt
@@ -9,14 +9,16 @@ set(ut_src
 		streams.cc
     )
 set(ut_libs Catch2::Catch2)
-#if (TARGET std::coroutine)
-    #list(APPEND ut_src fibonacci-coro.cc)
-    list(APPEND ut_src device_coro.cc)
-    if (TTG_HAVE_CUDA)
-        list(APPEND ut_src cuda_kernel.cu)
-    endif(TTG_HAVE_CUDA)
-    list(APPEND ut_libs std::coroutine)
-#endif()
+
+# coroutine tests
+# we definitely have TARGET std::coroutine
+list(APPEND ut_src fibonacci-coro.cc)
+list(APPEND ut_src device_coro.cc)
+if (TTG_HAVE_CUDA)
+    list(APPEND ut_src cuda_kernel.cu)
+endif(TTG_HAVE_CUDA)
+list(APPEND ut_libs std::coroutine)
+
 add_ttg_executable(core-unittests-ttg "${ut_src}" LINK_LIBRARIES "${ut_libs}" COMPILE_DEFINITIONS "CATCH_CONFIG_NO_POSIX_SIGNALS=1" )
 
 # serialization test: probes serialization via all supported serialization methods (MADNESS, Boost::serialization, cereal) that are available
diff --git a/tests/unit/fibonacci-coro.cc b/tests/unit/fibonacci-coro.cc
index dd4af6aa2..a464750f5 100644
--- a/tests/unit/fibonacci-coro.cc
+++ b/tests/unit/fibonacci-coro.cc
@@ -5,8 +5,6 @@
 #include "ttg/serialization/std/pair.h"
 #include "ttg/util/hash/std/pair.h"
 
-#include "ttg/util/coroutine.h"
-
 constexpr int64_t N = 1000;
 
 TEST_CASE("Fibonacci-coroutines", "[fib][core]") {
diff --git a/ttg/CMakeLists.txt b/ttg/CMakeLists.txt
index 307135ca0..e0785433e 100644
--- a/ttg/CMakeLists.txt
+++ b/ttg/CMakeLists.txt
@@ -116,7 +116,7 @@ if (TARGET std::coroutine)
   list(APPEND ttg-deps std::coroutine)
   list(APPEND ttg-defs "TTG_HAS_COROUTINE=1")
   list(APPEND ttg-util-headers
-          ${CMAKE_CURRENT_SOURCE_DIR}/ttg/util/coroutine.h
+          ${CMAKE_CURRENT_SOURCE_DIR}/ttg/coroutine.h
           )
 endif ()
 if (TTG_HAVE_CUDA)
diff --git a/ttg/ttg.h b/ttg/ttg.h
index 85e987e85..50a891c1d 100644
--- a/ttg/ttg.h
+++ b/ttg/ttg.h
@@ -40,7 +40,7 @@
 #include "ttg/parsec/ttg.h"
 #elif defined(TTG_USE_MADNESS)
 #include "ttg/madness/ttg.h"
-#endif  // TTG_USE_PARSEC|MADNESS
+#endif  // TTG_USE_{PARSEC|MADNESS}
 
 // these headers use the default backend
 #include "ttg/run.h"
diff --git a/ttg/ttg/util/coroutine.h b/ttg/ttg/coroutine.h
similarity index 89%
rename from ttg/ttg/util/coroutine.h
rename to ttg/ttg/coroutine.h
index 9de101e46..81d5b1657 100644
--- a/ttg/ttg/util/coroutine.h
+++ b/ttg/ttg/coroutine.h
@@ -2,8 +2,8 @@
 // Created by Eduard Valeyev on 10/31/22.
 //
 
-#ifndef TTG_UTIL_COROUTINE_H
-#define TTG_UTIL_COROUTINE_H
+#ifndef TTG_COROUTINE_H
+#define TTG_COROUTINE_H
 
 #include "ttg/config.h"
 #include TTG_CXX_COROUTINE_HEADER
@@ -205,6 +205,26 @@ namespace ttg {
 
   /// @}
 
+  /////////////////////////////////////////////////////////////////////////////
+  // describe all types of coroutine tasks known to TTG
+  /////////////////////////////////////////////////////////////////////////////
+
+  // fwd declare all coro promise types that have not been declared yet
+  namespace device::detail {
+    struct device_task_promise_type;
+  }  // namespace device::detail
+
+  /// describes all types of coroutine tasks known to TTG
+  /// @internal only exists to simplify metaprogramming in the backend code
+  enum class TaskCoroutineID {
+    /// not a coroutine, i.e. a standard task function, -> void
+    Invalid,
+    /// -> ttg::resumable_task
+    ResumableTask,
+    /// -> ttg::device::Task
+    DeviceTask
+  };
+
 }  // namespace ttg
 
-#endif  // TTG_UTIL_COROUTINE_H
+#endif  // TTG_COROUTINE_H
diff --git a/ttg/ttg/device/task.h b/ttg/ttg/device/task.h
index 1375e61ab..d95e0d1eb 100644
--- a/ttg/ttg/device/task.h
+++ b/ttg/ttg/device/task.h
@@ -12,11 +12,11 @@
 namespace ttg::device {
 
   namespace detail {
-    template<typename... Ts>
+    template <typename... Ts>
     struct to_device_t {
       std::tuple<std::add_lvalue_reference_t<Ts>...> ties;
     };
-  } // namespace detail
+  }  // namespace detail
 
   /**
    * Select a device to execute on based on the provided buffer and scratchspace objects.
@@ -25,9 +25,9 @@ namespace ttg::device {
    * \sa ttg::device::current_stream are available) and the buffers are available on the
    * selected device.
    */
-  template<typename... Args>
+  template <typename... Args>
   [[nodiscard]]
-  inline auto select(Args&&... args) {
+  inline auto select(Args &&...args) {
     return detail::to_device_t<std::remove_reference_t<Args>...>{std::tie(std::forward<Args>(args)...)};
   }
 
@@ -42,17 +42,16 @@ namespace ttg::device {
       TTG_DEVICE_CORO_COMPLETE
     };
 
-
-    template<typename... Ts>
+    template <typename... Ts>
     struct wait_kernel_t {
-      std::tuple<Ts&...> ties;
+      std::tuple<Ts &...> ties;
 
       /* always suspend */
       constexpr bool await_ready() const noexcept { return false; }
 
       /* always suspend */
       template <typename Promise>
-      constexpr void await_suspend( ttg::coroutine_handle<Promise> ) const noexcept {}
+      constexpr void await_suspend(ttg::coroutine_handle<Promise>) const noexcept {}
 
       void await_resume() noexcept {
         if constexpr (sizeof...(Ts) > 0) {
@@ -61,19 +60,20 @@ namespace ttg::device {
         }
       }
     };
-  } // namespace detail
+  }  // namespace detail
 
   /**
    * Wait for previously submitted kernels to complete and provided
    * ttg::Buffer and ttg::devicescratch to be transferred back to host.
    * Must only be called after awaiting \sa ttg::device::select has resumed.
    */
-  template<typename... Buffers>
+  template <typename... Buffers>
   [[nodiscard]]
-  inline auto wait(Buffers&&... args) {
-    static_assert(((ttg::meta::is_buffer_v<std::decay_t<Buffers>>
-                    ||ttg::meta::is_devicescratch_v<std::decay_t<Buffers>>)&&...),
-                  "Only ttg::Buffer and ttg::devicescratch can be waited on!");
+  inline auto wait(Buffers &&...args) {
+    static_assert(
+        ((ttg::meta::is_buffer_v<std::decay_t<Buffers>> || ttg::meta::is_devicescratch_v<std::decay_t<Buffers>>) &&
+         ...),
+        "Only ttg::Buffer and ttg::devicescratch can be waited on!");
     return detail::wait_kernel_t<std::remove_reference_t<Buffers>...>{std::tie(std::forward<Buffers>(args)...)};
   }
 
@@ -102,12 +102,10 @@ namespace ttg::device {
 
       send_coro_state(base_type base) : base_type(std::move(base)) {}
 
-      base_type& handle() { return *this; }
+      base_type &handle() { return *this; }
 
       /// @return true if ready to resume
-      inline bool ready() {
-        return true;
-      }
+      inline bool ready() { return true; }
 
       /// @return true if task completed and can be destroyed
       inline bool completed();
@@ -115,96 +113,86 @@ namespace ttg::device {
 
     /// the promise type for the send coroutine
     struct send_coro_promise_type {
-
       /* do not suspend the coroutine on first invocation, we want to run
       * the coroutine immediately and suspend only once.
-      */
-      ttg::suspend_never initial_suspend() {
-        return {};
-      }
+       */
+      ttg::suspend_never initial_suspend() { return {}; }
 
       /* we don't suspend the coroutine at the end.
       * it can be destroyed once the send/broadcast is done
-      */
-      ttg::suspend_never final_suspend() noexcept {
-        return {};
-      }
+       */
+      ttg::suspend_never final_suspend() noexcept { return {}; }
 
       send_coro_state get_return_object() { return send_coro_state{send_coro_handle_type::from_promise(*this)}; }
 
       /* the send coros only have an empty co_await */
-      ttg::suspend_always await_transform(ttg::Void) {
-        return {};
-      }
+      ttg::suspend_always await_transform(ttg::Void) { return {}; }
 
       void unhandled_exception() {
         std::cerr << "Send coroutine caught an unhandled exception!" << std::endl;
-        throw; // fwd
+        throw;  // fwd
       }
 
       void return_void() {}
-
     };
 
-    template<typename Key, typename Value, ttg::Runtime Runtime = ttg::ttg_runtime>
-    inline send_coro_state send_coro(const Key& key, Value&& value, ttg::Out<Key, std::decay_t<Value>> &t,
-                                     ttg::detail::value_copy_handler<Runtime>& ch) {
-      ttg::detail::value_copy_handler<Runtime> copy_handler = std::move(ch); // destroyed at the end of the coro
+    template <typename Key, typename Value, ttg::Runtime Runtime = ttg::ttg_runtime>
+    inline send_coro_state send_coro(const Key &key, Value &&value, ttg::Out<Key, std::decay_t<Value>> &t,
+                                     ttg::detail::value_copy_handler<Runtime> &ch) {
+      ttg::detail::value_copy_handler<Runtime> copy_handler = std::move(ch);  // destroyed at the end of the coro
       Key k = key;
       t.prepare_send(k, std::forward<Value>(value));
-      co_await ttg::Void{}; // we'll come back once the task is done
+      co_await ttg::Void{};  // we'll come back once the task is done
       t.send(k, std::forward<Value>(value));
     };
 
-    template<typename Value, ttg::Runtime Runtime = ttg::ttg_runtime>
-    inline send_coro_state sendv_coro(Value&& value, ttg::Out<void, std::decay_t<Value>> &t,
-                                      ttg::detail::value_copy_handler<Runtime>& ch) {
-      ttg::detail::value_copy_handler<Runtime> copy_handler = std::move(ch); // destroyed at the end of the coro
+    template <typename Value, ttg::Runtime Runtime = ttg::ttg_runtime>
+    inline send_coro_state sendv_coro(Value &&value, ttg::Out<void, std::decay_t<Value>> &t,
+                                      ttg::detail::value_copy_handler<Runtime> &ch) {
+      ttg::detail::value_copy_handler<Runtime> copy_handler = std::move(ch);  // destroyed at the end of the coro
       t.prepare_send(std::forward<Value>(value));
-      co_await ttg::Void{}; // we'll come back once the task is done
+      co_await ttg::Void{};  // we'll come back once the task is done
       t.sendv(std::forward<Value>(value));
     };
 
-    template<typename Key, ttg::Runtime Runtime = ttg::ttg_runtime>
-    inline send_coro_state sendk_coro(const Key& key, ttg::Out<Key, void> &t) {
+    template <typename Key, ttg::Runtime Runtime = ttg::ttg_runtime>
+    inline send_coro_state sendk_coro(const Key &key, ttg::Out<Key, void> &t) {
       // no need to prepare the send but we have to suspend once
       Key k = key;
-      co_await ttg::Void{}; // we'll come back once the task is done
+      co_await ttg::Void{};  // we'll come back once the task is done
       t.sendk(k);
     };
 
-    template<ttg::Runtime Runtime = ttg::ttg_runtime>
+    template <ttg::Runtime Runtime = ttg::ttg_runtime>
     inline send_coro_state send_coro(ttg::Out<void, void> &t) {
       // no need to prepare the send but we have to suspend once
-      co_await ttg::Void{}; // we'll come back once the task is done
+      co_await ttg::Void{};  // we'll come back once the task is done
       t.send();
     };
 
     struct send_t {
       send_coro_state coro;
     };
-  } // namespace detail
+  }  // namespace detail
 
   template <size_t i, typename keyT, typename valueT, typename... out_keysT, typename... out_valuesT,
             ttg::Runtime Runtime = ttg::ttg_runtime>
-  inline detail::send_t send(const keyT &key, valueT &&value,
-                                      std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
+  inline detail::send_t send(const keyT &key, valueT &&value, std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
     ttg::detail::value_copy_handler<Runtime> copy_handler;
-    return detail::send_t{detail::send_coro(key, copy_handler(std::forward<valueT>(value)), std::get<i>(t), copy_handler)};
+    return detail::send_t{
+        detail::send_coro(key, copy_handler(std::forward<valueT>(value)), std::get<i>(t), copy_handler)};
   }
 
   template <size_t i, typename valueT, typename... out_keysT, typename... out_valuesT,
             ttg::Runtime Runtime = ttg::ttg_runtime>
-  inline detail::send_t sendv(
-      valueT &&value, std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
+  inline detail::send_t sendv(valueT &&value, std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
     ttg::detail::value_copy_handler<Runtime> copy_handler;
     return detail::send_t{detail::sendv_coro(copy_handler(std::forward<valueT>(value)), std::get<i>(t), copy_handler)};
   }
 
   template <size_t i, typename Key, typename... out_keysT, typename... out_valuesT,
             ttg::Runtime Runtime = ttg::ttg_runtime>
-  inline detail::send_t sendk(
-      const Key& key, std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
+  inline detail::send_t sendk(const Key &key, std::tuple<ttg::Out<out_keysT, out_valuesT>...> &t) {
     return detail::send_t{detail::sendk_coro(key, std::get<i>(t))};
   }
 
@@ -419,7 +407,7 @@ namespace ttg::device {
 
   /// A device::Task is a coroutine (a callable that can be suspended and resumed).
 
-  /// \note Since task execution in TTG is not preempable, tasks should not block.
+  /// Since task execution in TTG is not preempable, tasks should not block.
   /// The purpose of suspending a task is to yield control back to the runtime until some events occur;
   /// in the meantime its executor (e.g., a user-space thread) can perform other work.
   /// Once the task function reaches a point where further progress is pending completion of one or more asynchronous
diff --git a/ttg/ttg/madness/ttg.h b/ttg/ttg/madness/ttg.h
index 86d05a15f..5fab6f6dd 100644
--- a/ttg/ttg/madness/ttg.h
+++ b/ttg/ttg/madness/ttg.h
@@ -24,7 +24,7 @@
 #include "ttg/util/void.h"
 #include "ttg/world.h"
 #ifdef TTG_HAS_COROUTINE
-#include "ttg/util/coroutine.h"
+#include "ttg/coroutine.h"
 #endif
 
 #include <array>
@@ -304,6 +304,7 @@ namespace ttg_madness {
       std::conditional_t<ttg::meta::is_void_v<keyT>, ttg::Void, keyT> key;  // Task key
 #ifdef TTG_HAS_COROUTINE
       void *suspended_task_address = nullptr;  // if not null the function is suspended
+      ttg::TaskCoroutineID coroutine_id = ttg::TaskCoroutineID::Invalid;
 #endif
 
       /// makes a tuple of references out of tuple of
@@ -344,17 +345,19 @@ namespace ttg_madness {
           if constexpr (!ttg::meta::is_void_v<keyT> && !ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
             TTG_PROCESS_TT_OP_RETURN(
                 suspended_task_address,
+                coroutine_id,
                 derived->op(key, this->make_input_refs(),
                             derived->output_terminals));  // !!! NOTE converting input values to refs
           } else if constexpr (!ttg::meta::is_void_v<keyT> && ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
-            TTG_PROCESS_TT_OP_RETURN(suspended_task_address, derived->op(key, derived->output_terminals));
+            TTG_PROCESS_TT_OP_RETURN(suspended_task_address, coroutine_id, derived->op(key, derived->output_terminals));
           } else if constexpr (ttg::meta::is_void_v<keyT> && !ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
             TTG_PROCESS_TT_OP_RETURN(
                 suspended_task_address,
+                coroutine_id,
                 derived->op(this->make_input_refs(),
                             derived->output_terminals));  // !!! NOTE converting input values to refs
           } else if constexpr (ttg::meta::is_void_v<keyT> && ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
-            TTG_PROCESS_TT_OP_RETURN(suspended_task_address, derived->op(derived->output_terminals));
+            TTG_PROCESS_TT_OP_RETURN(suspended_task_address, coroutine_id, derived->op(derived->output_terminals));
           } else  // unreachable
             ttg::abort();
         } else {  // resume suspended coroutine
@@ -384,9 +387,12 @@ namespace ttg_madness {
         if (suspended_task_address) {
           // TODO implement handling of suspended coroutines properly
 
+          // only resumable_task is recognized at the moment
+          assert(coroutine_id == ttg::TaskCoroutineID::ResumableTask);
+
           // right now can events are not properly implemented, we are only testing the workflow with dummy events
-          // so mark the events finished manually, parsec will rerun this task again and it should complete the second
-          // time
+          // so mark the events finished manually
+          // proper thing to do is to process event queue and resubmit this task again
           auto events =
               static_cast<ttg::resumable_task>(ttg::coroutine_handle<ttg::resumable_task_state>::from_address(suspended_task_address)).events();
           for (auto &event_ptr : events) {
diff --git a/ttg/ttg/parsec/task.h b/ttg/ttg/parsec/task.h
index 44148e1db..5b23d53af 100644
--- a/ttg/ttg/parsec/task.h
+++ b/ttg/ttg/parsec/task.h
@@ -187,6 +187,7 @@ namespace ttg_parsec {
       std::array<stream_info_t, num_streams> streams;
 #ifdef TTG_HAS_COROUTINE
       void* suspended_task_address = nullptr;  // if not null the function is suspended
+      ttg::TaskCoroutineID coroutine_id = ttg::TaskCoroutineID::Invalid;
 #endif
       device_state_t<TT::derived_has_device_op()> dev_state;
       ttg_data_copy_t *copies[num_copies] = { nullptr };  // the data copies tracked by this task
@@ -243,6 +244,7 @@ namespace ttg_parsec {
       std::array<stream_info_t, num_streams> streams;
 #ifdef TTG_HAS_COROUTINE
       void* suspended_task_address = nullptr;  // if not null the function is suspended
+      ttg::TaskCoroutineID coroutine_id = ttg::TaskCoroutineID::Invalid;
 #endif
       device_state_t<TT::derived_has_device_op()> dev_state;
       ttg_data_copy_t *copies[num_streams+1] = { nullptr };  // the data copies tracked by this task
diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 3f8771764..df02de725 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1610,38 +1610,46 @@ namespace ttg_parsec {
 
         if constexpr (!ttg::meta::is_void_v<keyT> && !ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
           auto input = make_tuple_of_ref_from_array(task, std::make_index_sequence<numinvals>{});
-          TTG_PROCESS_TT_OP_RETURN(suspended_task_address, baseobj->template op<Space>(task->key, std::move(input), obj->output_terminals));
+          TTG_PROCESS_TT_OP_RETURN(suspended_task_address, task->coroutine_id, baseobj->template op<Space>(task->key, std::move(input), obj->output_terminals));
         } else if constexpr (!ttg::meta::is_void_v<keyT> && ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
-          TTG_PROCESS_TT_OP_RETURN(suspended_task_address, baseobj->template op<Space>(task->key, obj->output_terminals));
+          TTG_PROCESS_TT_OP_RETURN(suspended_task_address, task->coroutine_id, baseobj->template op<Space>(task->key, obj->output_terminals));
         } else if constexpr (ttg::meta::is_void_v<keyT> && !ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
           auto input = make_tuple_of_ref_from_array(task, std::make_index_sequence<numinvals>{});
-          TTG_PROCESS_TT_OP_RETURN(suspended_task_address, baseobj->template op<Space>(std::move(input), obj->output_terminals));
+          TTG_PROCESS_TT_OP_RETURN(suspended_task_address, task->coroutine_id, baseobj->template op<Space>(std::move(input), obj->output_terminals));
         } else if constexpr (ttg::meta::is_void_v<keyT> && ttg::meta::is_empty_tuple_v<input_values_tuple_type>) {
-          TTG_PROCESS_TT_OP_RETURN(suspended_task_address, baseobj->template op<Space>(obj->output_terminals));
+          TTG_PROCESS_TT_OP_RETURN(suspended_task_address, task->coroutine_id, baseobj->template op<Space>(obj->output_terminals));
         } else {
           ttg::abort();
         }
         detail::parsec_ttg_caller = nullptr;
       }
       else {  // resume the suspended coroutine
+
+        assert(task->coroutine_id != ttg::TaskCoroutineID::Invalid);
+
+#ifdef TTG_HAS_COROUTINE
 #ifdef TTG_HAVE_DEVICE
-        ttg::device::Task coro = ttg::device::detail::device_task_handle_type::from_address(suspended_task_address);
-        assert(detail::parsec_ttg_caller == nullptr);
-        detail::parsec_ttg_caller = static_cast<detail::parsec_ttg_task_base_t*>(task);
-        // TODO: unify the outputs tls handling
+        if (task->coroutine_id == ttg::TaskCoroutineID::DeviceTask) {
+          ttg::device::Task coro = ttg::device::detail::device_task_handle_type::from_address(suspended_task_address);
+          assert(detail::parsec_ttg_caller == nullptr);
+          detail::parsec_ttg_caller = static_cast<detail::parsec_ttg_task_base_t*>(task);
+          // TODO: unify the outputs tls handling
+          auto old_output_tls_ptr = task->tt->outputs_tls_ptr_accessor();
+          task->tt->set_outputs_tls_ptr();
+          coro.resume();
+          if (coro.completed()) {
+            coro.destroy();
+            suspended_task_address = nullptr;
+          }
+          task->tt->set_outputs_tls_ptr(old_output_tls_ptr);
+          detail::parsec_ttg_caller = nullptr;
+        } else
+#endif  // TTG_HAVE_DEVICE
+      if (task->coroutine_id == ttg::TaskCoroutineID::ResumableTask) {
+        auto ret = static_cast<ttg::resumable_task>(ttg::coroutine_handle<ttg::resumable_task_state>::from_address(suspended_task_address));
+        assert(ret.ready());
         auto old_output_tls_ptr = task->tt->outputs_tls_ptr_accessor();
         task->tt->set_outputs_tls_ptr();
-        coro.resume();
-        if (coro.completed()) {
-          coro.destroy();
-          suspended_task_address = nullptr;
-        }
-        task->tt->set_outputs_tls_ptr(old_output_tls_ptr);
-        detail::parsec_ttg_caller = nullptr;
-#if 0
-#ifdef TTG_HAS_COROUTINE
-        auto ret = static_cast<ttg::resumable_task>(ttg::coroutine_handle<>::from_address(suspended_task_address));
-        assert(ret.ready());
         ret.resume();
         if (ret.completed()) {
           ret.destroy();
@@ -1649,15 +1657,23 @@ namespace ttg_parsec {
         }
         else { // not yet completed
           // leave suspended_task_address as is
+
+          // right now can events are not properly implemented, we are only testing the workflow with dummy events
+          // so mark the events finished manually, parsec will rerun this task again and it should complete the second time
+          auto events = static_cast<ttg::resumable_task>(ttg::coroutine_handle<ttg::resumable_task_state>::from_address(suspended_task_address)).events();
+          for (auto &event_ptr : events) {
+            event_ptr->finish();
+          }
+          assert(ttg::coroutine_handle<ttg::resumable_task_state>::from_address(suspended_task_address).promise().ready());
         }
-        task->suspended_task_address = suspended_task_address;
-#else
-#endif  // TTG_HAS_COROUTINE
-        ttg::abort();  // should not happen
-#endif  // 0
-#else  // TTG_HAVE_DEVICE
+        task->tt->set_outputs_tls_ptr(old_output_tls_ptr);
+        detail::parsec_ttg_caller = nullptr;
+      }
+      else
+        ttg::abort();  // unrecognized task id
+#else // TTG_HAS_COROUTINE
 ttg::abort();  // should not happen
-#endif  // TTG_HAVE_DEVICE
+#endif  // TTG_HAS_COROUTINE
       }
       task->suspended_task_address = suspended_task_address;
 
@@ -1672,30 +1688,7 @@ ttg::abort();  // should not happen
         }
       }
 
-// XXX the below code is not needed, should be removed once the fib test has been changed
-#if 0
-#ifdef TTG_HAS_COROUTINE
-      if (suspended_task_address) {
-        // right now can events are not properly implemented, we are only testing the workflow with dummy events
-        // so mark the events finished manually, parsec will rerun this task again and it should complete the second time
-        auto events = static_cast<ttg::resumable_task>(ttg::coroutine_handle<>::from_address(suspended_task_address)).events();
-        for (auto &event_ptr : events) {
-          event_ptr->finish();
-        }
-        assert(ttg::coroutine_handle<>::from_address(suspended_task_address).promise().ready());
-
-        // TODO: shove {ptr to parsec_task, ptr to this function} to the list of tasks suspended by this thread (hence stored in TLS)
-        // thread will loop over its list (after running every task? periodically? need a dedicated queue of ready tasks?)
-        // and resume the suspended tasks whose events are ready (N.B. ptr to parsec_task is enough to get the list of pending events)
-        // event clearance will in device case handled by host callbacks run by the dedicated device runtime thread
-
-        // TODO PARSEC_HOOK_RETURN_AGAIN -> PARSEC_HOOK_RETURN_ASYNC when event tracking and task resumption (by this thread) is ready
-        return PARSEC_HOOK_RETURN_AGAIN;
-      }
-      else
-#endif  // TTG_HAS_COROUTINE
-#endif // 0
-        return PARSEC_HOOK_RETURN_DONE;
+      return PARSEC_HOOK_RETURN_DONE;
     }
 
     template <ttg::ExecutionSpace Space>
@@ -1714,9 +1707,9 @@ ttg::abort();  // should not happen
         assert(detail::parsec_ttg_caller == NULL);
         detail::parsec_ttg_caller = task;
         if constexpr (!ttg::meta::is_void_v<keyT>) {
-          TTG_PROCESS_TT_OP_RETURN(suspended_task_address, baseobj->template op<Space>(task->key, obj->output_terminals));
+          TTG_PROCESS_TT_OP_RETURN(suspended_task_address, task->coroutine_id, baseobj->template op<Space>(task->key, obj->output_terminals));
         } else if constexpr (ttg::meta::is_void_v<keyT>) {
-          TTG_PROCESS_TT_OP_RETURN(suspended_task_address, baseobj->template op<Space>(obj->output_terminals));
+          TTG_PROCESS_TT_OP_RETURN(suspended_task_address, task->coroutine_id, baseobj->template op<Space>(obj->output_terminals));
         } else  // unreachable
           ttg:: abort();
         detail::parsec_ttg_caller = NULL;
@@ -3655,29 +3648,30 @@ ttg::abort();  // should not happen
 
       /* if we still have a coroutine handle we invoke it one more time to get the sends/broadcasts */
       if (task->suspended_task_address) {
+        assert(task->coroutine_id != ttg::TaskCoroutineID::Invalid);
 #ifdef TTG_HAVE_DEVICE
-        /* increment versions of all data we might have modified
-         * this must happen before we issue the sends */
-        //increment_data_versions(task, std::make_index_sequence<std::tuple_size_v<typename TT::input_values_tuple_type>>{});
-
-        // get the device task from the coroutine handle
-        auto dev_task = ttg::device::detail::device_task_handle_type::from_address(task->suspended_task_address);
-
-        // get the promise which contains the views
-        auto dev_data = dev_task.promise();
-
-        /* for now make sure we're waiting for the kernel to complete and the coro hasn't skipped this step */
-        assert(dev_data.state() == ttg::device::detail::TTG_DEVICE_CORO_SENDOUT);
-
-        /* execute the sends we stored */
-        if (dev_data.state() == ttg::device::detail::TTG_DEVICE_CORO_SENDOUT) {
-          /* set the current task, needed inside the sends */
-          detail::parsec_ttg_caller = task;
-          dev_data.do_sends();
-          detail::parsec_ttg_caller = nullptr;
+        if (task->coroutine_id == ttg::TaskCoroutineID::DeviceTask) {
+          /* increment versions of all data we might have modified
+           * this must happen before we issue the sends */
+          //increment_data_versions(task, std::make_index_sequence<std::tuple_size_v<typename TT::input_values_tuple_type>>{});
+
+          // get the device task from the coroutine handle
+          auto dev_task = ttg::device::detail::device_task_handle_type::from_address(task->suspended_task_address);
+
+          // get the promise which contains the views
+          auto dev_data = dev_task.promise();
+
+          /* for now make sure we're waiting for the kernel to complete and the coro hasn't skipped this step */
+          assert(dev_data.state() == ttg::device::detail::TTG_DEVICE_CORO_SENDOUT);
+
+          /* execute the sends we stored */
+          if (dev_data.state() == ttg::device::detail::TTG_DEVICE_CORO_SENDOUT) {
+            /* set the current task, needed inside the sends */
+            detail::parsec_ttg_caller = task;
+            dev_data.do_sends();
+            detail::parsec_ttg_caller = nullptr;
+          }
         }
-#else // TTG_HAVE_DEVICE
-        ttg::abort();  // should not happen
 #endif // TTG_HAVE_DEVICE
         /* the coroutine should have completed and we cannot access the promise anymore */
         task->suspended_task_address = nullptr;
diff --git a/ttg/ttg/tt.h b/ttg/ttg/tt.h
index 66f454bce..7024776aa 100644
--- a/ttg/ttg/tt.h
+++ b/ttg/ttg/tt.h
@@ -1,18 +1,19 @@
 #ifndef TTG_TT_H
 #define TTG_TT_H
 
-#include <memory>
-#include <vector>
-
 #include "ttg/fwd.h"
 
 #include "ttg/base/tt.h"
 #include "ttg/edge.h"
 
 #ifdef TTG_HAS_COROUTINE
-#include "ttg/util/coroutine.h"
+#include "ttg/coroutine.h"
 #endif
 
+#include <cassert>
+#include <memory>
+#include <vector>
+
 namespace ttg {
 
   // TODO describe TT concept (preferably as a C++20 concept)
@@ -176,19 +177,29 @@ namespace ttg {
 
 #ifndef TTG_PROCESS_TT_OP_RETURN
 #ifdef TTG_HAS_COROUTINE
-#define TTG_PROCESS_TT_OP_RETURN(result, invoke)       \
-  {                                                    \
-    using return_type = decltype(invoke);              \
-    if constexpr (std::is_same_v<return_type, void>) { \
-      invoke;                                          \
-    } else {                                           \
-      auto coro_return = invoke;                       \
-      result = coro_return.address();                  \
-    }                                                  \
+#define TTG_PROCESS_TT_OP_RETURN(result, id, invoke)                                                           \
+  {                                                                                                            \
+    using return_type = decltype(invoke);                                                                      \
+    if constexpr (std::is_same_v<return_type, void>) {                                                         \
+      invoke;                                                                                                  \
+      id = ttg::TaskCoroutineID::Invalid;                                                                      \
+    } else {                                                                                                   \
+      auto coro_return = invoke;                                                                               \
+      if constexpr (std::is_same_v<decltype(coro_return), ttg::coroutine_handle<ttg::resumable_task_state>>)   \
+        id = ttg::TaskCoroutineID::ResumableTask;                                                              \
+      else if constexpr (std::is_same_v<decltype(coro_return),                                                 \
+                                        ttg::coroutine_handle<ttg::device::detail::device_task_promise_type>>) \
+        id = ttg::TaskCoroutineID::DeviceTask;                                                                 \
+      else                                                                                                     \
+        std::abort();                                                                                          \
+      result = coro_return.address();                                                                          \
+    }                                                                                                          \
   }
 #else
 #define TTG_PROCESS_TT_OP_RETURN(result, invoke) invoke
 #endif
+#else
+#error "TTG_PROCESS_TT_OP_RETURN already defined in ttg/tt.h, check your header guards"
 #endif  // !defined(TTG_PROCESS_TT_OP_RETURN)
 
 #endif  // TTG_TT_H

From cd9f9d1742b37b299ddc33b4c3e7d089096f3d08 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Mon, 26 Feb 2024 10:28:40 -0500
Subject: [PATCH 256/259] purge Cereal-related code+mentions

---
 CMakeLists.txt                                |   2 -
 .../ExternalDependenciesVersions.cmake        |   1 -
 cmake/modules/FindOrFetchCereal.cmake         |  32 ------
 tests/unit/CMakeLists.txt                     |   2 +-
 tests/unit/serialization.cc                   | 106 +-----------------
 ttg/CMakeLists.txt                            |  13 ---
 ttg/ttg/madness/buffer.h                      |  19 ----
 ttg/ttg/parsec/buffer.h                       |  21 ----
 ttg/ttg/serialization/backends.h              |   1 -
 ttg/ttg/serialization/backends/cereal.h       |  98 ----------------
 ttg/ttg/serialization/data_descriptor.h       |  40 -------
 ttg/ttg/serialization/std/array.h             |  14 ---
 ttg/ttg/serialization/std/list.h              |  14 ---
 ttg/ttg/serialization/std/pair.h              |  14 ---
 ttg/ttg/serialization/std/tuple.h             |  14 ---
 ttg/ttg/serialization/std/vector.h            |  14 ---
 ttg/ttg/serialization/traits.h                |   3 +-
 17 files changed, 3 insertions(+), 405 deletions(-)
 delete mode 100644 cmake/modules/FindOrFetchCereal.cmake
 delete mode 100644 ttg/ttg/serialization/backends/cereal.h

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 59f2d583d..28490557f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -94,8 +94,6 @@ endif (BUILD_TESTING)
 ###########################
 # Boost
 include("${PROJECT_SOURCE_DIR}/cmake/modules/FindOrFetchBoost.cmake")
-# Cereal
-#include(FindOrFetchCereal)
 # C++ coroutines
 find_package(CXXStdCoroutine MODULE REQUIRED COMPONENTS Final Experimental)
 
diff --git a/cmake/modules/ExternalDependenciesVersions.cmake b/cmake/modules/ExternalDependenciesVersions.cmake
index 88cf52b11..fd96e5816 100644
--- a/cmake/modules/ExternalDependenciesVersions.cmake
+++ b/cmake/modules/ExternalDependenciesVersions.cmake
@@ -3,7 +3,6 @@
 
 set(TTG_TRACKED_VG_CMAKE_KIT_TAG 7ea2d4d3f8854b9e417f297fd74d6fc49aa13fd5)  # used to provide "real" FindOrFetchBoost
 set(TTG_TRACKED_CATCH2_VERSION 3.5.0)
-set(TTG_TRACKED_CEREAL_VERSION 1.3.0)
 set(TTG_TRACKED_MADNESS_TAG 2eb3bcf0138127ee2dbc651f1aabd3e9b0def4e3)
 set(TTG_TRACKED_PARSEC_TAG 0b3140f58ad9dc78a3d64da9fd73ecc7f443ece7)
 set(TTG_TRACKED_BTAS_TAG 4e8f5233aa7881dccdfcc37ce07128833926d3c2)
diff --git a/cmake/modules/FindOrFetchCereal.cmake b/cmake/modules/FindOrFetchCereal.cmake
deleted file mode 100644
index b1e6ab1c0..000000000
--- a/cmake/modules/FindOrFetchCereal.cmake
+++ /dev/null
@@ -1,32 +0,0 @@
-if (NOT TARGET cereal::cereal)
-  # find_package(cereal ${TTG_TRACKED_CEREAL_VERSION} QUIET)
-  # homebrew on macos provides cereal-config with version "unknown"
-  find_package(cereal)
-  if (cereal_FOUND AND NOT TARGET cereal::cereal)
-    if (TARGET cereal)
-      add_library(cereal::cereal ALIAS cereal)
-    else ()
-      message(FATAL_ERROR "cereal_FOUND=TRUE but no cereal target")
-    endif()
-  endif()
-endif(NOT TARGET cereal::cereal)
-
-if (TARGET cereal::cereal)
-  message(STATUS "Found cereal at ${cereal_CONFIG}")
-else (TARGET cereal::cereal)
-  # going hungry today
-endif()
-
-# fetchcontent is disabled for now
-if (FALSE)
-  FetchContent_Declare(
-          cereal
-          GIT_REPOSITORY      https://github.com/USCiLab/cereal
-          GIT_TAG v${TTG_TRACKED_CEREAL_VERSION})
-  FetchContent_MakeAvailable(cereal)
-  FetchContent_GetProperties(cereal
-          SOURCE_DIR CEREAL_SOURCE_DIR
-          BINARY_DIR CEREAL_BINARY_DIR
-          )
-
-endif(FALSE)
diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt
index 69378cbf3..275f3fdd8 100644
--- a/tests/unit/CMakeLists.txt
+++ b/tests/unit/CMakeLists.txt
@@ -21,7 +21,7 @@ list(APPEND ut_libs std::coroutine)
 
 add_ttg_executable(core-unittests-ttg "${ut_src}" LINK_LIBRARIES "${ut_libs}" COMPILE_DEFINITIONS "CATCH_CONFIG_NO_POSIX_SIGNALS=1" )
 
-# serialization test: probes serialization via all supported serialization methods (MADNESS, Boost::serialization, cereal) that are available
+# serialization test: probes serialization via all supported serialization methods (MADNESS, Boost::serialization) that are available
 add_ttg_executable(serialization serialization.cc unit_main.cpp
                    LINK_LIBRARIES Catch2::Catch2 ttg-serialization  $<TARGET_NAME_IF_EXISTS:BTAS::BTAS>
                    COMPILE_DEFINITIONS $<$<TARGET_EXISTS:BTAS::BTAS>:TTG_HAS_BTAS=1>)
diff --git a/tests/unit/serialization.cc b/tests/unit/serialization.cc
index 0c14cfaca..e12607de3 100644
--- a/tests/unit/serialization.cc
+++ b/tests/unit/serialization.cc
@@ -42,24 +42,14 @@ class POD {
 static_assert(std::is_trivially_copyable_v<POD>);
 static_assert(ttg::detail::is_memcpyable_v<POD>);
 
-#ifdef TTG_SERIALIZATION_SUPPORTS_CEREAL
-// WTF?! std::array of non-Serializable is Serializable
-static_assert(!cereal::traits::is_input_serializable<POD, cereal::BinaryInputArchive>::value);
-static_assert(!cereal::traits::is_output_serializable<POD, cereal::BinaryOutputArchive>::value);
-static_assert(!cereal::traits::is_input_serializable<std::array<POD, 3>, cereal::BinaryInputArchive>::value);
-static_assert(!cereal::traits::is_output_serializable<std::array<POD, 3>, cereal::BinaryOutputArchive>::value);
-#endif  // TTG_SERIALIZATION_SUPPORTS_CEREAL
-
 static_assert(!ttg::detail::is_madness_user_buffer_serializable_v<POD>);
 #ifdef TTG_SERIALIZATION_SUPPORTS_BOOST
 static_assert(!ttg::detail::is_boost_serializable_v<boost::archive::binary_oarchive, POD>);
 static_assert(ttg::detail::has_freestanding_boost_serialize_with_version_v<POD, boost::archive::binary_oarchive>);
 #endif  // TTG_SERIALIZATION_SUPPORTS_BOOST
 static_assert(!ttg::detail::is_boost_user_buffer_serializable_v<POD>);
-static_assert(!ttg::detail::is_cereal_user_buffer_serializable_v<POD>);
 static_assert(!ttg::detail::is_madness_user_buffer_serializable_v<std::array<POD, 3>>);
 static_assert(!ttg::detail::is_boost_user_buffer_serializable_v<std::array<POD, 3>>);
-static_assert(!ttg::detail::is_cereal_user_buffer_serializable_v<std::array<POD, 3>>);
 
 std::ostream& operator<<(std::ostream& s, const POD& f) {
   s << "POD(" << f.get() << ")";
@@ -161,7 +151,7 @@ namespace intrusive::symmetric::bc_v {
 
     int get() const { return value; }
 
-    // boost uses `unsigned int` for version, cereal uses `std::uint32_t`
+    // boost uses `unsigned int` for version
     template <typename Archive>
     void serialize(Archive& ar, const unsigned int version) {
       ar& value;
@@ -180,9 +170,6 @@ namespace intrusive_private::symmetric::bc_v {
 #ifdef TTG_SERIALIZATION_SUPPORTS_BOOST
     friend class boost::serialization::access;
 #endif
-#ifdef TTG_SERIALIZATION_SUPPORTS_CEREAL
-    friend class cereal::access;  // befriend the cereal version of access
-#endif
 
     template <typename Archive>
     void serialize(Archive& ar, const unsigned int version) {
@@ -201,72 +188,6 @@ namespace intrusive_private::symmetric::bc_v {
   static_assert(!std::is_trivially_copyable_v<NonPOD>);
 }  // namespace intrusive_private::symmetric::bc_v
 
-#ifdef TTG_SERIALIZATION_SUPPORTS_CEREAL
-namespace intrusive::symmetric::c {
-
-  class NonPOD {
-    int value;
-
-   public:
-    NonPOD() = default;
-    NonPOD(int value) : value(value) {}
-    NonPOD(const NonPOD& other) : value(other.value) {}
-
-    int get() const { return value; }
-
-    // versioned
-    template <class Archive>
-    std::enable_if_t<std::is_base_of_v<cereal::detail::InputArchiveBase, Archive> ||
-                     std::is_base_of_v<cereal::detail::OutputArchiveBase, Archive>>
-    serialize(Archive& ar) {
-      ar(value);
-    }
-  };
-  static_assert(!std::is_trivially_copyable_v<NonPOD>);
-
-  static_assert(ttg::detail::is_cereal_buffer_serializable_v<NonPOD>);
-  static_assert(!ttg::detail::is_boost_buffer_serializable_v<NonPOD>);
-  static_assert(!ttg::detail::is_madness_buffer_serializable_v<NonPOD>);
-  static_assert(ttg::detail::is_cereal_user_buffer_serializable_v<NonPOD>);
-  static_assert(!ttg::detail::is_boost_user_buffer_serializable_v<NonPOD>);
-  static_assert(!ttg::detail::is_madness_user_buffer_serializable_v<NonPOD>);
-
-}  // namespace intrusive::symmetric::c
-
-namespace intrusive::symmetric::c_v {
-
-  class NonPOD {
-    int value;
-
-   public:
-    NonPOD() = default;
-    NonPOD(int value) : value(value) {}
-    NonPOD(const NonPOD& other) : value(other.value) {}
-
-    int get() const { return value; }
-
-    // versioned
-    template <class Archive>
-    std::enable_if_t<std::is_base_of_v<cereal::detail::InputArchiveBase, Archive> ||
-                     std::is_base_of_v<cereal::detail::OutputArchiveBase, Archive>>
-    serialize(Archive& ar, std::uint32_t const version) {
-      ar(value);
-    }
-  };
-  static_assert(!std::is_trivially_copyable_v<NonPOD>);
-
-  static_assert(ttg::detail::is_cereal_buffer_serializable_v<NonPOD>);
-  static_assert(!ttg::detail::is_boost_serializable_v<boost::archive::binary_iarchive, NonPOD>);
-  static_assert(!ttg::detail::is_boost_buffer_serializable_v<NonPOD>);
-  static_assert(!ttg::detail::is_madness_buffer_serializable_v<NonPOD>);
-  static_assert(ttg::detail::is_cereal_user_buffer_serializable_v<NonPOD>);
-  static_assert(!ttg::detail::is_boost_user_buffer_serializable_v<NonPOD>);
-  static_assert(!ttg::detail::is_madness_user_buffer_serializable_v<NonPOD>);
-
-}  // namespace intrusive::symmetric::c_v
-
-#endif  // TTG_SERIALIZATION_SUPPORTS_CEREAL
-
 #ifdef TTG_SERIALIZATION_SUPPORTS_BOOST
 
 // boost serialization, with version (and object) tracking
@@ -372,10 +293,6 @@ static_assert(std::is_same_v<typex<boost::archive::binary_oarchive, intrusive::a
 
 #endif  // TTG_SERIALIZATION_SUPPORTS_BOOST
 
-#ifdef TTG_SERIALIZATION_SUPPORTS_CEREAL
-CEREAL_CLASS_VERSION(intrusive::symmetric::c_v::NonPOD, 17);
-#endif
-
 namespace nonintrusive::symmetric::m {
 
   class NonPOD {
@@ -468,17 +385,14 @@ namespace freestanding::symmetric::bc_v {
 static_assert(ttg::detail::is_madness_buffer_serializable_v<int>);
 static_assert(!ttg::detail::is_madness_user_buffer_serializable_v<int>);
 static_assert(!ttg::detail::is_boost_user_buffer_serializable_v<int>);
-static_assert(!ttg::detail::is_cereal_user_buffer_serializable_v<int>);
 static_assert(!ttg::detail::is_user_buffer_serializable_v<int>);
 static_assert(ttg::detail::is_madness_buffer_serializable_v<const int>);
 static_assert(!ttg::detail::is_madness_user_buffer_serializable_v<const int>);
 static_assert(!ttg::detail::is_boost_user_buffer_serializable_v<const int>);
-static_assert(!ttg::detail::is_cereal_user_buffer_serializable_v<const int>);
 static_assert(!ttg::detail::is_user_buffer_serializable_v<const int>);
 static_assert(ttg::detail::is_madness_buffer_serializable_v<int[4]>);
 static_assert(!ttg::detail::is_madness_user_buffer_serializable_v<int[4]>);
 static_assert(!ttg::detail::is_boost_user_buffer_serializable_v<int[4]>);
-static_assert(!ttg::detail::is_cereal_user_buffer_serializable_v<int[4]>);
 static_assert(!ttg::detail::is_user_buffer_serializable_v<int[4]>);
 static_assert(!ttg::detail::is_user_buffer_serializable_v<std::array<int, 4>>);
 
@@ -819,20 +733,6 @@ TEST_CASE("Boost Serialization", "[serialization]") {
 }
 #endif  // TTG_SERIALIZATION_SUPPORTS_BOOST
 
-#ifdef TTG_SERIALIZATION_SUPPORTS_CEREAL
-TEST_CASE("Cereal Serialization", "[serialization]") {
-  auto test = [](const auto& t) {
-    using T = std::remove_reference_t<decltype(t)>;
-    CHECK(ttg::detail::is_cereal_serializable_v<cereal::BinaryOutputArchive, T>);
-    using Tnc = std::remove_const_t<T>;
-    CHECK(ttg::detail::is_cereal_serializable_v<cereal::BinaryInputArchive, Tnc>);
-  };
-
-  test(intrusive::symmetric::bc_v::NonPOD{17});
-  test(freestanding::symmetric::bc_v::NonPOD{18});
-}
-#endif  // TTG_SERIALIZATION_SUPPORTS_CEREAL
-
 #if defined(TTG_SERIALIZATION_SUPPORTS_MADNESS) && defined(TTG_SERIALIZATION_SUPPORTS_BOOST)
 TEST_CASE("TTG Serialization", "[serialization]") {
   // Test code written as if calling from C
@@ -888,10 +788,6 @@ TEST_CASE("TTG Serialization", "[serialization]") {
   test_struct(intrusive::symmetric::bc_v::NonPOD{20});     // Boost
   test_struct(freestanding::symmetric::bc_v::NonPOD{21});  // Boost
 #endif
-#ifdef TTG_SERIALIZATION_SUPPORTS_CEREAL
-  test_struct(intrusive::symmetric::c::NonPOD{22});    // Cereal
-  test_struct(intrusive::symmetric::c_v::NonPOD{23});  // Cereal
-#endif
 
 #ifdef TTG_SERIALIZATION_SUPPORTS_BOOST
   // verify that turning off version and object tracking for Boost produces same archive since the TTG boost archives
diff --git a/ttg/CMakeLists.txt b/ttg/CMakeLists.txt
index e0785433e..900df6519 100644
--- a/ttg/CMakeLists.txt
+++ b/ttg/CMakeLists.txt
@@ -153,7 +153,6 @@ set(ttg-serialization-headers
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/serialization/traits.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/serialization/backends/boost.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/serialization/backends/boost/archive.h
-        ${CMAKE_CURRENT_SOURCE_DIR}/ttg/serialization/backends/cereal.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/serialization/backends/madness.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/serialization/std/allocator.h
         ${CMAKE_CURRENT_SOURCE_DIR}/ttg/serialization/std/array.h
@@ -180,10 +179,6 @@ if (TARGET Boost::serialization)
   endif()
   list(APPEND ttg-serialization-compile-definitions TTG_SERIALIZATION_SUPPORTS_BOOST=1)
 endif (TARGET Boost::serialization)
-if (TARGET cereal::cereal)
-  list(APPEND ttg-serialization-deps cereal::cereal)
-  list(APPEND ttg-serialization-compile-definitions TTG_SERIALIZATION_SUPPORTS_CEREAL=1)
-endif (TARGET cereal::cereal)
 
 add_ttg_library(ttg-serialization
                 "${ttg-serialization-sources}"
@@ -206,14 +201,6 @@ if (TARGET Boost::serialization)
           LINK_LIBRARIES "${ttg-serialization-boost-deps}"
           COMPILE_DEFINITIONS "TTG_SERIALIZATION_SUPPORTS_BOOST=1")
 endif(TARGET Boost::serialization)
-# make cereal-only serialization target
-if (TARGET cereal::cereal)
-  add_ttg_library(ttg-serialization-cereal
-          "${ttg-serialization-sources}"
-          PUBLIC_HEADER "${ttg-serialization-headers}"
-          LINK_LIBRARIES "cereal::cereal"
-          COMPILE_DEFINITIONS "TTG_SERIALIZATION_SUPPORTS_CEREAL=1")
-endif(TARGET cereal::cereal)
 
 #########################
 ####### MADNESS-specific
diff --git a/ttg/ttg/madness/buffer.h b/ttg/ttg/madness/buffer.h
index aac36c24b..8f210c1e7 100644
--- a/ttg/ttg/madness/buffer.h
+++ b/ttg/ttg/madness/buffer.h
@@ -273,25 +273,6 @@ struct Buffer : private Allocator {
   }
 #endif // TTG_SERIALIZATION_SUPPORTS_BOOST
 
-#if defined(TTG_SERIALIZATION_SUPPORTS_CEREAL) && 0
-  template <class Archive>
-  std::enable_if_t<std::is_base_of_v<cereal::detail::InputArchiveBase, Archive> ||
-                    std::is_base_of_v<cereal::detail::OutputArchiveBase, Archive>>
-  serialize(Archive& ar) {
-    if constexpr (ttg::detail::is_output_archive_v<Archive>)
-      std::size_t s = size();
-      ar(s);
-      /* TODO: how to serialize the array? */
-    else {
-      std::size_t s;
-      ar(s);
-      reset(s);
-      /* TODO: how to deserialize the array? */
-    }
-    ar(value);
-  }
-#endif // TTG_SERIALIZATION_SUPPORTS_CEREAL
-
 #if defined(TTG_SERIALIZATION_SUPPORTS_MADNESS)
   template <typename Archive>
   std::enable_if_t<std::is_base_of_v<madness::archive::BufferInputArchive, Archive> ||
diff --git a/ttg/ttg/parsec/buffer.h b/ttg/ttg/parsec/buffer.h
index 590a624f8..98b14eb12 100644
--- a/ttg/ttg/parsec/buffer.h
+++ b/ttg/ttg/parsec/buffer.h
@@ -350,27 +350,6 @@ struct Buffer : public detail::ttg_parsec_data_wrapper_t
   }
 #endif // TTG_SERIALIZATION_SUPPORTS_BOOST
 
-#ifdef TTG_SERIALIZATION_SUPPORTS_CEREAL
-  template <class Archive>
-  std::enable_if_t<std::is_base_of_v<cereal::detail::InputArchiveBase, Archive> ||
-                    std::is_base_of_v<cereal::detail::OutputArchiveBase, Archive>>
-  serialize(Archive& ar) {
-    if constexpr (ttg::detail::is_output_archive_v<Archive>)
-      std::size_t s = size();
-      assert(m_ttg_copy != nullptr); // only tracked objects allowed
-      m_ttg_copy->iovec_add(ttg::iovec{s*sizeof(T), current_device_ptr()});
-      ar(s);
-    else {
-      std::size_t s;
-      ar(s);
-      reset(s);
-      assert(m_ttg_copy != nullptr); // only tracked objects allowed
-      m_ttg_copy->iovec_add(ttg::iovec{s*sizeof(T), current_device_ptr()});
-    }
-    ar(value);
-  }
-#endif // TTG_SERIALIZATION_SUPPORTS_CEREAL
-
 #ifdef TTG_SERIALIZATION_SUPPORTS_MADNESS
   template <typename Archive>
   std::enable_if_t<std::is_base_of_v<madness::archive::BufferInputArchive, Archive> ||
diff --git a/ttg/ttg/serialization/backends.h b/ttg/ttg/serialization/backends.h
index 3cbc9cb42..97f88f5aa 100644
--- a/ttg/ttg/serialization/backends.h
+++ b/ttg/ttg/serialization/backends.h
@@ -6,7 +6,6 @@
 #define TTG_SERIALIZATION_BACKENDS_H
 
 #include "ttg/serialization/backends/boost.h"
-#include "ttg/serialization/backends/cereal.h"
 #include "ttg/serialization/backends/madness.h"
 
 #endif  // TTG_SERIALIZATION_ALL_H
diff --git a/ttg/ttg/serialization/backends/cereal.h b/ttg/ttg/serialization/backends/cereal.h
deleted file mode 100644
index f9e6731b5..000000000
--- a/ttg/ttg/serialization/backends/cereal.h
+++ /dev/null
@@ -1,98 +0,0 @@
-//
-// Created by Eduard Valeyev on 5/3/21.
-//
-
-#ifndef TTG_SERIALIZATION_CEREAL_H
-#define TTG_SERIALIZATION_CEREAL_H
-
-#include <type_traits>
-
-#ifdef TTG_SERIALIZATION_SUPPORTS_CEREAL
-#include <cereal/archives/binary.hpp>
-#include <cereal/cereal.hpp>
-#include <cereal/details/helpers.hpp>
-#include <cereal/details/traits.hpp>
-#endif
-
-namespace ttg::detail {
-
-  //////// is_cereal_serializable
-
-  template <typename Archive, typename T, class = void>
-  struct is_cereal_serializable : std::false_type {};
-
-#ifdef TTG_SERIALIZATION_SUPPORTS_CEREAL
-  template <typename Archive, typename T>
-  struct is_cereal_serializable<Archive, T,
-                                std::enable_if_t<cereal::traits::is_output_serializable<T, Archive>::value ||
-                                                 cereal::traits::is_input_serializable<T, Archive>::value>>
-      : std::true_type {};
-#endif  // TTG_SERIALIZATION_SUPPORTS_CEREAL
-
-  template <typename Archive, typename T>
-  inline static constexpr bool is_cereal_serializable_v = is_cereal_serializable<Archive, T>::value;
-
-  template <typename T, class = void>
-  struct is_cereal_buffer_serializable : std::false_type {};
-
-#ifdef TTG_SERIALIZATION_SUPPORTS_CEREAL
-  template <typename T>
-  struct is_cereal_buffer_serializable<T, std::enable_if_t<is_cereal_serializable_v<cereal::BinaryInputArchive, T> &&
-                                                           is_cereal_serializable_v<cereal::BinaryOutputArchive, T>>>
-      : std::true_type {};
-#endif  // TTG_SERIALIZATION_SUPPORTS_CEREAL
-
-  /// evaluates to true if can serialize @p T to/from buffer using Cereal serialization
-  template <typename T>
-  inline constexpr bool is_cereal_buffer_serializable_v = is_cereal_buffer_serializable<T>::value;
-
-  template <typename Archive, typename T, typename Enabler = void>
-  struct is_cereal_array_serializable;
-
-  template <typename Archive, typename T>
-  struct is_cereal_array_serializable<Archive, T, std::enable_if_t<!std::is_array_v<T>>> : std::false_type {};
-
-  template <typename Archive, typename T>
-  struct is_cereal_array_serializable<Archive, T, std::enable_if_t<std::is_array_v<T>>>
-      : std::bool_constant<is_cereal_serializable_v<Archive, std::remove_extent_t<T>>> {};
-
-  template <typename Archive, typename T>
-  inline static constexpr bool is_cereal_array_serializable_v = is_cereal_array_serializable<Archive, T>::value;
-
-  template <typename Archive, typename T>
-  inline static constexpr bool is_stlcontainer_cereal_serializable_v = false;
-
-  template <typename Archive, typename T, class = void>
-  struct is_cereal_user_serializable : std::false_type {};
-
-#ifdef TTG_SERIALIZATION_SUPPORTS_CEREAL
-  template <typename Archive, typename T>
-  struct is_cereal_user_serializable<
-      Archive, T,
-      std::enable_if_t<(cereal::traits::detail::count_specializations<T, Archive>::value != 0) ||
-                       ((cereal::traits::is_input_serializable<T, Archive>::value ||
-                         cereal::traits::is_output_serializable<T, Archive>::value) &&
-                        (!std::is_arithmetic_v<T> && !ttg::detail::is_cereal_array_serializable_v<Archive, T> &&
-                         !is_stlcontainer_cereal_serializable_v<Archive, T>))>> : std::true_type {};
-#endif
-
-  template <typename Archive, typename T>
-  inline constexpr bool is_cereal_user_serializable_v = is_cereal_user_serializable<Archive, T>::value;
-
-  template <typename T, class = void>
-  struct is_cereal_user_buffer_serializable : std::false_type {};
-
-#ifdef TTG_SERIALIZATION_SUPPORTS_CEREAL
-  template <typename T>
-  struct is_cereal_user_buffer_serializable<
-      T, std::enable_if_t<is_cereal_user_serializable_v<cereal::BinaryInputArchive, T> ||
-                          is_cereal_user_serializable_v<cereal::BinaryOutputArchive, T>>> : std::true_type {};
-#endif  // TTG_SERIALIZATION_SUPPORTS_CEREAL
-
-  /// evaluates to true if can serialize @p T to/from buffer using user-provided Cereal serialization
-  template <typename T>
-  inline constexpr bool is_cereal_user_buffer_serializable_v = is_cereal_user_buffer_serializable<T>::value;
-
-}  // namespace ttg::detail
-
-#endif  // TTG_SERIALIZATION_CEREAL_H
diff --git a/ttg/ttg/serialization/data_descriptor.h b/ttg/ttg/serialization/data_descriptor.h
index fb26b1d55..f14d00e72 100644
--- a/ttg/ttg/serialization/data_descriptor.h
+++ b/ttg/ttg/serialization/data_descriptor.h
@@ -236,46 +236,6 @@ namespace ttg {
 
 #endif  // has Boost serialization
 
-#if defined(TTG_SERIALIZATION_SUPPORTS_CEREAL)
-
-namespace ttg {
-
-  /// The default implementation for non-POD data types that are not directly copyable
-  /// do not support MADNESS or Boost serialization, and support Cereal serialization
-  template <typename T>
-  struct default_data_descriptor<
-      T, std::enable_if_t<(!detail::is_memcpyable_v<T> && !detail::is_madness_buffer_serializable_v<T> &&
-                           !detail::is_boost_buffer_serializable_v<T> && detail::is_cereal_buffer_serializable_v<T>) ||
-                          (!detail::is_madness_user_buffer_serializable_v<T> &&
-                           !detail::is_boost_user_buffer_serializable_v<T> &&
-                           detail::is_cereal_user_buffer_serializable_v<T>)>> {
-    static constexpr const bool serialize_size_is_const = false;
-
-    static uint64_t payload_size(const void *object) {
-      ttg::detail::counting_streambuf sbuf;
-      std::ostream os(&sbuf);
-      cereal::BinaryOutputArchive oa(os);
-      oa << (*(T *)object);
-      return sbuf.size();
-    }
-
-    /// object --- obj to be serialized
-    /// chunk_size --- inputs max amount of data to output, and on output returns amount actually output
-    /// pos --- position in the input buffer to resume serialization
-    /// buf[pos] --- place for output
-    static uint64_t pack_payload(const void *object, uint64_t chunk_size, uint64_t pos, void *_buf) { abort(); }
-
-    /// object --- obj to be deserialized
-    /// chunk_size --- amount of data for input
-    /// pos --- position in the input buffer to resume deserialization
-    /// object -- pointer to the object to fill up
-    static void unpack_payload(void *object, uint64_t chunk_size, uint64_t pos, const void *_buf) { abort(); }
-  };
-
-}  // namespace ttg
-
-#endif  // has Cereal serialization
-
 namespace ttg {
 
   // Returns a pointer to a constant static instance initialized
diff --git a/ttg/ttg/serialization/std/array.h b/ttg/ttg/serialization/std/array.h
index 60aa9fa93..05d008ac8 100644
--- a/ttg/ttg/serialization/std/array.h
+++ b/ttg/ttg/serialization/std/array.h
@@ -25,18 +25,4 @@ namespace ttg::detail {
 
 #endif  // TTG_SERIALIZATION_SUPPORTS_BOOST
 
-#ifdef TTG_SERIALIZATION_SUPPORTS_CEREAL
-#include <cereal/types/array.hpp>
-
-namespace ttg::detail {
-  template <typename Archive, typename T, std::size_t N>
-  inline static constexpr bool is_stlcontainer_cereal_serializable_v<Archive, std::array<T, N>> =
-      is_cereal_serializable_v<Archive, T>;
-  template <typename Archive, typename T, std::size_t N>
-  inline static constexpr bool is_stlcontainer_cereal_serializable_v<Archive, const std::array<T, N>> =
-      is_cereal_serializable_v<Archive, const T>;
-}  // namespace ttg::detail
-
-#endif
-
 #endif  // TTG_SERIALIZATION_STD_ARRAY_H
diff --git a/ttg/ttg/serialization/std/list.h b/ttg/ttg/serialization/std/list.h
index ebf24ab04..3c525b0f8 100644
--- a/ttg/ttg/serialization/std/list.h
+++ b/ttg/ttg/serialization/std/list.h
@@ -26,18 +26,4 @@ namespace ttg::detail {
 
 #endif  // TTG_SERIALIZATION_SUPPORTS_BOOST
 
-#ifdef TTG_SERIALIZATION_SUPPORTS_CEREAL
-#include <cereal/types/list.hpp>
-
-namespace ttg::detail {
-  template <typename Archive, typename T, typename A>
-  inline static constexpr bool is_stlcontainer_cereal_serializable_v<Archive, std::list<T, A>> =
-      is_cereal_serializable_v<Archive, T>;
-  template <typename Archive, typename T, typename A>
-  inline static constexpr bool is_stlcontainer_cereal_serializable_v<Archive, const std::list<T, A>> =
-      is_cereal_serializable_v<Archive, const T>;
-}  // namespace ttg::detail
-
-#endif
-
 #endif  // TTG_SERIALIZATION_STD_LIST_H
diff --git a/ttg/ttg/serialization/std/pair.h b/ttg/ttg/serialization/std/pair.h
index cca0076ed..ccfce52a9 100644
--- a/ttg/ttg/serialization/std/pair.h
+++ b/ttg/ttg/serialization/std/pair.h
@@ -25,18 +25,4 @@ namespace ttg::detail {
 
 #endif  // TTG_SERIALIZATION_SUPPORTS_BOOST
 
-#ifdef TTG_SERIALIZATION_SUPPORTS_CEREAL
-#include <cereal/types/utility.hpp>
-
-namespace ttg::detail {
-  template <typename Archive, typename T1, typename T2>
-  inline static constexpr bool is_stlcontainer_cereal_serializable_v<Archive, std::pair<T1, T2>> =
-      is_cereal_serializable_v<Archive, T1>&& is_cereal_serializable_v<Archive, T2>;
-  template <typename Archive, typename T1, typename T2>
-  inline static constexpr bool is_stlcontainer_cereal_serializable_v<Archive, const std::pair<T1, T2>> =
-      is_cereal_serializable_v<Archive, const T1>&& is_cereal_serializable_v<Archive, const T2>;
-}  // namespace ttg::detail
-
-#endif
-
 #endif  // TTG_SERIALIZATION_STD_PAIR_H
diff --git a/ttg/ttg/serialization/std/tuple.h b/ttg/ttg/serialization/std/tuple.h
index dd8e6aadb..048f64ec5 100644
--- a/ttg/ttg/serialization/std/tuple.h
+++ b/ttg/ttg/serialization/std/tuple.h
@@ -46,18 +46,4 @@ namespace ttg::detail {
 
 #endif  // TTG_SERIALIZATION_SUPPORTS_BOOST
 
-#ifdef TTG_SERIALIZATION_SUPPORTS_CEREAL
-#include <cereal/types/tuple.hpp>
-
-namespace ttg::detail {
-  template <typename Archive, typename... Ts>
-  inline static constexpr bool is_stlcontainer_cereal_serializable_v<Archive, std::tuple<Ts...>> =
-      (is_cereal_serializable_v<Archive, Ts> && ...);
-  template <typename Archive, typename... Ts>
-  inline static constexpr bool is_stlcontainer_cereal_serializable_v<Archive, const std::tuple<Ts...>> =
-      (is_cereal_serializable_v<Archive, const Ts> && ...);
-}  // namespace ttg::detail
-
-#endif
-
 #endif  // TTG_SERIALIZATION_STD_TUPLE_H
diff --git a/ttg/ttg/serialization/std/vector.h b/ttg/ttg/serialization/std/vector.h
index f939d24b4..90a05f072 100644
--- a/ttg/ttg/serialization/std/vector.h
+++ b/ttg/ttg/serialization/std/vector.h
@@ -26,18 +26,4 @@ namespace ttg::detail {
 
 #endif  // TTG_SERIALIZATION_SUPPORTS_BOOST
 
-#ifdef TTG_SERIALIZATION_SUPPORTS_CEREAL
-#include <cereal/types/vector.hpp>
-
-namespace ttg::detail {
-  template <typename Archive, typename T, typename A>
-  inline static constexpr bool is_stlcontainer_cereal_serializable_v<Archive, std::vector<T, A>> =
-      is_cereal_serializable_v<Archive, T>;
-  template <typename Archive, typename T, typename A>
-  inline static constexpr bool is_stlcontainer_cereal_serializable_v<Archive, const std::vector<T, A>> =
-      is_cereal_serializable_v<Archive, const T>;
-}  // namespace ttg::detail
-
-#endif
-
 #endif  // TTG_SERIALIZATION_STD_VECTOR_H
diff --git a/ttg/ttg/serialization/traits.h b/ttg/ttg/serialization/traits.h
index bf70298d6..d1b4905bd 100644
--- a/ttg/ttg/serialization/traits.h
+++ b/ttg/ttg/serialization/traits.h
@@ -179,8 +179,7 @@ namespace ttg::detail {
   //  is_boost_user_buffer_serializable_v<T>>> : std::true_type {};
   template <typename T>
   struct is_user_buffer_serializable<
-      T, std::enable_if_t<is_madness_user_buffer_serializable_v<T> || is_boost_user_buffer_serializable_v<T> ||
-                          is_cereal_user_buffer_serializable_v<T>>> : std::true_type {};
+      T, std::enable_if_t<is_madness_user_buffer_serializable_v<T> || is_boost_user_buffer_serializable_v<T>>> : std::true_type {};
 
   template <typename T>
   inline constexpr bool is_user_buffer_serializable_v = is_user_buffer_serializable<T>::value;

From accf8e9b5a04b03d3a1fa49c366846733f781ab7 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 26 Feb 2024 10:20:56 -0500
Subject: [PATCH 257/259] Remove referenes to op_cuda

Device tasks are identified by `have_*` members.

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/ttg.h | 18 +-----------------
 1 file changed, 1 insertion(+), 17 deletions(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index df02de725..63ae547aa 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -1274,13 +1274,12 @@ namespace ttg_parsec {
     ttg::World get_world() const override final { return world; }
 
    private:
-    /// dispatches a call to derivedT::op if Space == Host, otherwise to derivedT::op_cuda if Space == CUDA
+    /// dispatches a call to derivedT::op
     /// @return void if called a synchronous function, or ttg::coroutine_handle<> if called a coroutine (if non-null,
     ///    points to the suspended coroutine)
     template <ttg::ExecutionSpace Space, typename... Args>
     auto op(Args &&...args) {
       derivedT *derived = static_cast<derivedT *>(this);
-      // TODO: do we still distinguish op and op_cuda? How do we handle support for multiple devices?
       //if constexpr (Space == ttg::ExecutionSpace::Host) {
         using return_type = decltype(derived->op(std::forward<Args>(args)...));
         if constexpr (std::is_same_v<return_type,void>) {
@@ -1290,21 +1289,6 @@ namespace ttg_parsec {
         else {
           return derived->op(std::forward<Args>(args)...);
         }
-#if 0
-      }
-      else if constexpr (Space == ttg::ExecutionSpace::CUDA) {
-        using return_type = decltype(derived->op_cuda(std::forward<Args>(args)...));
-        if constexpr (std::is_same_v<return_type,void>) {
-          derived->op_cuda(std::forward<Args>(args)...);
-          return;
-        }
-        else {
-          return derived->op_cuda(std::forward<Args>(args)...);
-        }
-      }
-      else
-        ttg::abort();
-#endif // 0
     }
 
     template <std::size_t i, typename terminalT, typename Key>

From cdd22e3375c706f4267e5bb706177486bcd6399a Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 26 Feb 2024 18:16:08 -0500
Subject: [PATCH 258/259] PaRSEC backend: Fix race condition in copy management

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/ttg.h | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 63ae547aa..3533cf245 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -859,6 +859,7 @@ namespace ttg_parsec {
       /* if there was only one reader (the current task) or
        * a mutable copy and a successor, we release the copy */
       if (1 == readers || copy->is_mutable()) {
+        std::atomic_thread_fence(std::memory_order_acquire);
         if (nullptr != copy->get_next_task()) {
           /* Release the deferred task.
            * The copy was mutable and will be mutated by the released task,
@@ -943,9 +944,10 @@ namespace ttg_parsec {
            * no other readers, mark copy as mutable and defer the release
            * of the task
            */
-          copy_in->mark_mutable();
           assert(nullptr == copy_in->get_next_task());
           copy_in->set_next_task(&task->parsec_task);
+          std::atomic_thread_fence(std::memory_order_release);
+          copy_in->mark_mutable();
         } else {
           if (task->defer_writer && nullptr == copy_in->get_next_task()) {
             /* we're the first writer and want to wait for all readers to complete */
@@ -2386,7 +2388,7 @@ ttg::abort();  // should not happen
       if (reducer && 1 != task->streams[i].goal) {  // is this a streaming input? reduce the received value
         auto submit_reducer_task = [&](auto *parent_task){
           /* check if we need to create a task */
-          std::size_t c = parent_task->streams[i].reduce_count.fetch_add(1, std::memory_order_release);
+          std::size_t c = parent_task->streams[i].reduce_count.fetch_add(1, std::memory_order_acquire);
           //std::cout << "submit_reducer_task " << key << " c " << c << std::endl;
           if (0 == c) {
             /* we are responsible for creating the reduction task */

From f10a44e75ddf0c9d2dde26f63a56f1f431ff0e94 Mon Sep 17 00:00:00 2001
From: Joseph Schuchart <schuchart@icl.utk.edu>
Date: Mon, 26 Feb 2024 19:23:34 -0500
Subject: [PATCH 259/259] PaRSEC backend: fix race when checking for mutable
 tag in release_data_copy

We cannot call `copy->is_mutable()` because it reads the reader value
that we atomically decremented. If another thread took posession of the
copy we may try to do the same.

Signed-off-by: Joseph Schuchart <schuchart@icl.utk.edu>
---
 ttg/ttg/parsec/ttg.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ttg/ttg/parsec/ttg.h b/ttg/ttg/parsec/ttg.h
index 3533cf245..65e39991a 100644
--- a/ttg/ttg/parsec/ttg.h
+++ b/ttg/ttg/parsec/ttg.h
@@ -858,7 +858,7 @@ namespace ttg_parsec {
       }
       /* if there was only one reader (the current task) or
        * a mutable copy and a successor, we release the copy */
-      if (1 == readers || copy->is_mutable()) {
+      if (1 == readers || readers == copy->mutable_tag) {
         std::atomic_thread_fence(std::memory_order_acquire);
         if (nullptr != copy->get_next_task()) {
           /* Release the deferred task.