MPI RMA Backends, TCP Backend, Streaming Abstraction

.gitignore

@@ -1,2 +1,3 @@
 bin/*
 benchmark/results/*
+cmake-build-debug/*

CMakeLists.txt

@@ -0,0 +1,48 @@
+# Copyright (c) 2018 Marcel Ehrhardt <[email protected]>
+# Copyright (c) 2018 Matthias Noack <[email protected]>
+#
+# See accompanying file LICENSE and README for further information.
+
+project(ham LANGUAGES CXX)
+cmake_minimum_required(VERSION 3.2 FATAL_ERROR) # TODO verfify
+
+# set output directory
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
+message(STATUS "CMAKE_BINARY_DIR: " ${CMAKE_BINARY_DIR})
+
+### thirdparty dependencies
+
+# Boost
+find_package(Boost 1.40 COMPONENTS program_options system REQUIRED)
+add_library(boost_library INTERFACE)
+target_include_directories (boost_library INTERFACE ${Boost_INCLUDE_DIRS})
+target_link_libraries (boost_library INTERFACE ${Boost_LIBRARIES})
+
+# MPI
+find_package(MPI) # not required
+if (MPI_FOUND)
+	add_library (mpi_library INTERFACE)
+	target_include_directories (mpi_library INTERFACE ${MPI_CXX_INCLUDE_PATH})
+	target_compile_options (mpi_library INTERFACE ${MPI_CXX_COMPILE_FLAGS})
+	target_link_libraries (mpi_library INTERFACE ${MPI_CXX_LIBRARIES})
+endif ()
+
+# Intel SCIF (for Xeon Phi accelerators with KNC architecture)
+find_file(SCIF_HEADER_FILE "scif.h")
+if (SCIF_HEADER_FILE)
+	set(SCIF_FOUND ON)
+	get_filename_component(SCIF_INCLUDE_DIR "${SCIF_HEADER_FILE}" DIRECTORY)
+	message(STATUS "Found SCIF: ${SCIF_HEADER_FILE}")
+
+	add_library (scif_library INTERFACE)
+	target_include_directories (scif_library INTERFACE ${SCIF_INCLUDE_DIR})
+else ()
+	message(STATUS "Could NOT find SCIF (missing: scif.h)")
+endif ()
+
+# tell the compiler to be strict
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DHAM_DEBUG_ON")
+
+add_subdirectory(thirdparty/bmt ${CMAKE_CURRENT_BINARY_DIR}/build.noma_bmt)
+add_subdirectory(src)

Jamroot

@@ -24,22 +24,23 @@ rule get-boost-lib-path ( properties * )
 }
 
 lib boost_program_options : : <name>boost_program_options <conditional>@get-boost-lib-path ;
-lib scif : : <name>scif ;
+# lib scif : : <name>scif ;
 
 project HAM
 	: source-location $(SRC)
 	: requirements
 	<include>$(INC)
 	<include>$(BOOST_PATH)/include
 	<variant>debug:<define>HAM_DEBUG_ON
-	<variant>debug_mic:<define>HAM_DEBUG_ON 
+	<variant>debug_mic:<define>HAM_DEBUG_ON
 #	<toolset>intel:<cflags>"-static-intel"
 	<inlining>on # off, on, full
 	#<optimization>speed # off, speed, space
-	<cxxflags>"-std=c++11"
+	<cxxflags>"-hstd=c++11"
 	<threading>multi
-#	<link>static
-	: default-build debug release debug_mic release_mic
+	<link>static
+	: default-build release
+#	: default-build debug release debug_mic release_mic
 	;
 
 # Object files that are compiled the same for all targets
@@ -56,40 +57,59 @@ obj offload_obj_mpi : ham/offload/offload.cpp : <library>/mpi//mpi <define>HAM_C
 
 constant OBJ_FILES_MPI : communicator_obj_mpi runtime_obj_mpi offload_obj_mpi communicator_mpi_obj_mpi ;
 
-obj communicator_obj_scif : ham/net/communicator.cpp : <library>scif <define>HAM_COMM_SCIF ;
-obj communicator_scif_obj_scif : ham/net/communicator_scif.cpp : <library>scif <define>HAM_COMM_SCIF ;
-obj runtime_obj_scif : ham/offload/runtime.cpp : <library>scif <define>HAM_COMM_SCIF ;
-obj offload_obj_scif : ham/offload/offload.cpp : <library>scif <define>HAM_COMM_SCIF ;
+obj communicator_obj_mpi_rma_dyn : ham/net/communicator.cpp : <library>/mpi//mpi <define>HAM_COMM_MPI_RMA_DYNAMIC ;
+obj communicator_mpi_rma_dyn_obj_mpi_rma_dyn : ham/net/communicator_mpi_rma_dynamic.cpp : <library>/mpi//mpi <define>HAM_COMM_MPI_RMA_DYNAMIC ;
+obj runtime_obj_mpi_rma_dyn : ham/offload/runtime.cpp : <library>/mpi//mpi <define>HAM_COMM_MPI_RMA_DYNAMIC ;
+obj offload_obj_mpi_rma_dyn : ham/offload/offload.cpp : <library>/mpi//mpi <define>HAM_COMM_MPI_RMA_DYNAMIC ;
 
-constant OBJ_FILES_SCIF : communicator_obj_scif runtime_obj_scif offload_obj_scif communicator_scif_obj_scif ;
+constant OBJ_FILES_MPI_RMA_DYN : communicator_obj_mpi_rma_dyn communicator_mpi_rma_dyn_obj_mpi_rma_dyn runtime_obj_mpi_rma_dyn offload_obj_mpi_rma_dyn ;
+
+# obj communicator_obj_scif : ham/net/communicator.cpp : <library>scif <define>HAM_COMM_SCIF ;
+# obj communicator_scif_obj_scif : ham/net/communicator_scif.cpp : <library>scif <define>HAM_COMM_SCIF ;
+# obj runtime_obj_scif : ham/offload/runtime.cpp : <library>scif <define>HAM_COMM_SCIF ;
+# obj offload_obj_scif : ham/offload/offload.cpp : <library>scif <define>HAM_COMM_SCIF ;
+
+# constant OBJ_FILES_SCIF : communicator_obj_scif runtime_obj_scif offload_obj_scif communicator_scif_obj_scif ;
 
 # Libraries
 
 obj main_obj_mpi : ham/offload/main.cpp : <library>/mpi//mpi <define>HAM_COMM_MPI ;
-obj main_obj_scif : ham/offload/main.cpp : <library>scif <define>HAM_COMM_SCIF ;
+obj main_obj_mpi_rma_dyn : ham/offload/main.cpp : <library>/mpi//mpi <define>HAM_COMM_MPI_RMA_DYNAMIC ;
+# obj main_obj_scif : ham/offload/main.cpp : <library>scif <define>HAM_COMM_SCIF ;
 
 lib ham_offload_mpi
 	: $(OBJ_FILES_COMMON) $(OBJ_FILES_MPI) main_obj_mpi boost_program_options
 	: <library>/mpi//mpi <define>HAM_COMM_MPI
 	;
 
-lib ham_offload_scif
-	: $(OBJ_FILES_COMMON) $(OBJ_FILES_SCIF) main_obj_scif boost_program_options
-	: <library>scif <define>HAM_COMM_SCIF
-	;
+lib ham_offload_mpi_rma_dyn
+    : $(OBJ_FILES_COMMON) $(OBJ_FILES_MPI_RMA_DYN) main_obj_mpi_rma_dyn boost_program_options
+    : <library>/mpi//mpi <define>HAM_COMM_MPI_RMA_DYNAMIC
+    ;
+
+# lib ham_offload_scif
+#	: $(OBJ_FILES_COMMON) $(OBJ_FILES_SCIF) main_obj_scif boost_program_options
+#	: <library>scif <define>HAM_COMM_SCIF
+#	;
 
 obj main_explicit_obj_mpi : ham/offload/main_explicit.cpp : <library>/mpi//mpi <define>HAM_COMM_MPI ;
-obj main_explicit_obj_scif : ham/offload/main_explicit.cpp : <library>scif <define>HAM_COMM_SCIF ;
+obj main_explicit_obj_mpi_rma_dyn : ham/offload/main_explicit.cpp : <library>/mpi//mpi <define>HAM_COMM_MPI_RMA_DYNAMIC ;
+# obj main_explicit_obj_scif : ham/offload/main_explicit.cpp : <library>scif <define>HAM_COMM_SCIF ;
 
 lib ham_offload_mpi_explicit
 	: $(OBJ_FILES_COMMON) $(OBJ_FILES_MPI) main_explicit_obj_mpi boost_program_options
 	: <library>/mpi//mpi <define>HAM_COMM_MPI <define>HAM_EXPLICIT
 	;
 
-lib ham_offload_scif_explicit
-	: $(OBJ_FILES_COMMON) $(OBJ_FILES_SCIF) main_explicit_obj_scif boost_program_options
-	: <library>scif <define>HAM_COMM_SCIF <define>HAM_EXPLICIT
-	;
+lib ham_offload_mpi_rma_dyn_explicit
+    : $(OBJ_FILES_COMMON) $(OBJ_FILES_MPI_RMA_DYN) main_explicit_obj_mpi_rma_dyn boost_program_options
+    : <library>/mpi//mpi <define>HAM_COMM_MPI_RMA_DYNAMIC <define>HAM_EXPLICIT
+    ;
+
+# lib ham_offload_scif_explicit
+#	: $(OBJ_FILES_COMMON) $(OBJ_FILES_SCIF) main_explicit_obj_scif boost_program_options
+#	: <library>scif <define>HAM_COMM_SCIF <define>HAM_EXPLICIT
+#	;
 
 # Benchmarks
 
@@ -99,11 +119,17 @@ exe benchmark_ham_offload_mpi
 	: <library>/mpi//mpi <library>ham_offload_mpi
 	;	
 
-obj benchmark_ham_offload_scif_obj : benchmark_ham_offload.cpp : <library>scif <define>HAM_COMM_SCIF ;
-exe benchmark_ham_offload_scif
-	: benchmark_ham_offload_scif_obj boost_program_options ham_offload_scif
-	: <library>scif 
-	;	
+obj benchmark_ham_offload_mpi_rma_dyn_obj : benchmark_ham_offload.cpp : <library>/mpi//mpi <define>HAM_COMM_MPI_RMA_DYNAMIC ;
+exe benchmark_ham_offload_mpi_rma_dyn
+    : benchmark_ham_offload_mpi_rma_dyn_obj boost_program_options
+    : <library>/mpi//mpi <library>ham_offload_mpi_rma_dyn
+    ;
+
+# obj benchmark_ham_offload_scif_obj : benchmark_ham_offload.cpp : <library>scif <define>HAM_COMM_SCIF ;
+# exe benchmark_ham_offload_scif
+#	: benchmark_ham_offload_scif_obj boost_program_options ham_offload_scif
+#	: <library>scif
+#	;
 
 exe benchmark_intel_leo
 	: benchmark_intel_leo.cpp boost_program_options
@@ -116,51 +142,69 @@ exe active_msgs
 	: active_msgs.cpp
 	;
 
+exe active_msgs_over_file
+	: active_msgs_over_file.cpp boost_program_options
+	;
+
 exe ham_offload
-	: ham_offload.cpp ham_offload_scif boost_program_options
+	: ham_offload.cpp ham_offload_mpi_rma_dyn boost_program_options
 #	: <library>/mpi//mpi <define>HAM_COMM_MPI
-	: <library>scif <define>HAM_COMM_SCIF
+	: <library>/mpi//mpi <define>HAM_COMM_MPI_RMA_DYNAMIC
+#	: <library>scif <define>HAM_COMM_SCIF
 	;
 
 exe ham_offload_explicit
-	: ham_offload_explicit.cpp ham_offload_mpi_explicit boost_program_options
-	: <library>/mpi//mpi <define>HAM_COMM_MPI
+	: ham_offload_explicit.cpp ham_offload_mpi_rma_dyn_explicit boost_program_options
+#	: <library>/mpi//mpi <define>HAM_COMM_MPI
+	: <library>/mpi//mpi <define>HAM_COMM_MPI_RMA_DYNAMIC
 #	: <library>scif <define>HAM_COMM_SCIF
 	;
 
-exe inner_product_scif
-	: [ obj inner_product_obj : inner_product.cpp : <library>scif <define>HAM_COMM_SCIF ] ham_offload_scif boost_program_options
-	: <library>scif <define>HAM_COMM_SCIF
-	;
+# exe inner_product_scif
+#	: [ obj inner_product_obj : inner_product.cpp : <library>scif <define>HAM_COMM_SCIF ] ham_offload_scif boost_program_options
+#	: <library>scif <define>HAM_COMM_SCIF
+#	;
 
 exe inner_product_mpi
 	: [ obj inner_product_obj : inner_product.cpp : <library>/mpi//mpi <define>HAM_COMM_MPI ] ham_offload_mpi boost_program_options
 	: <library>/mpi//mpi <define>HAM_COMM_MPI
 	;
 
-#
-exe test_data_transfer_scif
-	: [ obj test_data_transfer_obj : test_data_transfer.cpp : <library>scif <define>HAM_COMM_SCIF ] ham_offload_scif boost_program_options
-	: <library>scif <define>HAM_COMM_SCIF
-	;
+exe inner_product_mpi_rma_dynamic
+    : [ obj inner_product_obj : inner_product.cpp : <library>/mpi//mpi <define>HAM_COMM_MPI_RMA_DYNAMIC ] ham_offload_mpi_rma_dyn boost_program_options
+    : <library>/mpi//mpi <cflags>-g <define>HAM_COMM_MPI_RMA_DYNAMIC
+    ;
+
+# exe test_data_transfer_scif
+#	: [ obj test_data_transfer_obj : test_data_transfer.cpp : <library>scif <define>HAM_COMM_SCIF ] ham_offload_scif boost_program_options
+#	: <library>scif <define>HAM_COMM_SCIF
+#	;
 
 exe test_data_transfer_mpi
 	: [ obj test_data_transfer_obj : test_data_transfer.cpp : <library>/mpi//mpi <define>HAM_COMM_MPI ] ham_offload_mpi boost_program_options
 	: <library>/mpi//mpi <define>HAM_COMM_MPI
 	;
 
-
-exe test_argument_transfer_scif
-	: [ obj test_argument_transfer_obj : test_argument_transfer.cpp : <library>scif <define>HAM_COMM_SCIF ] ham_offload_scif boost_program_options
-	: <library>scif <define>HAM_COMM_SCIF
+exe test_data_transfer_mpi_rma_dynamic
+	: [ obj test_data_transfer_obj : test_data_transfer.cpp : <library>/mpi//mpi <define>HAM_COMM_MPI_RMA_DYNAMIC ] ham_offload_mpi_rma_dyn boost_program_options
+	: <library>/mpi//mpi <define>HAM_COMM_MPI_RMA_DYNAMIC
 	;
 
+# exe test_argument_transfer_scif
+# 	: [ obj test_argument_transfer_obj : test_argument_transfer.cpp : <library>scif <define>HAM_COMM_SCIF ] ham_offload_scif boost_program_options
+#	: <library>scif <define>HAM_COMM_SCIF
+#	;
+
 exe test_argument_transfer_mpi
 	: [ obj test_argument_transfer_obj : test_argument_transfer.cpp : <library>/mpi//mpi <define>HAM_COMM_MPI ] ham_offload_mpi boost_program_options
 	: <library>/mpi//mpi <define>HAM_COMM_MPI
 	;
 
+exe test_argument_transfer_mpi_rma_dynamic
+	: [ obj test_argument_transfer_obj : test_argument_transfer.cpp : <library>/mpi//mpi <define>HAM_COMM_MPI_RMA_DYNAMIC ] ham_offload_mpi_rma_dyn boost_program_options
+	: <library>/mpi//mpi <define>HAM_COMM_MPI_RMA_DYNAMIC
+	;
 
 # Explicit targets (not built by default)
 explicit benchmark_intel_leo ;
-explicit test_data_transfer_scif ;
+# explicit test_data_transfer_scif ;

include/ham/misc/constants.hpp

@@ -17,6 +17,8 @@ namespace constants {
 enum net {
 	MSG_SIZE = HAM_MESSAGE_SIZE,
 	MSG_BUFFERS = 256,
+	DATA_PUT_CODE = 1,
+	DATA_GET_CODE = 2,
 };
 
 enum arch {

include/ham/misc/migratable.hpp

@@ -28,12 +28,12 @@ class migratable
 	 //: value(std::forward<T>(arg)) // NOTE: compatible types are allowed
 	 : value(std::forward<Compatible>(arg)) // NOTE: compatible types are allowed
 	{ 
-//		std::cout << "migratable-ctor: " << value << std::endl;
+		// std::cout << "migratable-ctor: " << value << std::endl;
 	}
 
 	operator const T& () const
 	{
-//		std::cout << "migratable-conversion: " << value << std::endl;
+		// std::cout << "migratable-conversion: " << value << std::endl;
 		return value;
 	}
 private:

include/ham/misc/types.hpp

@@ -9,9 +9,13 @@
 #include <algorithm>
 #include <cstddef>
 
+#include "ham/misc/migratable.hpp"
+
 namespace ham {
 
 typedef size_t node_t; // node type, e.g. MPI rank, identifies remote target process
+typedef size_t flag_t; // MPI RMA completion flag / buffer index
+typedef char byte_t;
 typedef char*  msg_buffer_t; // buffer type for messages
 
 namespace net {
@@ -29,7 +33,7 @@ class result_container
 	T get() { return T(std::move(res)); }
 
 private:
-	T res;
+	migratable<T> res;
 };
 
 template<>

include/ham/net/communicator.hpp

@@ -37,7 +37,14 @@ namespace net {
 	{
 		char data[constants::MSG_SIZE];
 	};
-
+
+	struct
+	msg_flag_buffer
+	{
+		// buffer for fully rma backend. windows will consist of several of those
+		char msg[constants::MSG_SIZE];
+		flag_t flag;
+	};
 
 	node_t this_node();
 }
@@ -49,8 +56,14 @@ namespace net {
 #elif defined HAM_COMM_SCIF
 #define HAM_COMM_ONE_SIDED
 #include "ham/net/communicator_scif.hpp"
+#elif defined HAM_COMM_MPI_RMA_DYNAMIC
+#include "ham/net/communicator_mpi_rma_dynamic.hpp"
+#elif defined HAM_COMM_MPI_RMA_DYNAMIC_DATA_ONLY
+#include "ham/net/communicator_mpi_rma_dynamic_data_only.hpp"
+#elif defined HAM_COMM_TCP
+#include "ham/net/communicator_tcp.hpp"
 #else
-static_assert(false, "Please define either HAM_COMM_MPI, or HAM_COMM_SCIF.");
+static_assert(false, "Please define either HAM_COMM_MPI, HAM_COMM_SCIF, HAM_COMM_MPI_RMA_DYNAMIC, HAM_COMM_MPI_RMA_DYNAMIC_DATA_ONLY or HAM_COMM_TCP");
 #endif
 
 #endif // ham_net_communicator_hpp