From 1343fa3da4c65d97cf654f9aaafcb8322d2845ad Mon Sep 17 00:00:00 2001 From: do-jason Date: Tue, 31 Oct 2023 23:46:45 -0700 Subject: [PATCH] Change MPI block size and minor SYCL document update --- README_sycl.md | 1 - src/mpi.cpp | 9 +++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README_sycl.md b/README_sycl.md index d6c791b4b..9c497185a 100644 --- a/README_sycl.md +++ b/README_sycl.md @@ -35,7 +35,6 @@ $ cmake \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_C_FLAGS="-O3" \ -DCMAKE_CXX_FLAGS="-O3 -march=native" \ --DCMAKE_EXE_LINKER_FLAGS="-O3 -march=native" \ .. $ #### This is Intel GPU Level Zero backend specific ##### diff --git a/src/mpi.cpp b/src/mpi.cpp index d97b16dd3..fc9b454a0 100644 --- a/src/mpi.cpp +++ b/src/mpi.cpp @@ -108,9 +108,10 @@ MpiNode::MpiNode(int &argc, char ** argv) MPI_Comm_create_group(MPI_COMM_WORLD, root_evenG, 1, &root_evenC); MPI_Comm_create_group(MPI_COMM_WORLD, root_oddG, 2, &root_oddC); - constexpr std::ptrdiff_t defBlock = RELION_MPI_MAX_SIZE; // default for MPI communication + constexpr std::ptrdiff_t defPtBlock = 4ULL*1024*1024*1024; // 4 GiB, default for MPI communication + constexpr std::ptrdiff_t defColBlock = 64*1024*1024; // 64 MiB, default for MPI communication constexpr std::ptrdiff_t maxPtBlock = 8ULL*1024*1024*1024; // 8 GiB, max for MPI pt-2-pt. Normal limit is sizeof(variable) * INT_MAX bytes - constexpr std::ptrdiff_t maxCollBlock = 4ULL*1024*1024*1024; // 4 GiB, max for MPI collective. Normal limit is sizeof(variable) * INT_MAX bytes + constexpr std::ptrdiff_t maxCollBlock = 1ULL*1024*1024*1024; // 1 GiB, max for MPI collective. Normal limit is sizeof(variable) * INT_MAX bytes // User input to change default MPI communication blocking size char* pEnvBlock = std::getenv("MAX_MPI_BLOCK"); // For both pt-2-pt and collective @@ -144,7 +145,7 @@ MpiNode::MpiNode(int &argc, char ** argv) p2p_blocksize = maxPtBlock; } else - p2p_blocksize = defBlock; + p2p_blocksize = defPtBlock; if (lColl > 0) // "MAX_MPI_COLL_BLOCK" is set { @@ -154,7 +155,7 @@ MpiNode::MpiNode(int &argc, char ** argv) coll_blocksize = maxCollBlock; } else - coll_blocksize = defBlock; + coll_blocksize = defColBlock; } // Make it multiple of 8 bytes p2p_blocksize = (p2p_blocksize / 8ULL) * 8ULL;