diff --git a/include/openPMD/ChunkInfo.hpp b/include/openPMD/ChunkInfo.hpp index 67ccf7f322..11206b2bdf 100644 --- a/include/openPMD/ChunkInfo.hpp +++ b/include/openPMD/ChunkInfo.hpp @@ -90,17 +90,87 @@ namespace chunk_assignment namespace host_info { + /** + * Methods for retrieving hostname / processor identifiers that openPMD-api + * is aware of. These can be used for locality-aware chunk distribution + * schemes in streaming setups. + */ enum class Method { - HOSTNAME + POSIX_HOSTNAME, + WINSOCKS_HOSTNAME, + MPI_PROCESSOR_NAME }; + /** + * @brief This defines the method identifiers used + * in `{"rank_table": "hostname"}` + * + * Currently recognized are: + * + * * posix_hostname + * * winsocks_hostname + * * mpi_processor_name + * + * For backwards compatibility reasons, "hostname" is also recognized as a + * deprecated alternative for "posix_hostname". + * + * @return Method enum identifier. The identifier is returned even if the + * method is not available on the system. This should by checked + * via methodAvailable(). + * @throws std::out_of_range If an unknown string identifier is passed. + */ + Method methodFromStringDescription(std::string const &descr); + + /** + * @brief Is the method available on the current system? + * + * @return true If it is available. + * @return false Otherwise. + */ + bool methodAvailable(Method); + + /** + * @brief Wrapper for the native hostname retrieval functions such as + * POSIX gethostname(). + * + * @return std::string The hostname / processor name returned by the native + * function. + */ std::string byMethod(Method); #if openPMD_HAVE_MPI + /** + * @brief Retrieve the hostname information on all MPI ranks and distribute + * a map of "rank -> hostname" to all ranks. + * + * This call is MPI collective. + * + * @return chunk_assignment::RankMeta Hostname / processor name information + * for all MPI ranks known to the communicator. + * The result is returned on all ranks. + */ chunk_assignment::RankMeta byMethodCollective(MPI_Comm, Method); #endif - std::string hostname(); +/* + * The following block contains one wrapper for each native hostname retrieval + * method. The purpose is to have the same function pointer type for all + * of them. + */ + +/* + * @todo Replace _WIN32 with proper Winsocks macro, + * add POSIX availability macro. + */ +#ifdef _WIN32 + std::string winsocks_hostname(); +#else + std::string posix_hostname(); +#endif +#if openPMD_HAVE_MPI + std::string mpi_processor_name(); +#endif + } // namespace host_info } // namespace openPMD diff --git a/src/ChunkInfo.cpp b/src/ChunkInfo.cpp index 9296e39e0a..fcae8a55bf 100644 --- a/src/ChunkInfo.cpp +++ b/src/ChunkInfo.cpp @@ -24,6 +24,11 @@ #include +/* + * @todo Replace _WIN32 with proper Winsocks macro, + * add POSIX availability macro. + */ + #ifdef _WIN32 #include #else @@ -59,13 +64,69 @@ bool WrittenChunkInfo::operator==(WrittenChunkInfo const &other) const namespace host_info { - constexpr size_t MAX_HOSTNAME_LENGTH = 200; + constexpr size_t MAX_HOSTNAME_LENGTH = 256; + + Method methodFromStringDescription(std::string const &descr) + { + static std::map const map{ + {"posix_hostname", Method::POSIX_HOSTNAME}, + {"hostname", Method::POSIX_HOSTNAME}, + {"winsocks_hostname", Method::WINSOCKS_HOSTNAME}, + {"mpi_processor_name", Method::MPI_PROCESSOR_NAME}}; + if (descr == "hostname") + { + std::cerr + << "[host_info::methodFromStringDescription] `hostname` is a " + "deprecated identifier for a hostname retrieval method. " + "Consider switching to `posix_hostname` instead." + << std::endl; + } + return map.at(descr); + } + +// @todo do this properly +#ifdef _WIN32 +#define openPMD_POSIX_AVAILABLE false +#else +#define openPMD_POSIX_AVAILABLE true +#endif + bool methodAvailable(Method method) + { + switch (method) + { + + case Method::POSIX_HOSTNAME: + return openPMD_POSIX_AVAILABLE; + case Method::WINSOCKS_HOSTNAME: + return !openPMD_POSIX_AVAILABLE; + case Method::MPI_PROCESSOR_NAME: + return openPMD_HAVE_MPI == 1; + } + throw std::runtime_error("Unreachable!"); + } std::string byMethod(Method method) { - static std::map map{ - {Method::HOSTNAME, &hostname}}; - return (*map[method])(); + static std::map const map{ +#ifdef _WIN32 + {Method::WINSOCKS_HOSTNAME, &winsocks_hostname} +#else + {Method::POSIX_HOSTNAME, &posix_hostname} +#endif +#if openPMD_HAVE_MPI + , + {Method::MPI_PROCESSOR_NAME, &mpi_processor_name} +#endif + }; + try + { + return (*map.at(method))(); + } + catch (std::out_of_range const &) + { + throw std::runtime_error( + "[hostname::byMethod] Specified method is not available."); + } } #if openPMD_HAVE_MPI @@ -81,18 +142,50 @@ namespace host_info } return res; } + + std::string mpi_processor_name() + { + std::string res; + res.resize(MPI_MAX_PROCESSOR_NAME); + int string_len; + if (MPI_Get_processor_name(res.data(), &string_len) != 0) + { + throw std::runtime_error( + "[mpi_processor_name] Could not inquire processor name."); + } + // MPI_Get_processor_name returns the string length without null + // terminator and std::string::resize() does not use null terminator + // either. So, no +-1 necessary. + res.resize(string_len); + res.shrink_to_fit(); + return res; + } #endif - std::string hostname() +#ifdef _WIN32 + std::string winsocks_hostname() { char hostname[MAX_HOSTNAME_LENGTH]; if (gethostname(hostname, MAX_HOSTNAME_LENGTH)) { throw std::runtime_error( - "[gethostname] Could not inquire hostname."); + "[winsocks_hostname] Could not inquire hostname."); } std::string res(hostname); return res; } +#else + std::string posix_hostname() + { + char hostname[MAX_HOSTNAME_LENGTH]; + if (gethostname(hostname, MAX_HOSTNAME_LENGTH)) + { + throw std::runtime_error( + "[posix_hostname] Could not inquire hostname."); + } + std::string res(hostname); + return res; + } +#endif } // namespace host_info } // namespace openPMD diff --git a/src/Series.cpp b/src/Series.cpp index 06260365cc..1f56683c3b 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -330,16 +330,19 @@ void Series::flushRankTable() -> std::optional { return std::nullopt; }, [](internal::SeriesData::SourceSpecifiedViaJSON &viaJson) -> std::optional { - if (viaJson.value == "hostname") + host_info::Method method; + try { - return host_info::hostname(); + method = + host_info::methodFromStringDescription(viaJson.value); } - else + catch (std::out_of_range const &) { throw error::WrongAPIUsage( "[Series] Wrong value for JSON option 'rank_table': '" + viaJson.value + "'."); - }; + } + return host_info::byMethod(method); }, [](internal::SeriesData::SourceSpecifiedManually &manually) -> std::optional { return manually.value; }}, diff --git a/src/binding/python/ChunkInfo.cpp b/src/binding/python/ChunkInfo.cpp index 3d5c046dad..6cc28e5aaf 100644 --- a/src/binding/python/ChunkInfo.cpp +++ b/src/binding/python/ChunkInfo.cpp @@ -76,7 +76,9 @@ void init_Chunk(py::module &m) })); py::enum_(m, "HostInfo") - .value("HOSTNAME", host_info::Method::HOSTNAME) + .value("POSIX_HOSTNAME", host_info::Method::POSIX_HOSTNAME) + .value("WINSOCKS_HOSTNAME", host_info::Method::WINSOCKS_HOSTNAME) + .value("MPI_PROCESSOR_NAME", host_info::Method::MPI_PROCESSOR_NAME) #if openPMD_HAVE_MPI .def( "get_collective", @@ -93,7 +95,12 @@ void init_Chunk(py::module &m) } }) #endif - .def("get", [](host_info::Method const &self) { - return host_info::byMethod(self); - }); + .def( + "get", + [](host_info::Method const &self) { + return host_info::byMethod(self); + }) + .def("available", &host_info::methodAvailable) + .def( + "from_string_description", &host_info::methodFromStringDescription); } diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 774c0c4323..5b9befaf28 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -1536,11 +1536,20 @@ inline void write_test(const std::string &backend) #ifdef _WIN32 WSADATA wsaData; WSAStartup(MAKEWORD(2, 0), &wsaData); + std::string jsonCfg = R"({"rank_table": "winsocks_hostname"})"; + chunk_assignment::RankMeta compare{ + {0, + host_info::byMethod( + host_info::methodFromStringDescription("winsocks_hostname"))}}; +#else + std::string jsonCfg = R"({"rank_table": "posix_hostname"})"; + chunk_assignment::RankMeta compare{ + {0, + host_info::byMethod( + host_info::methodFromStringDescription("posix_hostname"))}}; #endif - Series o = Series( - "../samples/serial_write." + backend, - Access::CREATE, - R"({"rank_table": "hostname"})"); + Series o = + Series("../samples/serial_write." + backend, Access::CREATE, jsonCfg); ParticleSpecies &e_1 = o.iterations[1].particles["e"]; @@ -1642,9 +1651,7 @@ inline void write_test(const std::string &backend) Series read("../samples/serial_write." + backend, Access::READ_ONLY); // need double parens here to avoid link errors to unprintableString // on Windows - REQUIRE( - (read.mpiRanksMetaInfo(/* collective = */ false) == - chunk_assignment::RankMeta{{0, host_info::hostname()}})); + REQUIRE((read.mpiRanksMetaInfo(/* collective = */ false) == compare)); #ifdef _WIN32 WSACleanup(); #endif @@ -1805,6 +1812,9 @@ inline void fileBased_write_test(const std::string &backend) #ifdef _WIN32 WSADATA wsaData; WSAStartup(MAKEWORD(2, 0), &wsaData); + std::string jsonCfg = R"({"rank_table": "winsocks_hostname"})"; +#else + std::string jsonCfg = R"({"rank_table": "posix_hostname"})"; #endif if (auxiliary::directory_exists("../samples/subdir")) auxiliary::remove_directory("../samples/subdir"); @@ -1813,7 +1823,7 @@ inline void fileBased_write_test(const std::string &backend) Series o = Series( "../samples/subdir/serial_fileBased_write%03T." + backend, Access::CREATE, - R"({"rank_table": "hostname"})"); + jsonCfg); ParticleSpecies &e_1 = o.iterations[1].particles["e"]; @@ -1933,7 +1943,7 @@ inline void fileBased_write_test(const std::string &backend) Series o = Series( "../samples/subdir/serial_fileBased_write%T." + backend, Access::READ_ONLY, - R"({"rank_table": "hostname"})"); + jsonCfg); REQUIRE(o.iterations.size() == 5); REQUIRE(o.iterations.count(1) == 1); @@ -2011,7 +2021,7 @@ inline void fileBased_write_test(const std::string &backend) Series o = Series( "../samples/subdir/serial_fileBased_write%T." + backend, Access::READ_WRITE, - R"({"rank_table": "hostname"})"); + jsonCfg); REQUIRE(o.iterations.size() == 5); o.iterations[6]; @@ -2053,7 +2063,7 @@ inline void fileBased_write_test(const std::string &backend) Series o = Series( "../samples/subdir/serial_fileBased_write%01T." + backend, Access::READ_WRITE, - R"({"rank_table": "hostname"})"); + jsonCfg); REQUIRE(o.iterations.size() == 1); /* @@ -2165,7 +2175,7 @@ inline void fileBased_write_test(const std::string &backend) throw std::system_error( std::error_code(errno, std::system_category())); } - chunk_assignment::RankMeta compare{{0, host_info::hostname()}}; + chunk_assignment::RankMeta compare{{0, host_info::posix_hostname()}}; dirent *entry; while ((entry = readdir(directory)) != nullptr) {