From b20cefc6cae770444032c9ea22ed7edecf85607c Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Thu, 7 Sep 2023 11:00:28 +0200 Subject: [PATCH 1/9] add HDF5 input and output files to eigensolver miniapp --- miniapp/miniapp_eigensolver.cpp | 55 +++++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/miniapp/miniapp_eigensolver.cpp b/miniapp/miniapp_eigensolver.cpp index a51ee0eb5c..bd6e0f9cee 100644 --- a/miniapp/miniapp_eigensolver.cpp +++ b/miniapp/miniapp_eigensolver.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -50,6 +51,10 @@ using dlaf::common::Ordering; using dlaf::matrix::MatrixMirror; using pika::this_thread::experimental::sync_wait; +#ifdef DLAF_WITH_HDF5 +using dlaf::matrix::internal::FileHDF5; +#endif + /// Check results of the eigensolver template void checkEigensolver(CommunicatorGrid comm_grid, blas::Uplo uplo, Matrix& A, @@ -60,12 +65,31 @@ struct Options SizeType m; SizeType mb; blas::Uplo uplo; +#ifdef DLAF_WITH_HDF5 + std::optional input_file; + std::optional output_file; +#endif Options(const pika::program_options::variables_map& vm) : MiniappOptions(vm), m(vm["matrix-size"].as()), mb(vm["block-size"].as()), uplo(dlaf::miniapp::parseUplo(vm["uplo"].as())) { DLAF_ASSERT(m > 0, m); DLAF_ASSERT(mb > 0, mb); + +#ifdef DLAF_WITH_HDF5 + if (vm.count("input-file") == 1) { + input_file = FileHDF5(vm["input-file"].as(), FileHDF5::FileMode::readonly); + + if (!vm["matrix-size"].defaulted()) { + std::cerr << "Warning! " + "Specified matrix size will be ignored because an input file has been specified." + << std::endl; + } + } + if (vm.count("output-file") == 1) { + output_file = FileHDF5(vm["output-file"].as(), FileHDF5::FileMode::readwrite); + } +#endif } Options(Options&&) = default; @@ -87,19 +111,25 @@ struct EigensolverMiniapp { Communicator world(MPI_COMM_WORLD); CommunicatorGrid comm_grid(world, opts.grid_rows, opts.grid_cols, Ordering::ColumnMajor); - // Allocate memory for the matrix - GlobalElementSize matrix_size(opts.m, opts.m); - TileElementSize block_size(opts.mb, opts.mb); - - ConstHostMatrixType matrix_ref = [matrix_size, block_size, comm_grid]() { + ConstHostMatrixType matrix_ref = [comm_grid, &opts]() { +#ifdef DLAF_WITH_HDF5 + if (opts.input_file) { + Matrix input = opts.input_file->read("/a", {opts.mb, opts.mb}); + return input; + } +#endif using dlaf::matrix::util::set_random_hermitian; - HostMatrixType hermitian(matrix_size, block_size, comm_grid); + HostMatrixType hermitian(GlobalElementSize(opts.m, opts.m), TileElementSize(opts.mb, opts.mb), + comm_grid); set_random_hermitian(hermitian); return hermitian; }(); + auto matrix_size = matrix_ref.size(); + auto block_size = matrix_ref.blockSize(); + for (int64_t run_index = -opts.nwarmups; run_index < opts.nruns; ++run_index) { if (0 == world.rank() && run_index >= 0) std::cout << "[" << run_index << "]" << std::endl; @@ -127,6 +157,15 @@ struct EigensolverMiniapp { DLAF_MPI_CHECK_ERROR(MPI_Barrier(world)); double elapsed_time = timeit.elapsed(); +#ifdef DLAF_WITH_HDF5 + if (run_index == opts.nruns - 1) { + if (opts.output_file) { + opts.output_file->write>(eigenvalues, "/evals"); + opts.output_file->write(eigenvectors, "/evecs"); + } + } +#endif + matrix.reset(); // print benchmark results @@ -176,6 +215,10 @@ int main(int argc, char** argv) { desc_commandline.add_options() ("matrix-size", value() ->default_value(4096), "Matrix size") ("block-size", value() ->default_value( 256), "Block cyclic distribution size") +#ifdef DLAF_WITH_HDF5 + ("input-file", value() , "Load matrix from given HDF5 file") + ("output-file", value() , "Save eigenvector and eigenvalue to given HDF5 file") +#endif ; // clang-format on dlaf::miniapp::addUploOption(desc_commandline); From 3c4e15646b8676490e894e0d4fdc66d20eebbbf3 Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Thu, 7 Sep 2023 11:14:39 +0200 Subject: [PATCH 2/9] Update miniapp/miniapp_eigensolver.cpp --- miniapp/miniapp_eigensolver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/miniapp/miniapp_eigensolver.cpp b/miniapp/miniapp_eigensolver.cpp index bd6e0f9cee..8a802ea7bc 100644 --- a/miniapp/miniapp_eigensolver.cpp +++ b/miniapp/miniapp_eigensolver.cpp @@ -217,7 +217,7 @@ int main(int argc, char** argv) { ("block-size", value() ->default_value( 256), "Block cyclic distribution size") #ifdef DLAF_WITH_HDF5 ("input-file", value() , "Load matrix from given HDF5 file") - ("output-file", value() , "Save eigenvector and eigenvalue to given HDF5 file") + ("output-file", value() , "Save eigenvectors and eigenvalues to given HDF5 file") #endif ; // clang-format on From d4c1c2787fb4166e42798baae655161c4a0b0ae3 Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Thu, 7 Sep 2023 17:50:53 +0200 Subject: [PATCH 3/9] support I/O for distributed matrices too --- miniapp/miniapp_eigensolver.cpp | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/miniapp/miniapp_eigensolver.cpp b/miniapp/miniapp_eigensolver.cpp index bd6e0f9cee..5c40b123cd 100644 --- a/miniapp/miniapp_eigensolver.cpp +++ b/miniapp/miniapp_eigensolver.cpp @@ -66,8 +66,8 @@ struct Options SizeType mb; blas::Uplo uplo; #ifdef DLAF_WITH_HDF5 - std::optional input_file; - std::optional output_file; + std::optional input_file; + std::optional output_file; #endif Options(const pika::program_options::variables_map& vm) @@ -87,7 +87,7 @@ struct Options } } if (vm.count("output-file") == 1) { - output_file = FileHDF5(vm["output-file"].as(), FileHDF5::FileMode::readwrite); + output_file = vm["output-file"].as(); } #endif } @@ -114,8 +114,10 @@ struct EigensolverMiniapp { ConstHostMatrixType matrix_ref = [comm_grid, &opts]() { #ifdef DLAF_WITH_HDF5 if (opts.input_file) { - Matrix input = opts.input_file->read("/a", {opts.mb, opts.mb}); - return input; + if (opts.local) + return opts.input_file->read("/a", {opts.mb, opts.mb}); + else + return opts.input_file->read("/a", {opts.mb, opts.mb}, comm_grid, {0, 0}); } #endif using dlaf::matrix::util::set_random_hermitian; @@ -160,8 +162,14 @@ struct EigensolverMiniapp { #ifdef DLAF_WITH_HDF5 if (run_index == opts.nruns - 1) { if (opts.output_file) { - opts.output_file->write>(eigenvalues, "/evals"); - opts.output_file->write(eigenvectors, "/evecs"); + auto outfile = [&]() { + if (opts.local) + return FileHDF5(*opts.output_file, FileHDF5::FileMode::readwrite); + else + return FileHDF5(world, *opts.output_file); + }(); + outfile.write(eigenvalues, "/evals"); + outfile.write(eigenvectors, "/evecs"); } } #endif From 22563fbfe926a238cf459538b4223377626b98b6 Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Thu, 7 Sep 2023 17:54:45 +0200 Subject: [PATCH 4/9] cleanup --- miniapp/miniapp_eigensolver.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/miniapp/miniapp_eigensolver.cpp b/miniapp/miniapp_eigensolver.cpp index fe96a48e83..aaf4ebb305 100644 --- a/miniapp/miniapp_eigensolver.cpp +++ b/miniapp/miniapp_eigensolver.cpp @@ -112,18 +112,18 @@ struct EigensolverMiniapp { CommunicatorGrid comm_grid(world, opts.grid_rows, opts.grid_cols, Ordering::ColumnMajor); ConstHostMatrixType matrix_ref = [comm_grid, &opts]() { + TileElementSize block_size(opts.mb, opts.mb); #ifdef DLAF_WITH_HDF5 if (opts.input_file) { if (opts.local) - return opts.input_file->read("/a", {opts.mb, opts.mb}); + return opts.input_file->read("/a", block_size); else - return opts.input_file->read("/a", {opts.mb, opts.mb}, comm_grid, {0, 0}); + return opts.input_file->read("/a", block_size, comm_grid, {0, 0}); } #endif using dlaf::matrix::util::set_random_hermitian; - HostMatrixType hermitian(GlobalElementSize(opts.m, opts.m), TileElementSize(opts.mb, opts.mb), - comm_grid); + HostMatrixType hermitian(GlobalElementSize(opts.m, opts.m), block_size, comm_grid); set_random_hermitian(hermitian); return hermitian; From 8505ef77a795351da4e67eadb04959cfe9edf80f Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Fri, 8 Sep 2023 14:28:12 +0200 Subject: [PATCH 5/9] use filesystem::path --- miniapp/miniapp_eigensolver.cpp | 26 ++++++++++++++------------ miniapp/miniapp_tridiag_solver.cpp | 12 +++++++----- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/miniapp/miniapp_eigensolver.cpp b/miniapp/miniapp_eigensolver.cpp index aaf4ebb305..99155d2988 100644 --- a/miniapp/miniapp_eigensolver.cpp +++ b/miniapp/miniapp_eigensolver.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -66,8 +67,8 @@ struct Options SizeType mb; blas::Uplo uplo; #ifdef DLAF_WITH_HDF5 - std::optional input_file; - std::optional output_file; + std::filesystem::path input_file; + std::filesystem::path output_file; #endif Options(const pika::program_options::variables_map& vm) @@ -78,7 +79,7 @@ struct Options #ifdef DLAF_WITH_HDF5 if (vm.count("input-file") == 1) { - input_file = FileHDF5(vm["input-file"].as(), FileHDF5::FileMode::readonly); + input_file = vm["input-file"].as(); if (!vm["matrix-size"].defaulted()) { std::cerr << "Warning! " @@ -87,7 +88,7 @@ struct Options } } if (vm.count("output-file") == 1) { - output_file = vm["output-file"].as(); + output_file = vm["output-file"].as(); } #endif } @@ -114,11 +115,12 @@ struct EigensolverMiniapp { ConstHostMatrixType matrix_ref = [comm_grid, &opts]() { TileElementSize block_size(opts.mb, opts.mb); #ifdef DLAF_WITH_HDF5 - if (opts.input_file) { + if (!opts.input_file.empty()) { + auto infile = FileHDF5(opts.input_file, FileHDF5::FileMode::readonly); if (opts.local) - return opts.input_file->read("/a", block_size); + return infile.read("/a", block_size); else - return opts.input_file->read("/a", block_size, comm_grid, {0, 0}); + return infile.read("/a", block_size, comm_grid, {0, 0}); } #endif using dlaf::matrix::util::set_random_hermitian; @@ -161,12 +163,12 @@ struct EigensolverMiniapp { #ifdef DLAF_WITH_HDF5 if (run_index == opts.nruns - 1) { - if (opts.output_file) { + if (!opts.output_file.empty()) { auto outfile = [&]() { if (opts.local) - return FileHDF5(*opts.output_file, FileHDF5::FileMode::readwrite); + return FileHDF5(opts.output_file, FileHDF5::FileMode::readwrite); else - return FileHDF5(world, *opts.output_file); + return FileHDF5(world, opts.output_file); }(); outfile.write(eigenvalues, "/evals"); outfile.write(eigenvectors, "/evecs"); @@ -224,8 +226,8 @@ int main(int argc, char** argv) { ("matrix-size", value() ->default_value(4096), "Matrix size") ("block-size", value() ->default_value( 256), "Block cyclic distribution size") #ifdef DLAF_WITH_HDF5 - ("input-file", value() , "Load matrix from given HDF5 file") - ("output-file", value() , "Save eigenvectors and eigenvalues to given HDF5 file") + ("input-file", value() , "Load matrix from given HDF5 file") + ("output-file", value() , "Save eigenvectors and eigenvalues to given HDF5 file") #endif ; // clang-format on diff --git a/miniapp/miniapp_tridiag_solver.cpp b/miniapp/miniapp_tridiag_solver.cpp index a6d3e27efa..79d21fc66a 100644 --- a/miniapp/miniapp_tridiag_solver.cpp +++ b/miniapp/miniapp_tridiag_solver.cpp @@ -9,6 +9,7 @@ // #include +#include #include #include #include @@ -59,7 +60,7 @@ struct Options SizeType m; SizeType mb; #ifdef DLAF_WITH_HDF5 - std::optional input_file; + std::filesystem::path input_file; #endif Options(const pika::program_options::variables_map& vm) @@ -69,7 +70,7 @@ struct Options #ifdef DLAF_WITH_HDF5 if (vm.count("input-file") == 1) { - input_file = FileHDF5(vm["input-file"].as(), FileHDF5::FileMode::readonly); + input_file = vm["input-file"].as(); if (!vm["matrix-size"].defaulted()) { std::cerr << "Warning! " @@ -100,8 +101,9 @@ struct TridiagSolverMiniapp { Matrix tridiag_ref = [&opts]() { #ifdef DLAF_WITH_HDF5 - if (opts.input_file) { - Matrix tridiag = opts.input_file->read("/tridiag", {opts.mb, 2}); + if (!opts.input_file.empty()) { + auto infile = FileHDF5(opts.input_file, FileHDF5::FileMode::readonly); + Matrix tridiag = infile.read("/tridiag", {opts.mb, 2}); return tridiag; } #endif @@ -196,7 +198,7 @@ int main(int argc, char** argv) { ("matrix-size", value() ->default_value(4096), "Matrix size") ("block-size", value() ->default_value( 256), "Block cyclic distribution size") #ifdef DLAF_WITH_HDF5 - ("input-file", value() , "Load matrix from given HDF5 file") + ("input-file", value() , "Load matrix from given HDF5 file") #endif ; // clang-format on From 1d81e2efb3ef2eec51f0561a7dd62fac6687e1e9 Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Fri, 8 Sep 2023 14:30:25 +0200 Subject: [PATCH 6/9] format --- miniapp/miniapp_eigensolver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/miniapp/miniapp_eigensolver.cpp b/miniapp/miniapp_eigensolver.cpp index 99155d2988..8dbf6bd362 100644 --- a/miniapp/miniapp_eigensolver.cpp +++ b/miniapp/miniapp_eigensolver.cpp @@ -9,9 +9,9 @@ // #include +#include #include #include -#include #include #include From 92812e34f84dc3e7a050d89952410fdb7d15279f Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Mon, 11 Sep 2023 08:31:50 +0200 Subject: [PATCH 7/9] Update miniapp/miniapp_tridiag_solver.cpp --- miniapp/miniapp_tridiag_solver.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/miniapp/miniapp_tridiag_solver.cpp b/miniapp/miniapp_tridiag_solver.cpp index 79d21fc66a..1b813c2e98 100644 --- a/miniapp/miniapp_tridiag_solver.cpp +++ b/miniapp/miniapp_tridiag_solver.cpp @@ -12,7 +12,6 @@ #include #include #include -#include #include #include From 04b16bd7c7719d288b8b82c157f0d2569ca8f153 Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Tue, 12 Sep 2023 08:49:47 +0200 Subject: [PATCH 8/9] allow input dataset selection and add input to output --- miniapp/miniapp_eigensolver.cpp | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/miniapp/miniapp_eigensolver.cpp b/miniapp/miniapp_eigensolver.cpp index 8dbf6bd362..0332b913b7 100644 --- a/miniapp/miniapp_eigensolver.cpp +++ b/miniapp/miniapp_eigensolver.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -68,6 +69,7 @@ struct Options blas::Uplo uplo; #ifdef DLAF_WITH_HDF5 std::filesystem::path input_file; + std::string input_dataset; std::filesystem::path output_file; #endif @@ -87,6 +89,7 @@ struct Options << std::endl; } } + input_dataset = vm["input-dataset"].as(); if (vm.count("output-file") == 1) { output_file = vm["output-file"].as(); } @@ -118,9 +121,9 @@ struct EigensolverMiniapp { if (!opts.input_file.empty()) { auto infile = FileHDF5(opts.input_file, FileHDF5::FileMode::readonly); if (opts.local) - return infile.read("/a", block_size); + return infile.read(opts.input_dataset, block_size); else - return infile.read("/a", block_size, comm_grid, {0, 0}); + return infile.read(opts.input_dataset, block_size, comm_grid, {0, 0}); } #endif using dlaf::matrix::util::set_random_hermitian; @@ -170,6 +173,7 @@ struct EigensolverMiniapp { else return FileHDF5(world, opts.output_file); }(); + outfile.write(matrix_ref, opts.input_dataset); outfile.write(eigenvalues, "/evals"); outfile.write(eigenvectors, "/evecs"); } @@ -223,11 +227,12 @@ int main(int argc, char** argv) { // clang-format off desc_commandline.add_options() - ("matrix-size", value() ->default_value(4096), "Matrix size") - ("block-size", value() ->default_value( 256), "Block cyclic distribution size") + ("matrix-size", value() ->default_value(4096), "Matrix size") + ("block-size", value() ->default_value( 256), "Block cyclic distribution size") #ifdef DLAF_WITH_HDF5 - ("input-file", value() , "Load matrix from given HDF5 file") - ("output-file", value() , "Save eigenvectors and eigenvalues to given HDF5 file") + ("input-file", value() , "Load matrix from given HDF5 file") + ("input-dataset", value() -> default_value("/input") , "Name of HDF5 dataset to load as matrix") + ("output-file", value() , "Save eigenvectors and eigenvalues to given HDF5 file") #endif ; // clang-format on From 6477f73c8e619a5de810dd862d4e3f68c1f2210d Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Tue, 12 Sep 2023 08:58:25 +0200 Subject: [PATCH 9/9] enable dataset name in tridiag solver too --- miniapp/miniapp_tridiag_solver.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/miniapp/miniapp_tridiag_solver.cpp b/miniapp/miniapp_tridiag_solver.cpp index 1b813c2e98..924a61567d 100644 --- a/miniapp/miniapp_tridiag_solver.cpp +++ b/miniapp/miniapp_tridiag_solver.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -60,6 +61,7 @@ struct Options SizeType mb; #ifdef DLAF_WITH_HDF5 std::filesystem::path input_file; + std::string input_dataset; #endif Options(const pika::program_options::variables_map& vm) @@ -77,6 +79,7 @@ struct Options << std::endl; } } + input_dataset = vm["input-dataset"].as(); #endif if (do_check != dlaf::miniapp::CheckIterFreq::None) { @@ -102,7 +105,7 @@ struct TridiagSolverMiniapp { #ifdef DLAF_WITH_HDF5 if (!opts.input_file.empty()) { auto infile = FileHDF5(opts.input_file, FileHDF5::FileMode::readonly); - Matrix tridiag = infile.read("/tridiag", {opts.mb, 2}); + Matrix tridiag = infile.read(opts.input_dataset, {opts.mb, 2}); return tridiag; } #endif @@ -197,7 +200,8 @@ int main(int argc, char** argv) { ("matrix-size", value() ->default_value(4096), "Matrix size") ("block-size", value() ->default_value( 256), "Block cyclic distribution size") #ifdef DLAF_WITH_HDF5 - ("input-file", value() , "Load matrix from given HDF5 file") + ("input-file", value() , "Load matrix from given HDF5 file") + ("input-dataset", value() ->default_value("/tridiag") , "Name of HDF5 dataset to load as matrix") #endif ; // clang-format on