From 6241686851797a18fb30c3493fd05f4384847f5d Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Thu, 31 Oct 2024 14:59:47 +0100 Subject: [PATCH 1/6] Test only hemm on cpu release CI pipelines --- ci/.gitlab-ci.yml | 20 ++++++++++---------- ci/ctest_to_gitlab.sh | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ci/.gitlab-ci.yml b/ci/.gitlab-ci.yml index 0204a001ee..b299ac75d9 100644 --- a/ci/.gitlab-ci.yml +++ b/ci/.gitlab-ci.yml @@ -1,19 +1,19 @@ include: - - local: 'ci/cpu/asan_ubsan_lsan.yml' + # - local: 'ci/cpu/asan_ubsan_lsan.yml' - local: 'ci/cpu/clang15_release_cxx20.yml' - local: 'ci/cpu/clang15_release_stdexec.yml' - local: 'ci/cpu/clang15_release.yml' - local: 'ci/cpu/clang16_release.yml' - local: 'ci/cpu/clang18_release.yml' - local: 'ci/cpu/gcc11_release_stdexec.yml' - - local: 'ci/cpu/gcc11_debug_stdexec.yml' + # - local: 'ci/cpu/gcc11_debug_stdexec.yml' - local: 'ci/cpu/gcc12_release_cxx20.yml' - - local: 'ci/cpu/gcc13_codecov.yml' + # - local: 'ci/cpu/gcc13_codecov.yml' - local: 'ci/cpu/gcc13_release.yml' - - local: 'ci/cuda/gcc11_release.yml' - - local: 'ci/cuda/gcc11_release_scalapack.yml' - - local: 'ci/cuda/gcc11_codecov.yml' - - local: 'ci/cuda/gcc11_debug_scalapack.yml' - - local: 'ci/cuda/gcc13_release_stdexec.yml' - - local: 'ci/rocm/clang14_release.yml' - - local: 'ci/rocm/clang14_release_stdexec.yml' + # - local: 'ci/cuda/gcc11_release.yml' + # - local: 'ci/cuda/gcc11_release_scalapack.yml' + # - local: 'ci/cuda/gcc11_codecov.yml' + # - local: 'ci/cuda/gcc11_debug_scalapack.yml' + # - local: 'ci/cuda/gcc13_release_stdexec.yml' + # - local: 'ci/rocm/clang14_release.yml' + # - local: 'ci/rocm/clang14_release_stdexec.yml' diff --git a/ci/ctest_to_gitlab.sh b/ci/ctest_to_gitlab.sh index cedf894f80..bb2465a7a7 100755 --- a/ci/ctest_to_gitlab.sh +++ b/ci/ctest_to_gitlab.sh @@ -61,7 +61,7 @@ JOB_TEMPLATE=" USE_MPI: 'YES' DISABLE_AFTER_SCRIPT: 'YES' DLAF_HDF5_TEST_OUTPUT_PATH: \$CI_PROJECT_DIR - script: mpi-ctest -L {{LABEL}} + script: mpi-ctest --timeout 300 --repeat-until-fail 50 -R test_multiplication_hermitian -L {{LABEL}} artifacts: paths: - codecov-reports/" @@ -100,7 +100,7 @@ JOB_TEMPLATE=" USE_MPI: 'YES' DISABLE_AFTER_SCRIPT: 'YES' DLAF_HDF5_TEST_OUTPUT_PATH: \$CI_PROJECT_DIR - script: mpi-ctest -L {{LABEL}}" + script: mpi-ctest --timeout 60 --repeat-until-fail 50 -R test_multiplication_hermitian -L {{LABEL}}" fi JOBS="" From 4875a0e0fbca811eb989fe89eaf3b2f7dfd6e836 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Thu, 31 Oct 2024 15:59:27 +0100 Subject: [PATCH 2/6] Add pika::wait after all hermitian matrix multiplication tests --- test/unit/multiplication/test_multiplication_hermitian.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/unit/multiplication/test_multiplication_hermitian.cpp b/test/unit/multiplication/test_multiplication_hermitian.cpp index 0f21eab622..e86b85e3d1 100644 --- a/test/unit/multiplication/test_multiplication_hermitian.cpp +++ b/test/unit/multiplication/test_multiplication_hermitian.cpp @@ -159,6 +159,7 @@ TYPED_TEST(HermitianMultiplicationTestMC, CorrectnessLocal) { testHermitianMultiplication(side, uplo, m, n, mb, nb, alpha, beta); + pika::wait(); } } } @@ -196,6 +197,7 @@ TYPED_TEST(HermitianMultiplicationTestGPU, CorrectnessLocal) { testHermitianMultiplication(side, uplo, m, n, mb, nb, alpha, beta); + pika::wait(); } } } From 9540415028e4add6b60a30d154efb61c4a21a691 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Mon, 4 Nov 2024 10:44:07 +0100 Subject: [PATCH 3/6] Debugging hemm test --- .../test_multiplication_hermitian.cpp | 122 +++++++++--------- 1 file changed, 64 insertions(+), 58 deletions(-) diff --git a/test/unit/multiplication/test_multiplication_hermitian.cpp b/test/unit/multiplication/test_multiplication_hermitian.cpp index e86b85e3d1..1e6191cea7 100644 --- a/test/unit/multiplication/test_multiplication_hermitian.cpp +++ b/test/unit/multiplication/test_multiplication_hermitian.cpp @@ -154,6 +154,12 @@ TYPED_TEST(HermitianMultiplicationTestMC, CorrectnessLocal) { if (side != blas::Side::Left || uplo != blas::Uplo::Lower) continue; for (const auto& [m, n, mb, nb] : sizes) { + std::cerr << "side: " << side; + std::cerr << ", uplo: " << uplo; + std::cerr << ", m: " << m; + std::cerr << ", n: " << n; + std::cerr << ", mb: " << mb; + std::cerr << ", nb: " << nb << std::endl; TypeParam alpha = TypeUtilities::element(-1.2, .7); TypeParam beta = TypeUtilities::element(1.12, -.1); @@ -165,61 +171,61 @@ TYPED_TEST(HermitianMultiplicationTestMC, CorrectnessLocal) { } } -TYPED_TEST(HermitianMultiplicationTestMC, CorrectnessDistributed) { - for (auto& comm_grid : this->commGrids()) { - for (const auto side : blas_sides) { - for (const auto uplo : blas_uplos) { - if (side != blas::Side::Left || uplo != blas::Uplo::Lower) - continue; - - for (const auto& [m, n, mb, nb] : sizes) { - TypeParam alpha = TypeUtilities::element(-1.2, .7); - TypeParam beta = TypeUtilities::element(1.12, -.1); - - testHermitianMultiplication(comm_grid, side, uplo, m, n, - mb, nb, alpha, beta); - pika::wait(); - } - } - } - } -} - -#ifdef DLAF_WITH_GPU -TYPED_TEST(HermitianMultiplicationTestGPU, CorrectnessLocal) { - for (const auto side : blas_sides) { - for (const auto uplo : blas_uplos) { - if (side != blas::Side::Left || uplo != blas::Uplo::Lower) - continue; - for (const auto& [m, n, mb, nb] : sizes) { - TypeParam alpha = TypeUtilities::element(-1.2, .7); - TypeParam beta = TypeUtilities::element(1.12, -.1); - - testHermitianMultiplication(side, uplo, m, n, mb, nb, - alpha, beta); - pika::wait(); - } - } - } -} - -TYPED_TEST(HermitianMultiplicationTestGPU, CorrectnessDistributed) { - for (auto& comm_grid : this->commGrids()) { - for (const auto side : blas_sides) { - for (const auto uplo : blas_uplos) { - if (side != blas::Side::Left || uplo != blas::Uplo::Lower) - continue; - - for (const auto& [m, n, mb, nb] : sizes) { - TypeParam alpha = TypeUtilities::element(-1.2, .7); - TypeParam beta = TypeUtilities::element(1.12, -.1); - - testHermitianMultiplication(comm_grid, side, uplo, m, n, - mb, nb, alpha, beta); - pika::wait(); - } - } - } - } -} -#endif +// TYPED_TEST(HermitianMultiplicationTestMC, CorrectnessDistributed) { +// for (auto& comm_grid : this->commGrids()) { +// for (const auto side : blas_sides) { +// for (const auto uplo : blas_uplos) { +// if (side != blas::Side::Left || uplo != blas::Uplo::Lower) +// continue; + +// for (const auto& [m, n, mb, nb] : sizes) { +// TypeParam alpha = TypeUtilities::element(-1.2, .7); +// TypeParam beta = TypeUtilities::element(1.12, -.1); + +// testHermitianMultiplication(comm_grid, side, uplo, m, n, +// mb, nb, alpha, beta); +// pika::wait(); +// } +// } +// } +// } +// } + +// #ifdef DLAF_WITH_GPU +// TYPED_TEST(HermitianMultiplicationTestGPU, CorrectnessLocal) { +// for (const auto side : blas_sides) { +// for (const auto uplo : blas_uplos) { +// if (side != blas::Side::Left || uplo != blas::Uplo::Lower) +// continue; +// for (const auto& [m, n, mb, nb] : sizes) { +// TypeParam alpha = TypeUtilities::element(-1.2, .7); +// TypeParam beta = TypeUtilities::element(1.12, -.1); + +// testHermitianMultiplication(side, uplo, m, n, mb, nb, +// alpha, beta); +// pika::wait(); +// } +// } +// } +// } + +// TYPED_TEST(HermitianMultiplicationTestGPU, CorrectnessDistributed) { +// for (auto& comm_grid : this->commGrids()) { +// for (const auto side : blas_sides) { +// for (const auto uplo : blas_uplos) { +// if (side != blas::Side::Left || uplo != blas::Uplo::Lower) +// continue; + +// for (const auto& [m, n, mb, nb] : sizes) { +// TypeParam alpha = TypeUtilities::element(-1.2, .7); +// TypeParam beta = TypeUtilities::element(1.12, -.1); + +// testHermitianMultiplication(comm_grid, side, uplo, m, n, +// mb, nb, alpha, beta); +// pika::wait(); +// } +// } +// } +// } +// } +// #endif From 61c8dd4133874dd1e086794afbb5b367deb95700 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Mon, 4 Nov 2024 17:33:34 +0100 Subject: [PATCH 4/6] More debugging --- test/unit/multiplication/test_multiplication_hermitian.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/unit/multiplication/test_multiplication_hermitian.cpp b/test/unit/multiplication/test_multiplication_hermitian.cpp index 1e6191cea7..01a7a28546 100644 --- a/test/unit/multiplication/test_multiplication_hermitian.cpp +++ b/test/unit/multiplication/test_multiplication_hermitian.cpp @@ -97,8 +97,8 @@ void testHermitianMultiplication(const blas::Side side, const blas::Uplo uplo, c } // SCOPED_TRACE cannot yield. - mat_ch.waitLocalTiles(); - SCOPED_TRACE(::testing::Message() << "m " << m << "n " << n << ", mb " << mb << ", nb " << nb); + // mat_ch.waitLocalTiles(); + // SCOPED_TRACE(::testing::Message() << "m " << m << "n " << n << ", mb " << mb << ", nb " << nb); CHECK_MATRIX_NEAR(res_c, mat_ch, 10 * (m + 1) * TypeUtilities::error, 10 * (m + 1) * TypeUtilities::error); } @@ -166,6 +166,7 @@ TYPED_TEST(HermitianMultiplicationTestMC, CorrectnessLocal) { testHermitianMultiplication(side, uplo, m, n, mb, nb, alpha, beta); pika::wait(); + std::cerr << "done\n"; } } } From e5063328ce30eeb465cce4c9d5a719b7528cedec Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 5 Nov 2024 13:23:49 +0100 Subject: [PATCH 5/6] Print worker thread index --- .../multiplication/test_multiplication_hermitian.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/test/unit/multiplication/test_multiplication_hermitian.cpp b/test/unit/multiplication/test_multiplication_hermitian.cpp index 01a7a28546..73edc5cd49 100644 --- a/test/unit/multiplication/test_multiplication_hermitian.cpp +++ b/test/unit/multiplication/test_multiplication_hermitian.cpp @@ -97,10 +97,18 @@ void testHermitianMultiplication(const blas::Side side, const blas::Uplo uplo, c } // SCOPED_TRACE cannot yield. - // mat_ch.waitLocalTiles(); - // SCOPED_TRACE(::testing::Message() << "m " << m << "n " << n << ", mb " << mb << ", nb " << nb); + mat_ch.waitLocalTiles(); + SCOPED_TRACE(::testing::Message() << "m " << m << "n " << n << ", mb " << mb << ", nb " << nb); + auto t_before = pika::get_worker_thread_num(); CHECK_MATRIX_NEAR(res_c, mat_ch, 10 * (m + 1) * TypeUtilities::error, 10 * (m + 1) * TypeUtilities::error); + auto t_after = pika::get_worker_thread_num(); + if (t_before != t_after) { + std::cerr << "worker thread index changed from " << t_before << " to " << t_after << std::endl; + } + else { + std::cerr << "worker thread index is still " << t_before << std::endl; + } } template From db382c68e474ca506a5c1ae91e112c4efad3e808 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 5 Nov 2024 16:06:36 +0100 Subject: [PATCH 6/6] Use custom pika branch --- ci/docker/common.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/docker/common.yaml b/ci/docker/common.yaml index 5ef667ae3c..690e2aef02 100644 --- a/ci/docker/common.yaml +++ b/ci/docker/common.yaml @@ -46,3 +46,6 @@ packages: # Force git as non-buildable to allow deprecated versions in environments # https://github.com/spack/spack/pull/30040 buildable: false + pika: + require: + - '@git.no-yield-async-rw-mutex=0.29.0'