From 5d0f37dea777053c34c7ac042f27ae35ef906f9d Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 28 May 2024 09:57:16 +0200 Subject: [PATCH 01/32] Use spack's blas/lapack libraries for MKL from version 0.5.X onwards --- spack/packages/dla-future/package.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/spack/packages/dla-future/package.py b/spack/packages/dla-future/package.py index b7ba7501ae..851696fde1 100644 --- a/spack/packages/dla-future/package.py +++ b/spack/packages/dla-future/package.py @@ -184,7 +184,7 @@ def cmake_args(self): args.append(self.define_from_variant("BUILD_SHARED_LIBS", "shared")) # BLAS/LAPACK - if spec["lapack"].name in INTEL_MATH_LIBRARIES: + if spec.version <= Version("0.4") and spec["lapack"].name in INTEL_MATH_LIBRARIES: mkl_provider = spec["lapack"].name vmap = { @@ -246,7 +246,8 @@ def cmake_args(self): ) ) else: - args.append(self.define("DLAF_WITH_MKL", False)) + # TODO: Auto-detect MKL or not? + args.append(self.define("DLAF_WITH_MKL", spec["lapack"].name in INTEL_MATH_LIBRARIES)) args.append( self.define( "LAPACK_LIBRARY", From 018ebcd4a842f0e468281f1185a76e5c066102de Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 28 May 2024 10:01:12 +0200 Subject: [PATCH 02/32] Remove DLAF_WITH_MKL_LEGACY CMake option --- BUILD.md | 1 - CMakeLists.txt | 38 +--- cmake/FindMKL.cmake | 313 ----------------------------- cmake/template/DLAFConfig.cmake.in | 43 +--- src/CMakeLists.txt | 1 - 5 files changed, 7 insertions(+), 389 deletions(-) delete mode 100644 cmake/FindMKL.cmake diff --git a/BUILD.md b/BUILD.md index a1c1a4fc5a..d65ae6a5a9 100644 --- a/BUILD.md +++ b/BUILD.md @@ -95,7 +95,6 @@ CMake option | Values | Note `blaspp_DIR` | CMAKE:PATH | Location of the blaspp CMake-config file `lapackpp_DIR` | CMAKE:PATH | Location of the lapackpp CMake-config file `DLAF_WITH_MKL` | `{ON,OFF}` (default: `OFF`) | if blaspp/lapackpp is built with oneMKL -`DLAF_WITH_LEGACY_MKL` | `{ON,OFF}` (default: `OFF`) | if blaspp/lapackpp is built with MKL (deprecated) `DLAF_WITH_SCALAPACK` | `{ON,OFF}` (default: `OFF`) | Enable ScaLAPACK-like API. `MKL_ROOT` | CMAKE:PATH | Location of the MKL library `DLAF_ASSERT_ENABLE` | `{ON,OFF}` (default: `ON`) | enable/disable cheap assertions diff --git a/CMakeLists.txt b/CMakeLists.txt index db5c37c6ed..5499b80bb2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,7 +35,6 @@ endif() option(BUILD_SHARED_LIBS "Build shared libraries." ON) option(DLAF_WITH_OPENMP "${DLAF_WITH_OPENMP_DESCRIPTION}" ${DLAF_WITH_OPENMP_DEFAULT}) option(DLAF_WITH_MKL "Enable oneMKL as provider for BLAS and LAPACK" OFF) -option(DLAF_WITH_MKL_LEGACY "Enable MKL as provider for BLAS and LAPACK" OFF) option(DLAF_WITH_CUDA "Enable CUDA support" OFF) option(DLAF_WITH_HIP "Enable HIP support" OFF) cmake_dependent_option( @@ -55,11 +54,7 @@ option(DLAF_BUILD_DOC "Build documentation" OFF) option(DLAF_WITH_PRECOMPILED_HEADERS "Use precompiled headers." OFF) option(DLAF_WITH_SCALAPACK "Build ScaLAPACK-like C API (requires ScaLAPACK)" OFF) -if(DLAF_WITH_MKL AND DLAF_WITH_MKL_LEGACY) - message(SEND_ERROR "DLAF_WITH_MKL and DLAF_WITH_MKL_LEGACY are mutually exclusive.") -endif() - -if(DLAF_WITH_MKL OR DLAF_WITH_MKL_LEGACY) +if(DLAF_WITH_MKL) # When using MKL there is no need to set the number of threads with # omp_set_num_threads; it's sufficient to use MKL's own mechanisms. set(DLAF_WITH_OPENMP OFF CACHE BOOL "${DLAF_WITH_OPENMP_DESCRIPTION}" FORCE) @@ -156,37 +151,6 @@ if(DLAF_WITH_MKL) # oneMKL set(SCALAPACK_FOUND TRUE) add_library(DLAF::SCALAPACK INTERFACE IMPORTED GLOBAL) endif() -elseif(DLAF_WITH_MKL_LEGACY) # MKL (deprecated) - find_package(MKL MODULE REQUIRED) - - set(MKL_LAPACK_TARGET "mkl::mkl_intel_32bit_seq_dyn" - CACHE STRING "MKL LAPACK target (see FindMKL for details)" - ) - - # Note: - # If specified targets are found, "standardize" them - - if(TARGET ${MKL_LAPACK_TARGET}) - set(LAPACK_FOUND TRUE) - add_library(DLAF::LAPACK INTERFACE IMPORTED GLOBAL) - target_link_libraries(DLAF::LAPACK INTERFACE ${MKL_LAPACK_TARGET}) - else() - message(FATAL_ERROR "${MKL_LAPACK_TARGET} as LAPACK target has not been found") - endif() - - if(DLAF_WITH_SCALAPACK) - set(MKL_SCALAPACK_TARGET "mkl::scalapack_intel_32bit_seq_dyn" - CACHE STRING "MKL ScaLAPACK target (see FindMKL for details)" - ) - - if(TARGET ${MKL_SCALAPACK_TARGET}) - set(SCALAPACK_FOUND TRUE) - add_library(DLAF::SCALAPACK INTERFACE IMPORTED GLOBAL) - target_link_libraries(DLAF::SCALAPACK INTERFACE ${MKL_SCALAPACK_TARGET}) - else() - message(FATAL_ERROR "${MKL_SCALAPACK_TARGET} as ScaLAPACK target has not been found") - endif() - endif() else() find_package(LAPACK REQUIRED) if(DLAF_WITH_SCALAPACK) diff --git a/cmake/FindMKL.cmake b/cmake/FindMKL.cmake deleted file mode 100644 index 8c1d121907..0000000000 --- a/cmake/FindMKL.cmake +++ /dev/null @@ -1,313 +0,0 @@ -# -# CMake recipes -# https://github.com/eth-cscs/cmake-recipes -# -# Copyright (c) 2018-2024, ETH Zurich -# BSD 3-Clause License. All rights reserved. -# -# Author: Teodor Nikolov (tnikolov@cscs.ch) -# - -# dlaf-no-license-check - -#[=======================================================================[.rst: -FindMKL -------- - -The following conventions are used: - -intel / INTEL - Bindings for everything except GNU Fortran -gf / GF - GNU Fortran bindings -seq / SEQ - sequential MKL -omp / OMP - threaded MKL with OpenMP back end -tbb / TBB - threaded MKL with TBB back end -32bit / 32BIT - MKL 32 bit integer interface (used most often) -64bit / 64BIT - MKL 64 bit integer interface -mpich / MPICH - MPICH / IntelMPI BLACS back end -ompi / OMPI - OpenMPI BLACS back end -st / ST - static libraries -dyn / DYN - dynamic libraries - -The module attempts to define a target for each MKL configuration. The -configuration will not be available if there are missing library files or a -missing dependency. - -MKL Link line advisor: - https://software.intel.com/en-us/articles/intel-mkl-link-line-advisor - -Note: Mixing GCC and Intel OpenMP backends is a bad idea. - -Search variables -^^^^^^^^^^^^^^^^ - -``MKLROOT`` - Environment variable set to MKL's root directory - -``MKL_ROOT`` - CMake variable set to MKL's root directory - -Example usage -^^^^^^^^^^^^^ - -To Find MKL: - - find_package(MKL REQUIRED) - -To check if target is available: - - if (TARGET mkl::scalapack_mpich_intel_32bit_omp_dyn) - ... - endif() - -To link to an available target (see list below): - - target_link_libraries(... mkl::scalapack_mpich_intel_32bit_omp_dyn) - -Note: dependencies are handled for you (MPI, OpenMP, ...) - -Imported targets -^^^^^^^^^^^^^^^^ - -MKL (BLAS, LAPACK, FFT) targets: - - mkl::mkl_[gf|intel]_[32bit|64bit]_[seq|omp|tbb]_[st|dyn] e.g. - - mkl::mkl_intel_32bit_omp_dyn - -BLACS targets: - - mkl::blacs_[mpich|ompi]_[gf|intel]_[32bit|64bit]_[seq|omp|tbb]_[st|dyn] e.g. - - mkl::blacs_intel_mpich_32bit_seq_st - -ScaLAPACK targets: - - mkl::scalapack_[mpich|ompi]_[gf|intel]_[32bit|64bit]_[seq|omp|tbb]_[st|dyn] e.g. - - mkl::scalapack_mpich_intel_64bit_omp_dyn - -Result variables -^^^^^^^^^^^^^^^^ - -MKL_FOUND - -Not supported -^^^^^^^^^^^^^ - -- F95 interfaces - -#]=======================================================================] - -cmake_minimum_required(VERSION 3.12) - -# check if compatible compiler is found -if(CMAKE_C_COMPILER_LOADED OR CMAKE_CXX_COMPILER_LOADED OR CMAKE_Fortran_COMPILER_LOADED) - set(_mkl_compiler_found TRUE) -else() - set(_mkl_compiler_found FALSE) -endif() - -# Dependencies -# -find_package(Threads) -find_package(MPI COMPONENTS CXX) -find_package(OpenMP COMPONENTS CXX) - -# If MKL_ROOT is not set, set it via the env variable MKLROOT. -# -if(NOT DEFINED MKL_ROOT) - set(MKL_ROOT $ENV{MKLROOT}) -endif() - -# Determine MKL's library folder -# -set(_mkl_libpath_suffix "lib/intel64") -if(CMAKE_SIZEOF_VOID_P EQUAL 4) # 32 bit - set(_mkl_libpath_suffix "lib/ia32") -endif() - -if(WIN32) - string(APPEND _mkl_libpath_suffix "_win") - set(_mkl_libname_prefix "") - set(_mkl_shared_lib "_dll.lib") - set(_mkl_static_lib ".lib") -elseif(APPLE) - string(APPEND _mkl_libpath_suffix "_mac") - set(_mkl_libname_prefix "lib") - set(_mkl_shared_lib ".dylib") - set(_mkl_static_lib ".a") -else() # LINUX - string(APPEND _mkl_libpath_suffix "_lin") - set(_mkl_libname_prefix "lib") - set(_mkl_shared_lib ".so") - set(_mkl_static_lib ".a") -endif() -set(_mkl_search_paths "${MKL_ROOT}" "${MKL_ROOT}/lib" "${MKL_ROOT}/mkl" "${MKL_ROOT}/compiler") - -# Functions: finds both static and shared MKL libraries -# -function(__mkl_find_library _varname _libname) - find_library( - ${_varname}_DYN - NAMES ${_mkl_libname_prefix}${_libname}${_mkl_shared_lib} - HINTS ${_mkl_search_paths} - PATH_SUFFIXES ${_mkl_libpath_suffix} - ) - mark_as_advanced(${_varname}_DYN) - find_library( - ${_varname}_ST - NAMES ${_mkl_libname_prefix}${_libname}${_mkl_static_lib} - HINTS ${_mkl_search_paths} - PATH_SUFFIXES ${_mkl_libpath_suffix} - ) - mark_as_advanced(${_varname}_ST) -endfunction() - -# Find MKL headers -# -find_path(MKL_INCLUDE_DIR mkl.h HINTS ${MKL_ROOT}/include ${MKL_ROOT}/mkl/include) -mark_as_advanced(MKL_INCLUDE_DIR) - -# Group flags for static libraries on Linux (GNU, PGI, ICC -> same linker) -# -if(UNIX AND NOT APPLE) - set(_mkl_linker_pre_flags_ST "-Wl,--start-group") - set(_mkl_linker_post_flags_ST "-Wl,--end-group") -endif() - -# Core MKL -# -__mkl_find_library(MKL_CORE_LIB mkl_core) - -# Interface -# -__mkl_find_library(MKL_INTERFACE_INTEL_32BIT_LIB mkl_intel_lp64) -__mkl_find_library(MKL_INTERFACE_INTEL_64BIT_LIB mkl_intel_ilp64) -if(NOT APPLE AND CMAKE_Fortran_COMPILER_LOADED AND CMAKE_Fortran_COMPILER_ID STREQUAL "GNU") - __mkl_find_library(MKL_INTERFACE_GF_32BIT_LIB mkl_gf_lp64) - __mkl_find_library(MKL_INTERFACE_GF_64BIT_LIB mkl_gf_ilp64) -endif() - -# Threading -# -__mkl_find_library(MKL_SEQ_LIB mkl_sequential) -if(NOT APPLE AND (CMAKE_C_COMPILER_ID STREQUAL "GNU" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU" - OR CMAKE_Fortran_COMPILER_ID STREQUAL "GNU") -) - __mkl_find_library(MKL_OMP_LIB mkl_gnu_thread) -else() - __mkl_find_library(MKL_OMP_LIB mkl_intel_thread) -endif() -__mkl_find_library(MKL_TBB_LIB mkl_tbb_thread) - -# BLACS -# -if(APPLE) - __mkl_find_library(MKL_BLACS_MPICH_32BIT_LIB mkl_blacs_mpich_lp64) - __mkl_find_library(MKL_BLACS_MPICH_64BIT_LIB mkl_blacs_mpich_ilp64) -else() - __mkl_find_library(MKL_BLACS_MPICH_32BIT_LIB mkl_blacs_intelmpi_lp64) - __mkl_find_library(MKL_BLACS_MPICH_64BIT_LIB mkl_blacs_intelmpi_ilp64) -endif() -__mkl_find_library(MKL_BLACS_OMPI_32BIT_LIB mkl_blacs_openmpi_lp64) -__mkl_find_library(MKL_BLACS_OMPI_64BIT_LIB mkl_blacs_openmpi_ilp64) - -# ScaLAPACK -# -__mkl_find_library(MKL_SCALAPACK_32BIT_LIB mkl_scalapack_lp64) -__mkl_find_library(MKL_SCALAPACK_64BIT_LIB mkl_scalapack_ilp64) - -# Check if core libs were found -# -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(MKL REQUIRED_VARS MKL_INCLUDE_DIR Threads_FOUND _mkl_compiler_found) - -# Sequential has no threading dependency. There is currently no TBB module -# shipped with CMake. The dependency is not accounted for. -# -set(_mkl_dep_found_SEQ TRUE) -set(_mkl_dep_found_TBB TRUE) -if(TARGET OpenMP::OpenMP_CXX) - set(_mkl_dep_OMP OpenMP::OpenMP_CXX) - set(_mkl_dep_found_OMP TRUE) -endif() - -# Define all blas, blacs and scalapack -# -foreach(_libtype "ST" "DYN") - set(_mkl_core_lib ${MKL_CORE_LIB_${_libtype}}) - foreach(_bits "32BIT" "64BIT") - set(_mkl_scalapack_lib ${MKL_SCALAPACK_${_bits}_LIB_${_libtype}}) - foreach(_iface "INTEL" "GF") - set(_mkl_interface_lib ${MKL_INTERFACE_${_iface}_${_bits}_LIB_${_libtype}}) - foreach(_threading "SEQ" "OMP" "TBB") - set(_mkl_threading_lib ${MKL_${_threading}_LIB_${_libtype}}) - - string(TOLOWER "${_iface}_${_bits}_${_threading}_${_libtype}" _tgt_config) - set(_mkl_tgt mkl::mkl_${_tgt_config}) - - if(MKL_FOUND - AND _mkl_interface_lib - AND _mkl_threading_lib - AND _mkl_core_lib - AND _mkl_dep_found_${_threading} - AND NOT TARGET ${_mkl_tgt} - ) - set(_mkl_libs - "${_mkl_linker_pre_flags_${_threading}}" - "${_mkl_interface_lib}" - "${_mkl_threading_lib}" - "${_mkl_core_lib}" - "${_mkl_linker_post_flags_${_threading}}" - "${_mkl_dep_${_threading}}" - "Threads::Threads" - ) - add_library(${_mkl_tgt} INTERFACE IMPORTED) - set_target_properties( - ${_mkl_tgt} PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${MKL_INCLUDE_DIR}" - INTERFACE_LINK_LIBRARIES "${_mkl_libs}" - ) - endif() - - foreach(_mpi_impl "MPICH" "OMPI") - set(_mkl_blacs_lib ${MKL_BLACS_${_mpi_impl}_${_bits}_LIB_${_libtype}}) - - string(TOLOWER "${_mpi_impl}_${_iface}_${_bits}_${_threading}_${_libtype}" _tgt_config) - set(_blacs_tgt mkl::blacs_${_tgt_config}) - set(_scalapack_tgt mkl::scalapack_${_tgt_config}) - - if(_mkl_blacs_lib - AND TARGET ${_mkl_tgt} - AND TARGET MPI::MPI_CXX - AND NOT TARGET ${_blacs_tgt} - ) - set(_blacs_libs - "${_mkl_linker_pre_flags_${_libtype}}" - "${_mkl_interface_lib}" - "${_mkl_threading_lib}" - "${_mkl_core_lib}" - "${_mkl_blacs_lib}" - "${_mkl_linker_post_flags_${_libtype}}" - "MPI::MPI_CXX" - "${_mkl_dep_${_threading}}" - "Threads::Threads" - ) - add_library(${_blacs_tgt} INTERFACE IMPORTED) - set_target_properties( - ${_blacs_tgt} PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${MKL_INCLUDE_DIR}" - INTERFACE_LINK_LIBRARIES "${_blacs_libs}" - ) - endif() - - if(_mkl_scalapack_lib AND TARGET ${_blacs_tgt} AND NOT TARGET ${_scalapack_tgt}) - set(_scalapack_libs "${_mkl_scalapack_lib}" "${_blacs_tgt}") - add_library(${_scalapack_tgt} INTERFACE IMPORTED) - set_target_properties( - ${_scalapack_tgt} PROPERTIES INTERFACE_LINK_LIBRARIES "${_scalapack_libs}" - ) - endif() - endforeach() - endforeach() - endforeach() - endforeach() -endforeach() diff --git a/cmake/template/DLAFConfig.cmake.in b/cmake/template/DLAFConfig.cmake.in index ef33e5599a..ad823fde2e 100644 --- a/cmake/template/DLAFConfig.cmake.in +++ b/cmake/template/DLAFConfig.cmake.in @@ -20,7 +20,6 @@ list(PREPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}) # ===== VARIABLES set(DLAF_WITH_OPENMP @DLAF_WITH_OPENMP@) set(DLAF_WITH_MKL @DLAF_WITH_MKL@) -set(DLAF_WITH_MKL_LEGACY @DLAF_WITH_MKL_LEGACY@) set(DLAF_WITH_CUDA @DLAF_WITH_CUDA@) set(DLAF_WITH_HIP @DLAF_WITH_HIP@) set(DLAF_WITH_GPU @DLAF_WITH_GPU@) @@ -53,42 +52,12 @@ if(DLAF_WITH_OPENMP) endif() # ----- LAPACK -if(DLAF_WITH_MKL) - set(MKL_INTERFACE "@MKL_INTERFACE@") - set(MKL_THREADING "@MKL_THREADING@") - set(MKL_MPI "@MKL_MPI@") - - find_dependency(MKL CONFIG) - - add_library(DLAF::LAPACK INTERFACE IMPORTED GLOBAL) - target_link_libraries(DLAF::LAPACK INTERFACE MKL::MKL) - - if(DLAF_WITH_SCALAPACK) - add_library(DLAF::SCALAPACK INTERFACE IMPORTED GLOBAL) - endif() -elseif(DLAF_WITH_MKL_LEGACY) - set(MKL_ROOT "@MKL_ROOT@") - set(MKL_CUSTOM_THREADING "@MKL_THREADING@") - - find_dependency(MKL) - - set(MKL_LAPACK_TARGET "@MKL_LAPACK_TARGET@") - add_library(DLAF::LAPACK INTERFACE IMPORTED GLOBAL) - target_link_libraries(DLAF::LAPACK INTERFACE ${MKL_LAPACK_TARGET}) - - if(DLAF_WITH_SCALAPACK) - set(MKL_SCALAPACK_TARGET "@MKL_SCALAPACK_TARGET@") - add_library(DLAF::SCALAPACK INTERFACE IMPORTED GLOBAL) - target_link_libraries(DLAF::SCALAPACK INTERFACE ${MKL_SCALAPACK_TARGET}) - endif() -else() - set(LAPACK_LIBRARY "@LAPACK_LIBRARY@") - find_dependency(LAPACK) - - if(DLAF_WITH_SCALAPACK) - set(SCALAPACK_LIBRARY "@SCALAPACK_LIBRARY@") - find_dependency(SCALAPACK) - endif() +set(LAPACK_LIBRARY "@LAPACK_LIBRARY@") +find_dependency(LAPACK) + +if(DLAF_WITH_SCALAPACK) + set(SCALAPACK_LIBRARY "@SCALAPACK_LIBRARY@") + find_dependency(SCALAPACK) endif() # ----- pika diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cdc1f1fd0c..dd38eb0a1d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -94,7 +94,6 @@ target_compile_definitions( $<$:DLAF_ASSERT_HEAVY_ENABLE> DLAF_FUNCTION_NAME=$,__PRETTY_FUNCTION__,__func__> $<$:DLAF_WITH_MKL> - $<$:DLAF_WITH_MKL> $<$:DLAF_WITH_OPENMP> $<$:DLAF_WITH_GPU> $<$:DLAF_WITH_CUDA> From 3f9ffc1ca7c04a1aaad135ac7f909cdce4b7c4eb Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 28 May 2024 10:34:04 +0200 Subject: [PATCH 03/32] Add auto-detection for MKL --- CMakeLists.txt | 10 ++++++++-- cmake/tests/mkl_set_num_threads.cpp | 15 +++++++++++++++ miniapp/CMakeLists.txt | 2 +- 3 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 cmake/tests/mkl_set_num_threads.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 5499b80bb2..178a134326 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,8 @@ # SPDX-License-Identifier: BSD-3-Clause # -cmake_minimum_required(VERSION 3.22) +# TODO: Bump to 3.25 or keep 3.22? +cmake_minimum_required(VERSION 3.25) project(DLAF VERSION 0.4.1) @@ -34,7 +35,6 @@ endif() option(BUILD_SHARED_LIBS "Build shared libraries." ON) option(DLAF_WITH_OPENMP "${DLAF_WITH_OPENMP_DESCRIPTION}" ${DLAF_WITH_OPENMP_DEFAULT}) -option(DLAF_WITH_MKL "Enable oneMKL as provider for BLAS and LAPACK" OFF) option(DLAF_WITH_CUDA "Enable CUDA support" OFF) option(DLAF_WITH_HIP "Enable HIP support" OFF) cmake_dependent_option( @@ -54,6 +54,12 @@ option(DLAF_BUILD_DOC "Build documentation" OFF) option(DLAF_WITH_PRECOMPILED_HEADERS "Use precompiled headers." OFF) option(DLAF_WITH_SCALAPACK "Build ScaLAPACK-like C API (requires ScaLAPACK)" OFF) +try_compile( + DLAF_WITH_MKL_TRY_COMPILE SOURCES "${PROJECT_SOURCE_DIR}/cmake/tests/mkl_set_num_threads.cpp" + LINK_OPTIONS "${LAPACK_LIBRARY}" +) +option(DLAF_WITH_MKL "Enable oneMKL as provider for BLAS and LAPACK" "${DLAF_WITH_MKL_TRY_COMPILE}") + if(DLAF_WITH_MKL) # When using MKL there is no need to set the number of threads with # omp_set_num_threads; it's sufficient to use MKL's own mechanisms. diff --git a/cmake/tests/mkl_set_num_threads.cpp b/cmake/tests/mkl_set_num_threads.cpp new file mode 100644 index 0000000000..a0eee9e1df --- /dev/null +++ b/cmake/tests/mkl_set_num_threads.cpp @@ -0,0 +1,15 @@ +// +// Distributed Linear Algebra with Future (DLAF) +// +// Copyright (c) 2018-2024, ETH Zurich +// All rights reserved. +// +// Please, refer to the LICENSE file in the root directory. +// SPDX-License-Identifier: BSD-3-Clause +// + +#include + +int main() { + [[maybe_unused]] auto mkl_num_threads = mkl_set_num_threads_local(1); +} diff --git a/miniapp/CMakeLists.txt b/miniapp/CMakeLists.txt index 9052ed64ec..042e915d0a 100644 --- a/miniapp/CMakeLists.txt +++ b/miniapp/CMakeLists.txt @@ -8,7 +8,7 @@ # SPDX-License-Identifier: BSD-3-Clause # -cmake_minimum_required(VERSION 3.22) +cmake_minimum_required(VERSION 3.25) project(DLAF-miniapps) list(PREPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) From dc0ffe34f068ea154c5609a2b6d17b38d4544145 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 28 May 2024 10:37:38 +0200 Subject: [PATCH 04/32] TEMP: Disable most CI configurations for testing --- ci/.gitlab-ci.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/ci/.gitlab-ci.yml b/ci/.gitlab-ci.yml index ca55102787..1e95751de9 100644 --- a/ci/.gitlab-ci.yml +++ b/ci/.gitlab-ci.yml @@ -1,18 +1,18 @@ include: - - local: 'ci/cpu/asan_ubsan_lsan.yml' - - local: 'ci/cpu/clang12_release.yml' - - local: 'ci/cpu/clang13_release_cxx20.yml' - - local: 'ci/cpu/clang14_release_stdexec.yml' - - local: 'ci/cpu/clang15_release.yml' - - local: 'ci/cpu/clang16_release.yml' - - local: 'ci/cpu/gcc11_codecov.yml' - - local: 'ci/cpu/gcc11_release.yml' - - local: 'ci/cpu/gcc11_release_stdexec.yml' - - local: 'ci/cpu/gcc11_debug_stdexec.yml' - - local: 'ci/cpu/gcc12_release_cxx20.yml' +# - local: 'ci/cpu/asan_ubsan_lsan.yml' +# - local: 'ci/cpu/clang12_release.yml' +# - local: 'ci/cpu/clang13_release_cxx20.yml' +# - local: 'ci/cpu/clang14_release_stdexec.yml' +# - local: 'ci/cpu/clang15_release.yml' +# - local: 'ci/cpu/clang16_release.yml' +# - local: 'ci/cpu/gcc11_codecov.yml' +# - local: 'ci/cpu/gcc11_release.yml' +# - local: 'ci/cpu/gcc11_release_stdexec.yml' +# - local: 'ci/cpu/gcc11_debug_stdexec.yml' +# - local: 'ci/cpu/gcc12_release_cxx20.yml' - local: 'ci/cuda/gcc11_release.yml' - local: 'ci/cuda/gcc11_release_scalapack.yml' - local: 'ci/cuda/gcc11_codecov.yml' - - local: 'ci/cuda/gcc11_debug_scalapack.yml' - - local: 'ci/rocm/clang14_release.yml' - - local: 'ci/rocm/clang14_release_stdexec.yml' +# - local: 'ci/cuda/gcc11_debug_scalapack.yml' +# - local: 'ci/rocm/clang14_release.yml' +# - local: 'ci/rocm/clang14_release_stdexec.yml' From 996750951fbe87696f41feca201242f5e8940177 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 28 May 2024 10:40:43 +0200 Subject: [PATCH 05/32] Update SPACK_SHA in CI --- ci/common-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/common-ci.yml b/ci/common-ci.yml index 626f81edc2..552e7fdc2f 100644 --- a/ci/common-ci.yml +++ b/ci/common-ci.yml @@ -34,7 +34,7 @@ stages: reports: dotenv: build.env variables: - SPACK_SHA: dcc4423a9d0219a26ceeb52e329ed956e41f4d2c + SPACK_SHA: fd3c18b6fd7eda8abbb72ee81b39aa81b053d9bd SPACK_DLAF_REPO: ./spack DOCKER_BUILD_ARGS: '[ "BASE_IMAGE", From f71272bc52b9409385d0658d36ee24c6f780ec4a Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 28 May 2024 10:40:53 +0200 Subject: [PATCH 06/32] Add libssl-dev to CI image for external openssl --- ci/docker/build.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/build.Dockerfile b/ci/docker/build.Dockerfile index 07e7b06fde..f50d178d8a 100644 --- a/ci/docker/build.Dockerfile +++ b/ci/docker/build.Dockerfile @@ -15,7 +15,7 @@ RUN apt-get -yqq update && \ apt-get -yqq install --no-install-recommends \ software-properties-common \ build-essential gfortran \ - autoconf automake ninja-build pkg-config \ + autoconf automake libssl-dev ninja-build pkg-config \ ${EXTRA_APTGET} \ gawk \ python3 python3-distutils \ From 910724d7ebca097a58131c069383eaeb5c494cbe Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 28 May 2024 11:16:41 +0200 Subject: [PATCH 07/32] Don't explicitly set DLAF_WITH_MKL in spack package --- spack/packages/dla-future/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack/packages/dla-future/package.py b/spack/packages/dla-future/package.py index 851696fde1..eec726c683 100644 --- a/spack/packages/dla-future/package.py +++ b/spack/packages/dla-future/package.py @@ -247,7 +247,7 @@ def cmake_args(self): ) else: # TODO: Auto-detect MKL or not? - args.append(self.define("DLAF_WITH_MKL", spec["lapack"].name in INTEL_MATH_LIBRARIES)) + # TODO: for now don't set explicitly, let dlaf auto-detect: args.append(self.define("DLAF_WITH_MKL", spec["lapack"].name in INTEL_MATH_LIBRARIES)) args.append( self.define( "LAPACK_LIBRARY", From e681b72d7dbfc16f7bb7af485dba7781061d9a47 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 28 May 2024 11:17:56 +0200 Subject: [PATCH 08/32] Print MKL try_compile result --- CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 178a134326..e015742524 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -58,6 +58,8 @@ try_compile( DLAF_WITH_MKL_TRY_COMPILE SOURCES "${PROJECT_SOURCE_DIR}/cmake/tests/mkl_set_num_threads.cpp" LINK_OPTIONS "${LAPACK_LIBRARY}" ) +# TODO: Maybe don't need to print this? Though maybe it's nice to have... +message(WARNING "DLAF_WITH_MKL_TRY_COMPILE=${DLAF_WITH_MKL_TRY_COMPILE}") option(DLAF_WITH_MKL "Enable oneMKL as provider for BLAS and LAPACK" "${DLAF_WITH_MKL_TRY_COMPILE}") if(DLAF_WITH_MKL) From bedcec4c227914ea69bdb97c311530a07b5fccd3 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 29 May 2024 10:25:52 +0200 Subject: [PATCH 09/32] Don't find_package(MKL) --- CMakeLists.txt | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e015742524..3698dece8f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -143,27 +143,9 @@ if(DLAF_WITH_OPENMP) find_package(OpenMP REQUIRED) endif() -# ----- LAPACK/SCALAPACK -if(DLAF_WITH_MKL) # oneMKL - set(MKL_INTERFACE "lp64" CACHE STRING "") - set(MKL_THREADING "sequential" CACHE STRING "") - set(MKL_MPI "mpich" CACHE STRING "") - - find_package(MKL CONFIG REQUIRED) - - set(LAPACK_FOUND TRUE) - add_library(DLAF::LAPACK INTERFACE IMPORTED GLOBAL) - target_link_libraries(DLAF::LAPACK INTERFACE MKL::MKL) - - if(DLAF_WITH_SCALAPACK) - set(SCALAPACK_FOUND TRUE) - add_library(DLAF::SCALAPACK INTERFACE IMPORTED GLOBAL) - endif() -else() - find_package(LAPACK REQUIRED) - if(DLAF_WITH_SCALAPACK) - find_package(SCALAPACK REQUIRED) - endif() +find_package(LAPACK REQUIRED) +if(DLAF_WITH_SCALAPACK) + find_package(SCALAPACK REQUIRED) endif() # ----- pika From 3ec281bde254d4945d1d2c877c867f86e16925de Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 29 May 2024 10:26:37 +0200 Subject: [PATCH 10/32] Don't use -Werror=all-warnings for CUDA in CI --- spack/packages/dla-future/package.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/spack/packages/dla-future/package.py b/spack/packages/dla-future/package.py index eec726c683..dda33f440b 100644 --- a/spack/packages/dla-future/package.py +++ b/spack/packages/dla-future/package.py @@ -287,7 +287,12 @@ def cmake_args(self): # Enable TESTS and setup CI specific parameters args.append(self.define("CMAKE_CXX_FLAGS", "-Werror")) if "+cuda" in spec: - args.append(self.define("CMAKE_CUDA_FLAGS", "-Werror=all-warnings")) + args.append( + self.define( + "CMAKE_CUDA_FLAGS", + "-Werror=cross-execution-space-call;-Werror=reorder;-Werror=deprecated-declarations;-Werror=default-stream-launch;-Werror=missing-launch-bounds;-Werror=ext-lambda-captures-this", + ) + ) if "+rocm" in spec: args.append(self.define("CMAKE_HIP_FLAGS", "-Werror")) args.append(self.define("BUILD_TESTING", True)) From a5cdae8ad8c8fa7eba9c5edb643867d64ab01d47 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 29 May 2024 10:26:51 +0200 Subject: [PATCH 11/32] Add TODO about enabling MKL and OpenMP --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3698dece8f..be30bd1879 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,6 +62,7 @@ try_compile( message(WARNING "DLAF_WITH_MKL_TRY_COMPILE=${DLAF_WITH_MKL_TRY_COMPILE}") option(DLAF_WITH_MKL "Enable oneMKL as provider for BLAS and LAPACK" "${DLAF_WITH_MKL_TRY_COMPILE}") +# TODO: Enable both, just in case? if(DLAF_WITH_MKL) # When using MKL there is no need to set the number of threads with # omp_set_num_threads; it's sufficient to use MKL's own mechanisms. From c54431bfec18c720d91123b6ae737fb9baa4e36d Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 29 May 2024 10:58:10 +0200 Subject: [PATCH 12/32] Remove -Werror=missing-launch-bounds from CUDA CI configuration --- spack/packages/dla-future/package.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spack/packages/dla-future/package.py b/spack/packages/dla-future/package.py index dda33f440b..22833290eb 100644 --- a/spack/packages/dla-future/package.py +++ b/spack/packages/dla-future/package.py @@ -287,10 +287,11 @@ def cmake_args(self): # Enable TESTS and setup CI specific parameters args.append(self.define("CMAKE_CXX_FLAGS", "-Werror")) if "+cuda" in spec: + # TODO: -Werror=missing-launch-bounds triggers warnings!? args.append( self.define( "CMAKE_CUDA_FLAGS", - "-Werror=cross-execution-space-call;-Werror=reorder;-Werror=deprecated-declarations;-Werror=default-stream-launch;-Werror=missing-launch-bounds;-Werror=ext-lambda-captures-this", + "-Werror=cross-execution-space-call;-Werror=reorder;-Werror=deprecated-declarations;-Werror=default-stream-launch;-Werror=ext-lambda-captures-this", ) ) if "+rocm" in spec: From ecb341d5709cf3b8ce32d329005c2f22c99340eb Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 29 May 2024 10:58:31 +0200 Subject: [PATCH 13/32] Use older try_compile signature --- CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index be30bd1879..d57c11ac23 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,8 +55,10 @@ option(DLAF_WITH_PRECOMPILED_HEADERS "Use precompiled headers." OFF) option(DLAF_WITH_SCALAPACK "Build ScaLAPACK-like C API (requires ScaLAPACK)" OFF) try_compile( - DLAF_WITH_MKL_TRY_COMPILE SOURCES "${PROJECT_SOURCE_DIR}/cmake/tests/mkl_set_num_threads.cpp" - LINK_OPTIONS "${LAPACK_LIBRARY}" + DLAF_WITH_MKL_TRY_COMPILE "${PROJECT_SOURCE_DIR}/cmake/tests" + "${PROJECT_SOURCE_DIR}/cmake/tests/mkl_set_num_threads.cpp" + CMAKE_FLAGS "-DINCLUDE_DIRECTORIES=${LAPACK_INCLUDE_DIR}" + LINK_LIBRARIES ${LAPACK_LIBRARY} ) # TODO: Maybe don't need to print this? Though maybe it's nice to have... message(WARNING "DLAF_WITH_MKL_TRY_COMPILE=${DLAF_WITH_MKL_TRY_COMPILE}") From ebe5106681a0eab65a28ccb53e53e2fe02e1ce3d Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 29 May 2024 11:07:36 +0200 Subject: [PATCH 14/32] Add LAPACK_INCLUDE_DIR and SCALAPACK_INCLUDE_DIR variables --- cmake/FindLAPACK.cmake | 11 ++++++++++- cmake/FindSCALAPACK.cmake | 4 ++++ cmake/template/DLAFConfig.cmake.in | 2 ++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/cmake/FindLAPACK.cmake b/cmake/FindLAPACK.cmake index 65f2fede49..ae46f85665 100644 --- a/cmake/FindLAPACK.cmake +++ b/cmake/FindLAPACK.cmake @@ -19,6 +19,9 @@ # LAPACK_LIBRARY # ;-list of {lib name, lib filepath, -Llibrary_folder} # +# LAPACK_INCLUDE_DIR +# ;-list of include folders +# # This module sets the following variables: # LAPACK_FOUND - set to true if a library implementing the LAPACK interface is found # @@ -62,9 +65,14 @@ mark_as_advanced(LAPACK_LIBRARY) _lapack_check_is_working() +# TODO: Checks? +mark_as_advanced(LAPACK_INCLUDE_DIR) + ### Package include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(LAPACK DEFAULT_MSG LAPACK_LIBRARY _LAPACK_CHECK _LAPACK_CHECK_BLAS) +find_package_handle_standard_args( + LAPACK DEFAULT_MSG LAPACK_LIBRARY LAPACK_INCLUDE_DIR _LAPACK_CHECK _LAPACK_CHECK_BLAS +) # Remove the placeholder if(LAPACK_LIBRARY STREQUAL "LAPACK_LIBRARIES-PLACEHOLDER-FOR-EMPTY-LIBRARIES") @@ -77,4 +85,5 @@ if(LAPACK_FOUND) endif() target_link_libraries(DLAF::LAPACK INTERFACE "${LAPACK_LIBRARY}" "${_DEPS}") + target_include_directories(DLAF::LAPACK INTERFACE "${LAPACK_INCLUDE_DIR}") endif() diff --git a/cmake/FindSCALAPACK.cmake b/cmake/FindSCALAPACK.cmake index f6f7eb1eab..d1d6a8c799 100644 --- a/cmake/FindSCALAPACK.cmake +++ b/cmake/FindSCALAPACK.cmake @@ -30,10 +30,14 @@ find_package_handle_standard_args(SCALAPACK DEFAULT_MSG SCALAPACK_LIBRARY) mark_as_advanced(SCALAPACK_LIBRARY) +# TODO: Checks? +mark_as_advanced(SCALAPACK_INCLUDE_DIR) + if(SCALAPACK_FOUND) if(NOT TARGET DLAF::SCALAPACK) add_library(DLAF::SCALAPACK INTERFACE IMPORTED GLOBAL) endif() target_link_libraries(DLAF::SCALAPACK INTERFACE "${SCALAPACK_LIBRARY}") + target_include_directories(DLAF::SCALAPACK INTERFACE "${SCALAPACK_INCLUDE_DIR}") endif() diff --git a/cmake/template/DLAFConfig.cmake.in b/cmake/template/DLAFConfig.cmake.in index ad823fde2e..a1a239a9d2 100644 --- a/cmake/template/DLAFConfig.cmake.in +++ b/cmake/template/DLAFConfig.cmake.in @@ -53,10 +53,12 @@ endif() # ----- LAPACK set(LAPACK_LIBRARY "@LAPACK_LIBRARY@") +set(LAPACK_INCLUDE_DIR "@LAPACK_INCLUDE_DIR@") find_dependency(LAPACK) if(DLAF_WITH_SCALAPACK) set(SCALAPACK_LIBRARY "@SCALAPACK_LIBRARY@") + set(SCALAPACK_INCLUDE_DIR "@SCALAPACK_INCLUDE_DIR@") find_dependency(SCALAPACK) endif() From 1631e5080428c11a29eb1cd95b1240119887c2c7 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 29 May 2024 11:32:49 +0200 Subject: [PATCH 15/32] Add placeholder for empty LAPACK_INCLUDE_DIR --- cmake/FindLAPACK.cmake | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cmake/FindLAPACK.cmake b/cmake/FindLAPACK.cmake index ae46f85665..2b70e6a347 100644 --- a/cmake/FindLAPACK.cmake +++ b/cmake/FindLAPACK.cmake @@ -66,6 +66,10 @@ mark_as_advanced(LAPACK_LIBRARY) _lapack_check_is_working() # TODO: Checks? +if(LAPACK_INCLUDE_DIR STREQUAL "" OR NOT LAPACK_INCLUDE_DIR) + set(LAPACK_INCLUDE_DIR "LAPACK_INCLUDE_DIR-PLACEHOLDER-FOR-EMPTY-INCLUDE-DIR") +endif() + mark_as_advanced(LAPACK_INCLUDE_DIR) ### Package @@ -74,11 +78,15 @@ find_package_handle_standard_args( LAPACK DEFAULT_MSG LAPACK_LIBRARY LAPACK_INCLUDE_DIR _LAPACK_CHECK _LAPACK_CHECK_BLAS ) -# Remove the placeholder +# Remove the placeholders if(LAPACK_LIBRARY STREQUAL "LAPACK_LIBRARIES-PLACEHOLDER-FOR-EMPTY-LIBRARIES") set(LAPACK_LIBRARY "") endif() +if(LAPACK_INCLUDE_DIR STREQUAL "LAPACK_INCLUDE_DIR-PLACEHOLDER-FOR-EMPTY-INCLUDE-DIR") + set(LAPACK_INCLUDE_DIR "") +endif() + if(LAPACK_FOUND) if(NOT TARGET DLAF::LAPACK) add_library(DLAF::LAPACK INTERFACE IMPORTED GLOBAL) From ed6f9da9138104561f3d7410d18b7157afbf033f Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 29 May 2024 11:43:53 +0200 Subject: [PATCH 16/32] Remove semicolons from CMAKE_CUDA_FLAGS list --- spack/packages/dla-future/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack/packages/dla-future/package.py b/spack/packages/dla-future/package.py index 22833290eb..8bafa43729 100644 --- a/spack/packages/dla-future/package.py +++ b/spack/packages/dla-future/package.py @@ -291,7 +291,7 @@ def cmake_args(self): args.append( self.define( "CMAKE_CUDA_FLAGS", - "-Werror=cross-execution-space-call;-Werror=reorder;-Werror=deprecated-declarations;-Werror=default-stream-launch;-Werror=ext-lambda-captures-this", + "-Werror=cross-execution-space-call -Werror=reorder -Werror=deprecated-declarations -Werror=default-stream-launch -Werror=ext-lambda-captures-this", ) ) if "+rocm" in spec: From 6e365c7e6955ac109377162d5d65907cc6b52065 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 29 May 2024 12:09:47 +0200 Subject: [PATCH 17/32] Disable all CUDA warnings in CI --- spack/packages/dla-future/package.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/spack/packages/dla-future/package.py b/spack/packages/dla-future/package.py index 8bafa43729..dcd805b48f 100644 --- a/spack/packages/dla-future/package.py +++ b/spack/packages/dla-future/package.py @@ -288,12 +288,14 @@ def cmake_args(self): args.append(self.define("CMAKE_CXX_FLAGS", "-Werror")) if "+cuda" in spec: # TODO: -Werror=missing-launch-bounds triggers warnings!? - args.append( - self.define( - "CMAKE_CUDA_FLAGS", - "-Werror=cross-execution-space-call -Werror=reorder -Werror=deprecated-declarations -Werror=default-stream-launch -Werror=ext-lambda-captures-this", - ) - ) + # TODO: Which to enable? + # args.append( + # self.define( + # "CMAKE_CUDA_FLAGS", + # "-Werror=cross-execution-space-call -Werror=reorder -Werror=deprecated-declarations -Werror=default-stream-launch -Werror=ext-lambda-captures-this", + # ) + # ) + pass if "+rocm" in spec: args.append(self.define("CMAKE_HIP_FLAGS", "-Werror")) args.append(self.define("BUILD_TESTING", True)) From 0e1c5446e2936f435fc99811a1706cfbe7741471 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 29 May 2024 12:42:40 +0200 Subject: [PATCH 18/32] Uncomment all CI configurations --- ci/.gitlab-ci.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/ci/.gitlab-ci.yml b/ci/.gitlab-ci.yml index 1e95751de9..ca55102787 100644 --- a/ci/.gitlab-ci.yml +++ b/ci/.gitlab-ci.yml @@ -1,18 +1,18 @@ include: -# - local: 'ci/cpu/asan_ubsan_lsan.yml' -# - local: 'ci/cpu/clang12_release.yml' -# - local: 'ci/cpu/clang13_release_cxx20.yml' -# - local: 'ci/cpu/clang14_release_stdexec.yml' -# - local: 'ci/cpu/clang15_release.yml' -# - local: 'ci/cpu/clang16_release.yml' -# - local: 'ci/cpu/gcc11_codecov.yml' -# - local: 'ci/cpu/gcc11_release.yml' -# - local: 'ci/cpu/gcc11_release_stdexec.yml' -# - local: 'ci/cpu/gcc11_debug_stdexec.yml' -# - local: 'ci/cpu/gcc12_release_cxx20.yml' + - local: 'ci/cpu/asan_ubsan_lsan.yml' + - local: 'ci/cpu/clang12_release.yml' + - local: 'ci/cpu/clang13_release_cxx20.yml' + - local: 'ci/cpu/clang14_release_stdexec.yml' + - local: 'ci/cpu/clang15_release.yml' + - local: 'ci/cpu/clang16_release.yml' + - local: 'ci/cpu/gcc11_codecov.yml' + - local: 'ci/cpu/gcc11_release.yml' + - local: 'ci/cpu/gcc11_release_stdexec.yml' + - local: 'ci/cpu/gcc11_debug_stdexec.yml' + - local: 'ci/cpu/gcc12_release_cxx20.yml' - local: 'ci/cuda/gcc11_release.yml' - local: 'ci/cuda/gcc11_release_scalapack.yml' - local: 'ci/cuda/gcc11_codecov.yml' -# - local: 'ci/cuda/gcc11_debug_scalapack.yml' -# - local: 'ci/rocm/clang14_release.yml' -# - local: 'ci/rocm/clang14_release_stdexec.yml' + - local: 'ci/cuda/gcc11_debug_scalapack.yml' + - local: 'ci/rocm/clang14_release.yml' + - local: 'ci/rocm/clang14_release_stdexec.yml' From 68519ff9a01084de9f04bbe2b63f34be0c102dbf Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 29 May 2024 14:18:25 +0200 Subject: [PATCH 19/32] Conditionally use separable compilation --- src/CMakeLists.txt | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index dd38eb0a1d..708175a9d0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -58,6 +58,16 @@ option(DLAF_ASSERT_ENABLE "Enable low impact assertions" ${DLAF_ASSERT_DEFAULT}) option(DLAF_ASSERT_MODERATE_ENABLE "Enable medium impact assertions" ${DLAF_ASSERT_MODERATE_DEFAULT}) option(DLAF_ASSERT_HEAVY_ENABLE "Enable high impact assertions" ${DLAF_ASSERT_HEAVY_DEFAULT}) +# We require separable compilation for CUDA with Umpire versions prior to 2024.02 (first 2024 version) +# because the libumpire.so library contains device code. In theory we only need it if +# UMPIRE_ENABLE_DEVICE_ALLOCATOR is enabled, but it's not exported by Umpire so we unconditionally use +# separable compilation before 2024 versions. Starting from 2024.02 the device code is in a separate +# library. +set(DLAF_WITH_SEPARABLE_COMPILATION OFF) +if(DLAF_WITH_CUDA AND Umpire_VERSION_MAJOR LESS 2024) + set(DLAF_WITH_SEPARABLE_COMPILATION ON) +endif() + # Define DLAF's PRIVATE properties add_library(dlaf.prop_private INTERFACE) target_compile_options(dlaf.prop_private INTERFACE -gz) @@ -174,7 +184,7 @@ function(DLAF_addSublibrary name) target_link_libraries(${lib_name} PUBLIC dlaf.prop) target_link_libraries(${lib_name} PUBLIC ${DLAF_ASL_LIBRARIES}) target_link_libraries(${lib_name} PRIVATE dlaf.prop_private) - if(DLAF_WITH_CUDA) + if(DLAF_WITH_SEPARABLE_COMPILATION) set_target_properties(${lib_name} PROPERTIES CUDA_SEPARABLE_COMPILATION ON) endif() endfunction() @@ -292,7 +302,7 @@ add_library( target_link_libraries(DLAF PUBLIC dlaf.prop) target_link_libraries(DLAF PRIVATE dlaf.prop_private) target_add_warnings(DLAF) -if(DLAF_WITH_CUDA) +if(DLAF_WITH_SEPARABLE_COMPILATION) set_target_properties(DLAF PROPERTIES CUDA_SEPARABLE_COMPILATION ON) endif() From 76aa1695cfd9b961ec9b35c5f51c53267dfb50c6 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 29 May 2024 14:19:38 +0200 Subject: [PATCH 20/32] Use -Werror=all-warnings again in CUDA CI --- spack/packages/dla-future/package.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/spack/packages/dla-future/package.py b/spack/packages/dla-future/package.py index dcd805b48f..eec726c683 100644 --- a/spack/packages/dla-future/package.py +++ b/spack/packages/dla-future/package.py @@ -287,15 +287,7 @@ def cmake_args(self): # Enable TESTS and setup CI specific parameters args.append(self.define("CMAKE_CXX_FLAGS", "-Werror")) if "+cuda" in spec: - # TODO: -Werror=missing-launch-bounds triggers warnings!? - # TODO: Which to enable? - # args.append( - # self.define( - # "CMAKE_CUDA_FLAGS", - # "-Werror=cross-execution-space-call -Werror=reorder -Werror=deprecated-declarations -Werror=default-stream-launch -Werror=ext-lambda-captures-this", - # ) - # ) - pass + args.append(self.define("CMAKE_CUDA_FLAGS", "-Werror=all-warnings")) if "+rocm" in spec: args.append(self.define("CMAKE_HIP_FLAGS", "-Werror")) args.append(self.define("BUILD_TESTING", True)) From bb54476402bde788f8e2f9692567ac27a134bef4 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 29 May 2024 14:20:31 +0200 Subject: [PATCH 21/32] Make minimum cmake version 3.22 again --- CMakeLists.txt | 3 +-- miniapp/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d57c11ac23..a4d541b45d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,8 +8,7 @@ # SPDX-License-Identifier: BSD-3-Clause # -# TODO: Bump to 3.25 or keep 3.22? -cmake_minimum_required(VERSION 3.25) +cmake_minimum_required(VERSION 3.22) project(DLAF VERSION 0.4.1) diff --git a/miniapp/CMakeLists.txt b/miniapp/CMakeLists.txt index 042e915d0a..9052ed64ec 100644 --- a/miniapp/CMakeLists.txt +++ b/miniapp/CMakeLists.txt @@ -8,7 +8,7 @@ # SPDX-License-Identifier: BSD-3-Clause # -cmake_minimum_required(VERSION 3.25) +cmake_minimum_required(VERSION 3.22) project(DLAF-miniapps) list(PREPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) From becc83074877aee1055caa5d14dfbee17c89051a Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 29 May 2024 14:21:20 +0200 Subject: [PATCH 22/32] Remove TODO --- CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a4d541b45d..37015df2a7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -63,7 +63,6 @@ try_compile( message(WARNING "DLAF_WITH_MKL_TRY_COMPILE=${DLAF_WITH_MKL_TRY_COMPILE}") option(DLAF_WITH_MKL "Enable oneMKL as provider for BLAS and LAPACK" "${DLAF_WITH_MKL_TRY_COMPILE}") -# TODO: Enable both, just in case? if(DLAF_WITH_MKL) # When using MKL there is no need to set the number of threads with # omp_set_num_threads; it's sufficient to use MKL's own mechanisms. From 7cbc1639df8ba04d9584b1af6169ae1f8e355a09 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 29 May 2024 14:22:18 +0200 Subject: [PATCH 23/32] Remove TODOs --- cmake/FindLAPACK.cmake | 1 - cmake/FindSCALAPACK.cmake | 2 -- 2 files changed, 3 deletions(-) diff --git a/cmake/FindLAPACK.cmake b/cmake/FindLAPACK.cmake index 2b70e6a347..08b1034773 100644 --- a/cmake/FindLAPACK.cmake +++ b/cmake/FindLAPACK.cmake @@ -65,7 +65,6 @@ mark_as_advanced(LAPACK_LIBRARY) _lapack_check_is_working() -# TODO: Checks? if(LAPACK_INCLUDE_DIR STREQUAL "" OR NOT LAPACK_INCLUDE_DIR) set(LAPACK_INCLUDE_DIR "LAPACK_INCLUDE_DIR-PLACEHOLDER-FOR-EMPTY-INCLUDE-DIR") endif() diff --git a/cmake/FindSCALAPACK.cmake b/cmake/FindSCALAPACK.cmake index d1d6a8c799..ad9ebc9d54 100644 --- a/cmake/FindSCALAPACK.cmake +++ b/cmake/FindSCALAPACK.cmake @@ -29,8 +29,6 @@ find_library( find_package_handle_standard_args(SCALAPACK DEFAULT_MSG SCALAPACK_LIBRARY) mark_as_advanced(SCALAPACK_LIBRARY) - -# TODO: Checks? mark_as_advanced(SCALAPACK_INCLUDE_DIR) if(SCALAPACK_FOUND) From fb0d48731d077b3510bcf819c78663dd033c28a5 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 29 May 2024 14:24:16 +0200 Subject: [PATCH 24/32] Remove TODO about auto-detecting MKL in spack package --- spack/packages/dla-future/package.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/spack/packages/dla-future/package.py b/spack/packages/dla-future/package.py index eec726c683..0a41fa4596 100644 --- a/spack/packages/dla-future/package.py +++ b/spack/packages/dla-future/package.py @@ -246,8 +246,7 @@ def cmake_args(self): ) ) else: - # TODO: Auto-detect MKL or not? - # TODO: for now don't set explicitly, let dlaf auto-detect: args.append(self.define("DLAF_WITH_MKL", spec["lapack"].name in INTEL_MATH_LIBRARIES)) + args.append(self.define("DLAF_WITH_MKL", spec["lapack"].name in INTEL_MATH_LIBRARIES)) args.append( self.define( "LAPACK_LIBRARY", From 460d24f4e41a20bb34481dae8a45664fdd755a3d Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 29 May 2024 14:25:56 +0200 Subject: [PATCH 25/32] Update DLAF_WITH_MKL option description --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 37015df2a7..85bd48225b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,7 +61,7 @@ try_compile( ) # TODO: Maybe don't need to print this? Though maybe it's nice to have... message(WARNING "DLAF_WITH_MKL_TRY_COMPILE=${DLAF_WITH_MKL_TRY_COMPILE}") -option(DLAF_WITH_MKL "Enable oneMKL as provider for BLAS and LAPACK" "${DLAF_WITH_MKL_TRY_COMPILE}") +option(DLAF_WITH_MKL "Enable MKL as provider for BLAS and LAPACK" "${DLAF_WITH_MKL_TRY_COMPILE}") if(DLAF_WITH_MKL) # When using MKL there is no need to set the number of threads with From b25b3e176283f53fa16349b5be7ce86230af4769 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 29 May 2024 17:33:24 +0200 Subject: [PATCH 26/32] Rerun MKL detection every time CMake is configured --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 85bd48225b..c8ee76ad00 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -53,6 +53,7 @@ option(DLAF_BUILD_DOC "Build documentation" OFF) option(DLAF_WITH_PRECOMPILED_HEADERS "Use precompiled headers." OFF) option(DLAF_WITH_SCALAPACK "Build ScaLAPACK-like C API (requires ScaLAPACK)" OFF) +unset(DLAF_WITH_MKL_TRY_COMPILE CACHE) try_compile( DLAF_WITH_MKL_TRY_COMPILE "${PROJECT_SOURCE_DIR}/cmake/tests" "${PROJECT_SOURCE_DIR}/cmake/tests/mkl_set_num_threads.cpp" From 672a700b90a1edbd9edd796cdd60aec5fac9e62a Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 29 May 2024 17:33:39 +0200 Subject: [PATCH 27/32] Fix MKL try_compile build directory --- CMakeLists.txt | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c8ee76ad00..afa2e5adb2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,13 +55,16 @@ option(DLAF_WITH_SCALAPACK "Build ScaLAPACK-like C API (requires ScaLAPACK)" OFF unset(DLAF_WITH_MKL_TRY_COMPILE CACHE) try_compile( - DLAF_WITH_MKL_TRY_COMPILE "${PROJECT_SOURCE_DIR}/cmake/tests" + DLAF_WITH_MKL_TRY_COMPILE "${PROJECT_BINARY_DIR}/cmake/tests" "${PROJECT_SOURCE_DIR}/cmake/tests/mkl_set_num_threads.cpp" CMAKE_FLAGS "-DINCLUDE_DIRECTORIES=${LAPACK_INCLUDE_DIR}" LINK_LIBRARIES ${LAPACK_LIBRARY} ) -# TODO: Maybe don't need to print this? Though maybe it's nice to have... -message(WARNING "DLAF_WITH_MKL_TRY_COMPILE=${DLAF_WITH_MKL_TRY_COMPILE}") +if(DLAF_WITH_MKL_TRY_COMPILE) + message(STATUS "Intel MKL detection: Found") +else() + message(STATUS "Intel MKL detection: Not found") +endif() option(DLAF_WITH_MKL "Enable MKL as provider for BLAS and LAPACK" "${DLAF_WITH_MKL_TRY_COMPILE}") if(DLAF_WITH_MKL) From 22b3f1877f09a5921d723a65473ef5bb57f5cf3a Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Thu, 30 May 2024 09:52:39 +0200 Subject: [PATCH 28/32] Refactor MKL auto-detection --- CMakeLists.txt | 32 +++++++++++++++++++++----------- src/CMakeLists.txt | 2 +- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index afa2e5adb2..205d8a9759 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -53,19 +53,29 @@ option(DLAF_BUILD_DOC "Build documentation" OFF) option(DLAF_WITH_PRECOMPILED_HEADERS "Use precompiled headers." OFF) option(DLAF_WITH_SCALAPACK "Build ScaLAPACK-like C API (requires ScaLAPACK)" OFF) -unset(DLAF_WITH_MKL_TRY_COMPILE CACHE) -try_compile( - DLAF_WITH_MKL_TRY_COMPILE "${PROJECT_BINARY_DIR}/cmake/tests" - "${PROJECT_SOURCE_DIR}/cmake/tests/mkl_set_num_threads.cpp" - CMAKE_FLAGS "-DINCLUDE_DIRECTORIES=${LAPACK_INCLUDE_DIR}" - LINK_LIBRARIES ${LAPACK_LIBRARY} -) -if(DLAF_WITH_MKL_TRY_COMPILE) - message(STATUS "Intel MKL detection: Found") +option(DLAF_WITH_MKL "Enable MKL as provider for BLAS and LAPACK" "${DLAF_WITH_MKL_TRY_COMPILE}") +if(DEFINED CACHE{DLAF_WITH_MKL}) + if(DLAF_WITH_MKL) + message(STATUS "Intel MKL support: Enabled by user") + else() + message(STATUS "Intel MKL support: Disabled by user") + endif() + set(DLAF_WITH_MKL_INTERNAL ${DLAF_WITH_MKL} CACHE BOOL "" FORCE) else() - message(STATUS "Intel MKL detection: Not found") + unset(DLAF_WITH_MKL_TRY_COMPILE CACHE) + try_compile( + DLAF_WITH_MKL_TRY_COMPILE "${PROJECT_BINARY_DIR}/cmake/tests" + "${PROJECT_SOURCE_DIR}/cmake/tests/mkl_set_num_threads.cpp" + CMAKE_FLAGS "-DINCLUDE_DIRECTORIES=${LAPACK_INCLUDE_DIR}" + LINK_LIBRARIES ${LAPACK_LIBRARY} + ) + if(DLAF_WITH_MKL_TRY_COMPILE) + message(STATUS "Intel MKL support: Found") + else() + message(STATUS "Intel MKL support: Not found") + endif() + set(DLAF_WITH_MKL_INTERNAL ${DLAF_WITH_MKL_TRY_COMPILE} CACHE BOOL "" FORCE) endif() -option(DLAF_WITH_MKL "Enable MKL as provider for BLAS and LAPACK" "${DLAF_WITH_MKL_TRY_COMPILE}") if(DLAF_WITH_MKL) # When using MKL there is no need to set the number of threads with diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 708175a9d0..4af59a4e25 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -103,7 +103,7 @@ target_compile_definitions( $<$:DLAF_ASSERT_MODERATE_ENABLE> $<$:DLAF_ASSERT_HEAVY_ENABLE> DLAF_FUNCTION_NAME=$,__PRETTY_FUNCTION__,__func__> - $<$:DLAF_WITH_MKL> + $<$:DLAF_WITH_MKL> $<$:DLAF_WITH_OPENMP> $<$:DLAF_WITH_GPU> $<$:DLAF_WITH_CUDA> From 9dc2f78d8ff707e8d6c64f517f5929bc83dcdf63 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Fri, 31 May 2024 11:26:50 +0200 Subject: [PATCH 29/32] Remove DLAF_WITH_MKL as an option It is only to be set by users, but should be undefined otherwise. --- CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 205d8a9759..3100730475 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -53,7 +53,6 @@ option(DLAF_BUILD_DOC "Build documentation" OFF) option(DLAF_WITH_PRECOMPILED_HEADERS "Use precompiled headers." OFF) option(DLAF_WITH_SCALAPACK "Build ScaLAPACK-like C API (requires ScaLAPACK)" OFF) -option(DLAF_WITH_MKL "Enable MKL as provider for BLAS and LAPACK" "${DLAF_WITH_MKL_TRY_COMPILE}") if(DEFINED CACHE{DLAF_WITH_MKL}) if(DLAF_WITH_MKL) message(STATUS "Intel MKL support: Enabled by user") From 1758a6694729dc0a5c12e3b0b94eba4df54c1d04 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Fri, 31 May 2024 11:30:04 +0200 Subject: [PATCH 30/32] Simplify checks for empty LAPACK_LIBRARY and LAPACK_INCLUDE_DIR --- cmake/FindLAPACK.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/FindLAPACK.cmake b/cmake/FindLAPACK.cmake index 08b1034773..283fa5f4dc 100644 --- a/cmake/FindLAPACK.cmake +++ b/cmake/FindLAPACK.cmake @@ -57,7 +57,7 @@ endmacro() # Dependencies set(_DEPS "") -if(LAPACK_LIBRARY STREQUAL "" OR NOT LAPACK_LIBRARY) +if(NOT LAPACK_LIBRARY) set(LAPACK_LIBRARY "LAPACK_LIBRARIES-PLACEHOLDER-FOR-EMPTY-LIBRARIES") endif() @@ -65,7 +65,7 @@ mark_as_advanced(LAPACK_LIBRARY) _lapack_check_is_working() -if(LAPACK_INCLUDE_DIR STREQUAL "" OR NOT LAPACK_INCLUDE_DIR) +if(NOT LAPACK_INCLUDE_DIR) set(LAPACK_INCLUDE_DIR "LAPACK_INCLUDE_DIR-PLACEHOLDER-FOR-EMPTY-INCLUDE-DIR") endif() From 5f5bb92d996ec30505f07de56936c21d2330e35b Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Fri, 31 May 2024 12:53:54 +0200 Subject: [PATCH 31/32] Remove DLAF_WITH_MKL from DLAFConfig.cmake.in --- cmake/template/DLAFConfig.cmake.in | 1 - 1 file changed, 1 deletion(-) diff --git a/cmake/template/DLAFConfig.cmake.in b/cmake/template/DLAFConfig.cmake.in index a1a239a9d2..99f39d8c2e 100644 --- a/cmake/template/DLAFConfig.cmake.in +++ b/cmake/template/DLAFConfig.cmake.in @@ -19,7 +19,6 @@ list(PREPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}) # ===== VARIABLES set(DLAF_WITH_OPENMP @DLAF_WITH_OPENMP@) -set(DLAF_WITH_MKL @DLAF_WITH_MKL@) set(DLAF_WITH_CUDA @DLAF_WITH_CUDA@) set(DLAF_WITH_HIP @DLAF_WITH_HIP@) set(DLAF_WITH_GPU @DLAF_WITH_GPU@) From f0fd524328f55a3b67720712b1c100b76a1253a8 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Fri, 31 May 2024 12:54:40 +0200 Subject: [PATCH 32/32] Update SPACK_SHA Co-authored-by: Rocco Meli --- ci/common-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/common-ci.yml b/ci/common-ci.yml index 552e7fdc2f..3a9e6f5fa5 100644 --- a/ci/common-ci.yml +++ b/ci/common-ci.yml @@ -34,7 +34,7 @@ stages: reports: dotenv: build.env variables: - SPACK_SHA: fd3c18b6fd7eda8abbb72ee81b39aa81b053d9bd + SPACK_SHA: 1184de8352281ce1b34a8f3f6b54d7fadc2b216a SPACK_DLAF_REPO: ./spack DOCKER_BUILD_ARGS: '[ "BASE_IMAGE",