Update AMGCL cuda solver and Hypre block solver

CMakeLists.txt

@@ -24,6 +24,7 @@ if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/PolySolveOptions.cmake)
     include(${CMAKE_CURRENT_SOURCE_DIR}/PolySolveOptions.cmake)
 endif()
 
+
 ################################################################################
 # CMake Policies
 ################################################################################
@@ -47,16 +48,19 @@ endif()
 # Polysolve options
 option(POLYSOLVE_WITH_SANITIZERS    "Enable sanitizers in compilation targets"          OFF)
 # Polysolve options for enabling/disabling optional libraries
+
 option(POLYSOLVE_WITH_CHOLMOD       "Enable Cholmod library"                             ON)
 option(POLYSOLVE_WITH_UMFPACK       "Enable UmfPack library"                             ON)
 option(POLYSOLVE_WITH_SUPERLU       "Enable SuperLU library"                             ON)
 option(POLYSOLVE_WITH_MKL           "Enable MKL library"  ${POLYSOLVE_NOT_ON_APPLE_SILICON})
 option(POLYSOLVE_WITH_CUSOLVER      "Enable cuSOLVER library"                           OFF)
-
+option(POLYSOLVE_WITH_CUSOLVER      "Enable cuSOLVER library"                           OFF)
+option(POLYSOLVE_WITH_CUDA          "Enable CUDA"                                        ON)
 option(POLYSOLVE_WITH_PARDISO       "Enable Pardiso library"                            OFF)
 option(POLYSOLVE_WITH_HYPRE         "Enable hypre"                                       ON)
 option(POLYSOLVE_WITH_AMGCL         "Use AMGCL"                                          ON)
 option(POLYSOLVE_WITH_SPECTRA       "Enable computing spectrum"                          ON)
+
 # Sanitizer options
 option(POLYSOLVE_SANITIZE_ADDRESS   "Sanitize Address"                                  OFF)
 option(POLYSOLVE_SANITIZE_MEMORY    "Sanitize Memory"                                   OFF)
@@ -76,6 +80,10 @@ if(POLYSOLVE_TOPLEVEL_PROJECT)
     set(CMAKE_CXX_STANDARD_REQUIRED ON)
     set(CMAKE_CXX_EXTENSIONS OFF)
 endif()
+if(NOT DEFINED CMAKE_CUDA_STANDARD)
+    set(CMAKE_CUDA_STANDARD 14)
+    set(CMAKE_CUDA_STANDARD_REQUIRED ON)
+endif()
 
 if (MSVC)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /bigobj")
@@ -92,6 +100,7 @@ include(polysolve_use_colors)
 # IPC Toolkit utils
 include(polysolve_prepend_current_path)
 include(polysolve_set_source_group)
+include(polysolve_target_link_system_libraries)
 
 # Sort projects inside the solution
 set_property(GLOBAL PROPERTY USE_FOLDERS ON)
@@ -143,11 +152,54 @@ endif()
 include(eigen)
 target_link_libraries(polysolve PUBLIC Eigen3::Eigen)
 
+# CUDA
+if(POLYSOLVE_WITH_CUDA)
+    include(CheckLanguage)
+    check_language(CUDA)
+    set(CMAKE_CUDA_ARCHITECTURES "61")
+    find_package(CUDAToolkit)    
+    if(CUDAToolkit_FOUND)
+        message(STATUS "Found CUDATooklkit")
+        set(CMAKE_CUDA_COMPILER ${CUDAToolkit_NVCC_EXECUTABLE})
+        enable_language(CUDA)
+        target_compile_definitions(polysolve PUBLIC -DPOLYSOLVE_WITH_CUDA)        
+    else()
+        message(FATAL_ERROR "No CUDA support found!")
+    endif()
+  # We need to explicitly state that we need all CUDA files in the particle
+  # library to be built with -dc as the member functions could be called by
+  # other libraries and executables.
+  set_target_properties(polysolve PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
+
+  # Nvidia RTX8000 -> compute_75
+  # Nvidia V100 -> compute_70
+  # Nvidia 1080/1080Ti -> compute_61
+  # Nvidia 3080Ti -> compute_86
+    if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
+        set(CMAKE_CUDA_ARCHITECTURES 70 75 86)
+    endif()
+    set_target_properties(polysolve PROPERTIES CUDA_ARCHITECTURES "70;75;86")
+
+    if(APPLE)
+        # We need to add the path to the driver (libcuda.dylib) as an rpath,
+        # so that the static cuda runtime can find it at runtime.
+        set_property(TARGET polysolve
+                    PROPERTY
+                    BUILD_RPATH ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES})
+    endif()
+    polysolve_target_link_system_libraries(polysolve PRIVATE CUDA::cudart)
+    polysolve_target_link_system_libraries(polysolve PRIVATE CUDA::cusparse)
+endif()
+
+
 # Hypre (GNU Lesser General Public License)
 if(POLYSOLVE_WITH_HYPRE)
     include(hypre)
     target_link_libraries(polysolve PUBLIC HYPRE::HYPRE)
     target_compile_definitions(polysolve PUBLIC -DPOLYSOLVE_WITH_HYPRE)
+    if(HYPRE_WITH_MPI)
+        target_compile_definitions(polysolve PUBLIC HYPRE_WITH_MPI)
+    endif()
 endif()
 
 # Json (MIT)
@@ -249,4 +301,11 @@ if(POLYSOLVE_WITH_TESTS)
     include("${catch2_SOURCE_DIR}/contrib/Catch.cmake")
 
     add_subdirectory(tests)
+
+    # Cuda test
+    if(POLYSOLVE_WITH_CUDA)
+        add_subdirectory(cudatest)
+    endif()
 endif()
+
+

README.md

@@ -43,35 +43,35 @@ Polysolve uses a json file to provide parameters to the individual solvers. The
 {
     "Eigen::LeastSquaresConjugateGradient": {
         "max_iter": 1000,
-        "tolerance": 1e-6
+        "tolerance": 1e-10
     },
     "Eigen::DGMRES": {
         "max_iter": 1000,
-        "tolerance": 1e-6
+        "tolerance": 1e-10
     },
     "Eigen::ConjugateGradient": {
         "max_iter": 1000,
-        "tolerance": 1e-6
+        "tolerance": 1e-10
     },
     "Eigen::BiCGSTAB": {
         "max_iter": 1000,
-        "tolerance": 1e-6
+        "tolerance": 1e-10
     },
     "Eigen::GMRES": {
         "max_iter": 1000,
-        "tolerance": 1e-6
+        "tolerance": 1e-10
     },
     "Eigen::MINRES": {
         "max_iter": 1000,
-        "tolerance": 1e-6
+        "tolerance": 1e-10
     },
     "Pardiso": {
         "mtype": -1
     },
     "Hypre": {
         "max_iter": 1000,
         "pre_max_iter": 1000,
-        "tolerance": 1e-6
+        "tolerance": 1e-10
     },
     "AMGCL": {
         "precond": {

cmake/polysolve/polysolve_target_link_system_libraries.cmake

@@ -0,0 +1,28 @@
+function(polysolve_target_link_system_libraries target)
+  set(options PRIVATE PUBLIC INTERFACE)
+  cmake_parse_arguments(TLLS "${options}" "" "" ${ARGN})
+  foreach(op ${options})
+    if(TLLS_${op})
+      set(scope ${op})
+    endif()
+  endforeach(op)
+  set(libs ${TLLS_UNPARSED_ARGUMENTS})
+
+  foreach(lib ${libs})
+    get_target_property(lib_include_dirs ${lib} INTERFACE_INCLUDE_DIRECTORIES)
+    if(lib_include_dirs)
+      if(scope)
+        target_include_directories(${target} SYSTEM ${scope} ${lib_include_dirs})
+      else()
+        target_include_directories(${target} SYSTEM PRIVATE ${lib_include_dirs})
+      endif()
+    else()
+      message(WARNING "${lib} doesn't set INTERFACE_INCLUDE_DIRECTORIES. No include_directories set.")
+    endif()
+    if(scope)
+      target_link_libraries(${target} ${scope} ${lib})
+    else()
+      target_link_libraries(${target} ${lib})
+    endif()
+  endforeach()
+endfunction(polysolve_target_link_system_libraries)

cmake/recipes/hypre.cmake

@@ -6,67 +6,23 @@ endif()
 
 message(STATUS "Third-party: creating target 'HYPRE::HYPRE'")
 
-include(FetchContent)
-FetchContent_Declare(
-    hypre
-    GIT_REPOSITORY https://github.com/hypre-space/hypre.git
-    GIT_TAG v2.15.1
-    GIT_SHALLOW TRUE
-)
-
-FetchContent_GetProperties(hypre)
-if(NOT hypre_POPULATED)
-    FetchContent_Populate(hypre)
-    file(REMOVE ${hypre_SOURCE_DIR}/src/utilities/version)
-endif()
-
-################################################################################
-
 set(HYPRE_SEQUENTIAL    ON CACHE INTERNAL "" FORCE)
 set(HYPRE_PRINT_ERRORS  ON CACHE INTERNAL "" FORCE)
 set(HYPRE_BIGINT        ON CACHE INTERNAL "" FORCE)
 set(HYPRE_USING_FEI    OFF CACHE INTERNAL "" FORCE)
 set(HYPRE_USING_OPENMP OFF CACHE INTERNAL "" FORCE)
 set(HYPRE_SHARED       OFF CACHE INTERNAL "" FORCE)
-# set(HYPRE_LONG_DOUBLE ON)
 
-set(HYPRE_BUILD_TYPE "${CMAKE_BUILD_TYPE}" CACHE INTERNAL "" FORCE)
 
-add_subdirectory(${hypre_SOURCE_DIR}/src ${hypre_BINARY_DIR})
-add_library(HYPRE::HYPRE ALIAS HYPRE)
-
-set_property(TARGET HYPRE PROPERTY FOLDER "dependencies")
-
-target_include_directories(HYPRE PUBLIC ${hypre_BINARY_DIR})
-target_include_directories(HYPRE PUBLIC ${hypre_SOURCE_DIR}/src)
-target_include_directories(HYPRE PUBLIC ${hypre_SOURCE_DIR}/src/blas)
-target_include_directories(HYPRE PUBLIC ${hypre_SOURCE_DIR}/src/lapack)
-target_include_directories(HYPRE PUBLIC ${hypre_SOURCE_DIR}/src/utilities)
-target_include_directories(HYPRE PUBLIC ${hypre_SOURCE_DIR}/src/multivector)
-target_include_directories(HYPRE PUBLIC ${hypre_SOURCE_DIR}/src/krylov)
-target_include_directories(HYPRE PUBLIC ${hypre_SOURCE_DIR}/src/seq_mv)
-target_include_directories(HYPRE PUBLIC ${hypre_SOURCE_DIR}/src/parcsr_mv)
-target_include_directories(HYPRE PUBLIC ${hypre_SOURCE_DIR}/src/parcsr_block_mv)
-target_include_directories(HYPRE PUBLIC ${hypre_SOURCE_DIR}/src/distributed_matrix)
-target_include_directories(HYPRE PUBLIC ${hypre_SOURCE_DIR}/src/IJ_mv)
-target_include_directories(HYPRE PUBLIC ${hypre_SOURCE_DIR}/src/matrix_matrix)
-target_include_directories(HYPRE PUBLIC ${hypre_SOURCE_DIR}/src/distributed_ls)
-target_include_directories(HYPRE PUBLIC ${hypre_SOURCE_DIR}/src/distributed_ls/Euclid)
-target_include_directories(HYPRE PUBLIC ${hypre_SOURCE_DIR}/src/distributed_ls/ParaSails)
-target_include_directories(HYPRE PUBLIC ${hypre_SOURCE_DIR}/src/parcsr_ls)
-target_include_directories(HYPRE PUBLIC ${hypre_SOURCE_DIR}/src/struct_mv)
-target_include_directories(HYPRE PUBLIC ${hypre_SOURCE_DIR}/src/struct_ls)
-target_include_directories(HYPRE PUBLIC ${hypre_SOURCE_DIR}/src/sstruct_mv)
-target_include_directories(HYPRE PUBLIC ${hypre_SOURCE_DIR}/src/sstruct_ls)
+include(FetchContent)
+FetchContent_Declare(
+    hypre
+    GIT_REPOSITORY https://github.com/hypre-space/hypre.git
+    GIT_TAG v2.25.0
+    GIT_SHALLOW TRUE
+)
 
-if(HYPRE_USING_OPENMP)
-	find_package(OpenMP QUIET REQUIRED)
-	target_link_libraries(HYPRE PUBLIC OpenMP::OpenMP_CXX)
-endif()
+FetchContent_MakeAvailable(hypre)
 
-if(NOT HYPRE_SEQUENTIAL)
-	find_package(MPI)
-	if(MPI_CXX_FOUND)
-		target_link_libraries(HYPRE PUBLIC MPI::MPI_CXX)
-	endif()
-endif()
+add_subdirectory("${hypre_SOURCE_DIR}/src")
+file(REMOVE "${hypre_SOURCE_DIR}/src/utilities/version")

cudatest/CMakeLists.txt

@@ -0,0 +1,37 @@
+################################################################################
+# Tests
+################################################################################
+set(test_sources
+	main.cpp
+	cudatest.cpp
+)
+add_executable(cuda_test ${test_sources})
+
+################################################################################
+# Required Libraries
+################################################################################
+include(catch2)
+target_link_libraries(cuda_test PUBLIC Catch2::Catch2)
+
+target_link_libraries(cuda_test PUBLIC polysolve::polysolve)
+
+include(polysolve_warnings)
+target_link_libraries(cuda_test PRIVATE polysolve::warnings)
+################################################################################
+# Register tests
+################################################################################
+foreach(source IN ITEMS ${test_sources})
+    source_group("tests" FILES "${source}")
+endforeach()
+
+# Register tests
+set(PARSE_CATCH_TESTS_ADD_TO_CONFIGURE_DEPENDS ON)
+catch_discover_tests(cuda_test)
+
+################################################################################
+# Data
+################################################################################
+set(DATA_DIR "${CMAKE_SOURCE_DIR}/tests/data")
+target_compile_definitions(cuda_test PUBLIC -DPOLYSOLVE_DATA_DIR=\"${DATA_DIR}\")
+polysolve_target_link_system_libraries(cuda_test PRIVATE CUDA::cudart)
+polysolve_target_link_system_libraries(cuda_test PRIVATE CUDA::cusparse)