Merge pull request #246 from Adrian-Diaz/Parallel-Explicit-Solver

FIerro Update

integrated-tests/Test-scripts/test_fierro_cuda.py

@@ -0,0 +1,152 @@
+#!/usr/bin/python3
+
+import os
+import os.path
+import sys
+import math
+
+
+solvers = ["fierro-parallel-explicit"]
+
+executables = []
+tests = []
+
+# Add paths to all tested executables
+for i in range(len(solvers)):
+    executables.append("./../../build-fierro-cuda/bin/"+solvers[i])
+
+# Check that each executable exists
+for i in range(len(solvers)):
+    if not os.path.exists(executables[i]):
+        raise ValueError(solvers[i]+" executable not found in build-fierro-openmp directory")
+\
+# Add names of each test
+parallel_explicit_tests = ["Noh", "Sedov", "Sod"]
+# parallel_implicit_tests = ["Beam"]
+
+inputs = []
+standard_results = []
+
+tests.append(parallel_explicit_tests)
+# tests.append(parallel_implicit_tests)
+
+position_keyword = "POINTS"
+
+for i in range(len(solvers)):
+    tmp1 = []
+    tmp2 = []
+    for j in range(len(tests[i])):
+        tmp1.append("Solver-Inputs/"+solvers[i]+"/"+tests[i][j]+".yaml")
+        tmp2.append("standard-results/"+solvers[i]+"/"+tests[i][j]+"/vtk/data/VTK0.vtk")
+    inputs.append(tmp1)
+    standard_results.append(tmp2)
+
+# Extract vector valued data from vtk output file
+def extract_vector_data(filename, keyword):
+    data = []
+    found_keyword = False
+
+    with open(filename, 'r') as file:
+        for line in file:
+            if found_keyword:
+                if line.strip():  # If the line is not blank
+                    # Split each line into fields.
+                    fields = line.split()
+                    xx = float(fields[0])
+                    yy = float(fields[1])
+                    zz = float(fields[2])
+                    # Group and append to results.
+                    vec = [xx, yy, zz]
+                    data.append(vec)
+                else:
+                    break  # Exit the loop when encountering a blank line
+            elif keyword in line:
+                found_keyword = True
+
+    return data
+
+# Extract scalar valued data from vtk output file
+def extract_scalar_data(filename, keyword):
+    data = []
+    found_keyword = False
+
+    with open(filename, 'r') as file:
+        for line in file:
+            if found_keyword:
+                if line.strip():  # If the line is not blank
+                    fields = line.split()
+                    if fields[0] != "LOOKUP_TABLE":
+                        data.append(float(fields[0]))
+                else:
+                    break  # Exit the loop when encountering a blank line
+            elif keyword in line:
+                found_keyword = True
+                # file.next() # skip first line "LOOKUP_TABLE"
+
+    return data
+
+# Calculate the percent difference between two arrays of scalars
+def percent_difference_scalars(array1, array2):
+    if len(array1) != len(array2):
+        raise ValueError("Arrays must have the same length")
+
+    percent_diff = []
+    for i in range(len(array1)):
+        diff = array2[i] - array1[i]
+        percent_diff.append((diff / array1[i]) * 100 if array1[i] != 0 else 0)
+
+    return percent_diff
+
+# Calculate the percent difference between two arrays of vectors
+def percent_difference_vectors(array1, array2):
+    if len(array1) != len(array2):
+        raise ValueError("Arrays must have the same length")
+
+    percent_diff = []
+    for i in range(len(array1)):
+        if len(array1[i]) != 3 or len(array2[i]) != 3:
+            raise ValueError("Subarrays must have length 3")
+
+        diff = [array2[i][j] - array1[i][j] for j in range(3)]
+        percent_diff.append([(diff[j] / array1[i][j]) * 100 if array1[i][j] != 0 else 0 for j in range(3)])
+
+    percent_diff_mag = []
+    for i in range(len(array1)):
+        percent_diff_mag.append(magnitude(percent_diff[i]))
+
+    return percent_diff_mag
+
+# Calculate the magnitude of a vector
+def magnitude(array):
+    mag = math.sqrt(sum(x**2 for x in array))
+    return mag
+
+# Run each test
+for i in range(len(solvers)):
+    for j in range(len(tests[i])):
+
+        # Run simulation
+        print("Running "+tests[i][j])
+        os.system(executables[i] + ' ' + inputs[i][j])
+
+        GT_positions = extract_vector_data(standard_results[i][j], position_keyword)
+
+        # Read simulation results
+        results_filename = "vtk/data/VTK0.vtk"
+
+        if os.path.exists(results_filename):
+            print("Simulation Finished")
+        else:
+            print("Simulation did not finish")
+            raise ValueError("Simulation did not finish")
+
+        results_positions = extract_vector_data(results_filename, position_keyword)
+        position_diff = percent_difference_vectors(GT_positions, results_positions)
+
+
+        for k in range(len(position_diff)):
+            if position_diff[k] >= 1.0e-6:
+                raise ValueError(" ****************** ERROR: Position difference out of range for "+tests[i][j]+" problem ****************** ")
+
+        print("Removing simulation outputs")
+        os.system('rm -rf  vtk' )

scripts/build-fierro.sh

@@ -47,6 +47,11 @@ show_help() {
     echo "          mac                         A Mac computer. This option does not allow for cuda and hip builds, and build_cores will be set to 1"
     echo "          msu                         A linux computer managed by the HPCC group at Mississippi State University"
     echo " "
+    echo "      --intel_mkl                     Decides whether to build Trilinos using the Intel MKL library"
+    echo " "
+    echo "          enabled                     Links and builds Trilinos with the Intel MKL library"
+    echo "          disabled                    Links and builds Trilinos using LAPACK and BLAS"
+    echo " "
     echo "      --heffte_build_type             The build type for the heffte installation. The default is 'fftw'"
     echo " "
     echo "          fftw                        General heffte run type"

scripts/trilinos-install.sh

@@ -26,6 +26,13 @@ then
     mkdir -p ${TRILINOS_BUILD_DIR} 
 fi
 
+if [ "$kokkos_build_type" = "cuda" ]; then
+    export OMPI_CXX=${TRILINOS_SOURCE_DIR}/packages/kokkos/bin/nvcc_wrapper
+    export CUDA_LAUNCH_BLOCKING=1
+elif [ "$kokkos_build_type" = *"hip"* ]; then
+    export OMPI_CXX=hipcc
+fi
+
 #check if Trilinos library files were installed, install them otherwise.
 [ -d "${TRILINOS_BUILD_DIR}/lib" ] && echo "Directory ${TRILINOS_BUILD_DIR}/lib exists, assuming successful installation; delete build folder and run build script again if there was an environment error that has been corrected."
 
@@ -50,6 +57,7 @@ CUDA_ADDITIONS=(
 -D Tpetra_ENABLE_CUDA=ON
 -D Xpetra_ENABLE_Kokkos_Refactor=ON
 -D MueLu_ENABLE_Kokkos_Refactor=ON
+-D Tpetra_ASSUME_GPU_AWARE_MPI:BOOL=FALSE
 )
 
 # Kokkos flags for Hip
@@ -64,6 +72,7 @@ export OMPI_CXX=hipcc
 -D KokkosKernels_ENABLE_TPL_CUSPARSE=OFF
 -D Tpetra_INST_HIP=ON
 -D Xpetra_ENABLE_Kokkos_Refactor=ON
+-D Tpetra_ASSUME_GPU_AWARE_MPI:BOOL=FALSE
 )
 
 # Kokkos flags for OpenMP
@@ -141,13 +150,10 @@ if [ "$kokkos_build_type" = "openmp" ]; then
         ${OPENMP_ADDITIONS[@]}
     )
 elif [ "$kokkos_build_type" = "cuda" ]; then
-    export OMPI_CXX=${TRILINOS_SOURCE_DIR}/packages/kokkos/bin/nvcc_wrapper
-    export CUDA_LAUNCH_BLOCKING=1
     cmake_options+=(
         ${CUDA_ADDITIONS[@]}
     )
 elif [ "$kokkos_build_type" = "hip" ]; then
-    export OMPI_CXX=hipcc
     cmake_options+=(
         ${HIP_ADDITIONS[@]}
     )

single-node-refactor/CMakeLists.txt

@@ -40,6 +40,44 @@ add_subdirectory(../lib/Elements/matar cbin)
 # include_directories(Mesh-Builder)
 # add_subdirectory(Mesh-Builder)
 
+if (FIERRO_ENABLE_TRILINOS)
+  find_package(Trilinos REQUIRED) #new
+  # Assume if the CXX compiler exists, the rest do too.
+  if (EXISTS ${Trilinos_CXX_COMPILER})
+    set(CMAKE_CXX_COMPILER ${Trilinos_CXX_COMPILER})
+    set(CMAKE_C_COMPILER ${Trilinos_C_COMPILER})
+    set(CMAKE_Fortran_COMPILER ${Trilinos_Fortran_COMPILER})
+  endif()
+  if(NOT DISTRIBUTION)
+    # Make sure to use same compilers and flags as Trilinos
+    set(CMAKE_CXX_FLAGS  "${Trilinos_CXX_COMPILER_FLAGS} ${CMAKE_CXX_FLAGS}")
+    set(CMAKE_C_FLAGS  "${Trilinos_C_COMPILER_FLAGS} ${CMAKE_C_FLAGS}")
+    set(CMAKE_Fortran_FLAGS  "${Trilinos_Fortran_COMPILER_FLAGS} ${CMAKE_Fortran_FLAGS}")
+  endif()
+
+  message("\nFound Trilinos!  Here are the details: ")
+  message("   Trilinos_DIR = ${Trilinos_DIR}")
+  message("   Trilinos_VERSION = ${Trilinos_VERSION}")
+  message("   Trilinos_PACKAGE_LIST = ${Trilinos_PACKAGE_LIST}")
+  message("   Trilinos_LIBRARIES = ${Trilinos_LIBRARIES}")
+  message("   Trilinos_INCLUDE_DIRS = ${Trilinos_INCLUDE_DIRS}")
+  message("   Trilinos_LIBRARY_DIRS = ${Trilinos_LIBRARY_DIRS}")
+  message("   Trilinos_TPL_LIST = ${Trilinos_TPL_LIST}")
+  message("   Trilinos_TPL_INCLUDE_DIRS = ${Trilinos_TPL_INCLUDE_DIRS}")
+  message("   Trilinos_TPL_LIBRARIES = ${Trilinos_TPL_LIBRARIES}")
+  message("   Trilinos_TPL_LIBRARY_DIRS = ${Trilinos_TPL_LIBRARY_DIRS}")
+  message("   Trilinos_BUILD_SHARED_LIBS = ${Trilinos_BUILD_SHARED_LIBS}")
+  message("End of Trilinos details\n")
+
+  include_directories(${Trilinos_INCLUDE_DIRS} ${Trilinos_TPL_INCLUDE_DIRS})
+  list(APPEND LINKING_LIBRARIES Trilinos::all_selected_libs)
+  add_definitions(-DTRILINOS_INTERFACE=1 -DHAVE_MPI=1)
+else()
+  find_package(Kokkos REQUIRED)
+  list(APPEND LINKING_LIBRARIES Kokkos::kokkos)
+endif()
+find_package(Matar REQUIRED)
+
 include_directories(src/material_models/artificial_viscosity)
 include_directories(src/material_models/eos)
 include_directories(src/material_models/erosion)

single-node-refactor/scripts/build-fierro.sh

@@ -7,6 +7,8 @@ show_help() {
     echo "  --build_action=<full-app|set-env|install-kokkos|fierro>. Default is 'full-app'"
     echo "  --machine=<darwin|chicoma|linux|mac>. Default is 'linux'"
     echo "  --build_cores=<Integers greater than 0>. Default is set 1"
+    echo "  --trilinos=<enabled|disabled>. Default is 'disabled'"
+    echo "  --intel_mkl=<enabled|disabled>. Default is 'disabled'"
     echo "  --help: Display this help message"
     echo " "
     echo " "
@@ -36,6 +38,16 @@ show_help() {
     echo "          linux                       A general linux machine (that does not use modules)"
     echo "          mac                         A Mac computer. This option does not allow for cuda and hip builds, and build_cores will be set to 1"
     echo " "
+    echo "      --trilinos                      Decides if Trilinos is available for certain MATAR functionality"
+    echo " "
+    echo "          disabled                    Trilinos is not being used"
+    echo "          enabled                     Trilinos will be linked with MATAR to enable relevant functionality"
+    echo " "
+    echo "      --intel_mkl                     Decides whether to build Trilinos using the Intel MKL library"
+    echo " "
+    echo "          enabled                     Links and builds Trilinos with the Intel MKL library"
+    echo "          disabled                    Links and builds Trilinos using LAPACK and BLAS"
+    echo " "
     echo "      --build_cores                   The number of build cores to be used by make and make install commands. The default is 1" 
     echo " "
     echo "      --debug                         Build with debug. Default is false." 
@@ -48,13 +60,17 @@ solver="SGH"
 machine="linux"
 kokkos_build_type="openmp"
 build_cores="1"
+trilinos="disabled"
+intel_mkl="disabled"
 debug="false"
 
 # Define arrays of valid options
 valid_build_action=("full-app" "set-env" "install-kokkos" "fierro")
 valid_solver=("SGH")
 valid_kokkos_build_types=("serial" "openmp" "pthreads" "cuda" "hip")
 valid_machines=("darwin" "chicoma" "linux" "mac")
+valid_trilinos=("disabled" "enabled")
+valid_intel_mkl=("disabled" "enabled")
 valid_debug=("true" "false")
 
 # Parse command line arguments
@@ -110,6 +126,26 @@ for arg in "$@"; do
                 return 1
             fi
             ;;
+        --trilinos=*)
+            option="${arg#*=}"
+            if [[ " ${valid_trilinos[*]} " == *" $option "* ]]; then
+                trilinos="$option"
+            else
+                echo "Error: Invalid --kokkos_build_type specified."
+                show_help
+                return 1
+            fi
+            ;;
+        --intel_mkl=*)
+            option="${arg#*=}"
+            if [[ " ${valid_intel_mkl[*]} " == *" $option "* ]]; then
+                intel_mkl="$option"
+            else
+                echo "Error: Invalid --intel_mkl specified."
+                show_help
+                return 1
+            fi
+            ;;
         --debug=*)
             option="${arg#*=}"
             if [[ " ${valid_debug[*]} " == *" $option "* ]]; then
@@ -155,6 +191,8 @@ echo "Building based on these argument options:"
 echo "Build action - ${build_action}"
 echo "Solver - ${solver}"
 echo "Kokkos backend - ${kokkos_build_type}"
+echo "Trilinos - ${trilinos}"
+echo "Intel MKL library - ${intel_mkl}"
 echo "make -j ${build_cores}"
 
 cd "$( dirname "${BASH_SOURCE[0]}" )"
@@ -164,13 +202,17 @@ source setup-env.sh ${machine} ${kokkos_build_type} ${build_cores}
 
 # Next, do action based on args
 if [ "$build_action" = "full-app" ]; then
-    source kokkos-install.sh ${kokkos_build_type} ${debug}
-    source matar-install.sh ${kokkos_build_type} ${debug}
-    source cmake_build.sh ${solver} ${debug}
+    if [ "$trilinos" = "disabled" ]; then    
+        source kokkos-install.sh ${kokkos_build_type} ${debug} 
+    elif [ "$trilinos" = "enabled" ]; then    
+        source trilinos-install.sh ${kokkos_build_type}  ${intel_mkl} ${debug}
+    fi
+    source matar-install.sh ${kokkos_build_type} ${debug} ${trilinos}
+    source cmake_build.sh ${solver} ${debug} ${trilinos}
 elif [ "$build_action" = "install-kokkos" ]; then
     source kokkos-install.sh ${kokkos_build_type}
 elif [ "$build_action" = "fierro" ]; then
-    source cmake_build.sh ${solver}
+    source cmake_build.sh ${solver} ${debug} ${trilinos}
 else
     echo "No build action, only setup the environment."
 fi

single-node-refactor/scripts/cmake_build.sh

@@ -2,16 +2,31 @@
 
 solver="${1}"
 debug="${2}"
+trilinos="${3}"
 
 echo "Removing old Kokkos build and installation directory"
 rm -rf ${SGH_BUILD_DIR}
 mkdir -p ${SGH_BUILD_DIR}
 
-cmake_options=(
--D BUILD_EXPLICIT_SOLVER=OFF
--D CMAKE_PREFIX_PATH="${MATAR_INSTALL_DIR};${KOKKOS_INSTALL_DIR}"
-#-D CMAKE_CXX_FLAGS="-I${matardir}/src"
-)
+
+if [ "$trilinos" = "enabled" ]; then
+    if [ ! -d "${TRILINOS_INSTALL_DIR}/lib" ]; then
+        Trilinos_DIR=${TRILINOS_INSTALL_DIR}/lib64/cmake/Trilinos
+    else
+        Trilinos_DIR=${TRILINOS_INSTALL_DIR}/lib/cmake/Trilinos
+    fi
+    cmake_options+=(
+        -D CMAKE_PREFIX_PATH="${MATAR_INSTALL_DIR}"
+        -D Trilinos_DIR="$Trilinos_DIR"
+        -D FIERRO_ENABLE_TRILINOS=ON
+    )
+else
+    cmake_options=(
+    -D BUILD_EXPLICIT_SOLVER=OFF
+    -D CMAKE_PREFIX_PATH="${MATAR_INSTALL_DIR};${KOKKOS_INSTALL_DIR}"
+    #-D CMAKE_CXX_FLAGS="-I${matardir}/src"
+    )
+fi
 
 if [ "$debug" = "true" ]; then
     echo "Setting debug to true for CMAKE build type"