Merge pull request #73 from lanl/Daniel

/approve

.gitignore

@@ -1,5 +1 @@
-build-kokkos-serial
-build-kokkos-openmp
-build-kokkos-cuda
-build-kokkos-cuda-amp
-build-kokkos-hip
+install-*

.gitmodules

@@ -1,9 +1,3 @@
-[submodule "src/Kokkos/kokkos"]
+[submodule "kokkos"]
 	path = src/Kokkos/kokkos
 	url = https://github.com/kokkos/kokkos
-[submodule "test/googletest"]
-	path = test/googletest
-	url = https://github.com/google/googletest.git
-[submodule "examples/phaseFieldMPI/heffte"]
-	path = examples/phaseFieldMPI/heffte
-	url = https://bitbucket.org/icl/heffte.git

CMakeLists.txt

@@ -9,6 +9,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED TRUE)
 project (MATAR)
 set (CMAKE_CXX_STANDARD 17)
 
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 
 # CMAKE_BUILD_TYPE:
 # 1. Release: `-O3 -DNDEBUG`
@@ -48,8 +49,52 @@ set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${VECTOR_C_FLAGS}")
 set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${VECTOR_CXX_FLAGS}")
 
 
-include_directories(src)
-add_subdirectory(src)
+add_library(matar INTERFACE)
+target_include_directories(matar INTERFACE 
+    $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>  
+    $<INSTALL_INTERFACE:include>
+)
+
+if(Matar_ENABLE_KOKKOS)
+    if("${Matar_KOKKOS_PACKAGE}" STREQUAL "Trilinos")
+        find_package(Trilinos REQUIRED)
+        target_link_libraries(matar INTERFACE ${Trilinos_LIBRARIES})
+        add_definitions(-DTRILINOS_INTERFACE=1)
+    else()
+        find_package(Kokkos REQUIRED)
+        target_link_libraries(matar INTERFACE Kokkos::kokkos)
+    endif()
+    add_definitions(-DHAVE_KOKKOS=1)
+endif()
+
+include(CMakePackageConfigHelpers)
+write_basic_package_version_file(
+    "${PROJECT_BINARY_DIR}/MatarConfigVersion.cmake"
+    VERSION 1.0
+    COMPATIBILITY AnyNewerVersion
+)
+
+install(TARGETS matar
+    EXPORT MatarTargets
+    LIBRARY DESTINATION lib COMPONENT Runtime
+    ARCHIVE DESTINATION lib COMPONENT Development
+    RUNTIME DESTINATION bin COMPONENT Runtime
+    PUBLIC_HEADER DESTINATION include COMPONENT Development
+    BUNDLE DESTINATION bin COMPONENT Runtime
+)
+
+include(CMakePackageConfigHelpers)
+configure_package_config_file(
+    "${PROJECT_SOURCE_DIR}/cmake/MatarConfig.cmake.in"
+    "${PROJECT_BINARY_DIR}/MatarConfig.cmake"
+    INSTALL_DESTINATION lib/cmake/matar
+)
+
+install(EXPORT MatarTargets DESTINATION lib/cmake/matar)
+install(FILES "${PROJECT_BINARY_DIR}/MatarConfigVersion.cmake"
+              "${PROJECT_BINARY_DIR}/MatarConfig.cmake"
+        DESTINATION lib/cmake/matar)
+install(DIRECTORY ${PROJECT_SOURCE_DIR}/src/include/ DESTINATION include)
 
 if (BUILD_EXAMPLES)
     include_directories(examples)

README.md

@@ -124,70 +124,37 @@ kokkos-install.sh
 backend-cmake-build.sh
 ```
 The word backend denotes cuda, hip, openMP, and so forth.  Scripts are also provided to build MATAR without Kokkos, and in that case there is no backend listed since it doesn't use Kokko.  The backend-cmake-build.sh script will run cmake and make for the project.  Afterwords, the user can just runs make inside the respective build directory to compile the project.  For clarity, running all the scripts is only necessary to set up and compile the code the first time, afterwards, the use can compile the code using make in the build directory.  The environment variables will need to be set when logging into a compute node or when changing to a different kokkos backend. For all builds, a single script is provided in each script folder to automate the entire build process, it runs the three aforementioned scripts sequentially. 
+The build-it script can take up to 3 arguments (with a minimum of 2)
 ```
-build-it.sh
+source build-it.sh <environment type> <parallelism> <build directory name (optional)>
 ```
-Before using the build-it.sh script, the user must verify that the settings in the other scripts that build MATAR with a Kokkos backend are correctly set.  After running the build-it.sh script, the entire project is compiled and stored in a directory that is named with the respective Kokkos backend e.g., build-kokkos-cuda.  Further details are provided on the three scripts to configure and build MATAR with a Kokkos backend.
-
-
-### Environment configuration script
-To start, the environment variables and modules must be configured by sourcing the following script
-```
-source sourceme-env.sh
-```
-This script is where the user will load the necessary module files for their given machine/architecture combination.  This script also creates the build directory for the project e.g., build-kokkos-cuda, build-kokkos-hip, build-kokkos-openmp, etc.
-
-
-### Install Kokkos script
-The next step is to install Kokkos, using the version that was cloned recursively within MATAR, and configure the Kokkos build for specific hardware and a backend.  
+environment has two options: 'hpc' or 'macos'
 ```
-source kokkos-install.sh
+    hpc: builds by loading modules and can perform parallel builds (make -j)
+    macos: does not load anything externally and expects the environment to be set up on your mac. Additionally, the builds will all be serial (make)
 ```
-Within this script, the user will need to set any Kokkos specific variables for their project. The architecture variables will need to be modified based on the architecture being used. The provided scripts are set for a particular hardware that might differ from what a user might be using.  CPU architecture information needs to be listed if running with the Kokkos serial or OpenMP backends; GPU architecture information must be listed if using a Kokkos GPU backend.  We refer the user to Kokkos compiling page to see the large list of compilation options,
+parallelism has six options: 'cuda', 'hip', 'openmp', 'pthreads', 'serial', 'none'
 ```
-https://github.com/kokkos/kokkos/wiki/Compiling
+    cuda: loads cuda module and a working gcc module pairing
+    hip: loads hip module and a working clang module pairing
+    openmp: loads gcc module and sets openmp environment variables
+    pthreads: loads gcc module and sets pthreads environment variables
+    serial and none: loads gcc module
 ```
+***Note*** - compiler can be changed with the appropriate variables in *setup-env.sh*, the ones provided are simply known to work together
 
+All other scripts will be called with the appropriate arguments as a result of running build-it.
+Before using the build-it.sh script, the user must verify that the settings in the other scripts that build MATAR with a Kokkos backend are correctly set.  After running the build-it.sh script, the entire project is compiled and stored in a directory that is named with the respective Kokkos backend e.g., build-kokkos-cuda.  Further details are provided on the three scripts to configure and build MATAR with a Kokkos backend.
 
-
-### CUDA compilation script
-To build the project with cuda, the last step is to type
-```
-source cuda-cmake-build.sh
-```
-
-
-### HIP compilation script
-To build the project with hip, the last step is to type
-```
-source hip-cmake-build.sh
-```
-
-
-### openMP compilation script
-To build the project with openMP, the last step is to type
-
-```
-source openmp-cmake-build.sh
+If you need to simply rebuild Fierro without making a new Kokkos installation, simply
 ```
-The sourceme-env.sh script (the first step) sets the number of threads to 16 by default.  Changing the number of threads used with openMP requires manually setting the environment variable OMP_NUM_THREADS.  
-
-
-
-### pthreads compilation script
-To build the project with ptheads, the last step is to type
+source cmake_build.sh <same args you passed to build-it>
 ```
-source pthreads-cmake-build.sh
-```    
-To specify number of threads when running a code with the Kokkos pthread backend, add the following command line arguments
+If you are getting back on to a machine or allocation to continue development, you will need to run
 ```
---kokkos-threads=4
+source setup-env.sh <same args you passed to build-it>
 ```
-
-
-### Automate build process
-A build-it.sh script is provided that runs all scripts sequentially for the user.  The build-it.sh script obviates the need to manually source each script.  The user must verify the settings are correct in each script prior to using the build-it.sh script.  If the build-it.sh script fails to build the project correctly, the user should carefully look at the loaded modules and settings for building Kokkos.   
-
+If the scripts fail to build, then carefully review the modules used and the computer architecture settings. 
 
 
 ## Contributing

cmake/MatarConfig.cmake.in

@@ -0,0 +1,4 @@
+@PACKAGE_INIT@
+
+include("${CMAKE_CURRENT_LIST_DIR}/MatarTargets.cmake")
+check_required_components("@Matar@")

examples/CMakeLists.txt

@@ -1,6 +1,11 @@
 cmake_minimum_required(VERSION 3.8)
 
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
+
+find_package(Matar REQUIRED)
+
 add_executable(mtest main.cpp)
+target_link_libraries(mtest matar)
 
 if (NOT KOKKOS)
    add_executable(test_for test_for.cpp)
@@ -14,14 +19,21 @@ if (KOKKOS)
   find_package(Kokkos REQUIRED) #new
 
   add_executable(mtestkokkos main_kokkos.cpp)
+  target_link_libraries(mtestkokkos matar Kokkos::kokkos)
+
   add_executable(test_kokkos_for kokkos_for.cpp)
+  target_link_libraries(test_kokkos_for matar Kokkos::kokkos)
+
   add_executable(test_dual_types test_dual_types.cpp)
+  target_link_libraries(test_dual_types matar Kokkos::kokkos)
+
   add_definitions(-DHAVE_KOKKOS=1)
 
   add_executable(kokkos_csr CSRKokkos.cpp)
-  target_link_libraries(kokkos_csr matar)
+  target_link_libraries(kokkos_csr matar Kokkos::kokkos)
+
   add_executable(kokkos_csc CSCKokkos.cpp)
-  target_link_libraries(kokkos_csc matar)
+  target_link_libraries(kokkos_csc matar Kokkos::kokkos)
 
 
   if (CUDA)
@@ -31,19 +43,14 @@ if (KOKKOS)
   elseif (OPENMP)
     add_definitions(-DHAVE_OPENMP=1)
     add_executable(parallel_hello_world parallel_hello_world.cpp)
-    target_link_libraries(parallel_hello_world matar)
+    target_link_libraries(parallel_hello_world matar Kokkos::kokkos)
   elseif (THREADS)
     add_definitions(-DHAVE_THREADS=1)
   endif()
 
-  target_link_libraries(mtestkokkos matar)
-  target_link_libraries(test_kokkos_for Kokkos::kokkos)
-  target_link_libraries(test_dual_types matar)
 endif()
 
 
-target_link_libraries(mtest matar)
-
 ### HIP Linking error, will add back in after fixed
 if (NOT HIP)
     include_directories(virtualFcnKokkos)

examples/halfspace_cooling/CMakeLists.txt

@@ -1,25 +1,13 @@
 cmake_minimum_required(VERSION 3.1.3)
 
+find_package(Matar REQUIRED)
+
 if (KOKKOS)
   find_package(Kokkos REQUIRED) #new
 
   add_executable(halfspace_cooling halfspace_cooling.cpp)
 
   add_definitions(-DHAVE_KOKKOS=1)
-  #set_target_properties(matar PROPERTIES COMPILE_DEFINITIONS HAVE_KOKKOS)
-  if (CUDA)
-    #add_definitions(-DHAVE_CUDA=1)
-    set_target_properties(matar PROPERTIES COMPILE_DEFINITIONS HAVE_CUDA)
-  elseif (HIP)
-    #add_definitions(-DHAVE_HIP=1)
-    set_target_properties(matar PROPERTIES COMPILE_DEFINITIONS HAVE_HIP)
-  elseif (OPENMP)
-    #add_definitions(-DHAVE_OPENMP=1)
-    set_target_properties(matar PROPERTIES COMPILE_DEFINITIONS HAVE_OPENMP)
-  elseif (THREADS)
-    add_definitions(-DHAVE_THREADS=1)
-    #set_target_properties(matar PROPERTIES COMPILE_DEFINITIONS HAVE_THREADS)
-  endif()
 
-  target_link_libraries(halfspace_cooling matar)
+  target_link_libraries(halfspace_cooling matar Kokkos::kokkos)
 endif()

examples/laplace/CMakeLists.txt

@@ -1,5 +1,7 @@
 cmake_minimum_required(VERSION 3.1.3)
 
+find_package(Matar REQUIRED)
+
 if (NOT KOKKOS)
   add_executable(carray_right main_carray_right.cpp)
   add_executable(carray_wrong main_carray_wrong.cpp)
@@ -22,24 +24,10 @@ if (KOKKOS)
   add_executable(farraykokkos_f_indexing main_farraykokkos_f_indexing.cpp)
 
   add_definitions(-DHAVE_KOKKOS=1)
-  #set_target_properties(matar PROPERTIES COMPILE_DEFINITIONS HAVE_KOKKOS)
-  if (CUDA)
-    #add_definitions(-DHAVE_CUDA=1)
-    set_target_properties(matar PROPERTIES COMPILE_DEFINITIONS HAVE_CUDA)
-  elseif (HIP)
-    #add_definitions(-DHAVE_HIP=1)
-    set_target_properties(matar PROPERTIES COMPILE_DEFINITIONS HAVE_HIP)
-  elseif (OPENMP)
-    #add_definitions(-DHAVE_OPENMP=1)
-    set_target_properties(matar PROPERTIES COMPILE_DEFINITIONS HAVE_OPENMP)
-  elseif (THREADS)
-    add_definitions(-DHAVE_THREADS=1)
-    #set_target_properties(matar PROPERTIES COMPILE_DEFINITIONS HAVE_THREADS)
-  endif()
 
-  target_link_libraries(kokkosview Kokkos::kokkos)
-  target_link_libraries(carraykokkos_default_indexing matar)
-  target_link_libraries(carraykokkos_c_indexing matar)
-  target_link_libraries(farraykokkos_default_indexing matar)
-  target_link_libraries(farraykokkos_f_indexing matar)
+  target_link_libraries(kokkosview matar Kokkos::kokkos)
+  target_link_libraries(carraykokkos_default_indexing matar Kokkos::kokkos)
+  target_link_libraries(carraykokkos_c_indexing matar Kokkos::kokkos)
+  target_link_libraries(farraykokkos_default_indexing matar Kokkos::kokkos)
+  target_link_libraries(farraykokkos_f_indexing matar Kokkos::kokkos)
 endif()

examples/main.cpp

@@ -10,5 +10,9 @@ int main() {
 
     auto test = CArray <int> (5, 5); 
 
+    test(3,3) = 10;
+
+    printf("Succesfully made and used a CArray\n");
+
     return 0;
 }

examples/main_kokkos.cpp

@@ -919,7 +919,7 @@ int main(int argc, char *argv[]) {
         CArrayDevice <int> a_carray_device(5);
         RUN({
             a_carray_device(0) = 0;
-            printf("\nalias name value [0] on divice = %d \n", a_carray_device(0));
+            printf("\nalias name value [0] on device = %d \n", a_carray_device(0));
         });
 
 

examples/sparsetests/CMakeLists.txt

@@ -8,10 +8,10 @@ if (KOKKOS)
     add_executable(powerIter powerIter.cpp)
     add_executable(spPower sparsePowerIter.cpp)
     add_executable(spatVec sparseMatVec.cpp)
-    target_link_libraries(spatVec matar)
-    target_link_libraries(matVec matar)
-    target_link_libraries(powerIter matar)
-    target_link_libraries(spPower matar)
+    target_link_libraries(spatVec matar Kokkos::kokkos)
+    target_link_libraries(matVec matar Kokkos::kokkos)
+    target_link_libraries(powerIter matar Kokkos::kokkos)
+    target_link_libraries(spPower matar Kokkos::kokkos)
 
   if (CUDA)
     add_definitions(-DHAVE_CUDA=1)

examples/test_rocm/CMakeLists.txt

@@ -24,5 +24,5 @@ if (KOKKOS)
 
   endif()
 
-  target_link_libraries(test_rocm matar)
+  target_link_libraries(test_rocm matar Kokkos::kokkos)
 endif()

examples/virtualFcnKokkos/CMakeLists.txt

@@ -2,26 +2,14 @@ cmake_minimum_required(VERSION 3.1.3)
 
 project (virttestkokkos)
 
+find_package(Matar REQUIRED)
+
 if (KOKKOS)
   find_package(Kokkos REQUIRED) #new
 
   add_executable(virttestkokkos child.cpp  child.hpp  inherited_inits.cpp  inherited_inits.hpp  kokkos_alias.h  main_kokkos_vfcn.cpp  parents.h)
 
   add_definitions(-DHAVE_KOKKOS=1)
-  #set_target_properties(matar PROPERTIES COMPILE_DEFINITIONS HAVE_KOKKOS)
-  if (CUDA)
-    #add_definitions(-DHAVE_CUDA=1)
-    set_target_properties(matar PROPERTIES COMPILE_DEFINITIONS HAVE_CUDA)
-  elseif (HIP)
-    #add_definitions(-DHAVE_HIP=1)
-    set_target_properties(matar PROPERTIES COMPILE_DEFINITIONS HAVE_HIP)
-  elseif (OPENMP)
-    #add_definitions(-DHAVE_OPENMP=1)
-    set_target_properties(matar PROPERTIES COMPILE_DEFINITIONS HAVE_OPENMP)
-  elseif (THREADS)
-    #add_definitions(-DHAVE_THREADS=1)
-    set_target_properties(matar PROPERTIES COMPILE_DEFINITIONS HAVE_THREADS)
-  endif()
 
   target_link_libraries(virttestkokkos matar Kokkos::kokkos)
 endif()

examples/virtualFcnMATAR/CMakeLists.txt

@@ -1,5 +1,6 @@
 cmake_minimum_required(VERSION 3.1.3)
 
+find_package(Matar REQUIRED)
 
 if (KOKKOS)
   add_definitions(-DHAVE_KOKKOS=1)
@@ -17,6 +18,6 @@ if (KOKKOS)
   endif()
 
   add_executable(virtual_fcn_test main.cpp classes.cpp)
-  target_link_libraries(virtual_fcn_test Kokkos::kokkos)
+  target_link_libraries(virtual_fcn_test matar Kokkos::kokkos)
 
 endif()

examples/watt-graph/CMakeLists.txt

@@ -10,7 +10,7 @@ if (KOKKOS)
     add_definitions(-DHAVE_KOKKOS=1)
 
     add_executable(test_kokkos_floyd kokkos_floyd.cpp)
-    target_link_libraries(test_kokkos_floyd matar)
+    target_link_libraries(test_kokkos_floyd matar Kokkos::kokkos)
 
   if (CUDA)
     add_definitions(-DHAVE_CUDA=1)