Adding tests

tests/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 AMD HPC Application Performance Team
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

tests/Pennant_BuildFiles/CMakeLists.txt

@@ -0,0 +1,62 @@
+cmake_minimum_required(VERSION 3.21 FATAL_ERROR)
+project(Pennant LANGUAGES CXX)
+include(CTest)
+
+set (CMAKE_CXX_STANDARD 14)
+
+if (NOT CMAKE_BUILD_TYPE)
+   set(CMAKE_BUILD_TYPE RelWithDebInfo)
+endif(NOT CMAKE_BUILD_TYPE)
+
+string(REPLACE -O2 -O3 CMAKE_CXX_FLAGS_RELWITHDEBINFO ${CMAKE_CXX_FLAGS_RELWITHDEBINFO})
+
+if (NOT CMAKE_GPU_RUNTIME)
+   set(GPU_RUNTIME "ROCM" CACHE STRING "Switches between ROCM and CUDA")
+else (NOT CMAKE_GPU_RUNTIME)
+   set(GPU_RUNTIME "${CMAKE_GPU_RUNTIME}" CACHE STRING "Switches between ROCM and CUDA")
+endif (NOT CMAKE_GPU_RUNTIME)
+# Really should only be ROCM or CUDA, but allowing HIP because it is the currently built-in option
+set(GPU_RUNTIMES "ROCM" "CUDA" "HIP")
+if(NOT "${GPU_RUNTIME}" IN_LIST GPU_RUNTIMES)
+    set(ERROR_MESSAGE "GPU_RUNTIME is set to \"${GPU_RUNTIME}\".\nGPU_RUNTIME must be either HIP, 
+        ROCM, or CUDA.")
+    message(FATAL_ERROR ${ERROR_MESSAGE})
+endif()
+# GPU_RUNTIME for AMD GPUs should really be ROCM, if selecting AMD GPUs 
+# so manually resetting to HIP if ROCM is selected
+if (${GPU_RUNTIME} MATCHES "ROCM")
+   set(GPU_RUNTIME "HIP")
+endif (${GPU_RUNTIME} MATCHES "ROCM")
+set_property(CACHE GPU_RUNTIME PROPERTY STRINGS ${GPU_RUNTIMES})
+
+enable_language(${GPU_RUNTIME})
+set(CMAKE_${GPU_RUNTIME}_EXTENSIONS OFF)
+set(CMAKE_${GPU_RUNTIME}_STANDARD_REQUIRED ON)
+
+set(PENNANT_CXX_SRCS src/Driver.cc src/ExportGold.cc src/GenMesh.cc src/Hydro.cc src/HydroBC.cc
+                     src/ImportGMV.cc src/InputFile.cc src/Mesh.cc src/Parallel.cc src/PolyGas.cc
+                     src/QCS.cc src/TTS.cc src/WriteXY.cc src/main.cc)
+
+set(PENNANT_HIP_SRCS src/HydroGPU.hip)
+
+add_executable(pennant ${PENNANT_CXX_SRCS} ${PENNANT_HIP_SRCS} )
+
+# Make example runnable using ctest
+add_test(NAME Pennant COMMAND pennant ../test/sedovbig/sedovbig.pnt )
+set_property(TEST Pennant
+             PROPERTY PASS_REGULAR_EXPRESSION "End cycle   3800, time = 9.64621e-01")
+
+set(ROCMCC_FLAGS "${ROCMCC_FLAGS} -munsafe-fp-atomics")
+set(CUDACC_FLAGS "${CUDACC_FLAGS} ")
+
+if (${GPU_RUNTIME} MATCHES "HIP")
+   set(HIPCC_FLAGS "${ROCMCC_FLAGS}")
+else (${GPU_RUNTIME} MATCHES "HIP")
+   set(HIPCC_FLAGS "${CUDACC_FLAGS}")
+endif (${GPU_RUNTIME} MATCHES "HIP")
+
+set_source_files_properties(${PENNANT_HIP_SRCS} PROPERTIES LANGUAGE ${GPU_RUNTIME})
+set_source_files_properties(HydroGPU.hip PROPERTIES COMPILE_FLAGS ${HIPCC_FLAGS})
+
+install(TARGETS pennant)
+

tests/Pennant_BuildFiles/Makefile

@@ -0,0 +1,62 @@
+EXECUTABLE = pennant
+BUILDDIR := build
+SRCDIR = src
+all: $(BUILDDIR)/$(EXECUTABLE) test
+
+.PHONY: test
+
+OBJECTS =  $(BUILDDIR)/Driver.o $(BUILDDIR)/GenMesh.o $(BUILDDIR)/HydroBC.o
+OBJECTS += $(BUILDDIR)/ImportGMV.o $(BUILDDIR)/Mesh.o $(BUILDDIR)/PolyGas.o
+OBJECTS += $(BUILDDIR)/TTS.o $(BUILDDIR)/main.o $(BUILDDIR)/ExportGold.o
+OBJECTS += $(BUILDDIR)/Hydro.o $(BUILDDIR)/HydroGPU.o $(BUILDDIR)/InputFile.o
+OBJECTS += $(BUILDDIR)/Parallel.o $(BUILDDIR)/QCS.o $(BUILDDIR)/WriteXY.o
+
+CXXFLAGS = -g -O3 -DNDEBUG
+HIPCC_FLAGS = -O3 -g -DNDEBUG -fPIC
+
+HIPCC ?= hipcc
+
+ifeq ($(HIPCC), nvcc)
+   HIPCC_FLAGS += -x cu
+   LDFLAGS = -lcudadevrt -lcudart_static -lrt -lpthread -ldl
+endif
+ifeq ($(HIPCC), hipcc)
+   HIPCC_FLAGS += -munsafe-fp-atomics
+   LDFLAGS = -L${ROCM_PATH}/hip/lib -lamdhip64
+endif
+
+$(BUILDDIR)/%.d : $(SRCDIR)/%.cc
+	@echo making depends for $<
+	$(maketargetdir)
+	@$(CXX) $(CXXFLAGS) $(CXXINCLUDES) -M $< | sed "1s![^ \t]\+\.o!$(@:.d=.o) $@!" >$@
+
+$(BUILDDIR)/%.d : $(SRCDIR)/%.hip
+	@echo making depends for $<
+	$(maketargetdir)
+	@$(HIPCC) $(HIPCCFLAGS) $(HIPCCINCLUDES) -M $< | sed "1s![^ \t]\+\.o!$(@:.d=.o) $@!" >$@
+
+$(BUILDDIR)/%.o : $(SRCDIR)/%.cc
+	@echo compiling $<
+	$(maketargetdir)
+	$(CXX) $(CXXFLAGS) $(CXXINCLUDES) -c -o $@ $<
+
+$(BUILDDIR)/%.o : $(SRCDIR)/%.hip
+	@echo compiling $<
+	$(maketargetdir)
+	$(HIPCC) $(HIPCC_FLAGS) -c $^ -o $@
+
+$(BUILDDIR)/$(EXECUTABLE) : $(OBJECTS)
+	@echo linking $@
+	$(maketargetdir)
+	$(CXX) $(OBJECTS) $(LDFLAGS) -o $@
+
+test : $(BUILDDIR)/$(EXECUTABLE)
+	$(BUILDDIR)/$(EXECUTABLE) test/sedovbig/sedovbig.pnt
+
+define maketargetdir
+	-@mkdir -p $(dir $@) > /dev/null 2>&1 
+endef
+
+clean :
+	rm -rf $(BUILDDIR)
+

tests/README.md

@@ -0,0 +1,2 @@
+# TestExamples
+Testing Scripts for Exercises

tests/advancedopenmp_memory1.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+module load amdclang
+
+cd ~/HPCTrainingExamples/Pragma_Examples/OpenMP/CXX/memory_pragmas
+
+export CXX=amdclang++
+export LIBOMPTARGET_INFO=-1
+export OMP_TARGET_OFFLOAD=MANDATORY
+
+mkdir build && cd build
+cmake ..
+make mem1
+./mem1
+
+cd ..
+rm -rf build

tests/advancedopenmp_memory2.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+module load amdclang
+
+cd ~/HPCTrainingExamples/Pragma_Examples/OpenMP/CXX/memory_pragmas
+
+export CXX=amdclang++
+export LIBOMPTARGET_INFO=-1
+export OMP_TARGET_OFFLOAD=MANDATORY
+
+mkdir build && cd build
+cmake ..
+make mem2
+./mem2
+
+cd ..
+rm -rf build

tests/affinity_mpi2_openmp2.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+module load gcc/11 rocm openmpi
+
+git clone https://code.ornl.gov/olcf/hello_mpi_omp.git
+cd hello_mpi_omp
+sed -i -e '/COMP/s/cc/mpicc/' Makefile
+make
+
+OMP_NUM_THREADS=2 OMP_PROC_BIND=close mpirun -np 2 -mca btl ^openib --map-by L3cache ./hello_mpi_omp
+
+cd ..
+rm -rf hello_mpi_omp

tests/affinity_mpi4_openmp2.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+module load gcc/11 rocm openmpi
+
+git clone https://code.ornl.gov/olcf/hello_mpi_omp.git
+cd hello_mpi_omp
+sed -i -e '/COMP/s/cc/mpicc/' Makefile
+make
+
+OMP_NUM_THREADS=2 OMP_PROC_BIND=close mpirun -np 4 -mca btl ^openib --map-by L3cache ./hello_mpi_omp
+
+cd ..
+rm -rf hello_mpi_omp

tests/affinity_mpi4_openmp2_bind.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+module load gcc/11 rocm openmpi
+
+git clone https://code.ornl.gov/olcf/hello_mpi_omp.git
+cd hello_mpi_omp
+sed -i -e '/COMP/s/cc/mpicc/' Makefile
+make
+
+OMP_NUM_THREADS=2 OMP_PROC_BIND=close mpirun -np 4 -mca btl ^openib --map-by L3cache --report-bindings ./hello_mpi_omp
+
+cd ..
+rm -rf hello_mpi_omp

tests/gpuawarempi_osubenchmark.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+mkdir OMB && cd OMB
+wget https://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-7.1-1.tar.gz
+tar -xf osu-micro-benchmarks-7.1-1.tar.gz
+
+module load gcc/12 rocm openmpi
+mkdir build
+cd osu-micro-benchmarks-7.1-1
+
+./configure --prefix=`pwd`/../build/ \
+	CC=`which mpicc` \
+        CXX=`which mpicxx` \
+	--enable-rocm \
+	--with-rocm=${ROCM_PATH}
+
+make -j 12
+make install
+
+ls -l ../build/libexec/osu-micro-benchmarks/mpi
+
+export HIP_VISIBLE_DEVICES=0,1
+
+mpirun -N 2 -n 2 ../build/libexec/osu-micro-benchmarks/mpi/pt2pt/osu_bw -m 10240000
+
+rm -rf ../../OMB

tests/gpuawarempi_pt2pt.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+module load openmpi rocm
+export OMPI_CXX=hipcc
+
+cd ~/HPCTrainingExamples/MPI-examples
+mpicxx -o ./pt2pt ./pt2pt.cpp
+
+mpirun -n 2 ./pt2pt
+
+make clean

tests/hip_jacobi_cmakelists.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+cd ~/HPCTrainingExamples/HIP/jacobi
+
+module load rocm
+module load openmpi
+
+mkdir build && cd build
+cmake ..
+make
+
+#salloc -p LocalQ --gpus=2 -n 2 -t 00:10:00
+mpirun -n 2 ./Jacobi_hip -g 2
+
+cd ..
+rm -rf build

tests/hip_saxpy_cmakelists.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+module load rocm
+
+cd ~/HPCTrainingExamples/HIP/saxpy
+
+mkdir build && cd build
+cmake ..
+make
+./saxpy
+cd ..
+rm -rf build

tests/hip_saxpy_makefile.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+module load rocm
+
+cd ~/HPCTrainingExamples/HIP/saxpy
+
+make saxpy
+./saxpy
+make clean

tests/hip_stream_cmakelists.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+module load rocm
+
+cd ~/HPCTrainingExamples/HIP/hip-stream
+
+mkdir build && cd build
+cmake ..
+make
+./stream
+cd ..
+rm -rf build

tests/hip_stream_makefile.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+module load rocm
+
+cd ~/HPCTrainingExamples/HIP/hip-stream
+
+make stream
+./stream
+make clean

tests/hip_vectoradd_cmakelists.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+module load rocm
+
+cd ~/HPCTrainingExamples/HIP/vectorAdd
+
+mkdir build && cd build
+cmake ..
+make
+./vectoradd
+cd ..
+rm -rf build

tests/hip_vectoradd_cmakelists_batch.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+sbatch --wait ~/HPCTrainingExamples/HIP/vectorAdd/hip_cmakelists_batch.sh
+
+grep PASSED! slurm-*.out
+ls
+rm  slurm-*.out

tests/hip_vectoradd_cmakelists_batch_direct.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+cp ~/HPCTrainingExamples/HIP/vectorAdd/hip_cmakelists_batch.sh hip_cmakelists_batch.sh
+chmod +x hip_cmakelists_batch.sh
+./hip_cmakelists_batch.sh
+
+rm -r hip_cmakelists_batch.sh

tests/hip_vectoradd_makefile.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+module load rocm
+
+cd ~/HPCTrainingExamples/HIP/vectorAdd
+
+make vectoradd
+./vectoradd
+make clean

tests/hip_vectoradd_makefile_batch.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+sbatch --wait ~/HPCTrainingExamples/HIP/vectorAdd/hip_makefile_batch.sh
+
+grep PASSED! slurm-*.out
+rm  slurm-*.out

tests/hip_vectoradd_makefile_batch_direct.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+cp ~/HPCTrainingExamples/HIP/vectorAdd/hip_makefile_batch.sh hip_makefile_batch.sh
+chmod +x hip_makefile_batch.sh
+./hip_makefile_batch.sh
+
+rm -r hip_makefile_batch.sh

tests/hipify_nbody.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+module load rocm
+
+cd ~/HPCTrainingExamples/HIPIFY/mini-nbody/cuda
+hipify-perl -examine nbody-orig.cu
+
+hipify-perl nbody-orig.cu > nbody-orig.cpp
+hipcc -DSHMOO -I../ nbody-orig.cpp -o nbody-orig
+
+./nbody-orig