tests: split netcdf tests to be able to run without fortran

The netcdf tests required netcdf, netcdf-cxx and netcdf-fortran. If we update netcdf the CI cannot handle the necessary rebuild of netcdf-cxx and netcdf-fortran. Especially for the GitHub CI the tests are now split to either test only netcdf or all three packages. Signed-off-by: Adrian Reber <[email protected]>
openhpc · Jun 20, 2023 · 98be98f · 98be98f
1 parent 62ce8af
commit 98be98f
Show file tree

Hide file tree

Showing 12 changed files with 171 additions and 61 deletions.
diff --git a/tests/Makefile.am b/tests/Makefile.am
@@ -180,6 +180,10 @@ if NETCDF_ENABLED
    TESTS += libs/netcdf/ohpc-tests/test_mpi_families
 endif
 
+if NETCDF_ENABLED_ALL
+   TESTS += libs/netcdf/ohpc-tests/netcdf_all_test_mpi_families
+endif
+
 if OPENBLAS_ENABLED
    TESTS += libs/openblas/ohpc-tests/test_compiler_families
 endif

diff --git a/tests/ci/Makefile b/tests/ci/Makefile
@@ -33,4 +33,6 @@ shellcheck-lint:
 		../../misc/get_source.sh \
 		../../misc/shell-functions \
 		../../tests/ci/prepare-ci-environment.sh \
-		../../tests/ci/setup_slurm_and_run_tests.sh
+		../../tests/ci/setup_slurm_and_run_tests.sh \
+		../../tests/libs/netcdf/ohpc-tests/netcdf_all_test_mpi_families \
+		../../tests/libs/netcdf/ohpc-tests/test_mpi_families
diff --git a/tests/ci/spec_to_test_mapping.py b/tests/ci/spec_to_test_mapping.py
@@ -99,6 +99,21 @@
         '',
         ''
     ],
+    'components/io-libs/netcdf/SPECS/netcdf.spec': [
+        'netcdf',
+        '',
+        ''
+    ],
+    'components/io-libs/netcdf-cxx/SPECS/netcdf-cxx4.spec': [
+        'netcdf-all',
+        '',
+        'netcdf-fortran-gnu12-mpich-ohpc netcdf-fortran-gnu12-openmpi4-ohpc'
+    ],
+    'components/io-libs/netcdf-fortran/SPECS/netcdf-fortran.spec': [
+        'netcdf-all',
+        '',
+        'netcdf-cxx-gnu12-mpich-ohpc netcdf-cxx-gnu12-openmpi4-ohpc'
+    ],
     'components/parallel-libs/scalapack/SPECS/scalapack.spec': [
         'scalapack',
         '',

diff --git a/tests/configure.ac b/tests/configure.ac
@@ -177,6 +177,12 @@ AC_ARG_ENABLE([netcdf],
    [AX_OPTION_DEFAULT(type=long,enable_netcdf,yes)])
 AM_CONDITIONAL(NETCDF_ENABLED,test "x$enable_netcdf" = "xyes" ) 
 #------------------------------------------------------------------------------------------
+# NetCDF all (including cxx fortran)
+AC_ARG_ENABLE([netcdf-all],
+   [AS_HELP_STRING([--enable-netcdf-all],[Enable all NetCDF tests (default=yes)])],[],
+   [AX_OPTION_DEFAULT(type=long,enable_netcdf_all,yes)])
+AM_CONDITIONAL(NETCDF_ENABLED_ALL,test "x$enable_netcdf_all" = "xyes" )
+#------------------------------------------------------------------------------------------
 # PNetCDF
 AC_ARG_ENABLE([pnetcdf],
    [AS_HELP_STRING([--enable-pnetcdf],[Enable PNetCDF tests (default=yes)])],[],
@@ -740,6 +746,12 @@ echo    Libraries:
        echo '    'NetCDF.................... : disabled
     fi
 
+    if test "x$enable_netcdf_all" = "xyes"; then
+       echo '    'NetCDF All................ : enabled
+    else
+       echo '    'NetCDF All................ : disabled
+    fi
+
     if test "x$enable_numpy" = "xyes"; then
        echo '    'Numpy..................... : enabled
     else

diff --git a/tests/libs/netcdf/Makefile.am b/tests/libs/netcdf/Makefile.am
@@ -1,4 +1,6 @@
-AUTOMAKE_OPTIONS = foreign
+AUTOMAKE_OPTIONS = foreign no-dependencies
 AM_MAKEFLAGS     = --no-print-directory
-SUBDIRS          = tests tests-parallel
-
+if PARALLEL_ENABLED
+PARALLEL_SUBDIR  = tests-parallel
+endif
+SUBDIRS          = tests $(PARALLEL_SUBDIR)
diff --git a/tests/libs/netcdf/configure.ac b/tests/libs/netcdf/configure.ac
@@ -8,20 +8,32 @@ AC_CONFIG_MACRO_DIR([m4])
 AM_INIT_AUTOMAKE([foreign])
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 
+AC_ARG_ENABLE([parallel],
+[AS_HELP_STRING([--enable-parallel],[Enable parallel tests (default=yes)])],
+         [],[enable_parallel=yes])
+AM_CONDITIONAL(PARALLEL_ENABLED,test "x$enable_parallel" = "xyes" )
+
+AC_ARG_ENABLE([all],
+[AS_HELP_STRING([--enable-all],[Enable all tests (c++, fortran)(default=no)])],
+         [],[enable_all=no])
+AM_CONDITIONAL(ALL_ENABLED,test "x$enable_all" = "xyes" )
+
 # verify we have necessary environment variables set
 
 CHECK_ENV([NETCDF_LIB],[netcdf])
 CHECK_ENV([NETCDF_INC],[netcdf])
+if test "x$enable_all" = "xyes"; then
 CHECK_ENV([NETCDF_CXX_LIB],[netcdf-cxx])
 CHECK_ENV([NETCDF_CXX_INC],[netcdf-cxx])
 CHECK_ENV([NETCDF_FORTRAN_LIB],[netcdf-fortran])
 CHECK_ENV([NETCDF_FORTRAN_INC],[netcdf-fortran])
+fi
 CHECK_ENV([HDF5_LIB],[hdf5])
 
 OHPC_COMPILER_FAMILY
 
 # Define include path and library linkage from environment variables
-# which should be provided via module loads. 
+# which should be provided via module loads.
 CFLAGS="-I${NETCDF_INC} ${CFLAGS}"
 CXXFLAGS="-I${NETCDF_INC} -I${NETCDF_CXX_INC} ${CXXFLAGS}"
 FCFLAGS="-I${NETCDF_FORTRAN_INC} ${FCFLAGS}"
@@ -56,20 +68,30 @@ echo
 echo '-------------------------------------------------- SUMMARY --------------------------------------------------'
 echo
 echo Package version............... : $PACKAGE-$VERSION
-echo OHPC compiler toolchain........ : $LMOD_FAMILY_COMPILER
-echo OHPC MPI toolchain............. : $LMOD_FAMILY_MPI
+echo OHPC compiler toolchain....... : $LMOD_FAMILY_COMPILER
+if test "x$enable_parallel" = "xyes"; then
+  echo OHPC MPI toolchain............ : $LMOD_FAMILY_MPI
+else
+  echo Parallel tests disabled
+fi
 echo
 echo C compiler.................... : `which $CC`
+if test "x$enable_all" = "xyes"; then
 echo C++ compiler.................. : `which $CXX`
 echo Fortran compiler ............. : `which $FC`
-echo 
+fi
+echo
 echo C compiler flags.............. : $CFLAGS
+if test "x$enable_all" = "xyes"; then
 echo C++ compiler flags............ : $CXXFLAGS
 echo Fortran compiler flags........ : $FCFLAGS
-echo 
+fi
+echo
 echo C linker flags................ : $C_LDFLAGS
+if test "x$enable_all" = "xyes"; then
 echo C++ linker flags.............. : $CXX_LDFLAGS
 echo Fortran linker flags.......... : $FC_LDFLAGS
+fi
 echo Linker flags.................. : $LDFLAGS $LIBS
-echo 
+echo
 echo '-------------------------------------------------------------------------------------------------------------'
diff --git a/tests/libs/netcdf/ohpc-tests/test_mpi_families b/tests/libs/netcdf/ohpc-tests/test_mpi_families
@@ -1,8 +1,10 @@
 #!/bin/bash
 # -*-sh-*-
 
-TEST_LOGS=""
-MAKEFLAGS=""
+# shellcheck disable=SC1091
+
+export MAKEFLAGS=""
+
 status=0
 
 source ./common/TEST_ENV  || exit 1
@@ -12,35 +14,62 @@ cd libs/netcdf            || exit 1
 export BATS_JUNIT_CLASS=NetCDF
 
 # bootstrap the local autotools project if necessary
-
 ./bootstrap || exit 1
 
-for compiler in $COMPILER_FAMILIES ; do
-    for mpi in $MPI_FAMILIES ; do
-
-        echo " "
-        echo " "
-        echo "----------------------------------------------------------"
-        echo "User Libraries: NetCDF tests: $compiler-$mpi"
-        echo "----------------------------------------------------------"
-
-        module purge                || exit 1
-        module load prun            || exit 1
-        module load $compiler       || exit 1
-        module load $mpi            || exit 1
-        module load netcdf          || exit 1
-        module load netcdf-cxx      || exit 1
-        module load netcdf-fortran  || exit 1
-
-        ./configure                 || exit 1
-        make clean                  || exit 1
-        make -k check               || status=1
-
-        save_logs_mpi_family tests          $compiler $mpi
-        save_logs_mpi_family tests-parallel $compiler $mpi
-
-        make distclean
-    done
+if [ -n "${SIMPLE_CI}" ]; then
+	export OMPI_MCA_btl="self,tcp"
+fi
+
+for compiler in ${COMPILER_FAMILIES} ; do
+	for mpi in ${MPI_FAMILIES} ; do
+		export PARALLEL_ENABLED=yes
+		export ALL_ENABLED=no
+		echo " "
+		echo " "
+		echo "----------------------------------------------------------"
+		echo "User Libraries: NetCDF tests: ${compiler}-${mpi}"
+		echo "----------------------------------------------------------"
+
+		module purge                || exit 1
+		module load prun            || exit 1
+		module load "${compiler}"   || exit 1
+		module load "${mpi}"        || exit 1
+		module load netcdf          || exit 1
+
+		./configure                 || exit 1
+		make clean                  || exit 1
+		make -k check               || status=1
+
+		save_logs_mpi_family tests          "${compiler}" "${mpi}"
+		save_logs_mpi_family tests-parallel "${compiler}" "${mpi}"
+
+		make distclean
+		unset PARALLEL_ENABLED
+		unset ALL_ENABLED
+	done
+
+	export PARALLEL_ENABLED=no
+	export ALL_ENABLED=no
+	echo " "
+	echo " "
+	echo "----------------------------------------------------------"
+	echo "User Libraries: NetCDF tests: ${compiler}"
+	echo "----------------------------------------------------------"
+
+	module purge                   || exit 1
+	module load prun               || exit 1
+	module load "${compiler}"      || exit 1
+	module load netcdf             || exit 1
+
+	./configure --disable-parallel || exit 1
+	make clean                     || exit 1
+	make -k check                  || status=1
+
+	save_logs_compiler_family tests "${compiler}"
+
+	make distclean
+	unset PARALLEL_ENABLED
+	unset ALL_ENABLED
 done
 
-exit ${status}
+exit "${status}"
diff --git a/tests/libs/netcdf/tests-parallel/Makefile.am b/tests/libs/netcdf/tests-parallel/Makefile.am
@@ -8,9 +8,11 @@ check_PROGRAMS       = C_parallel
 C_parallel_SOURCES   = err_macros.h  nc_tests.h  tst_parallel.c
 C_parallel_LDFLAGS   = $(C_LDFLAGS)
 
+
+if ALL_ENABLED
 check_PROGRAMS      += F90_parallel
 F90_parallel_SOURCES = f90tst_parallel.f90
 F90_parallel_LDFLAGS = $(FC_LDFLAGS)
+endif
 
 TESTS                = test_pnetcdf
-
diff --git a/tests/libs/netcdf/tests-parallel/test_pnetcdf b/tests/libs/netcdf/tests-parallel/test_pnetcdf
@@ -49,6 +49,9 @@ delimiter="$RESOURCE_MANAGER/$LMOD_FAMILY_COMPILER/$LMOD_FAMILY_MPI"
 
 
 @test "[$testname] Fortran parallel I/O ($delimiter)" {
+    if [ "$ALL_ENABLED" == "no" ]; then
+        skip "Skipping Fortran tests as requested"
+    fi
     if [ ! -x F90_parallel ];then
         flunk "F90_parallel binary not available"
     fi

diff --git a/tests/libs/netcdf/tests/Makefile.am b/tests/libs/netcdf/tests/Makefile.am
@@ -15,6 +15,9 @@ check_PROGRAMS   += C_read
 C_read_SOURCES   = C_simple_rd.c
 C_read_LDFLAGS   = $(C_LDFLAGS)
 
+TESTS             = test_C_module
+
+if ALL_ENABLED
 check_PROGRAMS   += F90_write
 F90_write_SOURCES = simple_xy_nc4_wr.f90
 F90_write_LDFLAGS = $(FC_LDFLAGS)
@@ -31,9 +34,9 @@ check_PROGRAMS   += CXX_read
 CXX_read_SOURCES  = simple_xy_rd.cpp
 CXX_read_LDFLAGS  = $(CXX_LDFLAGS)
 
-TESTS             = test_C_module
 TESTS            += test_CXX_module
 TESTS            += test_Fortran_module
+endif
 TESTS            += nctest
 TESTS            += compare_test_files.sh
 TESTS            += test_netcdf