diff --git a/.github/workflows/spack-dev-build.yml b/.github/workflows/spack-dev-build.yml new file mode 100644 index 0000000..77dc471 --- /dev/null +++ b/.github/workflows/spack-dev-build.yml @@ -0,0 +1,20 @@ +name: spack-dev-build + +on: + pull_request: + branches: [ main ] + workflow_dispatch: + schedule: + - cron: '0 12 1 * *' # 12:00 UTC on the 1st of every month and on manual dispatch + +jobs: + + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Build the Docker image with make checks + run: make spack-dev-build + diff --git a/Docker-dev-build b/Docker-dev-build new file mode 100644 index 0000000..d884a21 --- /dev/null +++ b/Docker-dev-build @@ -0,0 +1,134 @@ +FROM debian:bullseye + +# # which spack version are we using now? Default is develop +# # but other strings can be given to the docker build command +# # (for example docker build --build-arg SPACK_VERSION=v0.16.2) +ARG SPACK_VERSION=develop +ARG SPACK_OCT_VERSION=12.1 +RUN echo "Building with spack version ${SPACK_VERSION}" + +# Any extra packages to be installed in the host +ARG EXTRA_PACKAGES +RUN echo "Installing EXTRA_PACKAGES ${EXTRA_PACKAGES} on container host" + +# general environment for docker +ENV SPACK_ROOT=/home/user/spack \ + SPACK=/home/user/spack/bin/spack \ + FORCE_UNSAFE_CONFIGURE=1 + +RUN apt-get -y update +# Convenience tools, if desired for debugging etc +# RUN apt-get -y install wget time nano vim emacs git + +# From https://github.com/ax3l/dockerfiles/blob/master/spack/base/Dockerfile: +# install minimal spack dependencies +RUN apt-get install -y --no-install-recommends \ + autoconf \ + build-essential \ + ca-certificates \ + coreutils \ + curl \ + environment-modules \ + file \ + gfortran \ + git \ + openssh-server \ + python \ + unzip \ + vim \ + wget \ + && rm -rf /var/lib/apt/lists/* + +# load spack environment on login +RUN echo "source $SPACK_ROOT/share/spack/setup-env.sh" \ + > /etc/profile.d/spack.sh + +RUN adduser user +USER user +WORKDIR /home/user + +# install spack +RUN git clone https://github.com/spack/spack.git +# default branch is develop +RUN cd spack && git checkout $SPACK_VERSION + +# # show which version we use +RUN $SPACK --version + +# copy our package.py into the spack tree (and also example files) +COPY spack/package.py $SPACK_ROOT/var/spack/repos/builtin/packages/octopus/package.py +RUN ls -l $SPACK_ROOT/var/spack/repos/builtin/packages/octopus +COPY spack/test/ $SPACK_ROOT/var/spack/repos/builtin/packages/octopus/test +RUN ls -l $SPACK_ROOT/var/spack/repos/builtin/packages/octopus/test +COPY check_buildlog.py /home/user + +# Install and test serial and MPI versions of ocoptus via spack +# # serial version + +RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ + # make a dev-build direcotry for the serial version: + mkdir ~/dev-build-serial && cd ~/dev-build-serial && \ + # create a new environment for the serial version and activate it: + spack env create -d . && \ + spack env activate -d . && \ + # install the serial version of octopus: + # display specs of upcoming spack installation: + spack spec octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+arpack+cgal+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis && \ + # run the spack installation (adding it to the environment): + spack develop octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+arpack+cgal+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis && \ + spack add octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+arpack+cgal+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis && \ + spack install && \ + # run spack smoke tests for octopus. We get an error if any of the fails: + spack test run --alias test_serial octopus && \ + # display output from smoke tests (just for information): + spack test results -l test_serial && \ + # show which octopus version we use (for convenience): + spack load octopus && octopus --version && \ + # Run make checks + cd ~/dev-build-serial/octopus ; \ + echo "-+-+-+-+ Running make check-short" ; \ + make check-short | tee serial_check-short.log || /bin/true ; \ + echo "-+-+-+-+ Running make check-long" ; \ + make check-long | tee serial_check-long.log || /bin/true ; \ + echo "-+-+-+-+ Parsing serial_check-short.log" ; \ + python3 /home/user/check_buildlog.py serial_check-short.log foss2021a-serial; \ + echo "-+-+-+-+ Parsing serial_check-long.log" ; \ + python3 /home/user/check_buildlog.py serial_check-long.log foss2021a-serial; \ + # deactivate the environment. + spack env deactivate + +# # MPI version + +RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ + # create a new environment for the MPI version and activate it: + # make a dev-build direcotry for the parallel version: + mkdir ~/dev-build-parallel && cd ~/dev-build-parallel && \ + spack env create -d . && \ + spack env activate -d . && \ + # install the parallel version of octopus: + # display specs of upcoming spack installation: + spack spec octopus@${SPACK_OCT_VERSION} +mpi +netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ + # run the spack installation (adding it to the environment): + spack develop octopus@${SPACK_OCT_VERSION} +mpi +netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ + spack add octopus@${SPACK_OCT_VERSION} +mpi +netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ + spack install && \ + # run spack smoke tests for octopus. We get an error if any of the fails: + spack test run --alias test_MPI octopus && \ + # display output from smoke tests (just for information): + spack test results -l test_MPI && \ + # show which octopus version we use (for convenience): + spack load octopus && octopus --version && \ + # Run make checks + cd ~/dev-build-parallel/octopus ; \ + echo "+|+|+|+ Running make check-short" ; \ + make check-short | tee mpi_check-short.log || /bin/true ; \ + echo "+|+|+|+ Running make check-long" ; \ + make check-long | tee mpi_check-long.log || /bin/true ; \ + echo "+|+|+|+ Parsing mpi_check-short.log" ; \ + python3 /home/user/check_buildlog.py mpi_check-short.log foss2021a-mpi; \ + echo "+|+|+|+ Parsing mpi_check-long.log" ; \ + python3 /home/user/check_buildlog.py mpi_check-long.log foss2021a-mpi; \ + spack env deactivate + +# Provide bash in case the image is meant to be used interactively +CMD /bin/bash -l diff --git a/Dockerfile b/Dockerfile index 0f73552..a9c23a5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ FROM debian:bullseye # # but other strings can be given to the docker build command # # (for example docker build --build-arg SPACK_VERSION=v0.16.2) ARG SPACK_VERSION=develop -ARG OCT_VERSION=12.1 +ARG SPACK_OCT_VERSION=12.1 RUN echo "Building with spack version ${SPACK_VERSION}" # Any extra packages to be installed in the host @@ -68,9 +68,9 @@ RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env create octopus-serial && \ spack env activate octopus-serial && \ # display specs of upcoming spack installation: - spack spec octopus@${OCT_VERSION} ~mpi+netcdf+arpack+cgal+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis && \ + spack spec octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ # run the spack installation (adding it to the environment): - spack add octopus@${OCT_VERSION} ~mpi+netcdf+arpack+cgal+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis && \ + spack add octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ spack install && \ # run spack smoke tests for octopus. We get an error if any of the fails: spack test run --alias test_serial octopus && \ @@ -88,9 +88,9 @@ RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env create octopus-mpi && \ spack env activate octopus-mpi && \ # display specs of upcoming spack installation: - spack spec octopus@${OCT_VERSION} +mpi +netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ + spack spec octopus@${SPACK_OCT_VERSION} +mpi +netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ # run the spack installation (adding it to the environment): - spack add octopus@${OCT_VERSION} +mpi +netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ + spack add octopus@${SPACK_OCT_VERSION} +mpi +netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ spack install && \ # run spack smoke tests for octopus. We get an error if any of the fails: spack test run --alias test_MPI octopus && \ diff --git a/Makefile b/Makefile index baca8e6..9b5ceee 100644 --- a/Makefile +++ b/Makefile @@ -8,8 +8,10 @@ spack-latest: docker build -f Dockerfile --build-arg SPACK_VERSION=releases/latest -t octopus-spack . spack-latest-octopus-develop: - docker build -f Dockerfile --build-arg SPACK_VERSION=releases/latest --build-arg OCT_VERSION=develop -t octopus-spack . + docker build -f Dockerfile --build-arg SPACK_VERSION=releases/latest --build-arg SPACK_OCT_VERSION=develop -t octopus-spack . +spack-dev-build: + docker build -f Docker-dev-build --build-arg SPACK_VERSION=releases/latest --build-arg SPACK_OCT_VERSION=12.1 -t octopus-spack-dev-build . run-spack: docker run --rm -ti -v $PWD:/io octopus-spack diff --git a/check_buildlog.py b/check_buildlog.py new file mode 100644 index 0000000..5b788ef --- /dev/null +++ b/check_buildlog.py @@ -0,0 +1,88 @@ +# A simple script to check the log file of `make check or `make check-short` +# and report the number of failed tests. +# And if the failed tests are ignorable. +# (errors due to tight thresholds, for example) or needs attention. + +import sys +import re + +ignored_test_list = { + "foss2021a-mpi": [ + "finite_systems_2d/06-gdlib.test", + "finite_systems_3d/30-local_multipoles.test", + "periodic_systems/11-silicon_force.test", + ], + "foss2021a-serial": [ + "components/07-cholesky serial.test", + "finite_systems_2d/06-gdlib.test", + ], +} + + +def check_log(log_file): + with open(log_file, "r") as f: + logs = f.readlines() + # extract the last lines that has the format + + # |Status: 1 failures + # | Passed: 139 / 142 + # | Skipped: 0 / 142 + # | Failed: 3 / 142 + # | + # | testfile # failed testcases + # | ------------------------------------------------------------------------------ + # | finite_systems_2d/06-gdlib.test 1 + # | finite_systems_3d/30-local_multipoles.test 2 + # | periodic_systems/11-silicon_force.test 1 + # | + # |Total run-time of the testsuite: 03:27:12 + + # and extract the list of failed test files like + # finite_systems_2d/06-gdlib.test, + # finite_systems_3d/30-local_multipoles.test, + # periodic_systems/11-silicon_force.test + # from this example, by scanning from the end of the log file + # starting from the line that has the format + # `Total run-time of the testsuite:` + # and stop when the first line with the format + # `' -------------------------------------` + + test_lists = [] + reversed_logs = reversed(logs) + while not next(reversed_logs).startswith( + "Total run-time of the testsuite:" + ): + pass + while not re.match(r" +-+", (line := next(reversed_logs))): + if not line == "\n": # skip blank lines + test_lists.append(line.split()[0]) + + if not test_lists: + print("No failed tests") + sys.exit(0) + unexpected_failures = [ + test + for test in test_lists + if test not in ignored_test_list[used_toolchain] + ] + if unexpected_failures: + print("Found {} unexpected failures".format(len(unexpected_failures))) + print("Unexpected failures : \n" + ", \n".join(unexpected_failures)) + print("Built with tool chain : " + used_toolchain) + sys.exit(1) + print("All failed tests are ignored") + + +if __name__ == "__main__": + + try: + log_file = sys.argv[1] + except IndexError: + print("Usage: python3 check_buildlog.py log_file toolchain[optional]") + sys.exit(1) + try: + used_toolchain = sys.argv[2] + except IndexError: + print("No toolchain specified, assuming the default foss2021a-mpi") + used_toolchain = "foss2021a-mpi" + check_log(log_file)