From 709498d13756c82bf3bde09078d6e9d049754361 Mon Sep 17 00:00:00 2001 From: iamashwin99 Date: Thu, 17 Nov 2022 16:33:35 +0100 Subject: [PATCH 01/19] add spack-dev-build target --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index baca8e6..3123c41 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,8 @@ spack-latest: spack-latest-octopus-develop: docker build -f Dockerfile --build-arg SPACK_VERSION=releases/latest --build-arg OCT_VERSION=develop -t octopus-spack . +spack-dev-build: + docker build -f Docker-dev-build --build-arg SPACK_VERSION=dev-build -t octopus-spack-dev-build . run-spack: docker run --rm -ti -v $PWD:/io octopus-spack From 5c9791e9811360f5305a4da5a1d5872728b9b55b Mon Sep 17 00:00:00 2001 From: iamashwin99 Date: Thu, 17 Nov 2022 16:33:52 +0100 Subject: [PATCH 02/19] add minimal docker file --- Docker-dev-build | 113 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 Docker-dev-build diff --git a/Docker-dev-build b/Docker-dev-build new file mode 100644 index 0000000..bef622b --- /dev/null +++ b/Docker-dev-build @@ -0,0 +1,113 @@ +FROM debian:bullseye + +# # which spack version are we using now? Default is develop +# # but other strings can be given to the docker build command +# # (for example docker build --build-arg SPACK_VERSION=v0.16.2) +ARG SPACK_VERSION=develop +ARG OCT_VERSION=12.1 +RUN echo "Building with spack version ${SPACK_VERSION}" + +# Any extra packages to be installed in the host +ARG EXTRA_PACKAGES +RUN echo "Installing EXTRA_PACKAGES ${EXTRA_PACKAGES} on container host" + +# general environment for docker +ENV SPACK_ROOT=/home/user/spack \ + SPACK=/home/user/spack/bin/spack \ + FORCE_UNSAFE_CONFIGURE=1 + +RUN apt-get -y update +# Convenience tools, if desired for debugging etc +# RUN apt-get -y install wget time nano vim emacs git + +# From https://github.com/ax3l/dockerfiles/blob/master/spack/base/Dockerfile: +# install minimal spack dependencies +RUN apt-get install -y --no-install-recommends \ + autoconf \ + build-essential \ + ca-certificates \ + coreutils \ + curl \ + environment-modules \ + file \ + gfortran \ + git \ + openssh-server \ + python \ + unzip \ + vim \ + && rm -rf /var/lib/apt/lists/* + +# load spack environment on login +RUN echo "source $SPACK_ROOT/share/spack/setup-env.sh" \ + > /etc/profile.d/spack.sh + +RUN adduser user +USER user +WORKDIR /home/user + +# install spack +RUN git clone https://github.com/spack/spack.git +# default branch is develop +RUN cd spack && git checkout $SPACK_VERSION + +# # show which version we use +RUN $SPACK --version + +# copy our package.py into the spack tree (and also example files) +COPY spack/package.py $SPACK_ROOT/var/spack/repos/builtin/packages/octopus/package.py +RUN ls -l $SPACK_ROOT/var/spack/repos/builtin/packages/octopus +COPY spack/test/ $SPACK_ROOT/var/spack/repos/builtin/packages/octopus/test +RUN ls -l $SPACK_ROOT/var/spack/repos/builtin/packages/octopus/test + +# Install and test serial and MPI versions of ocoptus via spack +# # serial version + +RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ + # create a new environment for the serial version and activate it: + spack env create octopus-serial && \ + spack env activate octopus-serial && \ + # make a dev-build direcotry for the serial version: + mkdir ~/dev-build-serial && cd ~/dev-build-serial && \ + # install the serial version of octopus: + # display specs of upcoming spack installation: + spack spec octopus@${OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ + # run the spack installation (adding it to the environment): + spack dev-build octopus@${OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ + make check-short && \ + make check && \ + # run spack smoke tests for octopus. We get an error if any of the fails: + spack test run --alias test_serial octopus && \ + # display output from smoke tests (just for information): + spack test results -l test_serial && \ + # show which octopus version we use (for convenience): + spack load octopus && octopus --version && \ + # deactivate the environment. + spack env deactivate + +# # MPI version + +RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ + # create a new environment for the MPI version and activate it: + spack env create octopus-mpi && \ + spack env activate octopus-mpi && \ + # make a dev-build direcotry for the parallel version: + mkdir ~/dev-build-parallel && cd ~/dev-build-parallel && \ + # display specs of upcoming spack installation: + spack spec octopus@${OCT_VERSION} +mpi +netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ + # run the spack installation (adding it to the environment): + spack dev-build octopus@${OCT_VERSION} +mpi +netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ + make check-short && \ + make check && \ + # run spack smoke tests for octopus. We get an error if any of the fails: + spack test run --alias test_MPI octopus && \ + # display output from smoke tests (just for information): + spack test results -l test_MPI && \ + # show which octopus version we use (for convenience): + spack load octopus && octopus --version && \ + # deactivate the environment. + spack env deactivate + +# Provide bash in case the image is meant to be used interactively +CMD /bin/bash -l + From 6f864e898e57e685fd6b78279ed7d8b181865a76 Mon Sep 17 00:00:00 2001 From: iamashwin99 Date: Thu, 17 Nov 2022 16:34:01 +0100 Subject: [PATCH 03/19] add ci on manual trigger --- .github/workflows/spack-dev-build.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/workflows/spack-dev-build.yml diff --git a/.github/workflows/spack-dev-build.yml b/.github/workflows/spack-dev-build.yml new file mode 100644 index 0000000..c49ee9a --- /dev/null +++ b/.github/workflows/spack-dev-build.yml @@ -0,0 +1,18 @@ +name: spack-dev-build + +on: + workflow_dispatch: + schedule: + - cron: '0 12 1 * *' # 12:00 UTC on the 1st of every month and on manual dispatch + +jobs: + + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Build the Docker image with make checks + run: make spack-dev-build + From 5d5da807d7b45fa3960e64f5b4fa201a17f32bda Mon Sep 17 00:00:00 2001 From: iamashwin99 Date: Thu, 17 Nov 2022 16:53:14 +0100 Subject: [PATCH 04/19] trigger CI and correct SPACK_VERSION --- .github/workflows/spack-dev-build.yml | 2 ++ Makefile | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/spack-dev-build.yml b/.github/workflows/spack-dev-build.yml index c49ee9a..77dc471 100644 --- a/.github/workflows/spack-dev-build.yml +++ b/.github/workflows/spack-dev-build.yml @@ -1,6 +1,8 @@ name: spack-dev-build on: + pull_request: + branches: [ main ] workflow_dispatch: schedule: - cron: '0 12 1 * *' # 12:00 UTC on the 1st of every month and on manual dispatch diff --git a/Makefile b/Makefile index 3123c41..9a1ff9d 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ spack-latest-octopus-develop: docker build -f Dockerfile --build-arg SPACK_VERSION=releases/latest --build-arg OCT_VERSION=develop -t octopus-spack . spack-dev-build: - docker build -f Docker-dev-build --build-arg SPACK_VERSION=dev-build -t octopus-spack-dev-build . + docker build -f Docker-dev-build --build-arg SPACK_VERSION=releases/latest -t octopus-spack-dev-build . run-spack: docker run --rm -ti -v $PWD:/io octopus-spack From ebc1d7d22a3d98ccd5782a68dc9213b5e272e446 Mon Sep 17 00:00:00 2001 From: iamashwin99 Date: Thu, 17 Nov 2022 19:28:56 +0100 Subject: [PATCH 05/19] add octopus source --- Docker-dev-build | 2 ++ Makefile | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Docker-dev-build b/Docker-dev-build index bef622b..91566c2 100644 --- a/Docker-dev-build +++ b/Docker-dev-build @@ -69,6 +69,7 @@ RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env activate octopus-serial && \ # make a dev-build direcotry for the serial version: mkdir ~/dev-build-serial && cd ~/dev-build-serial && \ + git clone https://gitlab.com/octopus-code/octopus . && \ # install the serial version of octopus: # display specs of upcoming spack installation: spack spec octopus@${OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ @@ -93,6 +94,7 @@ RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env activate octopus-mpi && \ # make a dev-build direcotry for the parallel version: mkdir ~/dev-build-parallel && cd ~/dev-build-parallel && \ + git clone https://gitlab.com/octopus-code/octopus . && \ # display specs of upcoming spack installation: spack spec octopus@${OCT_VERSION} +mpi +netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ # run the spack installation (adding it to the environment): diff --git a/Makefile b/Makefile index 9a1ff9d..e3948e4 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ spack-latest-octopus-develop: docker build -f Dockerfile --build-arg SPACK_VERSION=releases/latest --build-arg OCT_VERSION=develop -t octopus-spack . spack-dev-build: - docker build -f Docker-dev-build --build-arg SPACK_VERSION=releases/latest -t octopus-spack-dev-build . + docker build -f Docker-dev-build --build-arg SPACK_VERSION=releases/latest --build-arg OCT_VERSION=develop -t octopus-spack-dev-build . run-spack: docker run --rm -ti -v $PWD:/io octopus-spack From b733bc03a8738f30235c7c3d6d5a19be373bb3fa Mon Sep 17 00:00:00 2001 From: iamashwin99 Date: Fri, 18 Nov 2022 11:08:12 +0100 Subject: [PATCH 06/19] run smoke tests before make check, ignore errors --- Docker-dev-build | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Docker-dev-build b/Docker-dev-build index 91566c2..2a26b49 100644 --- a/Docker-dev-build +++ b/Docker-dev-build @@ -75,14 +75,16 @@ RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack spec octopus@${OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ # run the spack installation (adding it to the environment): spack dev-build octopus@${OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ - make check-short && \ - make check && \ # run spack smoke tests for octopus. We get an error if any of the fails: spack test run --alias test_serial octopus && \ # display output from smoke tests (just for information): spack test results -l test_serial && \ # show which octopus version we use (for convenience): spack load octopus && octopus --version && \ + # Run make checks + cd ~/dev-build-serial ; \ + make check-short || /bin/true ; \ + make check || /bin/true ; \ # deactivate the environment. spack env deactivate @@ -99,14 +101,16 @@ RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack spec octopus@${OCT_VERSION} +mpi +netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ # run the spack installation (adding it to the environment): spack dev-build octopus@${OCT_VERSION} +mpi +netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ - make check-short && \ - make check && \ # run spack smoke tests for octopus. We get an error if any of the fails: spack test run --alias test_MPI octopus && \ # display output from smoke tests (just for information): spack test results -l test_MPI && \ # show which octopus version we use (for convenience): spack load octopus && octopus --version && \ + # Run make checks + cd ~/dev-build-parallel ; \ + make check-short || /bin/true ; \ + make check || /bin/true ; \ # deactivate the environment. spack env deactivate From ccf2ce233d640d033bd77af37111853f75d80ed7 Mon Sep 17 00:00:00 2001 From: iamashwin99 Date: Fri, 18 Nov 2022 22:54:29 +0100 Subject: [PATCH 07/19] change name from OCT_VERSION to SPACK_OCT_VERSION --- Dockerfile | 10 +++++----- Makefile | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index d45e8b0..a9c23a5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ FROM debian:bullseye # # but other strings can be given to the docker build command # # (for example docker build --build-arg SPACK_VERSION=v0.16.2) ARG SPACK_VERSION=develop -ARG OCT_VERSION=12.1 +ARG SPACK_OCT_VERSION=12.1 RUN echo "Building with spack version ${SPACK_VERSION}" # Any extra packages to be installed in the host @@ -68,9 +68,9 @@ RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env create octopus-serial && \ spack env activate octopus-serial && \ # display specs of upcoming spack installation: - spack spec octopus@${OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ + spack spec octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ # run the spack installation (adding it to the environment): - spack add octopus@${OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ + spack add octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ spack install && \ # run spack smoke tests for octopus. We get an error if any of the fails: spack test run --alias test_serial octopus && \ @@ -88,9 +88,9 @@ RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env create octopus-mpi && \ spack env activate octopus-mpi && \ # display specs of upcoming spack installation: - spack spec octopus@${OCT_VERSION} +mpi +netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ + spack spec octopus@${SPACK_OCT_VERSION} +mpi +netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ # run the spack installation (adding it to the environment): - spack add octopus@${OCT_VERSION} +mpi +netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ + spack add octopus@${SPACK_OCT_VERSION} +mpi +netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ spack install && \ # run spack smoke tests for octopus. We get an error if any of the fails: spack test run --alias test_MPI octopus && \ diff --git a/Makefile b/Makefile index e3948e4..df7b852 100644 --- a/Makefile +++ b/Makefile @@ -8,10 +8,10 @@ spack-latest: docker build -f Dockerfile --build-arg SPACK_VERSION=releases/latest -t octopus-spack . spack-latest-octopus-develop: - docker build -f Dockerfile --build-arg SPACK_VERSION=releases/latest --build-arg OCT_VERSION=develop -t octopus-spack . + docker build -f Dockerfile --build-arg SPACK_VERSION=releases/latest --build-arg SPACK_OCT_VERSION=develop -t octopus-spack . spack-dev-build: - docker build -f Docker-dev-build --build-arg SPACK_VERSION=releases/latest --build-arg OCT_VERSION=develop -t octopus-spack-dev-build . + docker build -f Docker-dev-build --build-arg SPACK_VERSION=releases/latest --build-arg SPACK_OCT_VERSION=develop -t octopus-spack-dev-build . run-spack: docker run --rm -ti -v $PWD:/io octopus-spack From 1c2ab34db1691004a18c2fe822415ad49dedf4f7 Mon Sep 17 00:00:00 2001 From: iamashwin99 Date: Mon, 21 Nov 2022 10:02:35 +0100 Subject: [PATCH 08/19] switch to v12.1 for testing --- Docker-dev-build | 6 ++++-- Makefile | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Docker-dev-build b/Docker-dev-build index 2a26b49..ffd300c 100644 --- a/Docker-dev-build +++ b/Docker-dev-build @@ -69,7 +69,8 @@ RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env activate octopus-serial && \ # make a dev-build direcotry for the serial version: mkdir ~/dev-build-serial && cd ~/dev-build-serial && \ - git clone https://gitlab.com/octopus-code/octopus . && \ + wget https://octopus-code.org/down.php?file=12.1/octopus-12.1.tar.gz -O octopus-12.1.tar.gz &&\ + tar --strip-components=1 -xf octopus-12.1.tar.gz -C ./&& \ # install the serial version of octopus: # display specs of upcoming spack installation: spack spec octopus@${OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ @@ -96,7 +97,8 @@ RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env activate octopus-mpi && \ # make a dev-build direcotry for the parallel version: mkdir ~/dev-build-parallel && cd ~/dev-build-parallel && \ - git clone https://gitlab.com/octopus-code/octopus . && \ + wget https://octopus-code.org/down.php?file=12.1/octopus-12.1.tar.gz -O octopus-12.1.tar.gz &&\ + tar --strip-components=1 -xf octopus-12.1.tar.gz -C ./&& \ # display specs of upcoming spack installation: spack spec octopus@${OCT_VERSION} +mpi +netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ # run the spack installation (adding it to the environment): diff --git a/Makefile b/Makefile index df7b852..9b5ceee 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ spack-latest-octopus-develop: docker build -f Dockerfile --build-arg SPACK_VERSION=releases/latest --build-arg SPACK_OCT_VERSION=develop -t octopus-spack . spack-dev-build: - docker build -f Docker-dev-build --build-arg SPACK_VERSION=releases/latest --build-arg SPACK_OCT_VERSION=develop -t octopus-spack-dev-build . + docker build -f Docker-dev-build --build-arg SPACK_VERSION=releases/latest --build-arg SPACK_OCT_VERSION=12.1 -t octopus-spack-dev-build . run-spack: docker run --rm -ti -v $PWD:/io octopus-spack From 64ba6c1636702579bd7b29da88aebd80708b7bd0 Mon Sep 17 00:00:00 2001 From: iamashwin99 Date: Mon, 21 Nov 2022 10:11:06 +0100 Subject: [PATCH 09/19] replace make check with check-long --- Docker-dev-build | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Docker-dev-build b/Docker-dev-build index ffd300c..c2c963e 100644 --- a/Docker-dev-build +++ b/Docker-dev-build @@ -36,6 +36,7 @@ RUN apt-get install -y --no-install-recommends \ python \ unzip \ vim \ + wget \ && rm -rf /var/lib/apt/lists/* # load spack environment on login @@ -69,6 +70,7 @@ RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env activate octopus-serial && \ # make a dev-build direcotry for the serial version: mkdir ~/dev-build-serial && cd ~/dev-build-serial && \ + wget https://octopus-code.org/down.php?file=12.1/octopus-12.1.tar.gz -O octopus-12.1.tar.gz &&\ tar --strip-components=1 -xf octopus-12.1.tar.gz -C ./&& \ # install the serial version of octopus: @@ -85,7 +87,7 @@ RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ # Run make checks cd ~/dev-build-serial ; \ make check-short || /bin/true ; \ - make check || /bin/true ; \ + make check-long || /bin/true ; \ # deactivate the environment. spack env deactivate @@ -112,7 +114,7 @@ RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ # Run make checks cd ~/dev-build-parallel ; \ make check-short || /bin/true ; \ - make check || /bin/true ; \ + make check-long || /bin/true ; \ # deactivate the environment. spack env deactivate From e074dcacc69338ac1db52528efdc51c70012e9e7 Mon Sep 17 00:00:00 2001 From: iamashwin99 Date: Tue, 22 Nov 2022 15:17:41 +0100 Subject: [PATCH 10/19] parse make check to find ignorable errors --- Docker-dev-build | 24 +++++++++---- check_buildlog.py | 87 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 6 deletions(-) create mode 100644 check_buildlog.py diff --git a/Docker-dev-build b/Docker-dev-build index c2c963e..3400b53 100644 --- a/Docker-dev-build +++ b/Docker-dev-build @@ -60,6 +60,7 @@ COPY spack/package.py $SPACK_ROOT/var/spack/repos/builtin/packages/octopus/packa RUN ls -l $SPACK_ROOT/var/spack/repos/builtin/packages/octopus COPY spack/test/ $SPACK_ROOT/var/spack/repos/builtin/packages/octopus/test RUN ls -l $SPACK_ROOT/var/spack/repos/builtin/packages/octopus/test +COPY check_buildlog.py /home/user # Install and test serial and MPI versions of ocoptus via spack # # serial version @@ -70,7 +71,7 @@ RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env activate octopus-serial && \ # make a dev-build direcotry for the serial version: mkdir ~/dev-build-serial && cd ~/dev-build-serial && \ - + wget https://octopus-code.org/down.php?file=12.1/octopus-12.1.tar.gz -O octopus-12.1.tar.gz &&\ tar --strip-components=1 -xf octopus-12.1.tar.gz -C ./&& \ # install the serial version of octopus: @@ -86,8 +87,14 @@ RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack load octopus && octopus --version && \ # Run make checks cd ~/dev-build-serial ; \ - make check-short || /bin/true ; \ - make check-long || /bin/true ; \ + echo "-+-+-+-+ Running make check-short" ; \ + make check-short | tee serial_check-short.log || /bin/true ; \ + echo "-+-+-+-+ Running make check-long" ; \ + make check-long | tee serial_check-long.log || /bin/true ; \ + echo "-+-+-+-+ Parsing serial_check-short.log" ; \ + python3 /home/user/check_buildlog.py serial_check-short.log foss2021a-serial; \ + echo "-+-+-+-+ Parsing serial_check-long.log" ; \ + python3 /home/user/check_buildlog.py serial_check-long.log foss2021a-serial; \ # deactivate the environment. spack env deactivate @@ -113,9 +120,14 @@ RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack load octopus && octopus --version && \ # Run make checks cd ~/dev-build-parallel ; \ - make check-short || /bin/true ; \ - make check-long || /bin/true ; \ - # deactivate the environment. + echo "+|+|+|+ Running make check-short" ; \ + make check-short | tee mpi_check-short.log || /bin/true ; \ + echo "+|+|+|+ Running make check-long" ; \ + make check-long | tee mpi_check-long.log || /bin/true ; \ + echo "+|+|+|+ Parsing mpi_check-short.log" ; \ + python3 /home/user/check_buildlog.py mpi_check-short.log foss2021a-mpi; \ + echo "+|+|+|+ Parsing mpi_check-long.log" ; \ + python3 /home/user/check_buildlog.py mpi_check-long.log foss2021a-mpi; \ spack env deactivate # Provide bash in case the image is meant to be used interactively diff --git a/check_buildlog.py b/check_buildlog.py new file mode 100644 index 0000000..2e9c4f0 --- /dev/null +++ b/check_buildlog.py @@ -0,0 +1,87 @@ +# A simple script to check the log file of `make check or `make check-short` +# and report the number of failed tests. +# And if the failed tests are ignorable. +# (errors due to tight thresholds, for example) or needs attention. + +import sys +import re + +ignored_test_list = { + "foss2021a-mpi": [ + "finite_systems_2d/06-gdlib.test", + "finite_systems_3d/30-local_multipoles.test", + "periodic_systems/11-silicon_force.test", + ], + "foss2021a-serial": [ + "components/07-cholesky serial.test", + "finite_systems_2d/06-gdlib.test", + ], +} + + +def check_log(log_file): + with open(log_file, "r") as f: + logs = f.readlines() + # extract the last lines that has the format + + # |Status: 1 failures + # | Passed: 139 / 142 + # | Skipped: 0 / 142 + # | Failed: 3 / 142 + # | + # | testfile # failed testcases + # | ------------------------------------------------------------------------------ + # | finite_systems_2d/06-gdlib.test 1 + # | finite_systems_3d/30-local_multipoles.test 2 + # | periodic_systems/11-silicon_force.test 1 + # | + # |Total run-time of the testsuite: 03:27:12 + + # and extract the list of failed test files like + # finite_systems_2d/06-gdlib.test, + # finite_systems_3d/30-local_multipoles.test, + # periodic_systems/11-silicon_force.test + # from this example, by scanning from the end of the log file + # starting from the line that has the format + # `Total run-time of the testsuite:` + # and stop when the first line with the format + # `' -------------------------------------` + + test_lists = [] + reversed_logs = reversed(logs) + while not next(reversed_logs).startswith( + "Total run-time of the testsuite:" + ): + pass + while not re.match(r" +-+", (line := next(reversed_logs))): + if not line == "\n": # skip blank lines + test_lists.append(line.split()[0]) + + if not test_lists: + print("No failed tests") + sys.exit(0) + unexpected_failures = [ + test + for test in test_lists + if test not in ignored_test_list[used_toolchain] + ] + if unexpected_failures: + print("Unexpected failures : \n" + ", ".join(unexpected_failures)) + print("Built with tool chain : " + used_toolchain) + sys.exit(1) + print("All failed tests are ignored") + + +if __name__ == "__main__": + + try: + log_file = sys.argv[1] + except IndexError: + print("Usage: python3 check_buildlog.py log_file toolchain[optional]") + sys.exit(1) + try: + used_toolchain = sys.argv[2] + except IndexError: + print("No toolchain specified, assuming the default foss2021a-mpi") + used_toolchain = "foss2021a-mpi" + check_log(log_file) From e1691a7b6811659c4a96aa38a0c61e1500733b24 Mon Sep 17 00:00:00 2001 From: iamashwin99 Date: Thu, 24 Nov 2022 15:34:26 +0100 Subject: [PATCH 11/19] dont stop on failed builds --- Docker-dev-build | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Docker-dev-build b/Docker-dev-build index 3400b53..8a5f520 100644 --- a/Docker-dev-build +++ b/Docker-dev-build @@ -92,9 +92,9 @@ RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ echo "-+-+-+-+ Running make check-long" ; \ make check-long | tee serial_check-long.log || /bin/true ; \ echo "-+-+-+-+ Parsing serial_check-short.log" ; \ - python3 /home/user/check_buildlog.py serial_check-short.log foss2021a-serial; \ + python3 /home/user/check_buildlog.py serial_check-short.log foss2021a-serial || /bin/true; \ echo "-+-+-+-+ Parsing serial_check-long.log" ; \ - python3 /home/user/check_buildlog.py serial_check-long.log foss2021a-serial; \ + python3 /home/user/check_buildlog.py serial_check-long.log foss2021a-serial || /bin/true; \ # deactivate the environment. spack env deactivate @@ -125,9 +125,9 @@ RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ echo "+|+|+|+ Running make check-long" ; \ make check-long | tee mpi_check-long.log || /bin/true ; \ echo "+|+|+|+ Parsing mpi_check-short.log" ; \ - python3 /home/user/check_buildlog.py mpi_check-short.log foss2021a-mpi; \ + python3 /home/user/check_buildlog.py mpi_check-short.log foss2021a-mpi || /bin/true; \ echo "+|+|+|+ Parsing mpi_check-long.log" ; \ - python3 /home/user/check_buildlog.py mpi_check-long.log foss2021a-mpi; \ + python3 /home/user/check_buildlog.py mpi_check-long.log foss2021a-mpi || /bin/true; \ spack env deactivate # Provide bash in case the image is meant to be used interactively From 7e8d09aa53cfb4c9c6e05a447ac015c4cd7b9182 Mon Sep 17 00:00:00 2001 From: iamashwin99 Date: Fri, 25 Nov 2022 12:16:24 +0100 Subject: [PATCH 12/19] clone with feature.manyFiles=true --- Docker-dev-build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker-dev-build b/Docker-dev-build index 8a5f520..4d5c56c 100644 --- a/Docker-dev-build +++ b/Docker-dev-build @@ -48,7 +48,7 @@ USER user WORKDIR /home/user # install spack -RUN git clone https://github.com/spack/spack.git +RUN git clone -c feature.manyFiles=true https://github.com/spack/spack.git # default branch is develop RUN cd spack && git checkout $SPACK_VERSION From 23221bdd714780a8fcef232f28ecbe9d402a4f5d Mon Sep 17 00:00:00 2001 From: iamashwin99 Date: Fri, 25 Nov 2022 13:33:16 +0100 Subject: [PATCH 13/19] Use SPACK_OCT_VERSION instead of 12.1 --- Docker-dev-build | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Docker-dev-build b/Docker-dev-build index 4d5c56c..c425440 100644 --- a/Docker-dev-build +++ b/Docker-dev-build @@ -4,7 +4,7 @@ FROM debian:bullseye # # but other strings can be given to the docker build command # # (for example docker build --build-arg SPACK_VERSION=v0.16.2) ARG SPACK_VERSION=develop -ARG OCT_VERSION=12.1 +ARG SPACK_OCT_VERSION=12.1 RUN echo "Building with spack version ${SPACK_VERSION}" # Any extra packages to be installed in the host @@ -72,13 +72,13 @@ RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ # make a dev-build direcotry for the serial version: mkdir ~/dev-build-serial && cd ~/dev-build-serial && \ - wget https://octopus-code.org/down.php?file=12.1/octopus-12.1.tar.gz -O octopus-12.1.tar.gz &&\ - tar --strip-components=1 -xf octopus-12.1.tar.gz -C ./&& \ + wget https://octopus-code.org/down.php?file=${SPACK_OCT_VERSION}/octopus-${SPACK_OCT_VERSION}.tar.gz -O octopus-${SPACK_OCT_VERSION}.tar.gz &&\ + tar --strip-components=1 -xf octopus-${SPACK_OCT_VERSION}.tar.gz -C ./&& \ # install the serial version of octopus: # display specs of upcoming spack installation: - spack spec octopus@${OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ + spack spec octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ # run the spack installation (adding it to the environment): - spack dev-build octopus@${OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ + spack dev-build octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ # run spack smoke tests for octopus. We get an error if any of the fails: spack test run --alias test_serial octopus && \ # display output from smoke tests (just for information): @@ -106,12 +106,12 @@ RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env activate octopus-mpi && \ # make a dev-build direcotry for the parallel version: mkdir ~/dev-build-parallel && cd ~/dev-build-parallel && \ - wget https://octopus-code.org/down.php?file=12.1/octopus-12.1.tar.gz -O octopus-12.1.tar.gz &&\ - tar --strip-components=1 -xf octopus-12.1.tar.gz -C ./&& \ + wget https://octopus-code.org/down.php?file=${SPACK_OCT_VERSION}/octopus-${SPACK_OCT_VERSION}.tar.gz -O octopus-${SPACK_OCT_VERSION}.tar.gz &&\ + tar --strip-components=1 -xf octopus-${SPACK_OCT_VERSION}.tar.gz -C ./&& \ # display specs of upcoming spack installation: - spack spec octopus@${OCT_VERSION} +mpi +netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ + spack spec octopus@${SPACK_OCT_VERSION} +mpi +netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ # run the spack installation (adding it to the environment): - spack dev-build octopus@${OCT_VERSION} +mpi +netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ + spack dev-build octopus@${SPACK_OCT_VERSION} +mpi +netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ # run spack smoke tests for octopus. We get an error if any of the fails: spack test run --alias test_MPI octopus && \ # display output from smoke tests (just for information): From 1ab25fcaa01351ce88670ff4a8abade8ae574ac9 Mon Sep 17 00:00:00 2001 From: iamashwin99 Date: Tue, 29 Nov 2022 14:11:17 +0100 Subject: [PATCH 14/19] expand the serial and mpi runs --- Docker-dev-build | 115 +++++++++++++++++++++++++++++++---------------- 1 file changed, 77 insertions(+), 38 deletions(-) diff --git a/Docker-dev-build b/Docker-dev-build index c425440..3028899 100644 --- a/Docker-dev-build +++ b/Docker-dev-build @@ -65,71 +65,110 @@ COPY check_buildlog.py /home/user # Install and test serial and MPI versions of ocoptus via spack # # serial version +# create a new environment for the serial version and activate it: + # make a dev-build direcotry for the serial version: +RUN mkdir ~/dev-build-serial && cd ~/dev-build-serial && \ + wget https://octopus-code.org/down.php?file=${SPACK_OCT_VERSION}/octopus-${SPACK_OCT_VERSION}.tar.gz -O octopus-${SPACK_OCT_VERSION}.tar.gz &&\ + tar --strip-components=1 -xf octopus-${SPACK_OCT_VERSION}.tar.gz -C ./ + RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ - # create a new environment for the serial version and activate it: spack env create octopus-serial && \ spack env activate octopus-serial && \ - # make a dev-build direcotry for the serial version: - mkdir ~/dev-build-serial && cd ~/dev-build-serial && \ - - wget https://octopus-code.org/down.php?file=${SPACK_OCT_VERSION}/octopus-${SPACK_OCT_VERSION}.tar.gz -O octopus-${SPACK_OCT_VERSION}.tar.gz &&\ - tar --strip-components=1 -xf octopus-${SPACK_OCT_VERSION}.tar.gz -C ./&& \ # install the serial version of octopus: # display specs of upcoming spack installation: - spack spec octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ + spack spec octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack + +RUN $SPACK_ROOT/share/spack/setup-env.sh && \ + spack env activate octopus-serial && \ + cd ~/dev-build-serial && \ # run the spack installation (adding it to the environment): - spack dev-build octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ + spack dev-build octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack + +RUN $SPACK_ROOT/share/spack/setup-env.sh && \ + spack env activate octopus-serial && \ # run spack smoke tests for octopus. We get an error if any of the fails: spack test run --alias test_serial octopus && \ # display output from smoke tests (just for information): - spack test results -l test_serial && \ + spack test results -l test_serial + # show which octopus version we use (for convenience): - spack load octopus && octopus --version && \ +RUN $SPACK_ROOT/share/spack/setup-env.sh && \ + spack env activate octopus-serial && \ + spack load octopus && octopus --version + # Run make checks +RUN $SPACK_ROOT/share/spack/setup-env.sh && \ + spack env activate octopus-serial && \ cd ~/dev-build-serial ; \ echo "-+-+-+-+ Running make check-short" ; \ - make check-short | tee serial_check-short.log || /bin/true ; \ + make check-short | tee serial_check-short.log || /bin/true + +RUN $SPACK_ROOT/share/spack/setup-env.sh && \ + spack env activate octopus-serial && \ + cd ~/dev-build-serial ; \ echo "-+-+-+-+ Running make check-long" ; \ - make check-long | tee serial_check-long.log || /bin/true ; \ - echo "-+-+-+-+ Parsing serial_check-short.log" ; \ - python3 /home/user/check_buildlog.py serial_check-short.log foss2021a-serial || /bin/true; \ - echo "-+-+-+-+ Parsing serial_check-long.log" ; \ - python3 /home/user/check_buildlog.py serial_check-long.log foss2021a-serial || /bin/true; \ - # deactivate the environment. - spack env deactivate + make check-long | tee serial_check-long.log || /bin/true + + # Parse the log files and check if the tests failed are ignorable +RUN echo "-+-+-+-+ Parsing serial_check-short.log" ; \ + python3 /home/user/check_buildlog.py serial_check-short.log foss2021a-serial || /bin/true + +RUN echo "-+-+-+-+ Parsing serial_check-long.log" ; \ + python3 /home/user/check_buildlog.py serial_check-long.log foss2021a-serial || /bin/true + # # MPI version +# create a new environment for the mpi version and activate it: + # make a dev-build direcotry for the mpi version: +RUN mkdir ~/dev-build-mpi && cd ~/dev-build-mpi && \ + wget https://octopus-code.org/down.php?file=${SPACK_OCT_VERSION}/octopus-${SPACK_OCT_VERSION}.tar.gz -O octopus-${SPACK_OCT_VERSION}.tar.gz &&\ + tar --strip-components=1 -xf octopus-${SPACK_OCT_VERSION}.tar.gz -C ./ + RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ - # create a new environment for the MPI version and activate it: spack env create octopus-mpi && \ spack env activate octopus-mpi && \ - # make a dev-build direcotry for the parallel version: - mkdir ~/dev-build-parallel && cd ~/dev-build-parallel && \ - wget https://octopus-code.org/down.php?file=${SPACK_OCT_VERSION}/octopus-${SPACK_OCT_VERSION}.tar.gz -O octopus-${SPACK_OCT_VERSION}.tar.gz &&\ - tar --strip-components=1 -xf octopus-${SPACK_OCT_VERSION}.tar.gz -C ./&& \ + # install the mpi version of octopus: # display specs of upcoming spack installation: - spack spec octopus@${SPACK_OCT_VERSION} +mpi +netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ + spack spec octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack + +RUN $SPACK_ROOT/share/spack/setup-env.sh && \ + spack env activate octopus-mpi && \ + cd ~/dev-build-mpi && \ # run the spack installation (adding it to the environment): - spack dev-build octopus@${SPACK_OCT_VERSION} +mpi +netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ + spack dev-build octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack + +RUN $SPACK_ROOT/share/spack/setup-env.sh && \ + spack env activate octopus-mpi && \ # run spack smoke tests for octopus. We get an error if any of the fails: - spack test run --alias test_MPI octopus && \ + spack test run --alias test_mpi octopus && \ # display output from smoke tests (just for information): - spack test results -l test_MPI && \ + spack test results -l test_mpi + # show which octopus version we use (for convenience): - spack load octopus && octopus --version && \ +RUN $SPACK_ROOT/share/spack/setup-env.sh && \ + spack env activate octopus-mpi && \ + spack load octopus && octopus --version + # Run make checks - cd ~/dev-build-parallel ; \ - echo "+|+|+|+ Running make check-short" ; \ - make check-short | tee mpi_check-short.log || /bin/true ; \ - echo "+|+|+|+ Running make check-long" ; \ - make check-long | tee mpi_check-long.log || /bin/true ; \ - echo "+|+|+|+ Parsing mpi_check-short.log" ; \ - python3 /home/user/check_buildlog.py mpi_check-short.log foss2021a-mpi || /bin/true; \ - echo "+|+|+|+ Parsing mpi_check-long.log" ; \ - python3 /home/user/check_buildlog.py mpi_check-long.log foss2021a-mpi || /bin/true; \ - spack env deactivate +RUN $SPACK_ROOT/share/spack/setup-env.sh && \ + spack env activate octopus-mpi && \ + cd ~/dev-build-mpi ; \ + echo "|+|+|+|+| Running make check-short" ; \ + make check-short | tee mpi_check-short.log || /bin/true +RUN $SPACK_ROOT/share/spack/setup-env.sh && \ + spack env activate octopus-mpi && \ + cd ~/dev-build-mpi ; \ + echo "|+|+|+|+| Running make check-long" ; \ + make check-long | tee mpi_check-long.log || /bin/true + + # Parse the log files and check if the tests failed are ignorable +RUN echo "|+|+|+|+| Parsing mpi_check-short.log" ; \ + python3 /home/user/check_buildlog.py mpi_check-short.log foss2021a-mpi || /bin/true + +RUN echo "|+|+|+|+| Parsing mpi_check-long.log" ; \ + python3 /home/user/check_buildlog.py mpi_check-long.log foss2021a-mpi || /bin/true # Provide bash in case the image is meant to be used interactively CMD /bin/bash -l From d5de3dee2a4daf2b95d5f45f8cf1992876372b46 Mon Sep 17 00:00:00 2001 From: iamashwin99 Date: Tue, 29 Nov 2022 14:12:12 +0100 Subject: [PATCH 15/19] print each tests in new line --- check_buildlog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check_buildlog.py b/check_buildlog.py index 2e9c4f0..bac5ed6 100644 --- a/check_buildlog.py +++ b/check_buildlog.py @@ -66,7 +66,7 @@ def check_log(log_file): if test not in ignored_test_list[used_toolchain] ] if unexpected_failures: - print("Unexpected failures : \n" + ", ".join(unexpected_failures)) + print("Unexpected failures : \n" + ", \n".join(unexpected_failures)) print("Built with tool chain : " + used_toolchain) sys.exit(1) print("All failed tests are ignored") From 53acd062f1b6cc43b00e8abcce949ef20bef5ac2 Mon Sep 17 00:00:00 2001 From: iamashwin99 Date: Tue, 29 Nov 2022 14:13:01 +0100 Subject: [PATCH 16/19] list the number of failures --- check_buildlog.py | 1 + 1 file changed, 1 insertion(+) diff --git a/check_buildlog.py b/check_buildlog.py index bac5ed6..5b788ef 100644 --- a/check_buildlog.py +++ b/check_buildlog.py @@ -66,6 +66,7 @@ def check_log(log_file): if test not in ignored_test_list[used_toolchain] ] if unexpected_failures: + print("Found {} unexpected failures".format(len(unexpected_failures))) print("Unexpected failures : \n" + ", \n".join(unexpected_failures)) print("Built with tool chain : " + used_toolchain) sys.exit(1) From 757d588edd1079ddd7af00322190c40a6996f638 Mon Sep 17 00:00:00 2001 From: iamashwin99 Date: Tue, 29 Nov 2022 14:26:50 +0100 Subject: [PATCH 17/19] fix typo --- Docker-dev-build | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Docker-dev-build b/Docker-dev-build index 3028899..baaa3e6 100644 --- a/Docker-dev-build +++ b/Docker-dev-build @@ -78,13 +78,13 @@ RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ # display specs of upcoming spack installation: spack spec octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack -RUN $SPACK_ROOT/share/spack/setup-env.sh && \ +RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env activate octopus-serial && \ cd ~/dev-build-serial && \ # run the spack installation (adding it to the environment): spack dev-build octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack -RUN $SPACK_ROOT/share/spack/setup-env.sh && \ +RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env activate octopus-serial && \ # run spack smoke tests for octopus. We get an error if any of the fails: spack test run --alias test_serial octopus && \ @@ -92,18 +92,18 @@ RUN $SPACK_ROOT/share/spack/setup-env.sh && \ spack test results -l test_serial # show which octopus version we use (for convenience): -RUN $SPACK_ROOT/share/spack/setup-env.sh && \ +RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env activate octopus-serial && \ spack load octopus && octopus --version # Run make checks -RUN $SPACK_ROOT/share/spack/setup-env.sh && \ +RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env activate octopus-serial && \ cd ~/dev-build-serial ; \ echo "-+-+-+-+ Running make check-short" ; \ make check-short | tee serial_check-short.log || /bin/true -RUN $SPACK_ROOT/share/spack/setup-env.sh && \ +RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env activate octopus-serial && \ cd ~/dev-build-serial ; \ echo "-+-+-+-+ Running make check-long" ; \ @@ -132,13 +132,13 @@ RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ # display specs of upcoming spack installation: spack spec octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack -RUN $SPACK_ROOT/share/spack/setup-env.sh && \ +RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env activate octopus-mpi && \ cd ~/dev-build-mpi && \ # run the spack installation (adding it to the environment): spack dev-build octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack -RUN $SPACK_ROOT/share/spack/setup-env.sh && \ +RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env activate octopus-mpi && \ # run spack smoke tests for octopus. We get an error if any of the fails: spack test run --alias test_mpi octopus && \ @@ -146,18 +146,18 @@ RUN $SPACK_ROOT/share/spack/setup-env.sh && \ spack test results -l test_mpi # show which octopus version we use (for convenience): -RUN $SPACK_ROOT/share/spack/setup-env.sh && \ +RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env activate octopus-mpi && \ spack load octopus && octopus --version # Run make checks -RUN $SPACK_ROOT/share/spack/setup-env.sh && \ +RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env activate octopus-mpi && \ cd ~/dev-build-mpi ; \ echo "|+|+|+|+| Running make check-short" ; \ make check-short | tee mpi_check-short.log || /bin/true -RUN $SPACK_ROOT/share/spack/setup-env.sh && \ +RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env activate octopus-mpi && \ cd ~/dev-build-mpi ; \ echo "|+|+|+|+| Running make check-long" ; \ From 66711c6c5e800dae731a254f6e8dc2e21cd4d23a Mon Sep 17 00:00:00 2001 From: iamashwin99 Date: Tue, 29 Nov 2022 15:00:36 +0100 Subject: [PATCH 18/19] add steps marker --- Docker-dev-build | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/Docker-dev-build b/Docker-dev-build index baaa3e6..06c1fb5 100644 --- a/Docker-dev-build +++ b/Docker-dev-build @@ -1,27 +1,36 @@ +# STEP 1 FROM debian:bullseye # # which spack version are we using now? Default is develop # # but other strings can be given to the docker build command # # (for example docker build --build-arg SPACK_VERSION=v0.16.2) +# STEP 2 ARG SPACK_VERSION=develop +# STEP 3 ARG SPACK_OCT_VERSION=12.1 +# STEP 4 RUN echo "Building with spack version ${SPACK_VERSION}" # Any extra packages to be installed in the host +# STEP 5 ARG EXTRA_PACKAGES +# STEP 6 RUN echo "Installing EXTRA_PACKAGES ${EXTRA_PACKAGES} on container host" # general environment for docker +# STEP 7 ENV SPACK_ROOT=/home/user/spack \ SPACK=/home/user/spack/bin/spack \ FORCE_UNSAFE_CONFIGURE=1 +# STEP 8 RUN apt-get -y update # Convenience tools, if desired for debugging etc # RUN apt-get -y install wget time nano vim emacs git # From https://github.com/ax3l/dockerfiles/blob/master/spack/base/Dockerfile: # install minimal spack dependencies +# STEP 9 RUN apt-get install -y --no-install-recommends \ autoconf \ build-essential \ @@ -40,26 +49,38 @@ RUN apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* # load spack environment on login +# STEP 10 RUN echo "source $SPACK_ROOT/share/spack/setup-env.sh" \ > /etc/profile.d/spack.sh +# STEP 11 RUN adduser user +# STEP 12 USER user +# STEP 13 WORKDIR /home/user # install spack +# STEP 14 RUN git clone -c feature.manyFiles=true https://github.com/spack/spack.git # default branch is develop +# STEP 15 RUN cd spack && git checkout $SPACK_VERSION # # show which version we use +# STEP 16 RUN $SPACK --version # copy our package.py into the spack tree (and also example files) +# STEP 17 COPY spack/package.py $SPACK_ROOT/var/spack/repos/builtin/packages/octopus/package.py +# STEP 18 RUN ls -l $SPACK_ROOT/var/spack/repos/builtin/packages/octopus +# STEP 19 COPY spack/test/ $SPACK_ROOT/var/spack/repos/builtin/packages/octopus/test +# STEP 20 RUN ls -l $SPACK_ROOT/var/spack/repos/builtin/packages/octopus/test +# STEP 21 COPY check_buildlog.py /home/user # Install and test serial and MPI versions of ocoptus via spack @@ -67,10 +88,12 @@ COPY check_buildlog.py /home/user # create a new environment for the serial version and activate it: # make a dev-build direcotry for the serial version: +# STEP 22 RUN mkdir ~/dev-build-serial && cd ~/dev-build-serial && \ wget https://octopus-code.org/down.php?file=${SPACK_OCT_VERSION}/octopus-${SPACK_OCT_VERSION}.tar.gz -O octopus-${SPACK_OCT_VERSION}.tar.gz &&\ tar --strip-components=1 -xf octopus-${SPACK_OCT_VERSION}.tar.gz -C ./ +# STEP 23 RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env create octopus-serial && \ spack env activate octopus-serial && \ @@ -78,12 +101,14 @@ RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ # display specs of upcoming spack installation: spack spec octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack +# STEP 24 RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env activate octopus-serial && \ cd ~/dev-build-serial && \ # run the spack installation (adding it to the environment): spack dev-build octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack +# STEP 25 RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env activate octopus-serial && \ # run spack smoke tests for octopus. We get an error if any of the fails: @@ -92,17 +117,20 @@ RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack test results -l test_serial # show which octopus version we use (for convenience): +# STEP 26 RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env activate octopus-serial && \ spack load octopus && octopus --version # Run make checks +# STEP 27 RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env activate octopus-serial && \ cd ~/dev-build-serial ; \ echo "-+-+-+-+ Running make check-short" ; \ make check-short | tee serial_check-short.log || /bin/true +# STEP 28 RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env activate octopus-serial && \ cd ~/dev-build-serial ; \ @@ -110,9 +138,11 @@ RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ make check-long | tee serial_check-long.log || /bin/true # Parse the log files and check if the tests failed are ignorable +# STEP 29 RUN echo "-+-+-+-+ Parsing serial_check-short.log" ; \ python3 /home/user/check_buildlog.py serial_check-short.log foss2021a-serial || /bin/true +# STEP 30 RUN echo "-+-+-+-+ Parsing serial_check-long.log" ; \ python3 /home/user/check_buildlog.py serial_check-long.log foss2021a-serial || /bin/true @@ -121,10 +151,12 @@ RUN echo "-+-+-+-+ Parsing serial_check-long.log" ; \ # create a new environment for the mpi version and activate it: # make a dev-build direcotry for the mpi version: +# STEP 31 RUN mkdir ~/dev-build-mpi && cd ~/dev-build-mpi && \ wget https://octopus-code.org/down.php?file=${SPACK_OCT_VERSION}/octopus-${SPACK_OCT_VERSION}.tar.gz -O octopus-${SPACK_OCT_VERSION}.tar.gz &&\ tar --strip-components=1 -xf octopus-${SPACK_OCT_VERSION}.tar.gz -C ./ +# STEP 32 RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env create octopus-mpi && \ spack env activate octopus-mpi && \ @@ -132,12 +164,14 @@ RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ # display specs of upcoming spack installation: spack spec octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack +# STEP 33 RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env activate octopus-mpi && \ cd ~/dev-build-mpi && \ # run the spack installation (adding it to the environment): spack dev-build octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack +# STEP 34 RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env activate octopus-mpi && \ # run spack smoke tests for octopus. We get an error if any of the fails: @@ -146,17 +180,20 @@ RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack test results -l test_mpi # show which octopus version we use (for convenience): +# STEP 35 RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env activate octopus-mpi && \ spack load octopus && octopus --version # Run make checks +# STEP 36 RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env activate octopus-mpi && \ cd ~/dev-build-mpi ; \ echo "|+|+|+|+| Running make check-short" ; \ make check-short | tee mpi_check-short.log || /bin/true +# STEP 37 RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ spack env activate octopus-mpi && \ cd ~/dev-build-mpi ; \ @@ -164,11 +201,14 @@ RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ make check-long | tee mpi_check-long.log || /bin/true # Parse the log files and check if the tests failed are ignorable +# STEP 38 RUN echo "|+|+|+|+| Parsing mpi_check-short.log" ; \ python3 /home/user/check_buildlog.py mpi_check-short.log foss2021a-mpi || /bin/true +# STEP 39 RUN echo "|+|+|+|+| Parsing mpi_check-long.log" ; \ python3 /home/user/check_buildlog.py mpi_check-long.log foss2021a-mpi || /bin/true # Provide bash in case the image is meant to be used interactively +# STEP 40 CMD /bin/bash -l From ee893cff2d8bd0a0246d3655887ab117b6edb003 Mon Sep 17 00:00:00 2001 From: iamashwin99 Date: Wed, 8 Mar 2023 13:53:04 +0100 Subject: [PATCH 19/19] update devbuild with develop command --- Docker-dev-build | 168 +++++++++++++---------------------------------- 1 file changed, 44 insertions(+), 124 deletions(-) diff --git a/Docker-dev-build b/Docker-dev-build index 06c1fb5..d884a21 100644 --- a/Docker-dev-build +++ b/Docker-dev-build @@ -1,36 +1,27 @@ -# STEP 1 FROM debian:bullseye # # which spack version are we using now? Default is develop # # but other strings can be given to the docker build command # # (for example docker build --build-arg SPACK_VERSION=v0.16.2) -# STEP 2 ARG SPACK_VERSION=develop -# STEP 3 ARG SPACK_OCT_VERSION=12.1 -# STEP 4 RUN echo "Building with spack version ${SPACK_VERSION}" # Any extra packages to be installed in the host -# STEP 5 ARG EXTRA_PACKAGES -# STEP 6 RUN echo "Installing EXTRA_PACKAGES ${EXTRA_PACKAGES} on container host" # general environment for docker -# STEP 7 ENV SPACK_ROOT=/home/user/spack \ SPACK=/home/user/spack/bin/spack \ FORCE_UNSAFE_CONFIGURE=1 -# STEP 8 RUN apt-get -y update # Convenience tools, if desired for debugging etc # RUN apt-get -y install wget time nano vim emacs git # From https://github.com/ax3l/dockerfiles/blob/master/spack/base/Dockerfile: # install minimal spack dependencies -# STEP 9 RUN apt-get install -y --no-install-recommends \ autoconf \ build-essential \ @@ -49,166 +40,95 @@ RUN apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* # load spack environment on login -# STEP 10 RUN echo "source $SPACK_ROOT/share/spack/setup-env.sh" \ > /etc/profile.d/spack.sh -# STEP 11 RUN adduser user -# STEP 12 USER user -# STEP 13 WORKDIR /home/user # install spack -# STEP 14 -RUN git clone -c feature.manyFiles=true https://github.com/spack/spack.git +RUN git clone https://github.com/spack/spack.git # default branch is develop -# STEP 15 RUN cd spack && git checkout $SPACK_VERSION # # show which version we use -# STEP 16 RUN $SPACK --version # copy our package.py into the spack tree (and also example files) -# STEP 17 COPY spack/package.py $SPACK_ROOT/var/spack/repos/builtin/packages/octopus/package.py -# STEP 18 RUN ls -l $SPACK_ROOT/var/spack/repos/builtin/packages/octopus -# STEP 19 COPY spack/test/ $SPACK_ROOT/var/spack/repos/builtin/packages/octopus/test -# STEP 20 RUN ls -l $SPACK_ROOT/var/spack/repos/builtin/packages/octopus/test -# STEP 21 COPY check_buildlog.py /home/user # Install and test serial and MPI versions of ocoptus via spack # # serial version -# create a new environment for the serial version and activate it: - # make a dev-build direcotry for the serial version: -# STEP 22 -RUN mkdir ~/dev-build-serial && cd ~/dev-build-serial && \ - wget https://octopus-code.org/down.php?file=${SPACK_OCT_VERSION}/octopus-${SPACK_OCT_VERSION}.tar.gz -O octopus-${SPACK_OCT_VERSION}.tar.gz &&\ - tar --strip-components=1 -xf octopus-${SPACK_OCT_VERSION}.tar.gz -C ./ - -# STEP 23 RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ - spack env create octopus-serial && \ - spack env activate octopus-serial && \ + # make a dev-build direcotry for the serial version: + mkdir ~/dev-build-serial && cd ~/dev-build-serial && \ + # create a new environment for the serial version and activate it: + spack env create -d . && \ + spack env activate -d . && \ # install the serial version of octopus: # display specs of upcoming spack installation: - spack spec octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack - -# STEP 24 -RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ - spack env activate octopus-serial && \ - cd ~/dev-build-serial && \ + spack spec octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+arpack+cgal+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis && \ # run the spack installation (adding it to the environment): - spack dev-build octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack - -# STEP 25 -RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ - spack env activate octopus-serial && \ + spack develop octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+arpack+cgal+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis && \ + spack add octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+arpack+cgal+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis && \ + spack install && \ # run spack smoke tests for octopus. We get an error if any of the fails: spack test run --alias test_serial octopus && \ # display output from smoke tests (just for information): - spack test results -l test_serial - + spack test results -l test_serial && \ # show which octopus version we use (for convenience): -# STEP 26 -RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ - spack env activate octopus-serial && \ - spack load octopus && octopus --version - + spack load octopus && octopus --version && \ # Run make checks -# STEP 27 -RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ - spack env activate octopus-serial && \ - cd ~/dev-build-serial ; \ + cd ~/dev-build-serial/octopus ; \ echo "-+-+-+-+ Running make check-short" ; \ - make check-short | tee serial_check-short.log || /bin/true - -# STEP 28 -RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ - spack env activate octopus-serial && \ - cd ~/dev-build-serial ; \ + make check-short | tee serial_check-short.log || /bin/true ; \ echo "-+-+-+-+ Running make check-long" ; \ - make check-long | tee serial_check-long.log || /bin/true - - # Parse the log files and check if the tests failed are ignorable -# STEP 29 -RUN echo "-+-+-+-+ Parsing serial_check-short.log" ; \ - python3 /home/user/check_buildlog.py serial_check-short.log foss2021a-serial || /bin/true - -# STEP 30 -RUN echo "-+-+-+-+ Parsing serial_check-long.log" ; \ - python3 /home/user/check_buildlog.py serial_check-long.log foss2021a-serial || /bin/true - + make check-long | tee serial_check-long.log || /bin/true ; \ + echo "-+-+-+-+ Parsing serial_check-short.log" ; \ + python3 /home/user/check_buildlog.py serial_check-short.log foss2021a-serial; \ + echo "-+-+-+-+ Parsing serial_check-long.log" ; \ + python3 /home/user/check_buildlog.py serial_check-long.log foss2021a-serial; \ + # deactivate the environment. + spack env deactivate # # MPI version -# create a new environment for the mpi version and activate it: - # make a dev-build direcotry for the mpi version: -# STEP 31 -RUN mkdir ~/dev-build-mpi && cd ~/dev-build-mpi && \ - wget https://octopus-code.org/down.php?file=${SPACK_OCT_VERSION}/octopus-${SPACK_OCT_VERSION}.tar.gz -O octopus-${SPACK_OCT_VERSION}.tar.gz &&\ - tar --strip-components=1 -xf octopus-${SPACK_OCT_VERSION}.tar.gz -C ./ - -# STEP 32 RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ - spack env create octopus-mpi && \ - spack env activate octopus-mpi && \ - # install the mpi version of octopus: + # create a new environment for the MPI version and activate it: + # make a dev-build direcotry for the parallel version: + mkdir ~/dev-build-parallel && cd ~/dev-build-parallel && \ + spack env create -d . && \ + spack env activate -d . && \ + # install the parallel version of octopus: # display specs of upcoming spack installation: - spack spec octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack - -# STEP 33 -RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ - spack env activate octopus-mpi && \ - cd ~/dev-build-mpi && \ + spack spec octopus@${SPACK_OCT_VERSION} +mpi +netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ # run the spack installation (adding it to the environment): - spack dev-build octopus@${SPACK_OCT_VERSION} ~mpi+netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack - -# STEP 34 -RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ - spack env activate octopus-mpi && \ + spack develop octopus@${SPACK_OCT_VERSION} +mpi +netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ + spack add octopus@${SPACK_OCT_VERSION} +mpi +netcdf+parmetis+arpack+cgal+pfft+python+likwid+libyaml+elpa+nlopt~debug~cuda~metis~scalapack && \ + spack install && \ # run spack smoke tests for octopus. We get an error if any of the fails: - spack test run --alias test_mpi octopus && \ + spack test run --alias test_MPI octopus && \ # display output from smoke tests (just for information): - spack test results -l test_mpi - + spack test results -l test_MPI && \ # show which octopus version we use (for convenience): -# STEP 35 -RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ - spack env activate octopus-mpi && \ - spack load octopus && octopus --version - + spack load octopus && octopus --version && \ # Run make checks -# STEP 36 -RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ - spack env activate octopus-mpi && \ - cd ~/dev-build-mpi ; \ - echo "|+|+|+|+| Running make check-short" ; \ - make check-short | tee mpi_check-short.log || /bin/true + cd ~/dev-build-parallel/octopus ; \ + echo "+|+|+|+ Running make check-short" ; \ + make check-short | tee mpi_check-short.log || /bin/true ; \ + echo "+|+|+|+ Running make check-long" ; \ + make check-long | tee mpi_check-long.log || /bin/true ; \ + echo "+|+|+|+ Parsing mpi_check-short.log" ; \ + python3 /home/user/check_buildlog.py mpi_check-short.log foss2021a-mpi; \ + echo "+|+|+|+ Parsing mpi_check-long.log" ; \ + python3 /home/user/check_buildlog.py mpi_check-long.log foss2021a-mpi; \ + spack env deactivate -# STEP 37 -RUN . $SPACK_ROOT/share/spack/setup-env.sh && \ - spack env activate octopus-mpi && \ - cd ~/dev-build-mpi ; \ - echo "|+|+|+|+| Running make check-long" ; \ - make check-long | tee mpi_check-long.log || /bin/true - - # Parse the log files and check if the tests failed are ignorable -# STEP 38 -RUN echo "|+|+|+|+| Parsing mpi_check-short.log" ; \ - python3 /home/user/check_buildlog.py mpi_check-short.log foss2021a-mpi || /bin/true - -# STEP 39 -RUN echo "|+|+|+|+| Parsing mpi_check-long.log" ; \ - python3 /home/user/check_buildlog.py mpi_check-long.log foss2021a-mpi || /bin/true # Provide bash in case the image is meant to be used interactively -# STEP 40 CMD /bin/bash -l -