Action with Pangea-3 installation reproduction and ppc64le emulation

.github/workflows/docker_build_tpls.yml

@@ -10,7 +10,6 @@ concurrency:
 jobs:
   build_images:
     name: ${{ matrix.name }}
-
     runs-on: ${{ matrix.RUNS_ON }}
 
     strategy:
@@ -22,46 +21,53 @@ jobs:
         include:
           - name: Ubuntu (20.04, gcc 9.4.0, open-mpi 4.0.3)
             DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc9
-            DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile
+            TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile
             DOCKER_ROOT_IMAGE: ubuntu:20.04
             DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=9"
             RUNS_ON: ubuntu-latest
           - name: Ubuntu (20.04, gcc 10.5.0, open-mpi 4.0.3) - github codespaces
             DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc10
-            DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile
+            TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile
             DOCKER_ROOT_IMAGE: mcr.microsoft.com/vscode/devcontainers/base:ubuntu-20.04
             DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=10"
             RUNS_ON: ubuntu-latest
           - name: Ubuntu (22.04, gcc 11.4.0, open-mpi 4.1.2)
             DOCKER_ROOT_IMAGE: ubuntu:22.04
             DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11
-            DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile
+            TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile
             DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=11"
             RUNS_ON: ubuntu-latest
           - name: Ubuntu (22.04, gcc 12.3.0, open-mpi 4.1.2)
             DOCKER_ROOT_IMAGE: ubuntu:22.04
             DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc12
-            DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile
+            TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile
             DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=12"
             RUNS_ON: ubuntu-latest
           - name: Ubuntu (22.04, clang 15.0.7 + gcc 11.4.0, open-mpi 4.1.2)
             DOCKER_ROOT_IMAGE: ubuntu:22.04
             DOCKER_REPOSITORY: geosx/ubuntu22.04-clang15
-            DOCKERFILE: docker/tpl-ubuntu-clang.Dockerfile
+            TPL_DOCKERFILE: docker/tpl-ubuntu-clang.Dockerfile
             DOCKER_COMPILER_BUILD_ARG: "--build-arg CLANG_MAJOR_VERSION=15 --build-arg GCC_MAJOR_VERSION=11"
             RUNS_ON: ubuntu-latest
           - name: Ubuntu (20.04, clang 10.0.0 + gcc 9.4.0, open-mpi 4.0.3, cuda-11.8.89)
             DOCKER_REPOSITORY: geosx/ubuntu20.04-clang10.0.0-cuda11.8.89
-            DOCKERFILE: docker/tpl-ubuntu-clang-cuda.Dockerfile
+            TPL_DOCKERFILE: docker/tpl-ubuntu-clang-cuda.Dockerfile
             RUNS_ON: ubuntu-latest
           - name: Rockylinux (8, gcc 8.5, cuda 12.5)
             DOCKER_REPOSITORY: geosx/rockylinux8-gcc8-cuda12.5
-            DOCKERFILE: docker/tpl-rockylinux-gcc-cuda-12.Dockerfile    
+            TPL_DOCKERFILE: docker/tpl-rockylinux-gcc-cuda-12.Dockerfile
             RUNS_ON: Runner_4core_16GB
           - name: Rockylinux (8, clang 17.0.6, cuda 12.5)
             DOCKER_REPOSITORY: geosx/rockylinux8-clang17-cuda12.5
-            DOCKERFILE: docker/tpl-rockylinux-clang-cuda-12.Dockerfile  
+            TPL_DOCKERFILE: docker/tpl-rockylinux-clang-cuda-12.Dockerfile
             RUNS_ON: Runner_4core_16GB
+          - name: Pangea 3 (AlmaLinux 8.8, gcc 9.4.0, open-mpi 4.1.2, cuda 11.5.0, openblas 0.3.10)
+            DOCKER_REPOSITORY: geosx/pangea3-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18
+            TPL_DOCKERFILE: docker/TotalEnergies/Pangea3.Dockerfile
+            HOST_CONFIG: host-configs/TOTAL/pangea3-gcc8.4.1-openmpi-4.1.2.cmake
+            DOCKER_ROOT_IMAGE: 7g8efcehpff/pangea-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18:4
+            HOST_ARCH: ppc64le
+            RUNS_ON: streak2
 
     steps:
     - name: Checkout
@@ -73,28 +79,42 @@ jobs:
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
 
-    - name: Login to DockerHub
-      uses: docker/login-action@v3
+    - name: Setup QEMU for ${{ matrix.HOST_ARCH }} emulation
+      if: ${{ matrix.HOST_ARCH }}
+      uses: docker/setup-qemu-action@v3
       with:
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
+        platforms: ${{ matrix.HOST_ARCH }}
 
     - name: Print environment
       run: printenv
 
-    - name: Run the docker build and push docker script
+    - name: Run the docker build docker script
       env:
-        DOCKERFILE: ${{ matrix.DOCKERFILE }}
+        TPL_DOCKERFILE: ${{ matrix.TPL_DOCKERFILE }}
         DOCKER_REPOSITORY: ${{ matrix.DOCKER_REPOSITORY }}
         DOCKER_COMPILER_BUILD_ARG: ${{ matrix.DOCKER_COMPILER_BUILD_ARG }}
         INSTALL_DIR_ROOT: ${{ matrix.INSTALL_DIR_ROOT || '/opt/GEOS' }}
         HOST_CONFIG: ${{ matrix.HOST_CONFIG || 'host-configs/environment.cmake'}}
         DOCKER_ROOT_IMAGE: ${{ matrix.DOCKER_ROOT_IMAGE || 'undefined' }}
-        PULL_REQUEST_NUMBER: ${{ github.event.number }}
-        BUILD_NUMBER: ${{ github.run_number }}
         COMMIT: ${{ github.sha }}
         BUILD_DIR: ${{ github.workspace }}
-      run: bash -x ./scripts/docker-build-and-push.sh
+        DOCKER_TAG: ${{ github.event.number }}-${{ github.run_number }}
+
+      run: bash -x ./scripts/docker-build.sh
+
+    - name: Login to DockerHub
+      uses: docker/login-action@v3
+      with:
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+        logout: ${{ matrix.RUNS_ON != 'streak2' }}
+
+    - name: Push docker image
+      env:
+        DOCKER_REPOSITORY: ${{ matrix.DOCKER_REPOSITORY }}
+        DOCKER_TAG: ${{ github.event.number }}-${{ github.run_number }}
+
+      run: docker push ${DOCKER_REPOSITORY}:${DOCKER_TAG}
 
   # Convenience job - passes when all docker images are built.
   check_that_all_images_built:

docker/TotalEnergies/Pangea3-base.Dockerfile

@@ -0,0 +1,191 @@
+# This Dockerfile is used to build a docker image reproducing the Pangea installation over a ppc64le architecture:
+# It is not directly callable by the TPL ci but the built image is.
+
+# syntax=docker/dockerfile:1
+FROM ppc64le/almalinux:8
+
+# Install other needed packages
+RUN dnf install -y \
+    # gcc deps \
+    libmpc-devel.ppc64le glibc-devel \
+    # mpirun deps  \
+    librdmacm hwloc \
+    git git-lfs \
+    python38-devel python38-numpy \
+    zlib-devel \
+    make \
+    bc \
+    file \
+    # Scotch deps \
+    bison \
+    flex \
+    # vtk deps \
+    patch && \
+    git-lfs install #&& alternatives --set python /usr/bin/python3
+
+# copy pangea tree for modules needed by TPLs and export environment variables
+
+## Temporary local variables needed buy several modules
+ARG MODULE_PATH="/data_local/sw"
+
+ARG SPACK_PATH="spack/0.17.0/opt/spack/linux-rhel8-power9le"
+
+ARG COMPILER="gcc"
+ARG DEFAULT_COMPILER_VER="8.4.1"
+ARG SPACK_COMPILER_VER=$DEFAULT_COMPILER_VER
+ARG MODULE_COMPILER_VER="9.4.0"
+
+ARG SPACK_COMPILER=$COMPILER-$SPACK_COMPILER_VER
+
+## liblustre
+COPY ./tarball/liblustreapi.so.1 /lib64/
+
+## CMake
+ADD ./tarball/cmake-*.tgz /
+
+### Environment variables to export
+ENV PATH="/data_local/appli_local/MTS/GEOSX/cmake/3.26.4/bin:${PATH}"
+
+## gcc
+ADD ./tarball/gcc-*.tgz /
+
+### Temporary local variables
+ARG GCC_VER=$COMPILER-$MODULE_COMPILER_VER
+ARG GCC_DIR="$GCC_VER-xe5cqnyajaqz75up3gflln5zlj2rue5v"
+
+### Environment variables to export
+ENV CPATH="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$GCC_DIR/include:${CPATH}" \
+    LD_LIBRARY_PATH="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$GCC_DIR/lib:\
+$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$GCC_DIR/lib64:${LD_LIBRARY_PATH}" \
+    LD_RUN_PATH="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$GCC_DIR/lib:\
+$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$GCC_DIR/lib64:${LD_RUN_PATH}" \
+    MANPATH="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$GCC_DIR/share/man:${MANPATH}" \
+    PATH="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$GCC_DIR/bin:${PATH}" \
+    CC=gcc \
+    CXX=g++ \
+    F77=gfortran \
+    F90=gfortran \
+    FC=gfortran \
+    GCC_ROOT=$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$GCC_DIR
+
+## ompi
+ADD ./tarball/ompi-*.tgz /
+
+### Temporary local variables
+ARG MPI_VER="4.1.2"
+ARG MPI_DIR="openmpi/$MPI_VER"
+ARG MPI_PREFIX="OMPI"
+
+### Environment variables to export
+ENV CPATH="$MODULE_PATH/$MPI_DIR/env/$SPACK_COMPILER/include:${CPATH}" \
+    LD_LIBRARY_PATH="$MODULE_PATH/$MPI_DIR/env/$SPACK_COMPILER/lib:${LD_LIBRARY_PATH}" \
+    MANPATH="$MODULE_PATH/$MPI_DIR/env/$SPACK_COMPILER/share/man:${MANPATH}" \
+    PATH="$MODULE_PATH/$MPI_DIR/env/$SPACK_COMPILER/bin:${PATH}" \
+    MPI_ROOT="$MODULE_PATH/$MPI_DIR/env/$SPACK_COMPILER" \
+    ${MPI_PREFIX}_CC="gcc" \
+    ${MPI_PREFIX}_CXX="g++" \
+    ${MPI_PREFIX}_F77="gfortran" \
+    ${MPI_PREFIX}_F90="gfortran" \
+    ${MPI_PREFIX}_FC="gfortran" \
+    OPAL_PREFIX="$MODULE_PATH/$MPI_DIR/env/$SPACK_COMPILER" \
+    OMPI_MCA_btl="self,vader,openib" \
+    OMPI_MCA_btl_openib_allow_ib="true" \
+    OMPI_MCA_btl_openib_warn_default_gid_prefix="0"
+
+## Cuda
+ADD ./tarball/cuda-*.tgz /
+
+### Temporary local variables
+ARG LIBCUDA_VER=450.156.00
+
+ARG CUDA_VER_MAJ="11"
+ARG CUDA_VER_MIN="5"
+ARG CUDA_VER_PATCH="0"
+ARG CUDA_VER="$CUDA_VER_MAJ.$CUDA_VER_MIN.$CUDA_VER_PATCH"
+ARG CUDA_DIR="cuda/$CUDA_VER"
+
+ARG CUPTI_DIR="$CUDA_DIR/extras/CUPTI"
+
+COPY ./tarball/libcuda.so.$LIBCUDA_VER /usr/lib64
+RUN ln -s /usr/lib64/libcuda.so.$LIBCUDA_VER /usr/lib64/libcuda.so.1
+
+### Environment variables to export
+ENV CPATH="$MODULE_PATH/$CUDA_DIR/include:\
+$MODULE_PATH/$CUPTI_DIR/include:${CPATH}" \
+    LIBRARY_PATH="$MODULE_PATH/$CUDA_DIR/lib64/stubs:\
+$MODULE_PATH/$CUDA_DIR/lib64:\
+$MODULE_PATH/$CUPTI_DIR/lib64:${LIBRARY_PATH}" \
+    LD_LIBRARY_PATH="$MODULE_PATH/$CUDA_DIR/lib64:\
+$MODULE_PATH/$CUPTI_DIR/lib64:${LD_LIBRARY_PATH}" \
+    LD_RUN_PATH="$MODULE_PATH/$CUDA_DIR/lib64:${LD_RUN_PATH}" \
+    MANPATH="$MODULE_PATH/$CUDA_DIR/doc/man:${MANPATH}" \
+    PATH="$MODULE_PATH/$CUDA_DIR/bin:\
+$MODULE_PATH/$CUDA_DIR/samples/bin/ppc64le/linux/release:${PATH}" \
+    CUBLAS_ROOT="$MODULE_PATH/$CUDA_DIR" \
+    CUDA_ROOT="$MODULE_PATH/$CUDA_DIR" \
+    CUDA_HOME="$MODULE_PATH/$CUDA_DIR" \
+    CUDA_PATH="$MODULE_PATH/$CUDA_DIR" \
+    CUDA_VERSION="$CUDA_VER_MAJ.$CUDA_VER_MIN" \
+    NVHPC_CUDA_HOME="$MODULE_PATH/$CUDA_DIR"
+
+## Openblas
+ADD ./tarball/openblas-*.tgz /
+
+### Temporary local variables
+ARG SPACK_COMPILER=$COMPILER-$MODULE_COMPILER_VER
+
+ARG BLAS_VER_MAJ="0"
+ARG BLAS_VER_MIN="3"
+ARG BLAS_VER_PATCH="18"
+ARG BLAS_DISTRIB="openblas"
+ARG BLAS_VER="$BLAS_VER_MAJ.$BLAS_VER_MIN.$BLAS_VER_PATCH"
+ARG BLAS_DIR="$BLAS_DISTRIB-$BLAS_VER-vk36pzksytuhylqesg4cca7667np5sjp"
+
+### Environment variables to export
+ENV LD_LIBRARY_PATH="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$BLAS_DIR/lib:${LD_LIBRARY_PATH}" \
+    LIBRARY_PATH="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$BLAS_DIR/lib:${LIBRARY_PATH}" \
+    PATH="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$BLAS_DIR/bin:${PATH}" \
+    OPENBLAS_ROOT="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$BLAS_DIR"
+
+## lsf
+ADD ./tarball/lsf-*.tgz /
+
+### Temporary local variables
+ARG GLIBC_DIR="linux3.10-glibc2.17-ppc64le"
+
+ARG LSF_VER_MAJ="10"
+ARG LSF_VER_MIN="1"
+ARG LSF_VER="$LSF_VER_MAJ.$LSF_VER_MIN"
+ARG LSF_DIR="lsf/$LSF_VER/$GLIBC_DIR"
+
+### Environment variables to export
+ENV LD_LIBRARY_PATH="$MODULE_PATH/$LSF_DIR/lib:${LD_LIBRARY_PATH}" \
+    PATH="$MODULE_PATH/$LSF_DIR/etc:$MODULE_PATH/$LSF_DIR/bin:${PATH}"
+
+# Uncomment the two following lines to test the TPLs build
+#
+#WORKDIR /root
+#RUN git clone --depth=1  https://github.com/GEOS-DEV/thirdPartyLibs.git && \
+#    cd thirdPartyLibs && git submodule init && git submodule update
+
+#RUN cd thirdPartyLibs && python3 scripts/config-build.py \
+#    --hostconfig=host-configs/TOTAL/pangea3-gcc8.4.1-openmpi-4.1.2.cmake \
+#    --buildtype=Release --installpath=/opt/tpls -DNUM_PROC=32 && \
+#    make -C build-pangea3-gcc8.4.1-openmpi-4.1.2-release -j && \
+#    cd .. && rm -rf thirdPartyLibs
+
+# Install tools needed by geos ci
+RUN dnf -y --enablerepo=powertools install \
+    ninja-build \
+    openssh-clients \
+    texlive \
+    graphviz \
+    libxml2
+
+# build sccache from source as prebuilt binary is not available for current archi / OS couple
+RUN dnf makecache --refresh && dnf -y install cargo openssl-devel
+RUN cargo install sccache --locked && mkdir -p /opt/sccache/ && cp -r /root/.cargo/bin /opt/sccache/
+RUN dnf remove -y cargo openssl-devel
+
+### Environment variables to export
+ENV SCCACHE=/opt/sccache/bin/sccache

docker/TotalEnergies/Pangea3.Dockerfile

@@ -0,0 +1,38 @@
+# Temporary local variables dedicated to the TPL build
+ARG TMP_DIR=/tmp
+ARG SRC_DIR=$TMP_DIR/thirdPartyLibs
+ARG BLD_DIR=$TMP_DIR/build
+
+# The docker base image has to be pangea3-almalinux8-*
+ARG DOCKER_ROOT_IMAGE
+
+FROM $DOCKER_ROOT_IMAGE as tpl_toolchain_intersect_geosx_toolchain
+ARG SRC_DIR
+ARG BLD_DIR
+
+# All the environment variables defined in this Dockerfile
+# (GEOSX_TPL_DIR but also compiler information like CC, CXX...)
+# are part of the image contract (otherwise ARG is used).
+# GEOSX use them so consider modifying their names with care.
+#
+# The installation directory is provided as a docker build argument.
+# We forward it using an environment variable.
+ARG INSTALL_DIR
+ENV GEOSX_TPL_DIR=$INSTALL_DIR
+
+# Get host config file from docker build arguments
+ARG HOST_CONFIG
+
+FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain
+# We now configure the build...
+RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/configure-tpl.sh
+# ... before we compile the TPLs!
+WORKDIR $BLD_DIR
+RUN --mount=src=.,dst=$SRC_DIR make
+
+# Extract only TPL's from previous stage
+FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain
+
+COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR
+
+ENV SCCACHE=/opt/sccache/bin/sccache

host-configs/TOTAL/pangea3-gcc8.4.1-openmpi-4.1.2.cmake

@@ -71,7 +71,7 @@ set(ENABLE_MATHPRESSO OFF CACHE BOOL "")
 # Silo configure script doesn't recognize systype
 set(SILO_BUILD_TYPE powerpc64-unknown-linux-gnu CACHE STRING "")
 
-set(GEOSX_BUILD_SHARED_LIBS ON CACHE BOOL "")
+set(GEOS_BUILD_SHARED_LIBS ON CACHE BOOL "")
 set(ENABLE_PAMELA ON CACHE BOOL "")
 set(ENABLE_PVTPackage ON CACHE BOOL "")
 

scripts/docker-build-and-push.sh → scripts/docker-build.sh

@@ -3,28 +3,26 @@ env
 
 # We save memory for the docker context
 echo .git > .dockerignore
-# This script will build and push a DOCKER_REPOSITORY:DOCKER_TAG image build from DOCKERFILE
+# This script will build an image from TPL_DOCKERFILE
 # with (optional) DOCKER_COMPILER_BUILD_ARG build arguments.
+# This image will be tagged with the DOCKER_REPOSITORY:DOCKER_TAG tag
 # A specific host-config file can be defined through variable HOST_CONFIG.
 # For the case of Total cluster only, DOCKER_ROOT_IMAGE is used to define docker base image.
 # Where the TPL are installed in the docker can be specified by parameter INSTALL_DIR.
-# Unlike DOCKER_TAG, these variables shall be defined by the "yaml derived classes" in a stage prior to `script` stage.
-DOCKER_TAG=${PULL_REQUEST_NUMBER}-${BUILD_NUMBER}
+# These variables shall be defined by the "yaml derived classes" in a stage prior to `script` stage.
 echo "Docker tag is ${DOCKER_REPOSITORY}:${DOCKER_TAG}"
 
-INSTALL_DIR=${INSTALL_DIR_ROOT}/GEOS_TPL-${PULL_REQUEST_NUMBER}-${BUILD_NUMBER}-${COMMIT:0:7}
+INSTALL_DIR=${INSTALL_DIR_ROOT}/GEOS_TPL-${DOCKER_TAG}-${COMMIT:0:7}
 echo "Installation directory is ${INSTALL_DIR}"
 
-docker build ${DOCKER_COMPILER_BUILD_ARG} \
+docker build --progress=plain ${DOCKER_COMPILER_BUILD_ARG} \
 --build-arg HOST_CONFIG=${HOST_CONFIG} \
 --build-arg DOCKER_ROOT_IMAGE=${DOCKER_ROOT_IMAGE} \
 --build-arg INSTALL_DIR=${INSTALL_DIR} \
 --tag ${DOCKER_REPOSITORY}:${DOCKER_TAG} \
---file ${DOCKERFILE} \
+--file ${TPL_DOCKERFILE} \
 --label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \
 --label "org.opencontainers.image.source=https://github.com/GEOS-DEV/thirdPartyLibs" \
 --label "org.opencontainers.image.revision=${COMMIT}" \
 --label "org.opencontainers.image.title=Building environment for GEOS" \
 .
-
-docker push ${DOCKER_REPOSITORY}:${DOCKER_TAG}