Feature/oneapi

.github/workflows/build-push-containers._yml

@@ -7,7 +7,7 @@ name:  Build and Push Images (Docker)
 on:
   # run it on push to the default repository branch
   push:
-    branches: [main]
+    branches: [feature/oneapi]
   # run it during pull request
   pull_request:
 
@@ -27,42 +27,60 @@ jobs:
       - name: Login to DockerHub Registry
         run: echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin    
 
-      - name: docker build tag noaaepic/ubuntu20.04-hpc-stack
+      - name: docker build tag noaaepic/ubuntu20.04-base
         run: | 
          echo ${{ steps.abc.outputs.all }}
           for file in ${{ steps.abc.outputs.all }}; do
-            if [[ $file == *"ubuntu20.04-gnu9.3-hpc-stack.docker"* ]]; then
-                echo "docker build tag noaaepic/ubuntu20.04-hpc-stack"
-                docker build -t noaaepic/ubuntu20.04-hpc-stack:0.1 -f ./docker/ubuntu20.04-gnu9.3-hpc-stack.docker .
+            if [[ $file == *"Dockerfile.ubuntu20.04-base"* ]]; then
+                echo "docker build tag noaaepic/ubuntu20.04-base"
+                docker build -t noaaepic/ubuntu20.04-base:1.0 -f ./Docker/Dockerfile.ubuntu20.04-base .
             fi           
           done
-      - name: docker push tag noaaepic/ubuntu20.04-hpc-stack
+      - name: docker push tag noaaepic/ubuntu20.04-base
         run: | 
           echo ${{ steps.abc.outputs.all }}
           for file in ${{ steps.abc.outputs.all }}; do
-            if [[ $file == *"ubuntu20.04-gnu9.3-hpc-stack.docker"* ]]; then
-                echo "docker push tag noaaepic/ubuntu20.04-hpc-stack:0.1"
-                docker push noaaepic/ubuntu20.04-hpc-stack:0.1
+            if [[ $file == *"Dockerfile.ubuntu20.04-base"* ]]; then
+                echo "docker push tag noaaepic/ubuntu20.04-base:1.0"
+                docker push noaaepic/ubuntu20.04-base:1.0
             fi
           done
-      - name: docker build tag noaaepic/ubuntu20.04-epic-srwapp
+      - name: docker build tag noaaepic/ubuntu20.04-spack
+        run: | 
+         echo ${{ steps.abc.outputs.all }}
+          for file in ${{ steps.abc.outputs.all }}; do
+            if [[ $file == *"Dockerfile.ubuntu20.04-spack"* ]]; then
+                echo "docker build tag noaaepic/ubuntu20.04-spack"
+                docker build -t noaaepic/ubuntu20.04-spack:1.0 -f ./Docker/Dockerfile.ubuntu20.04-spack .
+            fi           
+          done
+      - name: docker push tag noaaepic/ubuntu20.04-spack
         run: | 
           echo ${{ steps.abc.outputs.all }}
           for file in ${{ steps.abc.outputs.all }}; do
-            if [[ $file == *"ubuntu20.04-ufs-srwapp.docker"* ]]; then
-                echo "docker build tag noaaepic/ubuntu20.04-epic-srwapp"
-                docker build -t noaaepic/ubuntu20.04-epic-srwapp:1.0 -f ./docker/ubuntu20.04-ufs-srwapp.docker .
-            fi            
+            if [[ $file == *"Dockerfile.ubuntu20.04-spack"* ]]; then
+                echo "docker push tag noaaepic/ubuntu20.04-spack:1.0"
+                docker push noaaepic/ubuntu20.04-spack:1.0
+            fi
           done
-      - name: docker push tag noaaepic/ubuntu20.04-epic-srwapp
+      - name: docker build tag noaaepic/ubuntu20.04-intel22-ufs-srwapp
+        run: | 
+         echo ${{ steps.abc.outputs.all }}
+          for file in ${{ steps.abc.outputs.all }}; do
+            if [[ $file == *"Dockerfile.ubuntu20.04-intel22-ufs-srwapp"* ]]; then
+                echo "docker build tag noaaepic/ubuntu20.04-intel22-ufs-srwapp"
+                docker build -t noaaepic/ubuntu20.04-intel22-ufs-srwapp:1.0 -f ./Docker/Dockerfile.ubuntu20.04-intel22-ufs-srwapp .
+            fi           
+          done
+      - name: docker push tag noaaepic/ubuntu20.04-intel22-ufs-srwapp
         run: | 
           echo ${{ steps.abc.outputs.all }}
           for file in ${{ steps.abc.outputs.all }}; do
-            if [[ $file == *"ubuntu20.04-ufs-srwapp.docker"* ]]; then
-                echo "docker push tag noaaepic/ubuntu20.04-epic-srwapp:1.0"
-                docker push noaaepic/ubuntu20.04-epic-srwapp:1.0
-            fi            
-          done     
+            if [[ $file == *"Dockerfile.ubuntu20.04-intel22-ufs-srwapp"* ]]; then
+                echo "docker push tag noaaepic/ubuntu20.04-intel22-ufs-srwapp:1.0"
+                docker push noaaepic/ubuntu20.04-intel22-ufs-srwapp:1.0
+            fi
+          done
 
       - name: docker logout
         run: |

Docker/Dockerfile.ubuntu20.04-base

@@ -0,0 +1,24 @@
+FROM ubuntu:20.04 as build
+RUN mkdir -p /opt/build && mkdir -p /opt/dist
+RUN apt-get update &&   DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends     ca-certificates curl &&   rm -rf /var/lib/apt/lists/*
+# install cmake
+RUN cd /opt/build &&     curl -LO https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-linux-x86_64.sh && /bin/bash cmake-3.23.1-linux-x86_64.sh --prefix=/usr/local --skip-license
+RUN apt update -y --allow-unauthenticated
+RUN apt-get update -yq --allow-unauthenticated
+RUN apt install -y lmod
+RUN apt install -y tzdata
+RUN ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime
+RUN dpkg-reconfigure --frontend noninteractive tzdata
+RUN apt -y install build-essential git vim python3 wget libexpat1-dev lmod
+RUN apt -y install bc && apt -y install time
+RUN apt-get install -yq libtiff-dev
+RUN apt-get install -yq git-lfs
+RUN apt install -y python3-distutils
+RUN apt install -y python3-pip
+RUN apt install wget
+RUN apt-get install -y --no-install-recommends apt-utils
+RUN apt-get install -yq m4
+RUN echo "dash dash/sh boolean false" | debconf-set-selections
+RUN DEBIAN_FRONTEND=noninteractive dpkg-reconfigure dash
+RUN ls -l /bin/sh
+

Docker/Dockerfile.ubuntu20.04-intel22-ufs-srwapp

@@ -0,0 +1,47 @@
+#FROM noaaepic/ubuntu20.04-spack:release-public-v2
+FROM noaaepic/ubuntu20.04-spack:latest
+LABEL AUTHOR EPIC-AUS
+
+# install srw
+WORKDIR /opt
+RUN git clone -b release/public-v2 https://github.com/ufs-community/ufs-srweather-app.git
+#RUN git clone -b develop https://github.com/ufs-community/ufs-srweather-app.git
+WORKDIR /opt/ufs-srweather-app
+COPY build-srw.sh /opt/ufs-srweather-app
+RUN ./manage_externals/checkout_externals && \
+    mkdir /opt/ufs-srweather-app/build && \
+    /bin/bash -l /opt/ufs-srweather-app/build-srw.sh && \
+    mkdir /opt/ufs
+WORKDIR /opt/ufs
+ENV PATH=$PATH:/opt/miniconda/bin
+RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+    bash ./Miniconda3-latest-Linux-x86_64.sh -b -p /opt/miniconda && \
+    git clone --recursive https://github.com/NOAA-GSD/contrib_miniconda3.git miniconda3 && \
+    cd /opt/ufs/miniconda3/environments && \
+    sed -i '3 a - conda-forge' regional_workflow.yml  && \
+    sed -i 's/^-/  -/g' regional_workflow.yml  && \
+    sed -i 's/^  - https/#  - https/g' regional_workflow.yml  && \
+    cd /opt/ufs && \
+    conda env create -f /opt/ufs/miniconda3/environments/regional_workflow.yml  && \
+    mkdir -p /work && \
+    mkdir -p /contrib && \
+    mkdir -p /lustre  && \
+    mkdir -p /scratch1 && \
+    mkdir -p /scratch2 && \
+    mkdir -p /glade && \
+    mkdir -p /data && \
+    mkdir -p /scratch && \
+    mkdir -p /lfs && \
+    mkdir -p /u && \
+    mkdir -p /discover && \
+    mkdir -p /mnt && \
+    mkdir -p /lfs1 && \
+    mkdir -p /lfs2 && \
+    mkdir -p /lfs3 && \
+    mkdir -p /lfs4 && \
+    mkdir /opt/ufs-srweather-app/container-bin && \
+    mv /opt/ufs-srweather-app/bin/* /opt/ufs-srweather-app/container-bin && \
+    mkdir -p /opt/ufs-srweather-app/container-scripts     
+COPY srw.sh-template /opt/ufs-srweather-app/container-scripts
+COPY stage-srw.sh /opt/ufs-srweather-app/container-scripts
+

Docker/Dockerfile.ubuntu20.04-spack

@@ -0,0 +1,89 @@
+FROM noaaepic/ubuntu20.04-base:latest
+
+RUN mkdir -p /opt
+WORKDIR /opt
+RUN git clone -b feature/oneapi --recursive https://github.com/NOAA-EPIC/spack-stack.git
+WORKDIR spack-stack
+
+RUN pwd && ls -l /bin/sh && sed -i 's/source/./g' ./setup.sh && . ./setup.sh && spack install intel-oneapi-compilers && \
+    ls -l /bin/sh  && \
+    pwd && ls -l && . ./setup.sh && spack install intel-oneapi-compilers && \
+    spack install intel-oneapi-mpi && spack compiler list && spack find && \
+    spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/linux/bin/intel64 && spack compiler list && \
+    spack compiler rm [email protected]
+ENV PATH="${PATH}:/usr/local" 
+RUN mkdir -p /opt/spack-stack/configs/sites/ubuntu-intel && \
+    cp /root/.spack/linux/compilers.yaml /opt/spack-stack/configs/sites/ubuntu-intel && ls -l /opt/spack-stack/configs/sites/ubuntu-intel && \
+    loc=`find /opt/spack-stack/spack/opt/spack/ -iname intel-oneapi-mpi* | head -n 1` && \
+    echo "config:" > /opt/spack-stack/configs/sites/ubuntu-intel/config.yaml && \
+    echo "  build_jobs: 8" >> /opt/spack-stack/configs/sites/ubuntu-intel/config.yaml && \
+    echo "modules:" > /opt/spack-stack/configs/sites/ubuntu-intel/modules.yaml && \
+    echo "  default:" >> /opt/spack-stack/configs/sites/ubuntu-intel/modules.yaml && \
+    echo "    enable::" >> /opt/spack-stack/configs/sites/ubuntu-intel/modules.yaml && \
+    echo "    - lmod" >> /opt/spack-stack/configs/sites/ubuntu-intel/modules.yaml && \
+    echo "packages:" > /opt/spack-stack/configs/sites/ubuntu-intel/packages.yaml && \
+    echo "  all:" >> /opt/spack-stack/configs/sites/ubuntu-intel/packages.yaml && \
+    echo "    compiler:: [[email protected]]" >> /opt/spack-stack/configs/sites/ubuntu-intel/packages.yaml && \
+    echo "    providers:" >> /opt/spack-stack/configs/sites/ubuntu-intel/packages.yaml && \
+    echo "      mpi:: [[email protected]]" >> /opt/spack-stack/configs/sites/ubuntu-intel/packages.yaml && \
+    echo "  mpi:" >> /opt/spack-stack/configs/sites/ubuntu-intel/packages.yaml && \
+    echo "    buildable: False" >> /opt/spack-stack/configs/sites/ubuntu-intel/packages.yaml && \
+    echo "  intel-oneapi-mpi:" >> /opt/spack-stack/configs/sites/ubuntu-intel/packages.yaml && \
+    echo "    externals:" >> /opt/spack-stack/configs/sites/ubuntu-intel/packages.yaml && \
+    echo "    - spec: [email protected]" >> /opt/spack-stack/configs/sites/ubuntu-intel/packages.yaml && \
+    echo "      modules:" >> /opt/spack-stack/configs/sites/ubuntu-intel/packages.yaml && \
+    echo "      - impi/2021.6.0" >> /opt/spack-stack/configs/sites/ubuntu-intel/packages.yaml && \
+    echo "      prefix: $loc" >> /opt/spack-stack/configs/sites/ubuntu-intel/packages.yaml
+RUN cat configs/templates/ufs-srw-public-v2/spack.yaml && \
+    . ./setup.sh && \
+    spack compiler add && \ 
+    spack external find cmake && \ 
+    spack external find wget && \ 
+    spack external find m4 && \
+    spack external find git && \ 
+    spack external find git-lfs && \ 
+    spack external find openssl && \ 
+    spack external find libjpeg-turbo && \ 
+    spack external find perl && \
+    spack stack create env --site ubuntu-intel --template ufs-srw-public-v2 --name ufs-srw.intel && \
+    spack env activate envs/ufs-srw.intel && \ 
+    spack concretize && \ 
+    spack install
+RUN . ./setup.sh && \
+    spack env activate envs/ufs-srw.intel && \ 
+    spack add [email protected] && spack install
+RUN . ./setup.sh && \
+    spack env activate envs/ufs-srw.intel && \ 
+    spack add [email protected] && spack install
+ENV SHELL=/bin/bash
+RUN . ./setup.sh && \
+    /bin/bash && \
+    sed -i 's/3.9.12/3.8.10/g' envs/ufs-srw.intel/common/packages.yaml && \
+    ln -s /usr/bin/python3 /usr/bin/python && \
+    . /usr/share/lmod/lmod/init/bash && \
+    spack module tcl refresh -y --delete-tree && \
+    spack env activate envs/ufs-srw.intel && \  
+    spack external find python && \
+    spack remove fms && \
+    spack add [email protected] && \
+    spack install && \
+    spack stack setup-meta-modules && \
+    sed -i 's/impi/intel-oneapi-mpi/g' /opt/spack-stack/envs/ufs-srw.intel/install/modulefiles/intel/2021.6.0/stack-intel-oneapi-mpi/2021.6.0.lua && \
+    sed -i '/prerequisite modules/a load("intel-oneapi-compilers/2022.1.0")\nprereq("intel-oneapi-compilers/2022.1.0")' envs/ufs-srw.intel/install/modulefiles/Core/stack-intel/2021.6.0.lua && \
+    find /opt/spack-stack/envs/ufs-srw.intel/install/modulefiles -iname "*.lua" | xargs grep -l depends_on | xargs sed -i 's/depends_on/-- depends_on/g' 
+COPY modules.yaml spack/etc/spack/defaults/modules.yaml 
+RUN . ./setup.sh && spack module tcl refresh -y --delete-tree && \
+    ls -l /bin/sh && source /usr/share/lmod/lmod/init/bash && spack env activate envs/ufs-srw.intel && \
+    module use /opt/spack-stack/envs/ufs-srw.intel/install/modulefiles/Core &&  module avail && \
+    module load stack-intel stack-intel-oneapi-mpi && \
+    module avail && module load bacio crtm g2 g2tmpl gfsio gftl-shared ip jasper landsfcutil libpng sfcio sigio sp stack-python w3nco wgrib2 yafyaml zlib esmf fms/2022.01 hdf5 mapl nemsio nemsiogfs netcdf-c netcdf-fortran parallelio upp w3emc && \
+     module list && echo "env PATH=/opt/ufs-srweather-app/container-bin:$PATH" > locenvs && \
+     echo "ENV LD_LIBRARY_PATH=/opt/ufs-srweather-app/lib:$LD_LIBRARY_PATH" >> locenvs && \
+     echo "ENV MODULEPATH=$MODULEPATH" >> locenvs && \
+     echo "ENV FI_PROVIDER_PATH=$FI_PROVIDER_PATH" >> locenvs && \
+     echo "ENV LMOD_DEFAULT_MODULEPATH=$LMOD_DEFAULT_MODULEPATH" >> locenvs && \
+     echo "ENV LMOD_PKG=$LMOD_PKG" >> locenvs && \
+     echo "ENV LMOD_CMD=$LMOD_CMD" >> locenvs && \
+     echo "ENV LMOD_DIR=$LMOD_DIR" >> locenvs && \
+     echo "ENV MODULESHOME=$MODULESHOME" >> locenvs
+#    echo "DONE"

Docker/README.txt

@@ -0,0 +1,10 @@
+This is a series of Dockerfile recipes for building ubuntu20.04-based containers with intel compilers and MPI.
+The Dockerfile.ubuntu20.04-base will build a base container with ubuntu20.04 and some extra tools that are 
+needed by many UFS applications.
+The Dockerfile.ubuntu20.04-spack will use the ubuntu20.04 base image to start and then install spack-stack.
+Using spack-stack this recipe will then install intel oneapi compilers and MPI and use them to build
+the ufs-srw-public-v2 template of the spack stack
+The Dockerfile.ubuntu20.04-intel22-ufs-srwapp uses the ubuntu20.04-stack container as a base and then builds
+the ufs-srweather-app on top of it. It contains a script that can be used to stage the SRW for use on 
+Tier 1 platforms.
+

Docker/build-srw.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+source /usr/share/lmod/lmod/init/bash
+source /opt/spack-stack/setup.sh
+export PATH=$PATH:/opt/dist/usr/local/bin
+module use /opt/spack-stack/envs/ufs-srw.intel/install/modulefiles/Core
+module load stack-intel
+module load stack-intel-oneapi-mpi/2021.6.0
+module load netcdf-c netcdf-fortran libpng jasper
+module load sp zlib hdf5 netcdf-c netcdf-fortran parallelio esmf fms bacio crtm g2 g2tmpl ip w3nco upp gftl-shared yafyaml mapl gfsio landsfcutil nemsio nemsiogfs sfcio sigio w3emc wgrib2
+mkdir -p build
+cd build
+cmake -DCMAKE_INSTALL_PREFIX=/opt/ufs-srweather-app ..
+make -j 8
+