ci.yml

name: AMR-Wind-CI

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]

concurrency:
  group: ${{github.ref}}-${{github.head_ref}}-ci
  cancel-in-progress: true

jobs:
  Formatting:
    runs-on: ubuntu-latest
    steps:
    - name: Clone
      uses: actions/checkout@v4
    - name: Check formatting
      uses: DoozyX/clang-format-lint-action@v0.16.2
      with:
        source: './amr-wind ./unit_tests ./tools/utilities'
        exclude: '.'
        extensions: 'H,h,cpp'
        clangFormatVersion: 16
  CPU:
    needs: Formatting
    runs-on: ${{matrix.os}}
    strategy:
      matrix:
        os: [ubuntu-latest, macos-latest]
        build_type: [Release, Debug]
        include:
          - os: macos-latest
            install_deps: brew install mpich ccache
            comp: llvm
            procs: $(sysctl -n hw.ncpu)
            ccache_cache: /Users/runner/Library/Caches/ccache
          - os: ubuntu-latest
            install_deps: sudo apt-get install -y mpich libmpich-dev
            comp: gnu
            procs: $(nproc)
            ccache_cache: ~/.cache/ccache
          - build_type: Release
            ctest_args: -LE no_ci
            ccache_size: 150M
          - build_type: Debug
            ctest_args: -L unit
            ccache_size: 500M
        exclude:
          - os: macos-latest
            build_type: Debug
    steps:
    - name: Clone
      uses: actions/checkout@v4
      with: 
        submodules: true
    - name: Dependencies
      run: ${{matrix.install_deps}}
    - name: Setup
      run: |
        echo "NPROCS=${{matrix.procs}}" >> $GITHUB_ENV
        echo "CCACHE_COMPRESS=1" >> $GITHUB_ENV
        echo "CCACHE_COMPRESSLEVEL=1" >> $GITHUB_ENV
        echo "CCACHE_LOGFILE=${{github.workspace}}/ccache.log.txt" >> $GITHUB_ENV
        echo "CCACHE_MAXSIZE=${{matrix.ccache_size}}" >> $GITHUB_ENV
    - name: Install Ccache
      run: |
        if [ "${RUNNER_OS}" != "macOS" ]; then
          wget https://github.com/ccache/ccache/releases/download/v4.8/ccache-4.8-linux-x86_64.tar.xz
          tar xvf ccache-4.8-linux-x86_64.tar.xz
          sudo cp -f ccache-4.8-linux-x86_64/ccache /usr/local/bin/
        fi
    - name: Set Up Ccache
      uses: actions/cache@v4
      with:
        path: ${{matrix.ccache_cache}}
        key: ccache-${{github.workflow}}-${{github.job}}-${{matrix.os}}-${{matrix.build_type}}-git-${{github.sha}}
        restore-keys: |
             ccache-${{github.workflow}}-${{github.job}}-${{matrix.os}}-${{matrix.build_type}}-git-
    - name: Configure
      run: |
        cmake -B ${{runner.workspace}}/build-ci-${{matrix.build_type}} \
          -DCMAKE_INSTALL_PREFIX:PATH=${{runner.workspace}}/install-${{matrix.build_type}} \
          -DCMAKE_BUILD_TYPE:STRING=${{matrix.build_type}} \
          -DAMR_WIND_ENABLE_MPI:BOOL=ON \
          -DAMR_WIND_ENABLE_TESTS:BOOL=ON \
          -DAMR_WIND_TEST_WITH_FCOMPARE:BOOL=OFF \
          -DCMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache \
          ${{github.workspace}}
    - name: Build
      run: |
        echo "::add-matcher::.github/problem-matchers/gcc.json"
        ccache -z
        set -eu -o pipefail
        cmake --build ${{runner.workspace}}/build-ci-${{matrix.build_type}} --parallel ${{env.NPROCS}} 2>&1 | \
          tee -a ${{runner.workspace}}/build-ci-${{matrix.build_type}}/build-output.txt
    - name: Ccache Report
      run: |
        ccache -s
        ls ${{matrix.ccache_cache}}
        du -hs ${{matrix.ccache_cache}}
    - name: Report
      working-directory: ${{runner.workspace}}/build-ci-${{matrix.build_type}}
      run: |
        echo "::add-matcher::.github/problem-matchers/gcc.json"
        egrep "warning:|error:" build-output.txt | egrep -v "submods" \
          | egrep -v "ld: warning:" | egrep -v "lto-wrapper: warning:" | sort | uniq \
          | awk 'BEGIN{i=0}{print $0}{i++}END{print "Warnings: "i}' > build-output-warnings.txt
        cat build-output-warnings.txt
        exit $(tail -n 1 build-output-warnings.txt | awk '{print $2}')
    - name: Test
      working-directory: ${{runner.workspace}}/build-ci-${{matrix.build_type}}
      run: |
        ctest ${{matrix.ctest_args}} --output-on-failure
  GPU-Nvidia:
    name: GPU-CUDA
    needs: Formatting
    runs-on: ubuntu-20.04
    strategy:
      matrix:
        cuda_ver: [11.4]
        include:
          - cuda_ver: "11.4"
            cuda_pkg: 11-4
            cuda_extra: libcurand-dev-11-4 cuda-cupti-dev-11-4
    steps:
      - name: Clone
        uses: actions/checkout@v4
        with:
          submodules: true
      - name: Setup
        run: |
          echo "NPROCS=$(nproc)" >> $GITHUB_ENV
          echo "CCACHE_COMPRESS=1" >> $GITHUB_ENV
          echo "CCACHE_COMPRESSLEVEL=1" >> $GITHUB_ENV
          echo "CCACHE_LOGFILE=${{github.workspace}}/ccache.log.txt" >> $GITHUB_ENV
          echo "CCACHE_MAXSIZE=200M" >> $GITHUB_ENV
      - name: Install Ccache
        run: |
          wget https://github.com/ccache/ccache/releases/download/v4.8/ccache-4.8-linux-x86_64.tar.xz
          tar xvf ccache-4.8-linux-x86_64.tar.xz
          sudo cp -f ccache-4.8-linux-x86_64/ccache /usr/local/bin/
      - name: Set Up Ccache
        uses: actions/cache@v4
        with:
          path: ~/.cache/ccache
          key: ccache-${{github.workflow}}-${{github.job}}-git-${{github.sha}}
          restore-keys: |
               ccache-${{github.workflow}}-${{github.job}}-git-
      - name: Prepare CUDA environment
        run: |
          ${{github.workspace}}/submods/amrex/.github/workflows/dependencies/ubuntu_free_disk_space.sh
          sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub
          echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 /" | sudo tee /etc/apt/sources.list.d/cuda.list
          sudo apt-get update
          sudo apt-get install -y \
              cuda-command-line-tools-${{matrix.cuda_pkg}} cuda-compiler-${{matrix.cuda_pkg}} cuda-minimal-build-${{matrix.cuda_pkg}} cuda-nvml-dev-${{matrix.cuda_pkg}} cuda-nvtx-${{matrix.cuda_pkg}} ${{matrix.cuda_extra}}
      - name: Configure and build
        run: |
          export PATH=/usr/local/nvidia/bin:/usr/local/cuda-${{matrix.cuda_ver}}/bin:${PATH}
          export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda-${{matrix.cuda_ver}}/lib:${LD_LIBRARY_PATH}
          cmake -B build-${{matrix.cuda_pkg}} \
            -DCMAKE_BUILD_TYPE:STRING=Release \
            -DAMR_WIND_ENABLE_MPI:BOOL=OFF \
            -DAMR_WIND_ENABLE_CUDA:BOOL=ON \
            -DAMReX_CUDA_ERROR_CAPTURE_THIS:BOOL=ON \
            -DCMAKE_CUDA_COMPILE_SEPARABLE_COMPILATION:BOOL=ON \
            -DCMAKE_CUDA_ARCHITECTURES=70 \
            -DCMAKE_CUDA_COMPILER_LAUNCHER:STRING=ccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache \
            ${{github.workspace}}
          ccache -z
          cmake --build build-${{matrix.cuda_pkg}} --parallel ${{env.NPROCS}}
      - name: Ccache Report
        run: |
          ccache -s
          ls ~/.cache/ccache
          du -hs ~/.cache/ccache
  GPU-AMD:
    name: GPU-HIP
    needs: Formatting
    runs-on: ubuntu-20.04
    steps:
      - name: Clone
        uses: actions/checkout@v4
        with:
          submodules: true
      - name: Setup
        run: |
          echo "NPROCS=$(nproc)" >> $GITHUB_ENV
          echo "CCACHE_COMPRESS=1" >> $GITHUB_ENV
          echo "CCACHE_COMPRESSLEVEL=1" >> $GITHUB_ENV
          echo "CCACHE_LOGFILE=${{github.workspace}}/ccache.log.txt" >> $GITHUB_ENV
          echo "CCACHE_MAXSIZE=50M" >> $GITHUB_ENV
      - name: Install Ccache
        run: |
          wget https://github.com/ccache/ccache/releases/download/v4.8/ccache-4.8-linux-x86_64.tar.xz
          tar xvf ccache-4.8-linux-x86_64.tar.xz
          sudo cp -f ccache-4.8-linux-x86_64/ccache /usr/local/bin/
      - name: Set Up Ccache
        uses: actions/cache@v4
        with:
          path: ~/.cache/ccache
          key: ccache-${{github.workflow}}-${{github.job}}-git-${{github.sha}}
          restore-keys: |
               ccache-${{github.workflow}}-${{github.job}}-git-
      - name: Prepare HIP environment
        run: |
          ${{github.workspace}}/submods/amrex/.github/workflows/dependencies/ubuntu_free_disk_space.sh
          wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | sudo apt-key add -
          echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/5.6.1/ ubuntu main' | sudo tee /etc/apt/sources.list.d/rocm.list
          echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' | sudo tee -a /etc/profile.d/rocm.sh
          sudo apt-get update
          sudo apt-get install -y \
              build-essential rocm-dev rocrand-dev rocprim-dev
      - name: Configure and build
        run: |
          source /etc/profile.d/rocm.sh
          which clang
          which clang++
          cmake -B build \
            -DCMAKE_BUILD_TYPE:STRING=Release \
            -DAMR_WIND_ENABLE_MPI:BOOL=OFF \
            -DAMR_WIND_ENABLE_ROCM:BOOL=ON \
            -DCMAKE_CXX_COMPILER:STRING=$(which clang++) \
            -DCMAKE_C_COMPILER:STRING=$(which clang) \
            -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \
            -DAMReX_AMD_ARCH:STRING=gfx90a \
            -DCMAKE_HIP_ARCHITECTURES:STRING=gfx90a \
            -DAMDGPU_TARGETS:STRING=gfx90a \
            -DGPU_TARGETS:STRING=gfx90a \
            -DCMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache \
            ${{github.workspace}}
          ccache -z
          cmake --build build --parallel ${{env.NPROCS}}
      - name: Ccache Report
        run: |
          ccache -s
          ls ~/.cache/ccache
          du -hs ~/.cache/ccache
  GPU-Intel:
    name: GPU-SYCL
    needs: Formatting
    runs-on: ubuntu-20.04
    steps:
      - name: Clone
        uses: actions/checkout@v4
        with:
          submodules: true
      - name: Setup
        run: |
          echo "NPROCS=$(nproc)" >> $GITHUB_ENV
          echo "CCACHE_COMPRESS=1" >> $GITHUB_ENV
          echo "CCACHE_COMPRESSLEVEL=1" >> $GITHUB_ENV
          echo "CCACHE_LOGFILE=${{github.workspace}}/ccache.log.txt" >> $GITHUB_ENV
          echo "CCACHE_MAXSIZE=50M" >> $GITHUB_ENV
          echo "CCACHE_DEPEND=1" >> $GITHUB_ENV
      - name: Install Ccache
        run: |
          wget https://github.com/ccache/ccache/releases/download/v4.8/ccache-4.8-linux-x86_64.tar.xz
          tar xvf ccache-4.8-linux-x86_64.tar.xz
          sudo cp -f ccache-4.8-linux-x86_64/ccache /usr/local/bin/
      - name: Set Up Ccache
        uses: actions/cache@v4
        with:
          path: ~/.cache/ccache
          key: ccache-${{github.workflow}}-${{github.job}}-git-${{github.sha}}
          restore-keys: |
               ccache-${{github.workflow}}-${{github.job}}-git-
      - name: Prepare SyCL environment
        run: |
          export DEBIAN_FRONTEND=noninteractive
          ${{github.workspace}}/submods/amrex/.github/workflows/dependencies/ubuntu_free_disk_space.sh
          sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
          sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
          echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
          sudo apt-get update
          sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mkl-devel
      - name: Configure and build
        run: |
          set +e
          source /opt/intel/oneapi/setvars.sh
          set -e
          cmake -B build-SYCL \
            -DCMAKE_BUILD_TYPE:STRING=Release \
            -DCMAKE_CXX_COMPILER:STRING=$(which icpx) \
            -DCMAKE_C_COMPILER:STRING=$(which icx) \
            -DAMR_WIND_ENABLE_MPI:BOOL=OFF \
            -DAMR_WIND_ENABLE_SYCL:BOOL=ON \
            -DCMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache \
            ${{github.workspace}}
          ccache -z
          cmake --build build-SYCL --parallel ${{env.NPROCS}}
      - name: Ccache Report
        run: |
          ccache -s
          ls ~/.cache/ccache
          du -hs ~/.cache/ccache
#  Lint-cppcheck:
#    needs: Formatting
#    runs-on: macos-latest
#    steps:
#      - name: Clone
#        uses: actions/checkout@v4
#        with:
#          submodules: true
#      - name: Dependencies
#        run: brew install cppcheck
#      - name: Configure
#        run: |
#          echo "NPROCS=$(sysctl -n hw.ncpu)" >> $GITHUB_ENV
#          cmake \
#          -B ${{runner.workspace}}/build-cppcheck \
#          -DAMR_WIND_ENABLE_MPI:BOOL=OFF \
#          -DAMR_WIND_ENABLE_TESTS:BOOL=ON \
#          -DAMR_WIND_TEST_WITH_FCOMPARE:BOOL=OFF \
#          -DAMR_WIND_ENABLE_MASA:BOOL=OFF \
#          -DAMR_WIND_ENABLE_CPPCHECK:BOOL=ON \
#          ${{github.workspace}}
#      - name: Check
#        working-directory: ${{runner.workspace}}/build-cppcheck
#        run: make cppcheck-ci
#      - name: Full report
#        working-directory: ${{runner.workspace}}/build-cppcheck
#        run: cat cppcheck/cppcheck-full-report.txt
#      - name: Short report
#        working-directory: ${{runner.workspace}}/build-cppcheck
#        run: |
#          echo "::add-matcher::.github/problem-matchers/cppcheck.json"
#          cat cppcheck-ci-report.txt
#          exit $(tail -n 1 cppcheck-ci-report.txt | awk '{print $2}')
  Lint-clang-tidy:
    needs: Formatting
    runs-on: ubuntu-latest
    steps:
      - name: Clone
        uses: actions/checkout@v4
        with:
          submodules: true
      - name: Setup
        run: |
          echo "NPROCS=$(nproc)" >> $GITHUB_ENV
          echo "CCACHE_COMPRESS=1" >> $GITHUB_ENV
          echo "CCACHE_COMPRESSLEVEL=1" >> $GITHUB_ENV
          echo "CCACHE_LOGFILE=${{github.workspace}}/ccache.log.txt" >> $GITHUB_ENV
          echo "CCACHE_EXTRAFILES=${{github.workspace}}/.clang-tidy" >> $GITHUB_ENV
          echo "CCACHE_MAXSIZE=50M" >> $GITHUB_ENV
          echo "CTCACHE_DIR=~/.cache/ctcache" >> $GITHUB_ENV
      - name: Install Ccache
        run: |
          wget https://github.com/ccache/ccache/releases/download/v4.8/ccache-4.8-linux-x86_64.tar.xz
          sudo curl https://raw.githubusercontent.com/matus-chochlik/ctcache/7fd516e91c17779cbc6fc18bd119313d9532dd90/clang-tidy-cache -Lo /usr/bin/clang-tidy-cache
          tar xvf ccache-4.8-linux-x86_64.tar.xz
          sudo cp -f ccache-4.8-linux-x86_64/ccache /usr/local/bin/
          sudo chmod +x /usr/bin/clang-tidy-cache
          mkdir -p ~/.cache/ctcache
      - name: Set Up Ccache
        uses: actions/cache@v4
        with:
          path: ~/.cache
          key: ccache-${{github.workflow}}-${{github.job}}-git-${{github.sha}}
          restore-keys: |
               ccache-${{github.workflow}}-${{github.job}}-git-
      - name: Configure
        run: |
          cmake \
          -B ${{runner.workspace}}/build-clang-tidy \
          -DCMAKE_BUILD_TYPE:STRING=Release \
          -DCMAKE_CXX_COMPILER:STRING=clang++ \
          -DCMAKE_C_COMPILER:STRING=clang \
          -DAMR_WIND_ENABLE_MPI:BOOL=OFF \
          -DAMR_WIND_ENABLE_TESTS:BOOL=ON \
          -DAMR_WIND_TEST_WITH_FCOMPARE:BOOL=OFF \
          -DAMR_WIND_ENABLE_MASA:BOOL=OFF \
          -DAMR_WIND_ENABLE_ALL_WARNINGS:BOOL=ON \
          -DAMR_WIND_ENABLE_CLANG_TIDY:BOOL=ON \
          -DCMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache \
          ${{github.workspace}}
      - name: Check
        working-directory: ${{runner.workspace}}/build-clang-tidy
        run: |
          cmake --build . --parallel ${{env.NPROCS}} 2>&1 | tee -a clang-tidy-full-report.txt
          cat clang-tidy-full-report.txt | grep "warning:" | grep -v "submods" | sort | uniq | \
            awk 'BEGIN{i=0}{print $0}{i++}END{print "Warnings: "i}' > clang-tidy-ci-report.txt
      - name: Ccache Report
        run: |
          ls ~/.cache
          ls ~/.cache/ccache
          du -hs ~/.cache/ccache
          ls ~/.cache/ctcache
          du -hs ~/.cache/ctcache
          ccache -s
      - name: Full report
        working-directory: ${{runner.workspace}}/build-clang-tidy
        run: cat clang-tidy-full-report.txt
      - name: Short report
        working-directory: ${{runner.workspace}}/build-clang-tidy
        run: |
          echo "::add-matcher::.github/problem-matchers/gcc.json"
          cat clang-tidy-ci-report.txt
          exit $(tail -n 1 clang-tidy-ci-report.txt | awk '{print $2}')
  Lint-codeql:
    needs: Formatting
    runs-on: ubuntu-latest
    steps:
      - name: Clone
        uses: actions/checkout@v4
        with:
          submodules: true
      - name: Initialize CodeQL
        uses: github/codeql-action/init@v2
        with:
          languages: cpp
          queries: +security-and-quality
          config-file: ./.github/codeql/codeql-config.yml
      - name: Configure
        run: |
          echo "NPROCS=$(nproc)" >> $GITHUB_ENV
          cmake \
            -B ${{runner.workspace}}/build-ci-codeql \
            -DCMAKE_INSTALL_PREFIX:PATH=${{runner.workspace}}/install-codeql \
            -DCMAKE_BUILD_TYPE:STRING=Debug \
            -DCMAKE_CXX_COMPILER:STRING=clang++ \
            -DCMAKE_C_COMPILER:STRING=clang \
            -DAMR_WIND_ENABLE_MPI:BOOL=OFF \
            -DAMR_WIND_ENABLE_TESTS:BOOL=ON \
            -DAMR_WIND_TEST_WITH_FCOMPARE:BOOL=OFF \
            -DAMR_WIND_ENABLE_MASA:BOOL=OFF \
            ${{github.workspace}}
      - name: Build
        run: |
          cmake --build ${{runner.workspace}}/build-ci-codeql --parallel ${{env.NPROCS}}
      - name: Perform CodeQL Analysis
        uses: github/codeql-action/analyze@v2
        with:
          category: "/language:cpp"
          upload: False
          output: sarif-results
      - name: Filter CodeQL sarif
        uses: advanced-security/filter-sarif@v1
        with:
          patterns: |
            -**/submods/**
          input: sarif-results/cpp.sarif
          output: sarif-results/cpp.sarif
      - name: Upload CodeQL sarif
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: sarif-results/cpp.sarif
  Lint-codespell:
    needs: Formatting
    runs-on: ubuntu-latest
    steps:
      - name: Clone
        uses: actions/checkout@v4
        with:
          submodules: false
      - name: Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'
      - name: Dependencies
        run: |
          # Install Python packages
          python -m pip install --upgrade pip
          pip install codespell
      - name: Run codespell
        run: codespell
#  Docker:
#    needs: Formatting
#    runs-on: ubuntu-latest
#    container:
#      image: ecpe4s/exawind-snapshot
#      env:
#          SPACK_MANAGER: /spack-manager
#          E4S_MACHINE: true
#    steps:
#      - name: Clone
#        uses: actions/checkout@v4
#        with:
#          submodules: true
#      - name: Test
#        working-directory: /spack-manager/environments/exawind
#        run: |
#          /bin/bash -c " \
#            source ${SPACK_MANAGER}/start.sh && \
#            ln -s ${GITHUB_WORKSPACE} amr-wind && \
#            source ${SPACK_MANAGER}/start.sh && \
#            quick-develop -s amr-wind+ascent+hdf5+hypre+masa+netcdf~shared@main && \
#            spack install && \
#            spack cd -b amr-wind && \
#            spack build-env amr-wind ctest -j $(nproc) -L unit --output-on-failure \
#          "
  Save-PR-Number:
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    steps:
      - name: Save PR number
        env:
          PR_NUMBER: ${{ github.event.number }}
        run: |
          echo $PR_NUMBER > pr_number.txt
      - uses: actions/upload-artifact@v3
        with:
          name: pr_number
          path: pr_number.txt
          retention-days: 1