ci.yml

# Copyright 2021 ETH Zurich and University of Bologna.
# Solderpad Hardware License, Version 0.51, see LICENSE for details.
# SPDX-License-Identifier: SHL-0.51
#
# Author: Matheus Cavalcante <matheusd@iis.ee.ethz.ch>

# Run functional regression checks
name: ci
on: [push, pull_request]

# Only allow one workflow per PR/Branch (the last one)
# https://docs.github.com/en/actions/learn-github-actions/expressions
# https://docs.github.com/en/actions/learn-github-actions/contexts#github-context
# https://docs.github.com/en/actions/using-jobs/using-concurrency
concurrency:
  # github.workflow: name of the workflow
  # github.event.pull_request.number || github.ref: pull request number or branch name if not a pull request
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  # Cancel in-progress runs when a new workflow with the same group name is triggered
  cancel-in-progress: true

jobs:

#####################
#  Toolchain stage  #
#####################

  tc-llvm:
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v3
    - name: Recover the submodule commit hash
      id: recover_hash
      run: |
        git submodule status toolchain/riscv-llvm | cut -d' ' -f1
        echo "tc-llvm-hash=`git submodule status toolchain/riscv-llvm | cut -d' ' -f1`" >> $GITHUB_ENV
    - name: Cache the LLVM toolchain
      uses: actions/cache@v3
      id: tc-llvm-cache
      env:
        cache-name: cache-llvm
      with:
        path: install/riscv-llvm
        key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ env.tc-llvm-hash }}
        restore-keys:
          ${{ runner.os }}-build-${{ env.cache-name }}-${{ env.tc-llvm-hash }}
    - name: Download the LLVM toolchain
      if: steps.tc-llvm-cache.outputs.cache-hit != 'true'
      run: git submodule update --init --recursive -- toolchain/riscv-llvm
    - name: Download Newlib
      if: steps.tc-llvm-cache.outputs.cache-hit != 'true'
      run: git submodule update --init --recursive -- toolchain/newlib
    - name: Compile LLVM
      if: steps.tc-llvm-cache.outputs.cache-hit != 'true'
      run: |
        sudo apt-get install libmpc-dev
        sudo apt-get install -y ninja-build
        CC=gcc CXX=g++ make toolchain-llvm
    - name: Tar LLVM
      run: tar -cvf tc-llvm.tar install/riscv-llvm
    - name: Upload LLVM
      uses: actions/upload-artifact@v3
      with:
        name: tc-llvm
        path: tc-llvm.tar

  tc-gcc:
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v3
    - name: Recover the submodule commit hash
      id: recover_hash
      run: |
        git submodule status toolchain/riscv-gnu-toolchain | cut -d' ' -f1
        echo "tc-gcc-hash=`git submodule status toolchain/riscv-gnu-toolchain | cut -d' ' -f1`" >> $GITHUB_ENV
    - name: Cache the GCC toolchain
      uses: actions/cache@v3
      id: tc-gcc-cache
      env:
        cache-name: cache-gcc
      with:
        path: install/riscv-gcc
        key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ env.tc-gcc-hash }}
        restore-keys:
          ${{ runner.os }}-build-${{ env.cache-name }}-${{ env.tc-gcc-hash }}
    - name: Download the GCC toolchain
      if: steps.tc-gcc-cache.outputs.cache-hit != 'true'
      run: git submodule update --init --recursive -- toolchain/riscv-gnu-toolchain
    - name: Compile GCC
      if: steps.tc-gcc-cache.outputs.cache-hit != 'true'
      run: |
        sudo apt-get install libmpc-dev
        CC=gcc CXX=g++ make toolchain-gcc
    - name: Tar GCC
      run: tar -cvf tc-gcc.tar install/riscv-gcc
    - name: Upload GCC
      uses: actions/upload-artifact@v3
      with:
        name: tc-gcc
        path: tc-gcc.tar

  tc-isa-sim:
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v3
    - name: Recover the submodule commit hash
      id: recover_hash
      run: |
        git submodule status toolchain/riscv-isa-sim | cut -d' ' -f1
        echo "tc-isa-sim-hash=`git submodule status toolchain/riscv-isa-sim | cut -d' ' -f1`" >> $GITHUB_ENV
    - name: Cache Spike
      uses: actions/cache@v3
      id: tc-isa-sim-cache
      env:
        cache-name: cache-spike
      with:
        path: install/riscv-isa-sim
        key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ env.tc-isa-sim-hash }}
        restore-keys:
          ${{ runner.os }}-build-${{ env.cache-name }}-${{ env.tc-isa-sim-hash }}
    - name: Download Spike
      if: steps.tc-isa-sim-cache.outputs.cache-hit != 'true'
      run: |
        git submodule update --init --recursive -- toolchain/riscv-isa-sim
        git submodule foreach --recursive git reset --hard
    - name: Compile Spike
      if: steps.tc-isa-sim-cache.outputs.cache-hit != 'true'
      run: make riscv-isa-sim
    - name: Tar Spike
      run: tar -cvf tc-isa-sim.tar install/riscv-isa-sim
    - name: Upload Spike
      uses: actions/upload-artifact@v3
      with:
        name: tc-isa-sim
        path: tc-isa-sim.tar

  tc-verilator:
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v3
    - name: Recover the submodule commit hash
      id: recover_hash
      run: |
        git submodule status toolchain/verilator | cut -d' ' -f1
        echo "tc-verilator-hash=`git submodule status toolchain/verilator | cut -d' ' -f1`" >> $GITHUB_ENV
    - name: Cache Verilator
      uses: actions/cache@v3
      id: tc-verilator-cache
      env:
        cache-name: cache-verilator
      with:
        path: install/verilator
        key: ${{ runner.os }}-build-llvm10-${{ env.cache-name }}-${{ env.tc-verilator-hash }}
        restore-keys:
          ${{ runner.os }}-build-llvm10-${{ env.cache-name }}-${{ env.tc-verilator-hash }}
    - name: Download Verilator
      if: steps.tc-verilator-cache.outputs.cache-hit != 'true'
      run: |
        git submodule update --init --recursive -- toolchain/verilator
    - name: Compile Verilator
      if: steps.tc-verilator-cache.outputs.cache-hit != 'true'
      run: |
        sudo apt-get install flex libfl-dev help2man
        make verilator
    - name: Tar Verilator
      run: tar -cvf tc-verilator.tar install/verilator
    - name: Upload Verilator
      uses: actions/upload-artifact@v3
      with:
        name: tc-verilator
        path: tc-verilator.tar

###################
#  Compile stage  #
###################

  compile-apps:
    runs-on: ubuntu-20.04
    strategy:
      max-parallel: 1
      matrix:
        ara_config: [2_lanes, 4_lanes, 8_lanes, 16_lanes]
    needs: tc-llvm
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-python@v4
      with:
        python-version: 3.6
    - name: Install Python requirements
      run: pip install -r python-requirements.txt
    - name: Download Spike
      run: |
        git submodule update --init --recursive -- toolchain/riscv-isa-sim
        git submodule foreach --recursive git reset --hard
    - name: Download the LLVM toolchain
      uses: actions/download-artifact@v3
      with:
        name: tc-llvm
    - name: Untar LLVM
      run: tar xvf tc-llvm.tar
    - name: Compile applications
      run: config=${{ matrix.ara_config }} make -C apps
    - name: Upload applications
      uses: actions/upload-artifact@v3
      with:
        name: compile-apps-${{ matrix.ara_config }}
        path: apps/bin

  compile-riscv-tests:
    runs-on: ubuntu-20.04
    strategy:
      max-parallel: 1
      matrix:
        ara_config: [2_lanes, 4_lanes, 8_lanes, 16_lanes]
    needs: ["tc-llvm", "tc-gcc", "tc-isa-sim"]
    steps:
    - uses: actions/checkout@v3
    - name: Download Spike
      run: |
        git submodule update --init --recursive -- toolchain/riscv-isa-sim
        git submodule foreach --recursive git reset --hard
    - name: Get Spike artifacts
      uses: actions/download-artifact@v3
      with:
        name: tc-isa-sim
    - name: Untar Spike
      run: tar xvf tc-isa-sim.tar
    - name: Get LLVM toolchain artifacts
      uses: actions/download-artifact@v3
      with:
        name: tc-llvm
    - name: Untar LLVM
      run: tar xvf tc-llvm.tar
    - name: Get GCC toolchain artifacts
      uses: actions/download-artifact@v3
      with:
        name: tc-gcc
    - name: Untar GCC
      run: tar xvf tc-gcc.tar
    - name: Compile applications
      run: config=${{ matrix.ara_config }} make -C apps riscv_tests
    - name: Upload applications
      uses: actions/upload-artifact@v3
      with:
        name: compile-riscv-tests-${{ matrix.ara_config }}
        path: apps/bin

  compile-ara:
    runs-on: ubuntu-20.04
    strategy:
      max-parallel: 1
      matrix:
        ara_config: [2_lanes, 4_lanes, 8_lanes, 16_lanes]
    needs: ["tc-verilator", "tc-isa-sim"]
    steps:
    - uses: actions/checkout@v3
    - name: Download Spike
      run: |
        git submodule update --init --recursive -- toolchain/riscv-isa-sim
        git submodule foreach --recursive git reset --hard
    - name: Get Spike artifacts
      uses: actions/download-artifact@v3
      with:
        name: tc-isa-sim
    - name: Untar Spike
      run: tar xvf tc-isa-sim.tar
    - name: Get Verilator artifacts
      uses: actions/download-artifact@v3
      with:
        name: tc-verilator
    - name: Untar Verilator
      run: tar xvf tc-verilator.tar
    - name: Set the Verilator root directory
      run: |
        echo VERILATOR_ROOT="$GITHUB_WORKSPACE/install/verilator" >> $GITHUB_ENV
        VERILATOR_ROOT="$GITHUB_WORKSPACE/install/verilator"
        ln -s $VERILATOR_ROOT/share/verilator/include $VERILATOR_ROOT/include
        ln -s $VERILATOR_ROOT/share/verilator/bin/verilator_includer $VERILATOR_ROOT/bin/verilator_includer
    - name: Download RTL submodules
      run: make -C hardware update
    - name: Compile Verilated model of Ara
      run: |
        sudo apt-get install libelf-dev
        make -C hardware apply-patches
        config=${{ matrix.ara_config }} make -C hardware verilate
    - name: Tar Verilated model of Ara
      run: tar -cvf ara.tar hardware/build/verilator hardware/bender
    - name: Upload Ara Verilated model
      uses: actions/upload-artifact@v3
      with:
        name: compile-ara-${{ matrix.ara_config }}
        path: ara.tar

####################
#  Simulate stage  #
####################

  simulate:
    runs-on: ubuntu-20.04
    strategy:
      max-parallel: 1
      matrix:
        app:        [hello_world, imatmul, fmatmul, iconv2d, fconv2d, fconv3d, jacobi2d, dropout, fft, dwt, exp, softmax, dotproduct, fdotproduct, pathfinder, roi_align]
        ara_config: [2_lanes, 4_lanes, 8_lanes, 16_lanes]
    needs: ["compile-ara", "compile-apps"]
    steps:
    - uses: actions/checkout@v3
    - name: Get Spike artifacts
      uses: actions/download-artifact@v3
      with:
        name: tc-isa-sim
    - name: Untar Spike
      run: tar xvf tc-isa-sim.tar
    - name: Get Verilated model of Ara
      uses: actions/download-artifact@v3
      with:
        name: compile-ara-${{ matrix.ara_config }}
    - name: Untar Verilated model of Ara
      run: tar xvf ara.tar
    - name: Get applications
      uses: actions/download-artifact@v3
      with:
        name: compile-apps-${{ matrix.ara_config }}
        path: apps/bin
    - name: Run test
      run: config=${{ matrix.ara_config }} app=${{ matrix.app }} make -C hardware simv

########################
#  RISC-V Tests stage  #
########################

  riscv-tests-simv:
    runs-on: ubuntu-20.04
    strategy:
      max-parallel: 1
      matrix:
        ara_config: [2_lanes, 4_lanes, 8_lanes, 16_lanes]
    needs: ["compile-ara", "compile-riscv-tests"]
    steps:
    - uses: actions/checkout@v3
    - name: Get Spike artifacts
      uses: actions/download-artifact@v3
      with:
        name: tc-isa-sim
    - name: Untar Spike
      run: tar xvf tc-isa-sim.tar
    - name: Get Verilated model of Ara
      uses: actions/download-artifact@v3
      with:
        name: compile-ara-${{ matrix.ara_config }}
    - name: Untar Verilated model of Ara
      run: tar xvf ara.tar
    - name: Get RISC-V tests
      uses: actions/download-artifact@v3
      with:
        name: compile-riscv-tests-${{ matrix.ara_config }}
        path: apps/bin
    - name: Run tests
      run: config=${{ matrix.ara_config }} make -C hardware -j8 riscv_tests_simv

  riscv-tests-spike:
    runs-on: ubuntu-20.04
    needs: ["tc-isa-sim", "compile-riscv-tests"]
    steps:
    - uses: actions/checkout@v3
    - name: Get Spike artifacts
      uses: actions/download-artifact@v3
      with:
        name: tc-isa-sim
    - name: Untar Spike
      run: tar xvf tc-isa-sim.tar
    - name: Download the LLVM toolchain
      uses: actions/download-artifact@v3
      with:
        name: tc-llvm
    - name: Untar LLVM
      run: tar xvf tc-llvm.tar
    - name: Download the GCC toolchain
      uses: actions/download-artifact@v3
      with:
        name: tc-gcc
    - name: Untar GCC
      run: tar xvf tc-gcc.tar
    - name: Run tests
      run: |
        make -C apps/riscv-tests/isa clean
        make -C apps riscv_tests_spike
    - name: Upload dumps
      uses: actions/upload-artifact@v3
      with:
        name: riscv-tests-spike
        path: |
          apps/riscv-tests/isa/*.dump
          apps/riscv-tests/isa/*.out32

###################
#  Linting stage  #
###################

  check-license:
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-python@v4
      with:
        python-version: 3.6
    - name: Install Python requirements
      run: pip install -r python-requirements.txt
    - name: Check license
      run: python scripts/licence-checker.py --config scripts/licence-checker.hjson hardware

  check-clang-format:
    runs-on: ubuntu-20.04
    needs: ['tc-llvm']
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-python@v4
      with:
        python-version: 3.6
    - name: Install Python requirements
      run: pip install -r python-requirements.txt
    - name: Download the LLVM toolchain
      uses: actions/download-artifact@v3
      with:
        name: tc-llvm
    - name: Untar LLVM
      run: tar xvf tc-llvm.tar
    - name: Run clang-format
      run: |
        for file in `find apps -type f -name "*.[c|h|cpp|hpp]" | grep -vP "apps/riscv-tests"`; do
          ./scripts/run-clang-format.py --clang-format-executable install/riscv-llvm/bin/clang-format $file || EXIT_STATUS=$?
        done
        for file in `find apps/riscv-tests/isa/rv64uv -type f -name "*.[c|h|cpp|hpp]"`; do
          ./scripts/run-clang-format.py --clang-format-executable install/riscv-llvm/bin/clang-format $file || EXIT_STATUS=$?
        done
        exit $EXIT_STATUS

  check-trailing-whitespaces:
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - name: Determine base commit
      run: |
        if [[ -n $GITHUB_BASE_REF ]]; then
          # Make sure we have the latest version of the target branch
          git fetch origin $GITHUB_BASE_REF
          echo "base=origin/$GITHUB_BASE_REF" >> $GITHUB_ENV
        else
          echo "base=HEAD~1" >> $GITHUB_ENV
        fi
    # Don't check the `patches` directory! Trailing whitespaces
    # are mandatory there if they exist.
    - name: Check for trailing whitespaces and tabs
      run: |
        bash -O extglob -c                \
        "git diff --check $base HEAD --   \
        apps config .github *.md Bender.* \
        Makefile hardware/!(patches)"

#####################
#  Benchmark stage  #
#####################

  benchmark:
    runs-on: ubuntu-20.04
    strategy:
      max-parallel: 1
      matrix:
        ara_config: [2_lanes, 4_lanes, 8_lanes, 16_lanes]
    needs: ["compile-ara", "compile-apps"]
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-python@v4
      with:
        python-version: 3.6
    - name: Install Python requirements
      run: pip install -r python-requirements.txt
    - name: Download the LLVM toolchain
      uses: actions/download-artifact@v3
      with:
        name: tc-llvm
    - name: Untar LLVM
      run: tar xvf tc-llvm.tar
    - name: Get Spike artifacts
      uses: actions/download-artifact@v3
      with:
        name: tc-isa-sim
    - name: Untar Spike
      run: tar xvf tc-isa-sim.tar
    - name: Get Verilated model of Ara
      uses: actions/download-artifact@v3
      with:
        name: compile-ara-${{ matrix.ara_config }}
    - name: Untar Verilated model of Ara
      run: tar xvf ara.tar
    - name: Benchmark Ara
      run: config=${{ matrix.ara_config }} ./scripts/benchmark.sh ci
    - name: Tar [f]dotproduct runtime results
      run: |
        tar -cvf dotproducts-${{ matrix.ara_config }}.tar *dotproduct*.benchmark
        rm *dotproduct*.benchmark
    - name: Tar runtime results
      run: |
        tar -cvf benchmarks-${{ matrix.ara_config }}.tar *.benchmark
    - name: Upload [f]dotproduct runtime results
      uses: actions/upload-artifact@v3
      with:
        name: dotproducts-${{ matrix.ara_config }}
        path: dotproducts-${{ matrix.ara_config }}.tar
    - name: Upload runtime results
      uses: actions/upload-artifact@v3
      with:
        name: benchmark-${{ matrix.ara_config }}
        path: benchmarks-${{ matrix.ara_config }}.tar

  roofline:
    runs-on: ubuntu-20.04
    needs: benchmark
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-python@v4
      with:
        python-version: 3.6
    - name: Install Python requirements
      run: pip install -r python-requirements.txt
    - uses: actions/checkout@v3
    - name: Get [f]dotproduct results (2 lanes)
      uses: actions/download-artifact@v3
      with:
        name: dotproducts-2_lanes
    - name: Get [f]dotproduct results (4 lanes)
      uses: actions/download-artifact@v3
      with:
        name: dotproducts-4_lanes
    - name: Get [f]dotproduct results (8 lanes)
      uses: actions/download-artifact@v3
      with:
        name: dotproducts-8_lanes
    - name: Get [f]dotproduct results (16 lanes)
      uses: actions/download-artifact@v3
      with:
        name: dotproducts-16_lanes
    - name: Get benchmark results (2 lanes)
      uses: actions/download-artifact@v3
      with:
        name: benchmark-2_lanes
    - name: Get benchmark results (4 lanes)
      uses: actions/download-artifact@v3
      with:
        name: benchmark-4_lanes
    - name: Get benchmark results (8 lanes)
      uses: actions/download-artifact@v3
      with:
        name: benchmark-8_lanes
    - name: Get benchmark results (16 lanes)
      uses: actions/download-artifact@v3
      with:
        name: benchmark-16_lanes
    - name: Untar the [f]dotproduct results
      run: |
        tar xvf dotproducts-2_lanes.tar
        tar xvf dotproducts-4_lanes.tar
        tar xvf dotproducts-8_lanes.tar
        tar xvf dotproducts-16_lanes.tar
    - name: Untar the results
      run: |
        tar xvf benchmarks-2_lanes.tar
        tar xvf benchmarks-4_lanes.tar
        tar xvf benchmarks-8_lanes.tar
        tar xvf benchmarks-16_lanes.tar
    - name: Install gnuplot
      run: |
        sudo apt-get update
        sudo apt-get install gnuplot
    - name: Plot [f]dotproduct plots
      run: |
        for kernel in fdotproduct dotproduct; do
          > ${kernel}.benchmark
          for nr_lanes in 2 4 8 16; do
            cat ${kernel}_${nr_lanes}_bar_plots.benchmark >> ${kernel}.benchmark
          done
          python scripts/process_dotp.py ${kernel} ${kernel}.benchmark ${kernel}
          tar -cvf ${kernel}.tar $(find . -name "${kernel}_*.png")
        done
    - name: Plot the rooflines
      run: gnuplot -c scripts/benchmark.gnuplot
    - name: Upload the imatmul roofline
      uses: actions/upload-artifact@v3
      with:
        name: imatmul_roofline
        path: imatmul.png
    - name: Upload the fmatmul roofline
      uses: actions/upload-artifact@v3
      with:
        name: fmatmul_roofline
        path: fmatmul.png
    - name: Upload the iconv2d roofline
      uses: actions/upload-artifact@v3
      with:
        name: iconv2d_roofline
        path: iconv2d.png
    - name: Upload the fconv2d roofline
      uses: actions/upload-artifact@v3
      with:
        name: fconv2d_roofline
        path: fconv2d.png
    - name: Upload the fconv3d roofline
      uses: actions/upload-artifact@v3
      with:
        name: fconv3d_roofline
        path: fconv3d.png
    - name: Upload the jacobi2d roofline
      uses: actions/upload-artifact@v3
      with:
        name: jacobi2d_roofline
        path: jacobi2d.png
    - name: Upload the dropout roofline
      uses: actions/upload-artifact@v3
      with:
        name: dropout_roofline
        path: dropout.png
    - name: Upload the fft roofline
      uses: actions/upload-artifact@v3
      with:
        name: fft_roofline
        path: fft.png
    - name: Upload the dwt roofline
      uses: actions/upload-artifact@v3
      with:
        name: dwt_roofline
        path: dwt.png
    - name: Upload the exp roofline
      uses: actions/upload-artifact@v3
      with:
        name: exp_roofline
        path: exp.png
    - name: Upload the softmax roofline
      uses: actions/upload-artifact@v3
      with:
        name: softmax_roofline
        path: softmax.png
    - name: Upload the fdotproduct roofline
      uses: actions/upload-artifact@v3
      with:
        name: fdotproduct_plots
        path: fdotproduct.tar
    - name: Upload the dotproduct roofline
      uses: actions/upload-artifact@v3
      with:
        name: dotproduct_plots
        path: dotproduct.tar
    - name: Upload the pathfinder roofline
      uses: actions/upload-artifact@v3
      with:
        name: pathfinder_roofline
        path: pathfinder.png
    - name: Upload the roi_align roofline
      uses: actions/upload-artifact@v2
      with:
        name: roi_align_roofline
        path: roi_align.png

####################
#  Clean-up stage  #
####################

  clean-up:
    runs-on: ubuntu-20.04
    if: always()
    needs: ["simulate", "riscv-tests-spike", "riscv-tests-simv"]
    steps:
      - uses: actions/checkout@v3
      - name: Delete artifacts
        uses: geekyeggo/delete-artifact@v2
        with:
          name: |
            tc-llvm
            tc-gcc
            tc-isa-sim
            tc-verilator
            riscv-tests-spike

  clean-up-compile-runs:
    runs-on: ubuntu-20.04
    strategy:
      max-parallel: 1
      matrix:
        ara_config: [2_lanes, 4_lanes, 8_lanes, 16_lanes]
    if: always()
    needs: ["simulate", "riscv-tests-spike", "riscv-tests-simv"]
    steps:
      - uses: actions/checkout@v3
      - name: Delete artifacts
        uses: geekyeggo/delete-artifact@v2
        with:
          name: |
            compile-ara-${{ matrix.ara_config }}
            compile-apps-${{ matrix.ara_config }}
            compile-riscv-tests-${{ matrix.ara_config }}
            benchmark-${{ matrix.ara_config }}