ci-scheduled-fuzzing.yml

#
#  This workflow will normally run on a schedule.
#
#  It can also be invoked manually by pushing to a branch named as follows:
#
#      run-fuzzer[[-<protocol>]-<timeout>]
#
#      <timeout> is total run length, including setup.
#      <protocol> is the name of the unit to fuzz.
#
#  For example:
#
#    - 'run-fuzzer':             Start fuzzing all protocols (default timeout)
#    - 'run-fuzzer-3600':        Start fuzzing all protocols for one hour
#    - 'run-fuzzer-radius-7200': Start fuzzing RADIUS for two hours
#
#  Fuzzing failures (full log output including backtraces and reproducers) are
#  uploaded as "artifacts" for the GitHub Actions run.
#
#  The following script can be used to list fuzzer failures and download
#  reproducers for local reproduction:
#
#      scripts/build/fuzzer-fetch-artifacts
#
#  If local reproduction does not recreate the failure then you may wish to
#  attempt the reproduction within a GitHub Actions runner. To do this push to
#  the following branch:
#
#    - 'debug-fuzzer-<protocol>'
#
#  This will perform a fuzzer-enabled build and then instead of fuzzing will
#  launch a tmate session that can be used to access the environment. Further
#  details at the end of this file.
#

name: Scheduled fuzzing

on:
  push:
    branches:
      - 'run-fuzzer**'
      - 'debug-fuzzer-**'

  schedule:
    - cron: '0 4 * * *'

env:
  ASAN_OPTIONS: symbolize=1 detect_leaks=1 detect_stack_use_after_return=1
  LSAN_OPTIONS: fast_unwind_on_malloc=0:malloc_context_size=50
  M_PERTURB: "0x42"
  PANIC_ACTION: "gdb -batch -x raddb/panic.gdb %e %p 1>&0 2>&0"
  ANALYZE_C_DUMP: 1
  FR_GLOBAL_POOL: 4M
  TEST_CERTS: yes
  DO_BUILD: yes
  CI: 1
  GH_ACTIONS: 1
  CC: clang

jobs:

  #
  #  Constructs a matrix of protocols to fuzz as JSON that when set in the main
  #  fuzzer job is equivalent to the following YAML:
  #
  #    matrix:
  #      env:
  #        - { "PROTOCOL": "radius", "TOTAL_RUNTIME": "20000", "UPLOAD_SLOT": "0", "NUM_SLOTS": "15" }
  #        - { "PROTOCOL": "dhcpv4", "TOTAL_RUNTIME": "20000", "UPLOAD_SLOT": "3", "NUM_SLOTS": "15" }
  #        - ...
  #
  set-matrix:

    name: Setup build matrix

    runs-on: ubuntu-latest

    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
      starttimestamp: ${{ steps.starttimestamp.outputs.starttimestamp }}

    steps:

    - id: starttimestamp
      name: Record run start time
      run: |
        echo starttimestamp=`date +%s` >> $GITHUB_OUTPUT

    - uses: actions/checkout@v4
      with:
        lfs: false

    - id: set-matrix
      name: Setup the matrix
      run: |
        #
        #  By default we fuzz all protocols for 20000s (just short of the 6h
        #  GitHub Action run limit)
        #
        TOTAL_RUNTIME=20000
        SLOT_SPREAD=3  # secs
        read -r -a PROTOS <<< $(sed -ne 's/^FUZZER_PROTOCOLS\s\+=\s\+\(.*\)/\1/p' src/bin/all.mk)
        #
        if [[ "$GITHUB_REF" = refs/heads/run-fuzzer-*-* ]]; then
          PROTOS=${GITHUB_REF#refs/heads/run-fuzzer-}
          TOTAL_RUNTIME=${PROTOS##*-}
          PROTOS=( "${PROTOS%-*}" )
        elif [[ "$GITHUB_REF" = refs/heads/run-fuzzer-* ]]; then
          TOTAL_RUNTIME=${GITHUB_REF#refs/heads/run-fuzzer-}
        elif [[ "$GITHUB_REF" = refs/heads/debug-fuzzer-* ]]; then
          PROTOS=${GITHUB_REF#refs/heads/debug-fuzzer-}
          PROTOS=( "${PROTOS%-*}" )
        fi
        P=$(
            for i in ${!PROTOS[@]}; do
              echo "{ \"PROTOCOL\": \"${PROTOS[$i]}\", \"TOTAL_RUNTIME\": \"$TOTAL_RUNTIME\", \"UPLOAD_SLOT\": \"$((i * $SLOT_SPREAD))\", \"NUM_SLOTS\": \"$((${#PROTOS[@]} * $SLOT_SPREAD))\" },"
            done
          )
        M=$(cat <<EOF
        {
          "env": [
            ${P:0:-1}
          ]
        }
        EOF
          )
        echo "Matrix:"
        echo "$M"
        echo matrix=$M >> $GITHUB_OUTPUT


  fuzzer:

    needs:
      - set-matrix

    runs-on: ubuntu-24.04

    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.set-matrix.outputs.matrix) }}

    env: ${{ matrix.env }}

    name: Fuzzing ${{ matrix.env.PROTOCOL}}

    steps:

    #  Checkout, but defer pulling LFS objects until we've restored the cache
    #
    #  We include a bit of depth because we need to see when the corpus was
    #  last updated and because we will walk the tree until we find a commit
    #  that builds.
    #
    - uses: actions/checkout@v4
      with:
        lfs: false
        fetch-depth: 500

    #
    #  We push changes to the corpus to the GH Actions cache, and restore based
    #  on the commit date of the corpus tar file from the repo.
    #
    #  Therefore, if a new corpus is pushed to the repo then we will use it.
    #  Otherwise we will search the cache for a more recently merged version of
    #  the corpus in the repo.
    #
    - name: Get the corpus age
      id: corpusparams
      run: |
        CORPUSCT="$(git log -1 --format=%ct -- src/tests/fuzzer-corpus/$PROTOCOL.tar)"
        CORPUSAGE="$((`date +%s` - "$CORPUSCT"))"
        CORPUSDAYS="$(($CORPUSAGE / 86400))"
        echo "$PROTOCOL corpus age is $CORPUSAGE secs ($CORPUSDAYS days)"
        echo "corpusct=$CORPUSCT" >> $GITHUB_OUTPUT
        echo "corpusage=$CORPUSAGE" >> $GITHUB_OUTPUT

    - name: Restore the fuzzer corpus tar file from cache
      uses: actions/cache@v4
      id: corpus-cache
      with:
        path: src/tests/fuzzer-corpus/${{ matrix.env.PROTOCOL }}.tar
        key: corpus-${{ matrix.env.PROTOCOL }}-${{ steps.corpusparams.outputs.corpusct }}-${{ github.run_number }}
        restore-keys: |
          corpus-${{ matrix.env.PROTOCOL }}-${{ steps.corpusparams.outputs.corpusct }}-
      if: ${{ !startsWith(github.ref, 'refs/heads/debug-fuzzer-') }}

    - name: Install build dependencies
      uses: ./.github/actions/freeradius-deps
      with:
        use_docker: false
        cc: ${{ matrix.env.CC }}
        llvm_ver: 18

    - name: Show versions
      run: |
        $CC --version
        make --version
        krb5-config --all || :
        pcre-config --libs-posix --version 2>/dev/null || :
        pcre2-config --libs-posix --version 2>/dev/null || :

    #
    #  We walk up the tree if necessary to find a commit that builds so that we
    #  will fuzz something
    #
    - name: Find a commit that builds
      id: pick_commit
      run: |
        while : ; do
          echo "Testing commit:"
          git log -n1
          echo
          CFLAGS="-DWITH_EVAL_DEBUG -O2 -g3" ./configure -C \
            --enable-werror \
            --enable-address-sanitizer \
            --enable-undefined-behaviour-sanitizer \
            --enable-leak-sanitizer \
            --enable-fuzzer \
            --prefix=$HOME/freeradius \
          || cat ./config.log
          echo "Contents of src/include/autoconf.h"
          cat "./src/include/autoconf.h"
          mkdir -p build/tests/eapol_test
          : > build/tests/eapol_test/eapol_test.mk
          make -j `nproc` build/bin/fuzzer_$PROTOCOL && break || :
          git reset --hard HEAD^
          git clean -fxd
        done
        echo "commit_id=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
        make -j `nproc`
        make test.unit

    - name: Run fuzzer tests
      run: |
        REMAINING_TIME=$(( $TOTAL_RUNTIME + $START_TIMESTAMP - `date +%s` ))
        echo "Started at $START_TIMESTAMP for $TOTAL_RUNTIME secs. Fuzzing ${{ steps.pick_commit.outputs.commit_id }}:$PROTOCOL for $REMAINING_TIME secs"
        [[ "$REMAINING_TIME" -lt 1 ]] && exit 1
        make test.fuzzer.$PROTOCOL FUZZER_TIMEOUT="$REMAINING_TIME" FUZZER_ARGUMENTS="-jobs=`nproc` -workers=`nproc`" || :
        find build/fuzzer -type f ! -path 'build/fuzzer/*.log' | grep . && exit 1 || :
      env:
        GITHUB_REF: "${{ github.ref }}"
        START_TIMESTAMP: "${{ needs.set-matrix.outputs.starttimestamp }}"
      if: ${{ !startsWith(github.ref, 'refs/heads/debug-fuzzer-') }}

    - name: "Clang libFuzzer: Store assets on failure"
      uses: actions/upload-artifact@v4
      with:
        name: clang-fuzzer-${{ matrix.env.PROTOCOL }}-${{ steps.pick_commit.outputs.commit_id }}
        path: build/fuzzer
        retention-days: 30
      if: ${{ !startsWith(github.ref, 'refs/heads/debug-fuzzer') && failure() }}

    #
    #  Merge the corpus which will be stored in the cache for the next run
    #
    - name: Merge the corpus
      run: |
        make test.fuzzer.$PROTOCOL.merge
      if: ${{ !startsWith(github.ref, 'refs/heads/debug-fuzzer-') }}

    #
    #  We can push the LFS file directly, but we must use the GitHub API to
    #  create the actual commit due to the "signed-commits" branch protection
    #  rule for the master branch.
    #
    #  Force reinstall of pyOpenSSL is to work around Python
    #  cryptograpy package issue - PyGithub pulls in a newer
    #  version which clashes with older apt-installed pyOpenSSL
    #  https://github.com/pyca/pyopenssl/issues/1143
    #
    - name: Monthly push back of corpus
      run: |
        export FILE=src/tests/fuzzer-corpus/$PROTOCOL.tar
        if ! git diff --exit-code "$FILE"; then
          sudo apt-get install -y python3-venv
          python3 -m venv ~/.venv
          . ~/.venv/bin/activate
          pip install --force-reinstall -I -U pyOpenSSL
          pip install PyGithub
          git add "$FILE"
          OID="$(git lfs ls-files -l -I "$FILE" | cut -f1 -d ' ')"
          git lfs push --object-id origin "$OID"
          export CONTENTS="$(git show ":$FILE" | base64)"
          SLEEP_FOR=$(( ( $UPLOAD_SLOT - `date +%s` % $NUM_SLOTS + $NUM_SLOTS ) % $NUM_SLOTS ))
          echo Waiting $SLEEP_FOR secs for our upload slot...
          sleep $SLEEP_FOR
          python3 scripts/ci/commit_lfs_file_update.py
        fi
      env:
        GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
      if: ${{ !startsWith(github.ref, 'refs/heads/debug-fuzzer-') && steps.corpusparams.outputs.corpusage > 2592000 && github.repository_owner == 'FreeRADIUS' }}


    #
    #  If we are on the 'debug-fuzzer-*' branch then we start a tmate session to
    #  provide interactive shell access to the session so that the reproducers
    #  can be attempted in an identical environment to which the scheduled
    #  fuzzing occurred.
    #
    #  The SSH rendezvous point will be emited continuously in the job output,
    #  which will look something like:
    #
    #      SSH: ssh VfuX8SrNuU5pGPMyZcz7TpJTa@sfo2.tmate.io
    #
    #  For example:
    #
    #      git push origin debug-fuzzer-radius --force
    #
    #  Look at the job output in: https://github.com/FreeRADIUS/freeradius-server/actions
    #
    #      ssh VfuX8SrNuU5pGPMyZcz7TpJTa@sfo2.tmate.io
    #
    #  Access requires that you have the private key corresponding to the
    #  public key of the GitHub user that initiated the job.
    #
    #  Within this session you can use scripts/build/fuzzer-fetch-artifacts to
    #  download the reproducers just as you would do locally, e.g.
    #
    #      export GITHUB_TOKEN=<personal-access-token>
    #      scripts/build/fuzzer-fetch-artifacts
    #      scripts/build/fuzzer-fetch-artifacts https://api.github.com/repos/FreeRADIUS/freeradius-server/actions/artifacts/186571481/zip
    #      scripts/build/fuzzer build/fuzzer/radius/crash-f1536d0fa2de775038e5dab74d233487a7cde819
    #
    - name: "Debug: Start tmate"
      uses: mxschmitt/action-tmate@v3
      with:
        limit-access-to-actor: true
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      if: ${{ startsWith(github.ref, 'refs/heads/debug-fuzzer-') && always() }}