pr.yml

name: CTIA

on:
  # to enable pull requests from other repositories
  pull_request:
  # for deployment only
  push:
    branches:
      - master
      # Note: full regex not supported here, hardcoding vX.X and vX.X.X
      # See https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#filter-pattern-cheat-sheet
      - "v[0-9]+.[0-9]+"
      - "v[0-9]+.[0-9]+.[0-9]+"
  # Cron job: runs a matrix of JVM and Clojure versions, which can
  # be configured in the `cron-matrix` function in ./scripts/actions/print-matrix.clj
  # Note: automatically uses master branch https://stackoverflow.com/a/58800550
  schedule:
    # daily, 3am UTC
    - cron: "0 3 * * *"

env:
  # number of times to try possibly-flaky commands in this file
  ACTIONS_RETRY_TIMES: 10
  # workaround https://github.com/actions/cache/issues/2
  # to "delete" caches, increment this version
  ACTIONS_CACHE_VERSION: v17
  BIN_PATH: ${{github.workspace}}/bin
  LOG_PATH: ${{github.workspace}}/log
  # avoid the (somehow) default /usr/local/lib/lein/
  LEIN_HOME: ${{github.workspace}}/.lein
  LEIN_ROOT: 1
  LEIN_VERSION: 2.9.6
  CTIA_MAJOR_VERSION: 1
  SHELLCHECK_VERSION: v0.7.2
  BB_VERSION: 0.7.8
  DEFAULT_JAVA_VERSION: 11

jobs:
  setup:
    runs-on: "ubuntu-22.04"
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - uses: actions/checkout@v3
        with:
          # to read the latest commit message on a PR
          # https://github.com/zero88/gh-project-context/blob/26a3afdce1fe607734278e5ccec0851d50669ef5/README.md#latest-commit-message
          fetch-depth: 2
      # for grabbing the current commit msg (suprisingly hard!)
      - uses: zero88/gh-project-context@af8b62dd9d7ecbb2763268a040d42390ecfeb0f3
        id: gh-project-context
      - name: Binary Cache
        uses: actions/cache@v3
        with:
          path: ${{ env.BIN_PATH }}
          key: ctia-bin-${{ env.ACTIONS_CACHE_VERSION }}-${{ env.BB_VERSION }}
      - name: Setup PATH and directories
        run: |
          mkdir -p "${BIN_PATH}"
          echo "${BIN_PATH}" >> $GITHUB_PATH
      - name: Install babashka
        run: ./scripts/actions/install-bb.sh
      - name: Run tests for scripts
        run: ./scripts-test/test_runner.clj
      - run: ./scripts/actions/setup_env.clj
      # scan ctia's source for trojans. uberjar is scanned in deploy step.
      - run: ./scripts/actions/install-trojansourcefinder.sh
      - run: ./scripts/trojan-scan.sh
      - name: Setup test matrix splits
        id: set-matrix
        run: |
          echo "CTIA_COMMIT_MESSAGE=${CTIA_COMMIT_MESSAGE}"
          ./scripts/actions/print_matrix.clj
        env:
          CTIA_COMMIT_MESSAGE: ${{steps.gh-project-context.outputs.commitMsg}}
      # Setup previous timing information
      - name: Timing Cache
        id: get-timing
        uses: actions/cache@v3
        with:
          path: |
            target/test-results/all-test-timings.edn
          # cache will never match, will always restore from 'restore-keys'.
          # this is so we pick up timing information from the most recent build.
          key: writeonly-ctia-all-test-timings-${{ env.ACTIONS_CACHE_VERSION }}-${{ env.CTIA_TEST_SUITE }}-${{ github.ref }}-${{ github.sha }}
          # cache should match test timings from previous runs, if they exist.
          # pr's and pushes share timings via the "ci" setting of env.CTIA_TEST_SUITE,
          # and cron is separate as "cron".
          # timings shared between pull requests and cron only as last resort.
          restore-keys: |
            ctia-timings-${{ env.ACTIONS_CACHE_VERSION }}-${{ env.CTIA_TEST_SUITE }}-${{ github.ref }}-
            ctia-timings-${{ env.ACTIONS_CACHE_VERSION }}-${{ env.CTIA_TEST_SUITE }}-
            ctia-timings-${{ env.ACTIONS_CACHE_VERSION }}-
      - run: |
          if [ -f target/test-results/all-test-timings.edn ]; then
            echo "Found new timings"
            cat target/test-results/all-test-timings.edn
            cp target/test-results/all-test-timings.edn dev-resources/ctia_test_timings.edn
          else
            echo "Timings not found, creating empty dummy timings which will be ignored by tests"
            touch dev-resources/ctia_test_timings.edn
          fi
      - name: Upload current test timing
        uses: actions/upload-artifact@v4
        with:
          retention-days: 180
          name: current-test-timing
          path: dev-resources/ctia_test_timings.edn

      # Run shellcheck on CTIA's scripts
      - name: Install shellcheck
        run: |
          if ! command -v shellcheck &> /dev/null || ! shellcheck --version | grep "^version: ${SHELLCHECK_VERSION}$"
          then
            ( set -x && wget -qO- "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar -xJv )
            ( set -x && ls -al "${BIN_PATH}" )
            ( set -x && cp "shellcheck-${SHELLCHECK_VERSION}/shellcheck" "${BIN_PATH}" )
          fi
          shellcheck --version
      - name: Run shellcheck
        run: ./scripts/shellcheck-build.sh


      # warm dependency cache on project.clj change (ie., when cache-hit is not true).
      - name: Maven Cache
        id: maven-cache
        uses: actions/cache@v3
        with:
          path: |
            ~/.m2/repository
            ~/.gitlibs
          key: ctia-m2-cache-${{ env.ACTIONS_CACHE_VERSION }}-${{ hashFiles('project.clj') }}-${{ env.LEIN_VERSION }}-${{ env.DEFAULT_JAVA_VERSION }}
          # these will be considered a cache-miss by `cache-hit`
          restore-keys: |
            ctia-m2-cache-${{ env.ACTIONS_CACHE_VERSION }}-${{ hashFiles('project.clj') }}-${{ env.LEIN_VERSION }}-
            ctia-m2-cache-${{ env.ACTIONS_CACHE_VERSION }}-${{ hashFiles('project.clj') }}-
            ctia-m2-cache-${{ env.ACTIONS_CACHE_VERSION }}-
      - name: Setup Java
        if: steps.maven-cache.outputs.cache-hit != 'true'
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: ${{ env.DEFAULT_JAVA_VERSION }}
      - run: java -version
        if: steps.maven-cache.outputs.cache-hit != 'true'
      - name: Leiningen Cache
        if: steps.maven-cache.outputs.cache-hit != 'true'
        uses: actions/cache@v3
        id: lein-cache
        with:
          path: ${{env.LEIN_HOME}}
          key: ctia-lein-cache-${{ env.ACTIONS_CACHE_VERSION }}-${{ env.LEIN_VERSION }}
      - name: Install Leiningen
        if: steps.maven-cache.outputs.cache-hit != 'true' && steps.lein-cache.outputs.cache-hit != 'true'
        run: ./scripts/actions/install-lein.sh
      - name: Warm dependency cache
        if: steps.maven-cache.outputs.cache-hit != 'true'
        # poor man's travis_retry
        run: for i in {1..${ACTIONS_RETRY_TIMES}}; do lein warm-ci-deps && break; done
      - run: ./scripts/check-dependabot

  test-encoding:
    runs-on: ubuntu-20.04
    needs: [setup]
    steps:

      - name: Checkout
        uses: actions/checkout@v3

      - name: Start Docker environment
        run: docker compose -f containers/dev/docker-compose.yml up --no-color --quiet-pull -d

      - name: Maven Cache
        id: maven-cache
        uses: actions/cache@v3
        with:
          path: |
            ~/.m2/repository
            ~/.gitlibs
          key: ctia-m2-cache-${{ env.ACTIONS_CACHE_VERSION }}-${{ hashFiles('project.clj') }}-${{ env.LEIN_VERSION }}
          restore-keys: |
            ctia-m2-cache-${{ env.ACTIONS_CACHE_VERSION }}-${{ hashFiles('project.clj') }}-
            ctia-m2-cache-${{ env.ACTIONS_CACHE_VERSION }}-

      - name: Setup Java
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: ${{ env.DEFAULT_JAVA_VERSION }}

      # install lein using cache to avoid extra download and startup time
      - name: Install Leiningen from Cache
        uses: actions/cache@v3
        id: lein-cache
        with:
          path: ${{env.LEIN_HOME}}
          key: ctia-lein-cache-${{ env.ACTIONS_CACHE_VERSION }}-${{ env.LEIN_VERSION }}

      - name: Install Leiningen
        if: steps.lein-cache.outputs.cache-hit != 'true'
        run: ./scripts/actions/install-lein.sh

      # we should get a cache hit from maven here, since the `setup` job only succeeds if it exists.
      # in the unlikely event we don't, download deps for this specific job only.
      - name: Warm dependency cache
        if: steps.maven-cache.outputs.cache-hit != 'true'
        # poor man's travis_retry
        run: for i in {1..${ACTIONS_RETRY_TIMES}}; do lein warm-ci-deps && break; done

      - name: Run encoding tests
        run: lein with-profile +test-encoding test

  test-matrix:
    runs-on: ubuntu-20.04
    needs: [setup]
    strategy:
      matrix:
        this-split: ${{fromJson(needs.setup.outputs.matrix)}}
    env:
      CTIA_TEST_SUITE: ${{ matrix.this-split.test_suite }}
      CTIA_THIS_SPLIT: ${{ matrix.this-split.this_split }}
      CTIA_NSPLITS: ${{ matrix.this-split.total_splits }}
      CTIA_CI_PROFILES: ${{ matrix.this-split.ci_profiles }}
      JAVA_VERSION: ${{ matrix.this-split.java_version }}
    steps:
      - uses: actions/checkout@v3
      - name: Binary Cache
        uses: actions/cache@v3
        with:
          path: ${{ env.BIN_PATH }}
          key: ctia-bin-${{ env.ACTIONS_CACHE_VERSION }}-${{ env.BB_VERSION }}
      - name: Setup PATH and directories
        run: |
          mkdir -p "${BIN_PATH}"
          echo "${BIN_PATH}" >> $GITHUB_PATH
      - name: Install babashka
        run: ./scripts/actions/install-bb.sh
      - run: ./scripts/actions/setup_env.clj
      - name: Docker
        # depends on LOG_PATH and actions/checkout@v3
        run: docker compose -f containers/dev/docker-compose.yml up --no-color -d &> "${LOG_PATH}/docker-compose.log"
      - name: Download test timings
        uses: actions/download-artifact@v4
        with:
          name: current-test-timing
          path: target/current-test-timing
      - run: |
          # debugging
          ls -al target/current-test-timing
          if [ -f target/current-test-timing/ctia_test_timings.edn ]; then
            cat target/current-test-timing/ctia_test_timings.edn
          fi
          if [ -s target/current-test-timing/ctia_test_timings.edn ]; then
            echo "Updating dev-resources/ctia_test_timings.edn with new timing"
            cp target/current-test-timing/ctia_test_timings.edn dev-resources/ctia_test_timings.edn
          else
            echo "No previous timings found (empty file downloaded)"
          fi
      - run: |
          if [ -f dev-resources/ctia_test_timings.edn ]; then
            echo "Timing:"
            cat dev-resources/ctia_test_timings.edn
          else
            echo "No timing data, using slow-namespace heuristic"
          fi
      - name: ES setup
        run: sudo sysctl -w vm.max_map_count=262144

      - name: Maven Cache
        id: maven-cache
        uses: actions/cache@v3
        with:
          path: |
            ~/.m2/repository
            ~/.gitlibs
          key: ctia-m2-cache-${{ env.ACTIONS_CACHE_VERSION }}-${{ hashFiles('project.clj') }}-${{ env.LEIN_VERSION }}
          restore-keys: |
            ctia-m2-cache-${{ env.ACTIONS_CACHE_VERSION }}-${{ hashFiles('project.clj') }}-
            ctia-m2-cache-${{ env.ACTIONS_CACHE_VERSION }}-

      - uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: ${{ env.JAVA_VERSION }}
      - run: java -version

      # install lein using cache to avoid extra download and startup time
      - name: Leiningen Cache
        uses: actions/cache@v3
        id: lein-cache
        with:
          path: ${{env.LEIN_HOME}}
          key: ctia-lein-cache-${{ env.ACTIONS_CACHE_VERSION }}-${{ env.LEIN_VERSION }}
      - name: Install Leiningen
        if: steps.lein-cache.outputs.cache-hit != 'true'
        run: ./scripts/actions/install-lein.sh

      # we should get a cache hit from maven here, since the `setup` job only succeeds if it exists.
      # in the unlikely event we don't, download deps for this specific job only.
      - name: Warm dependency cache
        if: steps.maven-cache.outputs.cache-hit != 'true'
        # poor man's travis_retry
        run: for i in {1..${ACTIONS_RETRY_TIMES}}; do lein warm-ci-deps && break; done
      # Note: if this step fails because Leiningen is in offline mode, add more entries to the maven-cache key.
      # Most likely, whatever was changed to cause the build to fail needs to be appended to the cache key.
      # Note there are two Maven caches in this file, keep their keys synchronized.
      - name: Run CTIA tests
        run: ./build/run-tests.sh
        env:
          # offline to catch issues where `lein warm-ci-deps` doesn't download all
          # dependencies we need to run tests.
          LEIN_OFFLINE: true
          CTIA_WAIT_DOCKER: 1

      - name: Upload test timing
        uses: actions/upload-artifact@v4
        with:
          retention-days: 1
          name: test-timing-${{matrix.this-split.test_suite}}-${{matrix.this-split.java_version}}-${{matrix.this-split.this_split}}
          path: target/test-results/*.edn
      - name: Upload docker compose
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          retention-days: 10
          name: docker-compose-${{matrix.this-split.test_suite}}-${{matrix.this-split.java_version}}-${{matrix.this-split.this_split}}.log
          path: ${{env.LOG_PATH}}/docker-compose.log
  # fan-in tests so there's a single job we can add to protected branches.
  # otherwise, we'll have add all (range ${CTIA_NSPLITS}) jobs, and keep
  # them up to date
  # here's a GitHub Actions feature request that is relevant:
  # https://github.community/t/branch-protections-job-names-and-matrix-jobs/16317
  all-pr-checks:
    runs-on: ubuntu-20.04
    needs: [test-matrix]
    steps:
      - uses: actions/checkout@v3
      - name: Binary Cache
        uses: actions/cache@v3
        with:
          path: ${{ env.BIN_PATH }}
          key: ctia-bin-${{ env.ACTIONS_CACHE_VERSION }}-${{ env.BB_VERSION }}
      - name: Setup PATH and directories
        run: |
          mkdir -p "${BIN_PATH}"
          echo "${BIN_PATH}" >> $GITHUB_PATH
      - name: Install babashka
        run: ./scripts/actions/install-bb.sh
      - run: ./scripts/actions/setup_env.clj
      - name: Timing results Cache
        uses: actions/cache@v3
        with:
          path: target/test-results/all-test-timings.edn
          # cache should never hit
          key: ctia-timings-${{ env.ACTIONS_CACHE_VERSION }}-${{ env.CTIA_TEST_SUITE }}-${{ github.ref }}-${{ github.sha }}

      - name: Download test timings
        uses: actions/download-artifact@v4
        with:
          pattern: test-timing-*
          merge-multiple: true
          path: target/test-results

      - name: Print test timings
        run: ./scripts/summarize-tests.clj

      - name: Upload all test timings
        uses: actions/upload-artifact@v4
        with:
          retention-days: 180
          name: all-test-timings
          path: target/test-results/*.edn

      - run: echo "All tests pass!"
  deploy:
    runs-on: "ubuntu-22.04"
    outputs:
      build_version: ${{ steps.deploy.outputs.build_version }}
      build_type: ${{ steps.deploy.outputs.build_type }}
    needs: [all-pr-checks]
    # see on.push.branches for which branches are built on push
    if: github.event_name == 'push' && github.repository == 'threatgrid/ctia'
    steps:
      - uses: actions/checkout@v3
      - name: Binary Cache
        uses: actions/cache@v3
        with:
          path: ${{ env.BIN_PATH }}
          key: ctia-bin-${{ env.ACTIONS_CACHE_VERSION }}-${{ env.BB_VERSION }}
      - name: Setup PATH and directories
        run: |
          mkdir -p "${BIN_PATH}"
          echo "${BIN_PATH}" >> $GITHUB_PATH
      - name: Install babashka
        run: ./scripts/actions/install-bb.sh
      - run: ./scripts/actions/install-trojansourcefinder.sh
      - run: ./scripts/actions/setup_env.clj
      - name: Maven Cache
        id: maven-cache
        uses: actions/cache@v3
        with:
          path: ~/.m2/repository
          key: ctia-m2-cache-${{ env.ACTIONS_CACHE_VERSION }}-${{ hashFiles('project.clj') }}
          restore-keys: |
            ctia-m2-cache-${{ env.ACTIONS_CACHE_VERSION }}-

      - uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: ${{ env.DEFAULT_JAVA_VERSION }}
      - run: java -version

      # install lein using cache to avoid extra download and startup time
      - name: Leiningen Cache
        uses: actions/cache@v3
        id: lein-cache
        with:
          path: ${{env.LEIN_HOME}}
          key: ctia-lein-cache-${{ env.ACTIONS_CACHE_VERSION }}-${{ env.LEIN_VERSION }}
      - name: Install Leiningen
        if: steps.lein-cache.outputs.cache-hit != 'true'
        run: ./scripts/actions/install-lein.sh
      # we should get a cache hit from maven here, since the `setup` job only succeeds if it exists.
      # in the unlikely event we don't, download deps for this specific job only.
      - name: Warm dependency cache
        if: steps.maven-cache.outputs.cache-hit != 'true'
        # poor man's travis_retry
        run: for i in {1..${ACTIONS_RETRY_TIMES}}; do lein warm-ci-deps && break; done

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          # region matching buckets in ./build/deploy-actions.sh
          aws-region: us-east-1
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

      - id: deploy
        run: ./build/deploy-actions.sh
        env:
          # offline to catch issues where `lein warm-ci-deps` doesn't download all
          # dependencies we need to deploy.
          LEIN_OFFLINE: true

  update-tenzin:
    runs-on: ubuntu-22.04
    needs: deploy
    if: ${{ needs.deploy.outputs.build_type == 'int' }}
    steps:
    - name: Setup Git user
      run: |
       git config --global user.email "github-actions[bot]@users.noreply.github.com"
       git config --global user.name "GitHub Actions Bot"
    - name: Checkout tenzin
      uses: actions/checkout@v3
      with:
        repository: advthreat/tenzin
        ssh-key: ${{ secrets.TENZIN_PRIVATE_SSH }}
        path: tenzin
        ref: master
    - name: update ctia image tag
      env:
        BUILD_VERSION: ctia-${{ needs.deploy.outputs.build_version }}
      run: |
        cd tenzin/applications/ctia/kustomize/overlays/ctr-int-eks
        yq -i '.images[0].newTag = env(BUILD_VERSION)' kustomization.yaml
        git add kustomization.yaml
    - name: Commit and push changes if any
      env:
       BUILD_VERSION: ctia-${{ needs.deploy.outputs.build_version }}
      run: |
        cd tenzin
        if ! git diff --cached --quiet
        then
          git commit -m "Update ctia image tag to ${BUILD_VERSION}"
          git push
        else
          echo "No changes to commit"
        fi