Upgrade ctim (#1454) #5079
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CTIA | |
| on: | |
| # to enable pull requests from other repositories | |
| pull_request: | |
| # for deployment only | |
| push: | |
| branches: | |
| - master | |
| # Note: full regex not supported here, hardcoding vX.X and vX.X.X | |
| # See https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#filter-pattern-cheat-sheet | |
| - "v[0-9]+.[0-9]+" | |
| - "v[0-9]+.[0-9]+.[0-9]+" | |
| # Cron job: runs a matrix of JVM and Clojure versions, which can | |
| # be configured in the `cron-matrix` function in ./scripts/actions/print-matrix.clj | |
| # Note: automatically uses master branch https://stackoverflow.com/a/58800550 | |
| schedule: | |
| # daily, 3am UTC | |
| - cron: "0 3 * * *" | |
| env: | |
| # number of times to try possibly-flaky commands in this file | |
| ACTIONS_RETRY_TIMES: 10 | |
| # workaround https://github.com/actions/cache/issues/2 | |
| # to "delete" caches, increment this version | |
| ACTIONS_CACHE_VERSION: v18 | |
| BIN_PATH: ${{github.workspace}}/bin | |
| LOG_PATH: ${{github.workspace}}/log | |
| # avoid the (somehow) default /usr/local/lib/lein/ | |
| LEIN_HOME: ${{github.workspace}}/.lein | |
| LEIN_ROOT: 1 | |
| LEIN_VERSION: 2.11.2 | |
| CTIA_MAJOR_VERSION: 1 | |
| SHELLCHECK_VERSION: v0.10.0 | |
| BB_VERSION: 1.12.194 | |
| DEFAULT_JAVA_VERSION: 21 | |
| jobs: | |
| setup: | |
| runs-on: "ubuntu-22.04" | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| # to read the latest commit message on a PR | |
| # https://github.com/zero88/gh-project-context/blob/26a3afdce1fe607734278e5ccec0851d50669ef5/README.md#latest-commit-message | |
| fetch-depth: 2 | |
| # for grabbing the current commit msg (suprisingly hard!) | |
| - uses: zero88/gh-project-context@f18a0d15c231ca76c18ab18c131a19500bb7d88f | |
| id: gh-project-context | |
| - name: Binary Cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.BIN_PATH }} | |
| key: ctia-bin-${{ env.ACTIONS_CACHE_VERSION }}-${{ env.SHELLCHECK_VERSION }} | |
| - name: Setup PATH and directories | |
| run: | | |
| mkdir -p "${BIN_PATH}" | |
| echo "${BIN_PATH}" >> $GITHUB_PATH | |
| - name: Install clojure tools | |
| uses: DeLaGuardo/[email protected] | |
| with: | |
| bb: ${{env.BB_VERSION}} | |
| - name: Run tests for scripts | |
| run: ./scripts-test/test_runner.clj | |
| - run: ./scripts/actions/setup_env.clj | |
| # scan ctia's source for trojans. uberjar is scanned in deploy step. | |
| - run: ./scripts/actions/install-trojansourcefinder.sh | |
| - run: ./scripts/trojan-scan.sh | |
| - name: Setup test matrix splits | |
| id: set-matrix | |
| run: | | |
| echo "CTIA_COMMIT_MESSAGE=${CTIA_COMMIT_MESSAGE}" | |
| ./scripts/actions/print_matrix.clj | |
| env: | |
| CTIA_COMMIT_MESSAGE: ${{steps.gh-project-context.outputs.commitMsg}} | |
| # Retreive previous timing information | |
| # actions/cache/restore is used to avoid a 2m slowdown due to actions/cache deciding | |
| # whether to save a cache, which is not useful here. | |
| # https://github.com/actions/cache?tab=readme-ov-file#using-a-combination-of-restore-and-save-actions | |
| - name: Timing Cache | |
| id: get-timing | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: | | |
| target/test-results/all-test-timings.edn | |
| # unreachable key because each cache is saved with hashFiles('target/test-results/all-test-timings.edn'), | |
| # which we don't have yet. this ensures we always get the latest and most relevant timing information. | |
| key: writeonly-ctia-timings-${{ env.ACTIONS_CACHE_VERSION }}-${{ env.CTIA_TEST_SUITE }}-${{ github.ref }}-${{ github.sha }}-${{ github.run_number }}-${{ github.run_attempt }} | |
| # cache should match test timings from previous runs, if they exist. | |
| # pr's and pushes share timings via the "ci" setting of env.CTIA_TEST_SUITE, | |
| # and cron is separate as "cron". | |
| # timings shared between pull requests and cron only as last resort. | |
| restore-keys: | | |
| ctia-timings-${{ env.ACTIONS_CACHE_VERSION }}-${{ env.CTIA_TEST_SUITE }}-${{ github.ref }}-${{ github.sha }}- | |
| ctia-timings-${{ env.ACTIONS_CACHE_VERSION }}-${{ env.CTIA_TEST_SUITE }}-${{ github.ref }}- | |
| ctia-timings-${{ env.ACTIONS_CACHE_VERSION }}-${{ env.CTIA_TEST_SUITE }}- | |
| ctia-timings-${{ env.ACTIONS_CACHE_VERSION }}- | |
| - run: | | |
| if [ -f target/test-results/all-test-timings.edn ]; then | |
| echo "Found new timings" | |
| cat target/test-results/all-test-timings.edn | |
| cp target/test-results/all-test-timings.edn dev-resources/ctia_test_timings.edn | |
| else | |
| echo "Timings not found, creating empty dummy timings which will be ignored by tests" | |
| touch dev-resources/ctia_test_timings.edn | |
| fi | |
| - name: Upload current test timing | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| retention-days: 90 | |
| name: current-test-timing | |
| path: dev-resources/ctia_test_timings.edn | |
| # Run shellcheck on CTIA's scripts | |
| - name: Install shellcheck | |
| run: | | |
| if ! command -v shellcheck &> /dev/null || ! shellcheck --version | grep "^version: ${SHELLCHECK_VERSION}$" | |
| then | |
| ( set -x && wget -qO- "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar -xJv ) | |
| ( set -x && ls -al "${BIN_PATH}" ) | |
| ( set -x && cp "shellcheck-${SHELLCHECK_VERSION}/shellcheck" "${BIN_PATH}" ) | |
| fi | |
| shellcheck --version | |
| - name: Run shellcheck | |
| run: ./scripts/shellcheck-build.sh | |
| # warm dependency cache on project.clj change (ie., when cache-hit is not true). | |
| - name: Maven Cache | |
| id: maven-cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.m2/repository | |
| ~/.gitlibs | |
| key: ctia-m2-cache-${{ env.ACTIONS_CACHE_VERSION }}-${{ hashFiles('project.clj') }}--${{ env.DEFAULT_JAVA_VERSION }} | |
| # these will be considered a cache-miss by `cache-hit` | |
| restore-keys: | | |
| ctia-m2-cache-${{ env.ACTIONS_CACHE_VERSION }}-${{ hashFiles('project.clj') }}- | |
| ctia-m2-cache-${{ env.ACTIONS_CACHE_VERSION }}- | |
| - name: Setup Java | |
| if: steps.maven-cache.outputs.cache-hit != 'true' | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: ${{ env.DEFAULT_JAVA_VERSION }} | |
| - run: java -version | |
| if: steps.maven-cache.outputs.cache-hit != 'true' | |
| - name: Install clojure tools | |
| if: steps.maven-cache.outputs.cache-hit != 'true' && steps.lein-cache.outputs.cache-hit != 'true' | |
| uses: DeLaGuardo/[email protected] | |
| with: | |
| lein: ${{env.LEIN_VERSION}} | |
| - name: Warm dependency cache | |
| if: steps.maven-cache.outputs.cache-hit != 'true' | |
| # poor man's travis_retry | |
| run: for i in {1..${ACTIONS_RETRY_TIMES}}; do lein warm-ci-deps && break; done | |
| - run: ./scripts/check-dependabot | |
| test-encoding: | |
| runs-on: ubuntu-22.04 | |
| needs: [setup] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Start Docker environment | |
| run: docker compose -f containers/dev/docker-compose.yml up --no-color --quiet-pull -d | |
| - name: Maven Cache | |
| id: maven-cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.m2/repository | |
| ~/.gitlibs | |
| key: ctia-m2-cache-${{ env.ACTIONS_CACHE_VERSION }}-${{ hashFiles('project.clj') }} | |
| restore-keys: | | |
| ctia-m2-cache-${{ env.ACTIONS_CACHE_VERSION }}- | |
| - name: Setup Java | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: ${{ env.DEFAULT_JAVA_VERSION }} | |
| - name: Install clojure tools | |
| uses: DeLaGuardo/[email protected] | |
| with: | |
| lein: ${{env.LEIN_VERSION}} | |
| bb: ${{env.BB_VERSION}} | |
| # we should get a cache hit from maven here, since the `setup` job only succeeds if it exists. | |
| # in the unlikely event we don't, download deps for this specific job only. | |
| - name: Warm dependency cache | |
| if: steps.maven-cache.outputs.cache-hit != 'true' | |
| run: for i in {1..${ACTIONS_RETRY_TIMES}}; do lein warm-ci-deps && break; done | |
| - name: Run encoding tests | |
| run: lein with-profile +test-encoding test | |
| env: | |
| # offline to catch issues where `lein warm-ci-deps` doesn't download all | |
| # dependencies we need to run tests. | |
| LEIN_OFFLINE: true | |
| test-matrix: | |
| runs-on: ubuntu-22.04 | |
| needs: [setup] | |
| strategy: | |
| matrix: | |
| this-split: ${{fromJson(needs.setup.outputs.matrix)}} | |
| env: | |
| CTIA_TEST_SUITE: ${{ matrix.this-split.test_suite }} | |
| CTIA_THIS_SPLIT: ${{ matrix.this-split.this_split }} | |
| CTIA_NSPLITS: ${{ matrix.this-split.total_splits }} | |
| CTIA_CI_PROFILES: ${{ matrix.this-split.ci_profiles }} | |
| JAVA_VERSION: ${{ matrix.this-split.java_version }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Binary Cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.BIN_PATH }} | |
| key: ctia-bin-${{ env.ACTIONS_CACHE_VERSION }}-${{ env.SHELLCHECK_VERSION }} | |
| - name: Install clojure tools | |
| uses: DeLaGuardo/[email protected] | |
| with: | |
| lein: ${{env.LEIN_VERSION}} | |
| bb: ${{env.BB_VERSION}} | |
| - name: Setup PATH and directories | |
| run: | | |
| mkdir -p "${BIN_PATH}" | |
| echo "${BIN_PATH}" >> $GITHUB_PATH | |
| - run: ./scripts/actions/setup_env.clj | |
| - name: Docker | |
| # depends on LOG_PATH and actions/checkout@v4 | |
| run: docker compose -f containers/dev/docker-compose.yml up --no-color -d &> "${LOG_PATH}/docker-compose.log" | |
| - name: Download test timings | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: current-test-timing | |
| path: target/current-test-timing | |
| - run: | | |
| # debugging | |
| ls -al target/current-test-timing | |
| if [ -f target/current-test-timing/ctia_test_timings.edn ]; then | |
| cat target/current-test-timing/ctia_test_timings.edn | |
| fi | |
| if [ -s target/current-test-timing/ctia_test_timings.edn ]; then | |
| echo "Updating dev-resources/ctia_test_timings.edn with new timing" | |
| cp target/current-test-timing/ctia_test_timings.edn dev-resources/ctia_test_timings.edn | |
| else | |
| echo "No previous timings found (empty file downloaded)" | |
| fi | |
| - run: | | |
| if [ -f dev-resources/ctia_test_timings.edn ]; then | |
| echo "Timing:" | |
| cat dev-resources/ctia_test_timings.edn | |
| else | |
| echo "No timing data, using slow-namespace heuristic" | |
| fi | |
| - name: ES setup | |
| run: sudo sysctl -w vm.max_map_count=262144 | |
| - name: Maven Cache | |
| id: maven-cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.m2/repository | |
| ~/.gitlibs | |
| key: ctia-m2-cache-${{ env.ACTIONS_CACHE_VERSION }}-${{ hashFiles('project.clj') }}-${{ env.LEIN_VERSION }} | |
| restore-keys: | | |
| ctia-m2-cache-${{ env.ACTIONS_CACHE_VERSION }}-${{ hashFiles('project.clj') }}- | |
| ctia-m2-cache-${{ env.ACTIONS_CACHE_VERSION }}- | |
| - uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: ${{ env.JAVA_VERSION }} | |
| - run: java -version | |
| # we should get a cache hit from maven here, since the `setup` job only succeeds if it exists. | |
| # in the unlikely event we don't, download deps for this specific job only. | |
| - name: Warm dependency cache | |
| if: steps.maven-cache.outputs.cache-hit != 'true' | |
| # poor man's travis_retry | |
| run: for i in {1..${ACTIONS_RETRY_TIMES}}; do lein warm-ci-deps && break; done | |
| # Note: if this step fails because Leiningen is in offline mode, add more entries to the maven-cache key. | |
| # Most likely, whatever was changed to cause the build to fail needs to be appended to the cache key. | |
| # Note there are two Maven caches in this file, keep their keys synchronized. | |
| - name: Run CTIA tests | |
| run: ./build/run-tests.sh | |
| env: | |
| # offline to catch issues where `lein warm-ci-deps` doesn't download all | |
| # dependencies we need to run tests. | |
| LEIN_OFFLINE: true | |
| CTIA_WAIT_DOCKER: 1 | |
| - name: Upload test timing | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| retention-days: 1 | |
| name: test-timing-${{matrix.this-split.test_suite}}-${{matrix.this-split.java_version}}-${{matrix.this-split.this_split}} | |
| path: target/test-results/*.edn | |
| - name: Upload docker compose | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| retention-days: 10 | |
| name: docker-compose-${{matrix.this-split.test_suite}}-${{matrix.this-split.java_version}}-${{matrix.this-split.this_split}}.log | |
| path: ${{env.LOG_PATH}}/docker-compose.log | |
| # fan-in tests so there's a single job we can add to protected branches. | |
| # otherwise, we'll have add all (range ${CTIA_NSPLITS}) jobs, and keep | |
| # them up to date | |
| # here's a GitHub Actions feature request that is relevant: | |
| # https://github.community/t/branch-protections-job-names-and-matrix-jobs/16317 | |
| all-pr-checks: | |
| runs-on: ubuntu-22.04 | |
| needs: [test-matrix] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Binary Cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.BIN_PATH }} | |
| key: ctia-bin-${{ env.ACTIONS_CACHE_VERSION }}-${{ env.SHELLCHECK_VERSION }} | |
| - name: Setup PATH and directories | |
| run: | | |
| mkdir -p "${BIN_PATH}" | |
| echo "${BIN_PATH}" >> $GITHUB_PATH | |
| - name: Install clojure tools | |
| uses: DeLaGuardo/[email protected] | |
| with: | |
| bb: ${{env.BB_VERSION}} | |
| - run: ./scripts/actions/setup_env.clj | |
| - name: Download test timings | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: test-timing-* | |
| merge-multiple: true | |
| path: target/test-results | |
| - name: Print test timings | |
| run: ./scripts/summarize-tests.clj | |
| # save if new timing results for this sha | |
| - name: Ensure timing results exist | |
| run: ls target/test-results/all-test-timings.edn | |
| - name: Cache timing results | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: target/test-results/all-test-timings.edn | |
| # unique enough to always keep our timing information up-to-date | |
| key: ctia-timings-${{ env.ACTIONS_CACHE_VERSION }}-${{ env.CTIA_TEST_SUITE }}-${{ github.ref }}-${{ github.sha }}-${{ hashFiles('target/test-results/all-test-timings.edn') }} | |
| - name: Upload all test timings | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| retention-days: 90 | |
| name: all-test-timings | |
| path: target/test-results/*.edn | |
| - run: echo "All tests pass!" | |
| deploy: | |
| runs-on: "ubuntu-22.04" | |
| needs: [all-pr-checks] | |
| # see on.push.branches for which branches are built on push | |
| if: github.event_name == 'push' && github.repository == 'threatgrid/ctia' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Binary Cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.BIN_PATH }} | |
| key: ctia-bin-${{ env.ACTIONS_CACHE_VERSION }}-${{ env.SHELLCHECK_VERSION }} | |
| - name: Setup PATH and directories | |
| run: | | |
| mkdir -p "${BIN_PATH}" | |
| echo "${BIN_PATH}" >> $GITHUB_PATH | |
| - name: Install clojure tools | |
| uses: DeLaGuardo/[email protected] | |
| with: | |
| lein: ${{env.LEIN_VERSION}} | |
| bb: ${{env.BB_VERSION}} | |
| - run: ./scripts/actions/install-trojansourcefinder.sh | |
| - run: ./scripts/actions/setup_env.clj | |
| - name: Maven Cache | |
| id: maven-cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.m2/repository | |
| key: ctia-m2-cache-${{ env.ACTIONS_CACHE_VERSION }}-${{ hashFiles('project.clj') }} | |
| restore-keys: | | |
| ctia-m2-cache-${{ env.ACTIONS_CACHE_VERSION }}- | |
| - uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: ${{ env.DEFAULT_JAVA_VERSION }} | |
| - run: java -version | |
| # we should get a cache hit from maven here, since the `setup` job only succeeds if it exists. | |
| # in the unlikely event we don't, download deps for this specific job only. | |
| - name: Warm dependency cache | |
| if: steps.maven-cache.outputs.cache-hit != 'true' | |
| # poor man's travis_retry | |
| run: for i in {1..${ACTIONS_RETRY_TIMES}}; do lein warm-ci-deps && break; done | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| # region matching buckets in ./build/deploy-actions.sh | |
| aws-region: us-east-1 | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| - id: deploy | |
| run: ./build/deploy-actions.sh | |
| env: | |
| # offline to catch issues where `lein warm-ci-deps` doesn't download all | |
| # dependencies we need to deploy. | |
| LEIN_OFFLINE: true |