All Tests #6034
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: All Tests | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - main | |
| - release/** | |
| - cloud/** | |
| workflow_dispatch: | |
| inputs: | |
| commit: | |
| description: "Commit SHA" | |
| required: true | |
| run_single_functional_test: | |
| description: "Run a single functional test" | |
| type: boolean | |
| default: false | |
| run_single_unit_test: | |
| description: "Run a single unit test (INSTEAD of functional test)" | |
| type: boolean | |
| default: false | |
| unit_test_directory: | |
| description: "[Unit Test Only] Directory to run unit tests in" | |
| type: string | |
| default: "./temporal" | |
| n_runs: | |
| description: "Number of times to repeat the single test per database type" | |
| type: number | |
| default: 1 | |
| test_name: | |
| description: "Name of the test to run (i.e. 'TestAcquireShard_DeadlineExceededErrorSuite' or 'TestFunctionalSuite/TestUpdateWorkflow')" | |
| type: string | |
| timeout_minutes: | |
| description: "Test timeout in minutes" | |
| type: number | |
| default: 120 | |
| test_runner: | |
| description: "Which runner to use. Choose higher RAM if your n_runs is high." | |
| type: choice | |
| default: "16GB RAM (ubuntu-20.04)" | |
| options: | |
| - "16GB RAM (ubuntu-20.04)" | |
| - "64GB RAM (ubuntu-22.04)" | |
| cassandra: | |
| description: "cassandra" | |
| type: boolean | |
| default: false | |
| sqlite: | |
| description: "sqlite" | |
| type: boolean | |
| default: false | |
| # mysql8: | |
| # description: "mysql8" | |
| # type: boolean | |
| # default: false | |
| # postgres12: | |
| # description: "postgres12" | |
| # type: boolean | |
| # default: false | |
| # if [[ "${{ inputs.mysql8 }}" == "true" ]]; then dbs_arr+=("mysql8"); fi | |
| # if [[ "${{ inputs.postgres12 }}" == "true" ]]; then dbs_arr+=("postgres12"); fi | |
| concurrency: # Auto-cancel existing runs in the PR when a new commit is pushed | |
| group: run-tests-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| env: | |
| # For workflow_dispatch: use the given commit. | |
| # For pull_request: use the head of the PR branch (not the merge branch which is the default!) | |
| # For push: use the pushed commit. | |
| COMMIT: ${{ github.event.inputs.commit || github.event.pull_request.head.sha || github.sha }} | |
| PR_BASE_COMMIT: ${{ github.event.pull_request.base.sha }} | |
| DOCKER_COMPOSE_FILE: ./develop/github/docker-compose.yml | |
| TEMPORAL_VERSION_CHECK_DISABLED: 1 | |
| BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }} | |
| jobs: | |
| set-up-single-test: | |
| name: Set up single test | |
| runs-on: ubuntu-20.04 | |
| outputs: | |
| shard_indices: ${{ steps.generate_output.outputs.shard_indices }} | |
| total_shards: ${{ steps.generate_output.outputs.shards }} | |
| github_timeout: ${{ steps.generate_output.outputs.github_timeout }} | |
| test_timeout: ${{ steps.generate_output.outputs.test_timeout }} | |
| single_test_args: ${{ steps.generate_output.outputs.single_test_args }} | |
| runs_on: ${{ steps.generate_output.outputs.runs_on }} | |
| dbs: ${{ steps.generate_output.outputs.dbs }} | |
| steps: | |
| - id: generate_output | |
| run: | | |
| shards=3 | |
| timeout=30 | |
| runs_on='["ubuntu-20.04"]' | |
| if [[ "${{ inputs.run_single_functional_test }}" == "true" || "${{ inputs.run_single_unit_test }}" == "true" ]]; then | |
| shards=1 | |
| timeout=${{ inputs.timeout_minutes }} | |
| single_test_args="-run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}" | |
| if [[ "${{ inputs.test_runner }}" == "64GB RAM (ubuntu-22.04)" ]]; then | |
| runs_on='[ "ubuntu-latest-16-cores" ]' | |
| fi | |
| dbs_arr=() | |
| if [[ "${{ inputs.cassandra }}" == "true" ]]; then dbs_arr+=("cassandra"); fi | |
| if [[ "${{ inputs.sqlite }}" == "true" ]]; then dbs_arr+=("sqlite"); fi | |
| if [[ ${#dbs_arr[@]} == 0 ]]; then | |
| dbs='["sqlite"]' | |
| else | |
| dbs=$(jq -c -n '$ARGS.positional' --args "${dbs_arr[@]}") | |
| fi | |
| fi | |
| { | |
| echo "shard_indices=[ $(seq -s, 0 $((shards-1))) ]" | |
| echo "shards=$shards" | |
| echo "github_timeout=$((timeout+5))" | |
| echo "test_timeout=${timeout}m" | |
| echo "single_test_args=$single_test_args" | |
| echo "runs_on=$runs_on" | |
| echo "dbs=$dbs" | |
| } >> "$GITHUB_OUTPUT" | |
| - id: cat_output | |
| run: | | |
| cat "$GITHUB_OUTPUT" | |
| misc-checks: | |
| name: Misc checks | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runs-on: [ubuntu-20.04] | |
| runs-on: ${{ matrix.runs-on }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| ref: ${{ env.COMMIT }} | |
| # buf-breaking tries to compare HEAD against merge base so we need to be able to find it | |
| fetch-depth: 100 | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: 'go.mod' | |
| check-latest: true | |
| - uses: arduino/setup-protoc@v3 | |
| - run: GOOS=windows GOARCH=amd64 make clean-bins bins | |
| - run: GOOS=darwin GOARCH=arm64 make clean-bins bins | |
| - run: make clean-bins ci-build-misc | |
| - run: make build-tests | |
| unit-test: | |
| if: ${{ inputs.run_single_functional_test != true }} | |
| name: Unit test | |
| needs: [misc-checks, set-up-single-test] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runs-on: [ubuntu-20.04] | |
| runs-on: ${{ matrix.runs-on }} | |
| env: | |
| BUILDKITE_MESSAGE: "{\"job\": \"unit-test\"}" | |
| SINGLE_TEST_ARGS: ${{ needs.set-up-single-test.outputs.single_test_args }} | |
| UNIT_TEST_DIR: ${{ inputs.unit_test_directory }} | |
| TEST_TIMEOUT: ${{ needs.set-up-single-test.outputs.test_timeout }} | |
| RUN_SINGLE_UNIT_TEST: ${{ inputs.run_single_unit_test }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| ref: ${{ env.COMMIT }} | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: 'go.mod' | |
| check-latest: true | |
| - name: Run unit tests | |
| if: ${{ !cancelled() && !inputs.run_single_unit_test }} | |
| timeout-minutes: 15 | |
| run: make unit-test-coverage | |
| - name: Run single unit test | |
| if: ${{ !cancelled() && inputs.run_single_unit_test }} | |
| timeout-minutes: 15 | |
| run: UNIT_TEST_DIRS=$UNIT_TEST_DIR make unit-test-coverage | |
| - name: Upload test results | |
| if: ${{ !cancelled() }} | |
| run: make upload-test-results | |
| integration-test: | |
| if: ${{ inputs.run_single_functional_test != true && inputs.run_single_unit_test != true }} | |
| name: Integration test | |
| needs: misc-checks | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runs-on: [ubuntu-20.04] | |
| runs-on: ${{ matrix.runs-on }} | |
| env: | |
| BUILDKITE_MESSAGE: "{\"job\": \"integration-test\"}" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| ref: ${{ env.COMMIT }} | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: 'go.mod' | |
| check-latest: true | |
| - name: Start containerized dependencies | |
| uses: hoverkraft-tech/[email protected] | |
| with: | |
| compose-file: ${{ env.DOCKER_COMPOSE_FILE }} | |
| services: | | |
| cassandra | |
| mysql | |
| postgresql | |
| down-flags: -v | |
| - name: Run integration test | |
| timeout-minutes: 15 | |
| run: make integration-test-coverage | |
| - name: Upload test results | |
| if: ${{ !cancelled() }} | |
| run: make upload-test-results | |
| - name: Tear down docker compose | |
| if: ${{ always() }} | |
| run: | | |
| docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v | |
| functional-test: | |
| if: ${{ inputs.run_single_unit_test != true }} | |
| name: Functional test | |
| needs: [misc-checks, set-up-single-test] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runs-on: ${{ fromJson(needs.set-up-single-test.outputs.runs_on) }} | |
| shard_index: ${{ fromJson(needs.set-up-single-test.outputs.shard_indices) }} | |
| name: | |
| - cass_es | |
| - cass_es8 | |
| - sqlite | |
| - mysql8 | |
| - postgres12 | |
| - postgres12_pgx | |
| include: | |
| - name: cass_es | |
| persistence_type: nosql | |
| persistence_driver: cassandra | |
| containers: [cassandra, elasticsearch] | |
| es_version: v7 | |
| - name: cass_es8 | |
| persistence_type: nosql | |
| persistence_driver: cassandra | |
| containers: [cassandra, elasticsearch8] | |
| es_version: v8 | |
| - name: sqlite | |
| persistence_type: sql | |
| persistence_driver: sqlite | |
| containers: [] | |
| - name: mysql8 | |
| persistence_type: sql | |
| persistence_driver: mysql8 | |
| containers: [mysql] | |
| - name: postgres12 | |
| persistence_type: sql | |
| persistence_driver: postgres12 | |
| containers: [postgresql] | |
| - name: postgres12_pgx | |
| persistence_type: sql | |
| persistence_driver: postgres12_pgx | |
| containers: [postgresql] | |
| runs-on: ${{ matrix.runs-on }} | |
| env: | |
| TEST_TOTAL_SHARDS: ${{ needs.set-up-single-test.outputs.total_shards }} | |
| TEST_SHARD_INDEX: ${{ matrix.shard_index }} | |
| PERSISTENCE_TYPE: ${{ matrix.persistence_type }} | |
| PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }} | |
| SINGLE_TEST_ARGS: ${{ needs.set-up-single-test.outputs.single_test_args }} | |
| TEST_TIMEOUT: ${{ needs.set-up-single-test.outputs.test_timeout }} | |
| BUILDKITE_MESSAGE: "{\"job\": \"functional-test\", \"db\": \"${{ matrix.persistence_driver }}\"}" | |
| steps: | |
| - name: Print functional test | |
| run: echo "${{ needs.set-up-single-test.outputs.dbs }}" && echo "$SINGLE_TEST_ARGS" | |
| - uses: ScribeMD/[email protected] | |
| if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }} | |
| with: | |
| key: docker-${{ runner.os }}-${{ hashFiles(env.DOCKER_COMPOSE_FILE) }} | |
| - uses: actions/checkout@v4 | |
| if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }} | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| ref: ${{ env.COMMIT }} | |
| - uses: actions/setup-go@v5 | |
| if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }} | |
| with: | |
| go-version-file: 'go.mod' | |
| check-latest: true | |
| - name: Start containerized dependencies | |
| if: ${{ toJson(matrix.containers) != '[]' && (inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER))) }} | |
| uses: hoverkraft-tech/[email protected] | |
| with: | |
| compose-file: ${{ env.DOCKER_COMPOSE_FILE }} | |
| services: "${{ join(matrix.containers, '\n') }}" | |
| down-flags: -v | |
| - name: Run functional test | |
| if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }} | |
| timeout-minutes: ${{ fromJSON(needs.set-up-single-test.outputs.github_timeout) }} # make sure this is larger than the test timeout in the Makefile | |
| run: make functional-test-coverage | |
| - name: Upload test results | |
| if: ${{ !cancelled() && (inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER))) }} | |
| run: make upload-test-results | |
| functional-test-xdc: | |
| if: ${{ inputs.run_single_functional_test != true && inputs.run_single_unit_test != true }} | |
| name: Functional test xdc | |
| needs: misc-checks | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runs-on: [ubuntu-20.04] | |
| name: [cass_es] | |
| # name: [cass_es, cass_es8, mysql8, postgres12, postgres12_pgx] | |
| include: | |
| - name: cass_es | |
| persistence_type: nosql | |
| persistence_driver: elasticsearch | |
| containers: [cassandra, elasticsearch] | |
| # - name: cass_es8 | |
| # persistence_type: nosql | |
| # persistence_driver: elasticsearch | |
| # containers: [cassandra, elasticsearch8] | |
| # - name: mysql8 | |
| # persistence_type: sql | |
| # persistence_driver: mysql8 | |
| # containers: [mysql] | |
| # - name: postgres12 | |
| # persistence_type: sql | |
| # persistence_driver: postgres12 | |
| # containers: [postgresql] | |
| # - name: postgres12_pgx | |
| # persistence_type: sql | |
| # persistence_driver: postgres12_pgx | |
| # containers: [postgresql] | |
| runs-on: ${{ matrix.runs-on }} | |
| env: | |
| PERSISTENCE_TYPE: ${{ matrix.persistence_type }} | |
| PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }} | |
| BUILDKITE_MESSAGE: "{\"job\": \"functional-test-xdc\", \"db\": \"${{ matrix.persistence_driver }}\"}" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| ref: ${{ env.COMMIT }} | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: 'go.mod' | |
| check-latest: true | |
| - name: Start containerized dependencies | |
| if: ${{ toJson(matrix.containers) != '[]' }} | |
| uses: hoverkraft-tech/[email protected] | |
| with: | |
| compose-file: ${{ env.DOCKER_COMPOSE_FILE }} | |
| services: "${{ join(matrix.containers, '\n') }}" | |
| down-flags: -v | |
| - name: Run functional test xdc | |
| timeout-minutes: 15 | |
| run: make functional-test-xdc-coverage | |
| - name: Upload test results | |
| if: ${{ !cancelled() }} | |
| run: make upload-test-results | |
| functional-test-ndc: | |
| if: ${{ inputs.run_single_functional_test != true && inputs.run_single_unit_test != true }} | |
| name: Functional test ndc | |
| needs: misc-checks | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runs-on: [ubuntu-20.04] | |
| name: | |
| - cass_es | |
| - cass_es8 | |
| - mysql8 | |
| - postgres12 | |
| - postgres12_pgx | |
| include: | |
| - name: cass_es | |
| persistence_type: nosql | |
| persistence_driver: elasticsearch | |
| containers: [cassandra, elasticsearch] | |
| es_version: v7 | |
| - name: cass_es8 | |
| persistence_type: nosql | |
| persistence_driver: elasticsearch | |
| containers: [cassandra, elasticsearch8] | |
| es_version: v8 | |
| - name: mysql8 | |
| persistence_type: sql | |
| persistence_driver: mysql8 | |
| containers: [mysql] | |
| - name: postgres12 | |
| persistence_type: sql | |
| persistence_driver: postgres12 | |
| containers: [postgresql] | |
| - name: postgres12_pgx | |
| persistence_type: sql | |
| persistence_driver: postgres12_pgx | |
| containers: [postgresql] | |
| runs-on: ${{ matrix.runs-on }} | |
| env: | |
| PERSISTENCE_TYPE: ${{ matrix.persistence_type }} | |
| PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }} | |
| ES_VERSION: ${{ matrix.es_version }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| ref: ${{ env.COMMIT }} | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: 'go.mod' | |
| check-latest: true | |
| - name: Start containerized dependencies | |
| if: ${{ toJson(matrix.containers) != '[]' }} | |
| uses: hoverkraft-tech/[email protected] | |
| with: | |
| compose-file: ${{ env.DOCKER_COMPOSE_FILE }} | |
| services: "${{ join(matrix.containers, '\n') }}" | |
| down-flags: -v | |
| - name: Run functional test ndc | |
| timeout-minutes: 15 | |
| run: make functional-test-ndc-coverage | |
| test-status: | |
| if: ${{ inputs.run_single_functional_test != true && inputs.run_single_unit_test != true }} | |
| name: Test Status | |
| needs: | |
| - unit-test | |
| - integration-test | |
| - functional-test | |
| - functional-test-xdc | |
| - functional-test-ndc | |
| runs-on: ubuntu-20.04 | |
| env: | |
| RESULTS: ${{ toJSON(needs.*.result) }} | |
| steps: | |
| - name: Check results | |
| run: | | |
| if [[ -n $(echo "$RESULTS" | jq '.[] | select (. != "success")') ]]; then | |
| exit 1 | |
| fi |