Skip to content

Commit

Permalink
Migrate on-demand-ocr-soak-test.yml (WIP)
Browse files Browse the repository at this point in the history
  • Loading branch information
@lukaszcl
lukaszcl committed Sep 9, 2024
1 parent 969cdf8 commit 58390ae
Show file tree
Hide file tree
Showing 3 changed files with 37 additions and 123 deletions.
10 changes: 4 additions & 6 deletions .github/e2e-tests.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,22 +21,20 @@ runner-test-matrix:
pyroscope_env: ci-smoke-ocr-evm-simulated

# Example of a configuration for running a single soak test in Kubernetes Remote Runner
- id: soak/ocr_test.go:^TestOCRv1Soak$
- id: soak/ocr_test.go:TestOCRv1Soak
path: integration-tests/soak/ocr_test.go
test_env_type: k8s-remote-runner
runs_on: ubuntu-latest
test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRv1Soak$ -test.parallel=1 -timeout 30m -count=1 -json
test_config_override_required: true
test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRv1Soak$ -test.parallel=1 -timeout 900h -count=1 -json
test_secrets_required: true
test_env_vars:
TEST_SUITE: soak

- id: soak/ocr_test.go:^TestOCRv2Soak$
- id: soak/ocr_test.go:TestOCRv2Soak
path: integration-tests/soak/ocr_test.go
test_env_type: k8s-remote-runner
runs_on: ubuntu-latest
test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRv2Soak$ -test.parallel=1 -timeout 30m -count=1 -json
test_config_override_required: true
test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run TestOCRv2Soak -test.parallel=1 -timeout 900h -count=1 -json
test_secrets_required: true
test_env_vars:
TEST_SUITE: soak
Expand Down
127 changes: 33 additions & 94 deletions .github/workflows/on-demand-ocr-soak-test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,25 +8,20 @@ on:
default: TestOCRSoak
type: choice
options:
- TestOCRv1Soak
- TestOCRv2Soak
- TestForwarderOCRv1Soak
- TestForwarderOCRv2Soak
- TestOCRSoak_GethReorgBelowFinality_FinalityTagDisabled
- TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled
- TestOCRSoak_GasSpike
- TestOCRSoak_ChangeBlockGasLimit
- TestOCRSoak_RPCDownForAllCLNodes
- TestOCRSoak_RPCDownForHalfCLNodes
- soak/ocr_test.go:TestOCRv1Soak
- soak/ocr_test.go:TestOCRv2Soak
# - TestForwarderOCRv1Soak
# - TestForwarderOCRv2Soak
# - TestOCRSoak_GethReorgBelowFinality_FinalityTagDisabled
# - TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled
# - TestOCRSoak_GasSpike
# - TestOCRSoak_ChangeBlockGasLimit
# - TestOCRSoak_RPCDownForAllCLNodes
# - TestOCRSoak_RPCDownForHalfCLNodes
test_config_override_path:
description: Path to a test config file used to override the default test config
required: false
type: string
slackMemberID:
description: Slack Member ID (Not your @)
required: true
default: U01A2B2C3D4
type: string
test_secrets_override_key:
description: 'Key to run tests with custom test secrets'
required: false
Expand All @@ -38,84 +33,28 @@ on:
type: string

jobs:
ocr_soak_test:
name: OCR Soak Test
environment: integration
runs-on: ubuntu-latest
permissions:
checks: write
pull-requests: write
id-token: write
contents: read
env:
CHAINLINK_ENV_USER: ${{ github.actor }}
run-e2e-tests-workflow:
name: Run E2E Tests
uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
with:
test_ids: ${{ inputs.testToRun}}
test_config_override_path: ${{ inputs.test_config_override_path }}
chainlink_version: ${{ inputs.chainlink_version }}
secrets:
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
AWS_REGION: ${{ secrets.QA_AWS_REGION }}
AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
TEST_SECRETS_OVERRIDE_BASE64: ${{ secrets[inputs.test_secrets_override_key] }}
SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }}
SLACK_CHANNEL: ${{ secrets.QA_SLACK_CHANNEL }}
TEST_LOG_LEVEL: debug
REF_NAME: ${{ github.head_ref || github.ref_name }}
ENV_JOB_IMAGE_BASE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests
steps:
- name: Collect Metrics
id: collect-gha-metrics
uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
with:
id: on-demand-ocr-soak-test
org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
this-job-name: ${{ inputs.network }} OCR Soak Test
continue-on-error: true
- name: Checkout the repo
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
with:
ref: ${{ env.REF_NAME }}
- name: Get Slack config
run: |
SLACK_USER=$(jq -r '.inputs.slackMemberID' $GITHUB_EVENT_PATH)
echo ::add-mask::$SLACK_USER
echo SLACK_USER=$SLACK_USER >> $GITHUB_ENV
- name: Show test config override path in summary
if: ${{ inputs.test_config_override_path }}
run: |
echo "### Test config override path" >> $GITHUB_STEP_SUMMARY
echo "[${{ inputs.test_config_override_path }}]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/${{ github.sha }}/${{ inputs.test_config_override_path }})" >> $GITHUB_STEP_SUMMARY
- name: Show chainlink version in summary
run: |
echo "### Chainlink version" >> $GITHUB_STEP_SUMMARY
echo "${{ inputs.chainlink_version }}" >> $GITHUB_STEP_SUMMARY
- name: Build Image
id: build_test_image
uses: ./.github/actions/build-test-image
with:
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
- name: Run Tests
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@94cb11f4bd545607a2f221c6685052b3abee723d # v2.3.32
env:
DETACH_RUNNER: true
TEST_SUITE: soak
TEST_ARGS: -test.timeout 900h -test.memprofile memprofile.out -test.cpuprofile profile.out
ENV_JOB_IMAGE: ${{ steps.build_test_image.outputs.test_image }}
# We can comment these out when we have a stable soak test and aren't worried about resource consumption
TEST_UPLOAD_CPU_PROFILE: true
TEST_UPLOAD_MEM_PROFILE: true
E2E_TEST_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
E2E_TEST_CHAINLINK_VERSION: ${{ inputs.chainlink_version }}
E2E_TEST_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
E2E_TEST_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
E2E_TEST_GRAFANA_BASE_URL: "http://localhost:8080/primary"
E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
E2E_TEST_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
with:
test_command_to_run: cd ./integration-tests && go test -v -count=1 -run ^${{ github.event.inputs.testToRun }}$ ./soak
test_download_vendor_packages_command: make gomod
test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }}
test_config_override_path: ${{ inputs.test_config_override_path }}
token: ${{ secrets.GITHUB_TOKEN }}
should_cleanup: false
go_mod_path: ./integration-tests/go.mod
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
23 changes: 0 additions & 23 deletions .github/workflows/run-e2e-tests-reusable-workflow.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -352,29 +352,6 @@ jobs:
fi
shell: bash

- name: Check if test config override is required for any test
shell: bash
run: |
# Parse the JSON to check for test_config_override_required in Docker matrix
DOCKER_TESTS_REQUIRING_CONFIG_OVERRIDE=$(echo '${{ steps.set-docker-matrix.outputs.matrix }}' | jq 'if .tests then .tests[] | select(has("test_config_override_required") and .test_config_override_required) | .id else empty end' -r)
# Parse the JSON to check for test_config_override_required in Kubernetes matrix
K8S_TESTS_REQUIRING_CONFIG_OVERRIDE=$(echo '${{ steps.set-k8s-runner-matrix.outputs.matrix }}' | jq 'if .tests then .tests[] | select(has("test_config_override_required") and .test_config_override_required) | .id else empty end' -r)
# Determine if any tests require a configuration override
if [ ! -z "$DOCKER_TESTS_REQUIRING_CONFIG_OVERRIDE" ] || [ ! -z "$K8S_TESTS_REQUIRING_CONFIG_OVERRIDE" ]; then
echo "Tests in .github/e2e-tests.yml requiring test config override:"
if [ ! -z "$DOCKER_TESTS_REQUIRING_CONFIG_OVERRIDE" ]; then
echo $DOCKER_TESTS_REQUIRING_CONFIG_OVERRIDE
fi
if [ ! -z "$K8S_TESTS_REQUIRING_CONFIG_OVERRIDE" ]; then
echo $K8S_TESTS_REQUIRING_CONFIG_OVERRIDE
fi
echo "::error::Error: Some of the tests require a test config override. Please see workflow logs and set 'test_config_override_path' to run these tests."
exit 1
else
echo "No tests require a configuration override. Proceeding without overrides."
fi
- name: Check if test secrets are required for any test
shell: bash
run: |
Expand Down

0 comments on commit 58390ae

Please sign in to comment.