From d772c9976e0a60d176efaceb8014cc3bbc0a0b78 Mon Sep 17 00:00:00 2001 From: Makram Kamaleddine Date: Mon, 12 Aug 2024 12:13:36 +0300 Subject: [PATCH 1/5] wip --- .github/actions/setup-postgres/.env | 5 ++ .github/actions/setup-postgres/action.yml | 18 +++++ .github/actions/setup-postgres/bin/pg_dump | 23 +++++++ .../actions/setup-postgres/docker-compose.yml | 15 +++++ .../wait-for-healthy-postgres.sh | 25 +++++++ .github/workflows/ccip-integration-test.yml | 65 +++++++++++++++++++ 6 files changed, 151 insertions(+) create mode 100644 .github/actions/setup-postgres/.env create mode 100644 .github/actions/setup-postgres/action.yml create mode 100755 .github/actions/setup-postgres/bin/pg_dump create mode 100644 .github/actions/setup-postgres/docker-compose.yml create mode 100755 .github/actions/setup-postgres/wait-for-healthy-postgres.sh create mode 100644 .github/workflows/ccip-integration-test.yml diff --git a/.github/actions/setup-postgres/.env b/.github/actions/setup-postgres/.env new file mode 100644 index 00000000..47ed8d9b --- /dev/null +++ b/.github/actions/setup-postgres/.env @@ -0,0 +1,5 @@ +POSTGRES_USER=postgres +POSTGRES_OPTIONS="-c max_connections=1000 -c shared_buffers=2GB -c log_lock_waits=true" +POSTGRES_PASSWORD=postgres +POSTGRES_DB=chainlink_test +POSTGRES_HOST_AUTH_METHOD=trust diff --git a/.github/actions/setup-postgres/action.yml b/.github/actions/setup-postgres/action.yml new file mode 100644 index 00000000..45bfba59 --- /dev/null +++ b/.github/actions/setup-postgres/action.yml @@ -0,0 +1,18 @@ +name: Setup Postgresql +description: Setup postgres docker container via docker-compose, allowing usage of a custom command, see https://github.com/orgs/community/discussions/26688 +inputs: + base-path: + description: Path to the base of the repo + required: false + default: . +runs: + using: composite + steps: + - name: Start postgres service + run: docker compose up -d + shell: bash + working-directory: ${{ inputs.base-path }}/.github/actions/setup-postgres + - name: Wait for postgres service to be healthy + run: ./wait-for-healthy-postgres.sh + shell: bash + working-directory: ${{ inputs.base-path }}/.github/actions/setup-postgres diff --git a/.github/actions/setup-postgres/bin/pg_dump b/.github/actions/setup-postgres/bin/pg_dump new file mode 100755 index 00000000..d8135ad8 --- /dev/null +++ b/.github/actions/setup-postgres/bin/pg_dump @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# +# This script acts as a docker replacement around pg_dump so that developers can +# run DB involved tests locally without having postgres installed. +# +# Installation: +# - Make sure that your PATH doesn't already contain a postgres installation +# - Add this script to your PATH +# +# Usage: +# You should be able to setup your test db via: +# - go build -o chainlink.test . # Build the chainlink binary to run test db prep commands from +# - export CL_DATABASE_URL="postgresql://postgres:postgres@localhost:5432/chainlink_test?sslmode=disable" +# - pushd .github/actions/setup-postgres/ # Navigate to the setup-postgres action so we can spin up a docker postgres +# instance +# - docker compose up # Spin up postgres +# - ./chainlink.test local db preparetest # Run the db migration, which will shell out to our pg_dump wrapper too. +# - popd +# - go test -timeout 30s ./core/services/workflows/... -v # Run tests that use the database + +cd "$(dirname "$0")" || exit + +docker compose exec -T postgres pg_dump "$@" diff --git a/.github/actions/setup-postgres/docker-compose.yml b/.github/actions/setup-postgres/docker-compose.yml new file mode 100644 index 00000000..23f8d82b --- /dev/null +++ b/.github/actions/setup-postgres/docker-compose.yml @@ -0,0 +1,15 @@ +name: gha_postgres +services: + postgres: + ports: + - "5432:5432" + container_name: cl_pg + image: postgres:14-alpine + command: postgres ${POSTGRES_OPTIONS} + env_file: + - .env + healthcheck: + test: "pg_isready -d ${POSTGRES_DB} -U ${POSTGRES_USER}" + interval: 2s + timeout: 5s + retries: 5 diff --git a/.github/actions/setup-postgres/wait-for-healthy-postgres.sh b/.github/actions/setup-postgres/wait-for-healthy-postgres.sh new file mode 100755 index 00000000..438cfbaf --- /dev/null +++ b/.github/actions/setup-postgres/wait-for-healthy-postgres.sh @@ -0,0 +1,25 @@ +#!/bin/bash +RETRIES=10 + +until [ $RETRIES -eq 0 ]; do + DOCKER_OUTPUT=$(docker compose ps postgres --status running --format json) + JSON_TYPE=$(echo "$DOCKER_OUTPUT" | jq -r 'type') + + if [ "$JSON_TYPE" == "array" ]; then + HEALTH_STATUS=$(echo "$DOCKER_OUTPUT" | jq -r '.[0].Health') + elif [ "$JSON_TYPE" == "object" ]; then + HEALTH_STATUS=$(echo "$DOCKER_OUTPUT" | jq -r '.Health') + else + HEALTH_STATUS="Unknown JSON type: $JSON_TYPE" + fi + + echo "postgres health status: $HEALTH_STATUS" + if [ "$HEALTH_STATUS" == "healthy" ]; then + exit 0 + fi + + echo "Waiting for postgres server, $((RETRIES--)) remaining attempts..." + sleep 2 +done + +exit 1 diff --git a/.github/workflows/ccip-integration-test.yml b/.github/workflows/ccip-integration-test.yml new file mode 100644 index 00000000..9609c31f --- /dev/null +++ b/.github/workflows/ccip-integration-test.yml @@ -0,0 +1,65 @@ +name: "Run CCIP OCR3 Integration Test" + +on: + pull_request: + push: + branches: + - 'ccip-develop' + +jobs: + integration-test-ccip-ocr3: + env: + # We explicitly have this env var not be "CL_DATABASE_URL" to avoid having it be used by core related tests + # when they should not be using it, while still allowing us to DRY up the setup + DB_URL: postgresql://postgres:postgres@localhost:5432/chainlink_test?sslmode=disable + + runs-on: ubuntu-latest + strategy: + matrix: + go-version: ['1.22.5'] + steps: + - name: Checkout the repo + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + - name: Setup Go ${{ matrix.go-version }} + uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + with: + go-version: ${{ matrix.go-version }} + - name: Display Go version + run: go version + - name: Clone Repository B + run: | + git clone https://github.com/smartcontractkit/ccip.git + cd ccip + git checkout ccip-develop + - name: Update chainlink-ccip dependency in ccip + run: | + cd ccip + go get github.com/smartcontractkit/chainlink-ccip@$GITHUB_SHA + make gomodtidy + - name: Setup Postgres + uses: ./.github/actions/setup-postgres + - name: Download Go vendor packages + run: | + cd ccip + go mod download + - name: Build binary + run: | + cd ccip + go build -o ccip.test . + - name: Setup DB + run: | + cd ccip + ./ccip.test local db preparetest + env: + CL_DATABASE_URL: ${{ env.DB_URL }} + - name: Run ccip ocr3 integration test + run: | + cd ccip + go test -v -timeout 3m -run "^TestIntegration_OCR3Nodes$" ./core/capabilities/ccip/ccip_integration_tests + EXITCODE=${PIPESTATUS[0]} + if [ $EXITCODE -ne 0 ]; then + echo "Integration test failed" + else + echo "Integration test passed!" + fi + exit $EXITCODE From 827f21fe56fae5501e833fed99c45e9d920377f2 Mon Sep 17 00:00:00 2001 From: Makram Kamaleddine Date: Mon, 12 Aug 2024 12:16:48 +0300 Subject: [PATCH 2/5] fix go get? --- .github/workflows/ccip-integration-test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ccip-integration-test.yml b/.github/workflows/ccip-integration-test.yml index 9609c31f..72267d96 100644 --- a/.github/workflows/ccip-integration-test.yml +++ b/.github/workflows/ccip-integration-test.yml @@ -26,16 +26,16 @@ jobs: go-version: ${{ matrix.go-version }} - name: Display Go version run: go version - - name: Clone Repository B + - name: Clone CCIP repo run: | git clone https://github.com/smartcontractkit/ccip.git cd ccip git checkout ccip-develop - name: Update chainlink-ccip dependency in ccip run: | - cd ccip - go get github.com/smartcontractkit/chainlink-ccip@$GITHUB_SHA - make gomodtidy + cd ccip + go get github.com/smartcontractkit/chainlink-ccip@${{ github.event.pull_request.head.sha }} + make gomodtidy - name: Setup Postgres uses: ./.github/actions/setup-postgres - name: Download Go vendor packages From 01ed8ca1fd9dd468ed80a5b6b4348095bc2a4776 Mon Sep 17 00:00:00 2001 From: Makram Kamaleddine Date: Mon, 12 Aug 2024 13:58:00 +0300 Subject: [PATCH 3/5] checkout mk/bump-chainlink-ccip branch on ccip --- .github/workflows/ccip-integration-test.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ccip-integration-test.yml b/.github/workflows/ccip-integration-test.yml index 72267d96..00f80701 100644 --- a/.github/workflows/ccip-integration-test.yml +++ b/.github/workflows/ccip-integration-test.yml @@ -30,12 +30,14 @@ jobs: run: | git clone https://github.com/smartcontractkit/ccip.git cd ccip - git checkout ccip-develop - - name: Update chainlink-ccip dependency in ccip - run: | - cd ccip - go get github.com/smartcontractkit/chainlink-ccip@${{ github.event.pull_request.head.sha }} - make gomodtidy + git fetch + git checkout mk/bump-chainlink-ccip + # TODO: remove this temporarily just to test the integration test run, which should be passing. + # - name: Update chainlink-ccip dependency in ccip + # run: | + # cd ccip + # go get github.com/smartcontractkit/chainlink-ccip@${{ github.event.pull_request.head.sha }} + # make gomodtidy - name: Setup Postgres uses: ./.github/actions/setup-postgres - name: Download Go vendor packages From acd756d03ec6c827363c4201f2f8903bd27e784c Mon Sep 17 00:00:00 2001 From: Makram Kamaleddine Date: Mon, 12 Aug 2024 14:04:08 +0300 Subject: [PATCH 4/5] add CL_DATABASE_URL env var --- .github/workflows/ccip-integration-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ccip-integration-test.yml b/.github/workflows/ccip-integration-test.yml index 00f80701..7ebec0e2 100644 --- a/.github/workflows/ccip-integration-test.yml +++ b/.github/workflows/ccip-integration-test.yml @@ -65,3 +65,5 @@ jobs: echo "Integration test passed!" fi exit $EXITCODE + env: + CL_DATABASE_URL: ${{ env.DB_URL }} From 6cbccd243356bcb218bc140b3cb7c7a5f04185a4 Mon Sep 17 00:00:00 2001 From: Makram Kamaleddine Date: Mon, 12 Aug 2024 20:39:14 +0300 Subject: [PATCH 5/5] revert TODO --- .github/workflows/ccip-integration-test.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ccip-integration-test.yml b/.github/workflows/ccip-integration-test.yml index 7ebec0e2..15dc0e3e 100644 --- a/.github/workflows/ccip-integration-test.yml +++ b/.github/workflows/ccip-integration-test.yml @@ -31,13 +31,12 @@ jobs: git clone https://github.com/smartcontractkit/ccip.git cd ccip git fetch - git checkout mk/bump-chainlink-ccip - # TODO: remove this temporarily just to test the integration test run, which should be passing. - # - name: Update chainlink-ccip dependency in ccip - # run: | - # cd ccip - # go get github.com/smartcontractkit/chainlink-ccip@${{ github.event.pull_request.head.sha }} - # make gomodtidy + git checkout ccip-develop + - name: Update chainlink-ccip dependency in ccip + run: | + cd ccip + go get github.com/smartcontractkit/chainlink-ccip@${{ github.event.pull_request.head.sha }} + make gomodtidy - name: Setup Postgres uses: ./.github/actions/setup-postgres - name: Download Go vendor packages