diff --git a/.github/workflows/periodic-cover.yml b/.github/workflows/periodic-cover.yml new file mode 100644 index 00000000..b8982fcc --- /dev/null +++ b/.github/workflows/periodic-cover.yml @@ -0,0 +1,27 @@ +name: Periodically capture coverage +on: + workflow_dispatch: {} + schedule: + # Run every 12 hours. + - cron: '0 */12 * * *' + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + +jobs: + test: + uses: ./.github/workflows/stage-test.yml + with: + live-test: true + coverage: true + secrets: + PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_CI_ROLE_ARN: ${{ secrets.AWS_CI_ROLE_ARN }} + ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }} + ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }} + ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }} + ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/pr-tests-command.yaml b/.github/workflows/pr-tests-command.yaml index d8752c27..f121f68e 100644 --- a/.github/workflows/pr-tests-command.yaml +++ b/.github/workflows/pr-tests-command.yaml @@ -28,6 +28,7 @@ jobs: with: live-test: true commit-ref: refs/pull/${{ github.event.client_payload.pull_request.number }}/merge + coverage: true secrets: PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -37,3 +38,4 @@ jobs: ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }} ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }} ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/pr-tests.yaml b/.github/workflows/pr-tests.yaml index e5e4ea55..8f84c4f7 100644 --- a/.github/workflows/pr-tests.yaml +++ b/.github/workflows/pr-tests.yaml @@ -13,6 +13,7 @@ jobs: uses: ./.github/workflows/stage-test.yml with: live-test: true + coverage: true secrets: PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -22,3 +23,4 @@ jobs: ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }} ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }} ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/stage-test.yml b/.github/workflows/stage-test.yml index 3acbebb0..4f15a8b2 100644 --- a/.github/workflows/stage-test.yml +++ b/.github/workflows/stage-test.yml @@ -30,6 +30,11 @@ on: default: '' required: false type: string + coverage: + description: "Collects coverage stats" + default: false + required: false + type: boolean secrets: PULUMI_ACCESS_TOKEN: { required: false } AWS_ACCESS_KEY_ID: { required: false } @@ -39,6 +44,7 @@ on: ARM_CLIENT_SECRET: { required: false } ARM_SUBSCRIPTION_ID: { required: false } ARM_TENANT_ID: { required: false } + CODECOV_TOKEN: { required: false } env: @@ -121,7 +127,22 @@ jobs: path: ${{ steps.list-schemas.outputs.schemas }} - name: Test + if: ${{ !inputs.coverage }} run: make test + - name: Test with Coverage + if: ${{ inputs.coverage }} + run: make test_cover + + - name: Upload coverage data + if: ${{ inputs.coverage }} + uses: codecov/codecov-action@v3 + with: + directory: coverage + files: "*" + fail_ci_if_error: false + verbose: true + token: ${{ secrets.CODECOV_TOKEN }} + strategy: fail-fast: false matrix: diff --git a/.gitignore b/.gitignore index 203091f2..4778bd73 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ command-output/ obj/ __pycache__/ node_modules/ +/coverage diff --git a/Makefile b/Makefile index 9d9ccbda..dfb7d36d 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,8 @@ PLUGIN_VERSION_AZURE_NATIVE := 1.99.1 PLUGIN_VERSION_RANDOM := 4.12.1 GO := go +BUILD_FLAGS ?= + .phony: .EXPORT_ALL_VARIABLES .EXPORT_ALL_VARIABLES: @@ -58,15 +60,32 @@ lint-copyright: build:: ensure mkdir -p ./bin - ${GO} build -o ./bin -p ${CONCURRENCY} ./cmd/... + ${GO} build $(BUILD_FLAGS) -o ./bin -p ${CONCURRENCY} ./cmd/... # Ensure that in tests, the language server is accessible test:: build get_plugins get_schemas PATH="${PWD}/bin:${PATH}" PULUMI_LIVE_TEST="${PULUMI_LIVE_TEST}" \ - ${GO} test -v --timeout 30m -count 1 -race -parallel ${CONCURRENCY} ./pkg/... + ${GO} test -v --timeout 30m -count 1 -race -parallel ${CONCURRENCY} ./... + +# Runs tests with code coverage tracking. +# Two output files are generated in the coverage/ directory: +# +# unit.out unit test coverage +# integration.out integration test coverage +# +# Integration test coverage is only generated for tests that invoke the +# pulumi-language-yaml executable. +.phony: test_cover +test_cover: get_plugins get_schemas + make build BUILD_FLAGS="-cover -coverpkg=github.com/pulumi/pulumi-yaml/..." + rm -rf coverage && mkdir -p coverage + $(eval COVERDIR := $(shell mktemp -d)) + PATH="${PWD}/bin:${PATH}" PULUMI_LIVE_TEST="${PULUMI_LIVE_TEST}" GOCOVERDIR=$(COVERDIR) \ + ${GO} test -v -coverprofile=coverage/unit.out -coverpkg=./... ./... + go tool covdata textfmt -i=$(COVERDIR) -o=coverage/integration.out test_short:: - ${GO} test --timeout 30m -short -count 1 -parallel ${CONCURRENCY} ./pkg/... + ${GO} test --timeout 30m -short -count 1 -parallel ${CONCURRENCY} ./... test_live:: PULUMI_LIVE_TEST = true test_live:: test_live_prereq test