From d03a6a809db1b1b9ba5fcb968c6fd707a821e8f4 Mon Sep 17 00:00:00 2001 From: Rick Elrod Date: Thu, 31 Aug 2023 00:06:11 -0500 Subject: [PATCH] Enable collection integration tests on GHA There are a number of changes here: - Abstract out a GHA composite action for running the dev environment - Update the e2e tests to use that new abstracted action - Introduce a new (matrixed) job for running collection integration tests. This splits the jobs up based on filename. - Collect coverage info and generate an html report that people can download easily to see collection coverage info. - Do some hacks to delete the intermediary coverage file artifacts which aren't needed after the job finishes. Signed-off-by: Rick Elrod --- .github/actions/run_awx_devel/action.yml | 89 ++++++++++++ .../actions/upload_awx_devel_logs/action.yml | 19 +++ .github/workflows/ci.yml | 132 ++++++++++++++++++ .github/workflows/e2e_test.yml | 50 ++----- Makefile | 2 +- .../ad_hoc_command_cancel/tasks/main.yml | 2 +- .../targets/demo_data/tasks/main.yml | 1 + .../integration/targets/group/tasks/main.yml | 32 +++-- .../targets/job_cancel/tasks/main.yml | 9 +- .../targets/project_update/tasks/main.yml | 4 + .../integration/targets/user/tasks/main.yml | 1 + 11 files changed, 278 insertions(+), 63 deletions(-) create mode 100644 .github/actions/run_awx_devel/action.yml create mode 100644 .github/actions/upload_awx_devel_logs/action.yml diff --git a/.github/actions/run_awx_devel/action.yml b/.github/actions/run_awx_devel/action.yml new file mode 100644 index 000000000000..e7c05ca9958f --- /dev/null +++ b/.github/actions/run_awx_devel/action.yml @@ -0,0 +1,89 @@ +# This currently *always* uses the "warm build cache" image +# We should do something to allow forcing a rebuild, probably by looking for +# some string in the commit message or something. + +name: Run AWX (devel environment) +description: Runs AWX with `make docker-compose` +inputs: + github-token: + description: GitHub Token for registry access + required: true + build-ui: + description: Should the UI be built? + required: false + default: false + type: boolean +outputs: + ip: + description: The IP of the tools_awx_1 container + value: ${{ steps.data.outputs.ip }} + admin-token: + description: OAuth token for admin user + value: ${{ steps.data.outputs.admin_token }} +runs: + using: composite + steps: + - name: Get python version from Makefile + shell: bash + run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV + + - name: Upgrade ansible-core + shell: bash + run: python3 -m pip install --upgrade ansible-core + + - name: Install system deps + shell: bash + run: sudo apt-get install -y gettext + + - name: Log in to registry + shell: bash + run: | + echo "${{ inputs.github-token }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Pre-pull latest available devel image and build HEAD on top of it + shell: bash + run: | + docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} + DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} \ + COMPOSE_TAG=${{ github.base_ref }} \ + make docker-compose-build + + - name: Start AWX + shell: bash + run: | + DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} \ + COMPOSE_TAG=${{ github.base_ref }} \ + COMPOSE_UP_OPTS="-d" \ + make docker-compose + + - name: Update default AWX password + shell: bash + run: | + while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' -k https://localhost:8043/api/v2/ping/)" != "200" ]] + do + echo "Waiting for AWX..." + sleep 5 + done + echo "AWX is up, updating the password..." + docker exec -i tools_awx_1 sh <<-EOSH + awx-manage update_password --username=admin --password=password + EOSH + + - name: Build UI + # This must be a string comparison in composite actions: + # https://github.com/actions/runner/issues/2238 + if: ${{ inputs.build-ui == 'true' }} + shell: bash + run: | + docker exec -i tools_awx_1 sh <<-EOSH + make ui-devel + EOSH + + - name: Get instance data + id: data + shell: bash + run: | + AWX_IP=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' tools_awx_1) + ADMIN_TOKEN=$(docker exec -i tools_awx_1 awx-manage create_oauth2_token --user admin) + echo "ip=$AWX_IP" >> $GITHUB_OUTPUT + echo "admin_token=$ADMIN_TOKEN" >> $GITHUB_OUTPUT diff --git a/.github/actions/upload_awx_devel_logs/action.yml b/.github/actions/upload_awx_devel_logs/action.yml new file mode 100644 index 000000000000..e8b80bc0a2d2 --- /dev/null +++ b/.github/actions/upload_awx_devel_logs/action.yml @@ -0,0 +1,19 @@ +name: Upload logs +description: Upload logs from `make docker-compose` devel environment to GitHub as an artifact +inputs: + log-filename: + description: "*Unique* name of the log file" + required: true +runs: + using: composite + steps: + - name: Get AWX logs + shell: bash + run: | + docker logs tools_awx_1 > ${{ inputs.log-filename }} + + - name: Upload AWX logs as artifact + uses: actions/upload-artifact@v3 + with: + name: docker-compose-logs + path: ${{ inputs.log-filename }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 406c3763673f..43978da5b847 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,3 +117,135 @@ jobs: # needed due to cgroupsv2. This is fixed, but a stable release # with the fix has not been made yet. ANSIBLE_TEST_PREFER_PODMAN: 1 + + collection-integration: + name: awx_collection integration + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + target-regex: + - name: a-h + regex: ^[a-h] + - name: i-p + regex: ^[i-p] + - name: r-z0-9 + regex: ^[r-z0-9] + steps: + - uses: actions/checkout@v3 + + - uses: ./.github/actions/run_awx_devel + id: awx + with: + build-ui: false + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install dependencies for running tests + run: | + python3 -m pip install -e ./awxkit/ + python3 -m pip install -r awx_collection/requirements.txt + + - name: Run integration tests + run: | + echo "::remove-matcher owner=python::" # Disable annoying annotations from setup-python + echo '[general]' > ~/.tower_cli.cfg + echo 'host = https://${{ steps.awx.outputs.ip }}:8043' >> ~/.tower_cli.cfg + echo 'oauth_token = ${{ steps.awx.outputs.admin-token }}' >> ~/.tower_cli.cfg + echo 'verify_ssl = false' >> ~/.tower_cli.cfg + TARGETS="$(ls awx_collection/tests/integration/targets | grep '${{ matrix.target-regex.regex }}' | tr '\n' ' ')" + make COLLECTION_VERSION=100.100.100-git COLLECTION_TEST_TARGET="--coverage --requirements $TARGETS" test_collection_integration + env: + ANSIBLE_TEST_PREFER_PODMAN: 1 + + # Upload coverage report as artifact + - uses: actions/upload-artifact@v3 + with: + name: coverage-${{ matrix.target-regex.name }} + path: ~/.ansible/collections/ansible_collections/awx/awx/tests/output/coverage/ + + - uses: ./.github/actions/upload_awx_devel_logs + with: + log-filename: collection-integration-${{ matrix.target-regex.name }}.log + + collection-integration-coverage-combine: + name: combine awx_collection integration coverage + runs-on: ubuntu-latest + needs: + - collection-integration + strategy: + fail-fast: false + steps: + - uses: actions/checkout@v3 + + - name: Upgrade ansible-core + run: python3 -m pip install --upgrade ansible-core + + - name: Download coverage artifacts + uses: actions/download-artifact@v3 + with: + path: coverage + + - name: Combine coverage + run: | + make COLLECTION_VERSION=100.100.100-git install_collection + mkdir -p ~/.ansible/collections/ansible_collections/awx/awx/tests/output/coverage + cd coverage + for i in coverage-*; do + cp -rv $i/* ~/.ansible/collections/ansible_collections/awx/awx/tests/output/coverage/ + done + cd ~/.ansible/collections/ansible_collections/awx/awx + ansible-test coverage combine --requirements + ansible-test coverage html + echo '## AWX Collection Integration Coverage' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + ansible-test coverage report >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo >> $GITHUB_STEP_SUMMARY + echo '## AWX Collection Integration Coverage HTML' >> $GITHUB_STEP_SUMMARY + echo 'Download the HTML artifacts to view the coverage report.' >> $GITHUB_STEP_SUMMARY + + # This is a huge hack, there's no official action for removing artifacts currently. + # Also ACTIONS_RUNTIME_URL and ACTIONS_RUNTIME_TOKEN aren't available in normal run + # steps, so we have to use github-script to get them. + # + # The advantage of doing this, though, is that we save on artifact storage space. + + - name: Get secret artifact runtime URL + uses: actions/github-script@v6 + id: get-runtime-url + with: + result-encoding: string + script: | + const { ACTIONS_RUNTIME_URL } = process.env; + return ACTIONS_RUNTIME_URL; + + - name: Get secret artifact runtime token + uses: actions/github-script@v6 + id: get-runtime-token + with: + result-encoding: string + script: | + const { ACTIONS_RUNTIME_TOKEN } = process.env; + return ACTIONS_RUNTIME_TOKEN; + + - name: Remove intermediary artifacts + env: + ACTIONS_RUNTIME_URL: ${{ steps.get-runtime-url.outputs.result }} + ACTIONS_RUNTIME_TOKEN: ${{ steps.get-runtime-token.outputs.result }} + run: | + echo "::add-mask::${ACTIONS_RUNTIME_TOKEN}" + artifacts=$( + curl -H "Authorization: Bearer $ACTIONS_RUNTIME_TOKEN" \ + ${ACTIONS_RUNTIME_URL}_apis/pipelines/workflows/${{ github.run_id }}/artifacts?api-version=6.0-preview \ + | jq -r '.value | .[] | select(.name | startswith("coverage-")) | .url' + ) + + for artifact in $artifacts; do + curl -i -X DELETE -H "Accept: application/json;api-version=6.0-preview" -H "Authorization: Bearer $ACTIONS_RUNTIME_TOKEN" "$artifact" + done + + - name: Upload coverage report as artifact + uses: actions/upload-artifact@v3 + with: + name: awx-collection-integration-coverage-html + path: ~/.ansible/collections/ansible_collections/awx/awx/tests/output/reports/coverage diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml index 22affd08ef7f..8a03db9acbe9 100644 --- a/.github/workflows/e2e_test.yml +++ b/.github/workflows/e2e_test.yml @@ -21,32 +21,12 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Get python version from Makefile - run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV - - - name: Install python ${{ env.py_version }} - uses: actions/setup-python@v4 + - uses: ./.github/actions/run_awx_devel + id: awx with: - python-version: ${{ env.py_version }} - - - name: Install system deps - run: sudo apt-get install -y gettext - - - name: Log in to registry - run: | - echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - - name: Pre-pull image to warm build cache - run: | - docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} - - - name: Build UI - run: | - DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ github.base_ref }} make ui-devel - - - name: Start AWX - run: | - DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ github.base_ref }} make docker-compose &> make-docker-compose-output.log & + build-ui: true + github-token: ${{ secrets.GITHUB_TOKEN }} + log-filename: e2e-${{ matrix.job }}.log - name: Pull awx_cypress_base image run: | @@ -65,18 +45,6 @@ jobs: cd ${{ secrets.E2E_PROJECT }}/ui-tests/awx-pf-tests docker build -t awx-pf-tests . - - name: Update default AWX password - run: | - while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' -k https://localhost:8043/api/v2/ping/)" != "200" ]] - do - echo "Waiting for AWX..." - sleep 5; - done - echo "AWX is up, updating the password..." - docker exec -i tools_awx_1 sh <<-EOSH - awx-manage update_password --username=admin --password=password - EOSH - - name: Run E2E tests env: CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} @@ -86,7 +54,7 @@ jobs: export COMMIT_INFO_SHA=$GITHUB_SHA export COMMIT_INFO_REMOTE=$GITHUB_REPOSITORY_OWNER cd ${{ secrets.E2E_PROJECT }}/ui-tests/awx-pf-tests - AWX_IP=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' tools_awx_1) + AWX_IP=${{ steps.awx.outputs.ip }} printenv > .env echo "Executing tests:" docker run \ @@ -102,8 +70,6 @@ jobs: -w /e2e \ awx-pf-tests run --project . - - name: Save AWX logs - uses: actions/upload-artifact@v2 + - uses: ./.github/actions/upload_awx_devel_logs with: - name: AWX-logs-${{ matrix.job }} - path: make-docker-compose-output.log + log-filename: e2e-${{ matrix.job }}.log diff --git a/Makefile b/Makefile index 52cde438f79d..6fb4f69a1021 100644 --- a/Makefile +++ b/Makefile @@ -383,7 +383,7 @@ test_collection_sanity: cd $(COLLECTION_INSTALL) && ansible-test sanity $(COLLECTION_SANITY_ARGS) test_collection_integration: install_collection - cd $(COLLECTION_INSTALL) && ansible-test integration $(COLLECTION_TEST_TARGET) + cd $(COLLECTION_INSTALL) && ansible-test integration -vvv $(COLLECTION_TEST_TARGET) test_unit: @if [ "$(VENV_BASE)" ]; then \ diff --git a/awx_collection/tests/integration/targets/ad_hoc_command_cancel/tasks/main.yml b/awx_collection/tests/integration/targets/ad_hoc_command_cancel/tasks/main.yml index 7cdb7457d100..55c1803db976 100644 --- a/awx_collection/tests/integration/targets/ad_hoc_command_cancel/tasks/main.yml +++ b/awx_collection/tests/integration/targets/ad_hoc_command_cancel/tasks/main.yml @@ -49,8 +49,8 @@ - name: Cancel the command ad_hoc_command_cancel: command_id: "{{ command.id }}" + request_timeout: 60 register: results - ignore_errors: true - assert: that: diff --git a/awx_collection/tests/integration/targets/demo_data/tasks/main.yml b/awx_collection/tests/integration/targets/demo_data/tasks/main.yml index db152671bd1d..3052a0ebfd74 100644 --- a/awx_collection/tests/integration/targets/demo_data/tasks/main.yml +++ b/awx_collection/tests/integration/targets/demo_data/tasks/main.yml @@ -33,6 +33,7 @@ name: "localhost" inventory: "Demo Inventory" state: present + enabled: true variables: ansible_connection: local register: result diff --git a/awx_collection/tests/integration/targets/group/tasks/main.yml b/awx_collection/tests/integration/targets/group/tasks/main.yml index 0a73a73534ba..c9a492911c8a 100644 --- a/awx_collection/tests/integration/targets/group/tasks/main.yml +++ b/awx_collection/tests/integration/targets/group/tasks/main.yml @@ -21,14 +21,14 @@ name: "{{ inv_name }}" organization: Default state: present - register: result + register: inv_result - name: Create a Host host: name: "{{ host_name4 }}" inventory: "{{ inv_name }}" state: present - register: result + register: host_result - name: Add Host to Group group: @@ -37,16 +37,18 @@ hosts: - "{{ host_name4 }}" preserve_existing_hosts: true - register: result + register: group_result - assert: that: - - "result is changed" + - inv_result is changed + - host_result is changed + - group_result is changed - name: Create Group 1 group: name: "{{ group_name1 }}" - inventory: "{{ result.id }}" + inventory: "{{ inv_result.id }}" state: present variables: foo: bar @@ -165,21 +167,21 @@ that: - group1_host_count == "3" -- name: Delete Group 2 +- name: Delete Group 3 group: - name: "{{ group_name2 }}" + name: "{{ group_name3 }}" inventory: "{{ inv_name }}" state: absent register: result -# In this case, group 2 was last a child of group1 so deleting group1 deleted group2 - assert: that: - - "result is not changed" + - "result is changed" -- name: Delete Group 3 +# If we delete group 1 first it will delete group 2 and 3 +- name: Delete Group 1 group: - name: "{{ group_name3 }}" + name: "{{ group_name1 }}" inventory: "{{ inv_name }}" state: absent register: result @@ -188,17 +190,17 @@ that: - "result is changed" -# If we delete group 1 first it will delete group 2 and 3 -- name: Delete Group 1 +- name: Delete Group 2 group: - name: "{{ group_name1 }}" + name: "{{ group_name2 }}" inventory: "{{ inv_name }}" state: absent register: result +# In this case, group 2 was last a child of group1 so deleting group1 deleted group2 - assert: that: - - "result is changed" + - "result is not changed" - name: Check module fails with correct msg group: diff --git a/awx_collection/tests/integration/targets/job_cancel/tasks/main.yml b/awx_collection/tests/integration/targets/job_cancel/tasks/main.yml index 254ea89ce756..deaab76f06da 100644 --- a/awx_collection/tests/integration/targets/job_cancel/tasks/main.yml +++ b/awx_collection/tests/integration/targets/job_cancel/tasks/main.yml @@ -11,6 +11,7 @@ - name: Cancel the job job_cancel: job_id: "{{ job.id }}" + request_timeout: 60 register: results - assert: @@ -23,10 +24,10 @@ fail_if_not_running: true register: results ignore_errors: true - -- assert: - that: - - results is failed + # This test can be flaky, so we retry it a few times + until: results is failed and results.msg == 'Job is not running' + retries: 6 + delay: 5 - name: Check module fails with correct msg job_cancel: diff --git a/awx_collection/tests/integration/targets/project_update/tasks/main.yml b/awx_collection/tests/integration/targets/project_update/tasks/main.yml index 4b08e685fedb..65191be65d24 100644 --- a/awx_collection/tests/integration/targets/project_update/tasks/main.yml +++ b/awx_collection/tests/integration/targets/project_update/tasks/main.yml @@ -61,6 +61,10 @@ organization: Default state: absent register: result + until: result is changed # wait for the project update to settle + retries: 6 + delay: 5 + - assert: that: diff --git a/awx_collection/tests/integration/targets/user/tasks/main.yml b/awx_collection/tests/integration/targets/user/tasks/main.yml index 2d88bb199ab5..a3fae666b0e9 100644 --- a/awx_collection/tests/integration/targets/user/tasks/main.yml +++ b/awx_collection/tests/integration/targets/user/tasks/main.yml @@ -220,6 +220,7 @@ user: controller_username: "{{ username }}-orgadmin" controller_password: "{{ username }}-orgadmin" + controller_oauthtoken: false # Hack for CI where we use oauth in config file username: "{{ username }}" first_name: Joe password: "{{ 65535 | random | to_uuid }}"