Sanath migration testing #918
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Pull Request Workflow | |
| on: [pull_request] | |
| # So that only one PR workflow will be run simultaneously. This ensures that there will not be multiple runners accessing | |
| # the same Terraform state at the same time. | |
| concurrency: pr-workflow-${{ github.event.pull_request.number }} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| PRE_RELEASE_NAME: tmp-pr-${{ github.event.pull_request.number }} | |
| # Used to re-sign Linux artifacts | |
| GPG_MAIL: ${{ secrets.LOGGING_GPG_MAIL }} | |
| GPG_PASSPHRASE: ${{ secrets.OHAI_GPG_PASSPHRASE }} | |
| GPG_PRIVATE_KEY_BASE64: ${{ secrets.OHAI_GPG_PRIVATE_KEY_BASE64 }} # base64 encoded | |
| jobs: | |
| # Empties the GH pre-relase | |
| # Generates strategy matrices that can be used by other jobs to run the build or testing of all supported packages | |
| setup_environment: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| pre_release_name: ${{ steps.set_vars.outputs.pre_release_name }} | |
| sles_matrix: ${{ steps.set-matrices.outputs.sles_matrix }} | |
| linux_and_windows_matrix: ${{ steps.set-matrices.outputs.linux_and_windows_matrix }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| # This is a workaround required to pass environment variables to the run_e2e_tests workflow | |
| # See: https://stackoverflow.com/a/73305536 and https://github.com/orgs/community/discussions/26671#discussioncomment-3252793 | |
| - name: Make dynamically-generated env variables available to be passed down to reusable workflows | |
| id: set_vars | |
| run: | | |
| echo "pre_release_name=$PRE_RELEASE_NAME" >> $GITHUB_OUTPUT | |
| - name: (Re)create pre-release | |
| run: | | |
| pre_release_exists=$(gh release view $PRE_RELEASE_NAME &>/dev/null && echo "true" || echo "false") | |
| if [[ $pre_release_exists == "true" ]]; then | |
| echo "Deleting existing pre-release" | |
| gh release delete ${{ env.PRE_RELEASE_NAME }} -y --cleanup-tag | |
| fi | |
| pre_release_tag=$PRE_RELEASE_NAME | |
| pre_release_title="Temporary release to build and test artifacts from PR#${{ github.event.pull_request.number }}" | |
| pre_release_notes="Created by PR: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/pull/${{ github.event.pull_request.number }}" | |
| # Releases created from a runner are always DRAFT | |
| echo "Creating release: $pre_release_tag" | |
| gh release create "$pre_release_tag" --title "$pre_release_title" --notes "$pre_release_notes" | |
| # We need the pre-release to NOT be a draft, otherwise it won't be visible to download packages from the Ansible-managed hosts | |
| echo "Updating release to be a pre-release" | |
| gh release edit "$pre_release_tag" --draft=false --prerelease | |
| - name: Install python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Compute and upload matrices | |
| id: set-matrices | |
| run: | | |
| make versions/generateMatrices | |
| echo "linux_and_windows_matrix=$( cat versions/linuxAndWindowsMatrix.json )" >> "$GITHUB_OUTPUT" | |
| echo "sles_matrix=$( cat versions/slesMatrix.json )" >> "$GITHUB_OUTPUT" | |
| gh release upload ${{ env.PRE_RELEASE_NAME }} versions/linuxAndWindowsMatrix.json --repo newrelic/fluent-bit-package | |
| gh release upload ${{ env.PRE_RELEASE_NAME }} versions/slesMatrix.json --repo newrelic/fluent-bit-package | |
| - name: Compute and upload schemas | |
| run: | | |
| make schemas/generateSchemas; | |
| gh release upload ${{ env.PRE_RELEASE_NAME }} schemas/generated-linux-schema-staging.yaml --repo newrelic/fluent-bit-package | |
| gh release upload ${{ env.PRE_RELEASE_NAME }} schemas/generated-linux-schema-production.yaml --repo newrelic/fluent-bit-package | |
| # Downloads all Fluent Bit packages that are officially supported, preferably from the New Relic Infrastructure Agent | |
| # repository (Linux packages, already re-signed by NR) or Logging's S3 bucket (Windows packages, already packaged for the NRIA). | |
| # If these are not available, they are downloaded from the official Fluent Bit repository and repackaged to be used by the NRIA. | |
| download_official_packages: | |
| needs: [setup_environment] | |
| runs-on: ubuntu-latest | |
| if: ${{ needs.setup_environment.outputs.linux_and_windows_matrix != '[]' }} | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| include: ${{ fromJson(needs.setup_environment.outputs.linux_and_windows_matrix) }} | |
| name: ${{ matrix.osDistro }}-${{ matrix.osVersion }}-${{ matrix.arch }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v3 | |
| - name: Attempt downloading package from New Relic repository | |
| id: download_package_from_nr | |
| run: | | |
| mkdir -p packages | |
| if wget --directory-prefix=packages "${{ matrix.nrPackageUrl }}"; then | |
| echo "result=success" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "result=failure" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Download, rename and resign Linux package | |
| if: ${{ steps.download_package_from_nr.outputs.result == 'failure' && matrix.osDistro != 'windows-server' }} | |
| run: | | |
| curl ${{ matrix.packageUrl }} -o packages/${{ matrix.targetPackageName }} | |
| sudo apt-get install -y debsigs | |
| bash ./scripts/sign.sh | |
| - name: Download and re-zip Windows package | |
| if: ${{ steps.download_package_from_nr.outputs.result == 'failure' && matrix.osDistro == 'windows-server' }} | |
| run: | | |
| wget ${{ matrix.packageUrl }} | |
| unzip fluent-bit-${{ matrix.fbVersion }}-${{ matrix.arch }}.zip | |
| zip -r -j packages/${{ matrix.targetPackageName }} \ | |
| fluent-bit-${{ matrix.fbVersion }}-${{ matrix.arch }}/bin/fluent-bit.exe \ | |
| fluent-bit-${{ matrix.fbVersion }}-${{ matrix.arch }}/bin/fluent-bit.dll | |
| # gh release upload can have issues if multiple jobs attempt uploading the same file concurrently (it can happen | |
| # for those distros using the same package, such as Windows). To avoid this, we first push all the files to a | |
| # shared filesystem and let the "prepare_prerelease" step below upload them later, sequentially. This GH action | |
| # ensures that if two jobs attempt pushing the same file, they get overwritten (last one prevails). | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| # Artifacts are pushed to *shared network folders* that have this name and that contain | |
| # the artifact inside of them. Example: fluent-bit-2.1.8-386.exe/fluent-bit-2.1.8-386.exe | |
| name: ${{ matrix.targetPackageName }} | |
| path: packages/${{ matrix.targetPackageName }} | |
| upload_official_packages_to_prerelease: | |
| needs: [ download_official_packages ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Download all artifacts from shared filesystem | |
| uses: actions/download-artifact@v3 | |
| with: | |
| path: packages | |
| - name: Push all artifacts to pre-release | |
| run: | |
| # To understand the need for /*/*, see comment in "upload artifacts" step above | |
| gh release upload ${{ env.PRE_RELEASE_NAME }} packages/*/* --repo newrelic/fluent-bit-package | |
| spin_up_suse: | |
| needs: setup_environment | |
| if: ${{ needs.setup_environment.outputs.sles_matrix != '[]' }} | |
| uses: ./.github/workflows/run_task.yml | |
| with: | |
| container_make_target: "terraform/ec2-suse-builders/provision PRE_RELEASE_NAME=${{ needs.setup_environment.outputs.pre_release_name }}" | |
| secrets: inherit | |
| build_suse_packages: | |
| needs: [setup_environment,spin_up_suse] | |
| uses: ./.github/workflows/run_task.yml | |
| with: | |
| container_make_target: "ansible/build-fb-suse/run PRE_RELEASE_NAME=${{ needs.setup_environment.outputs.pre_release_name }}" | |
| secrets: inherit | |
| tear_down_suse: | |
| needs: [setup_environment,build_suse_packages] | |
| uses: ./.github/workflows/run_task.yml | |
| with: | |
| container_make_target: "terraform/ec2-suse-builders/clean PRE_RELEASE_NAME=${{ needs.setup_environment.outputs.pre_release_name }}" | |
| secrets: inherit | |
| sign_suse_packages: | |
| needs: [setup_environment, build_suse_packages] | |
| runs-on: ubuntu-latest | |
| if: ${{ needs.setup_environment.outputs.sles_matrix != '[]' }} | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| include: ${{ fromJson(needs.setup_environment.outputs.sles_matrix) }} | |
| name: ${{ matrix.osDistro }}-${{ matrix.osVersion }}-${{ matrix.arch }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v3 | |
| - name: Fetch package from GitHub pre-release | |
| run: | | |
| mkdir -p packages | |
| gh release download ${{ env.PRE_RELEASE_NAME }} --pattern ${{ matrix.targetPackageName }} --dir packages | |
| - name: Sign package | |
| run: | | |
| sudo apt-get install -y debsigs | |
| bash ./scripts/sign.sh | |
| - name: Upload signed asset | |
| run: | |
| gh release upload ${{ env.PRE_RELEASE_NAME }} packages/* --clobber | |
| # Runs E2E tests using the packages available in the tmp-pr-#PR prerelease | |
| run_e2e_tests_prerelease: | |
| needs: [ setup_environment, sign_suse_packages, upload_official_packages_to_prerelease ] | |
| name: Run E2E tests by installing NRIA from Production and installing Fluent Bit from the PR pre-release | |
| uses: ./.github/workflows/run_e2e_tests.yml | |
| with: | |
| gh_release_name: ${{ needs.setup_environment.outputs.pre_release_name }} | |
| infra_agent_version: latest | |
| infra_agent_env: prerelease | |
| test_report_filename: test-report-prerelease.xml | |
| secrets: inherit | |
| publish_linux_to_staging: | |
| name: Publish linux packages to staging | |
| needs: [ setup_environment, run_e2e_tests_prerelease ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Publish linux packages to staging | |
| uses: newrelic/infrastructure-publish-action@v1 | |
| with: | |
| app_name: fluent-bit | |
| tag: ${{ env.PRE_RELEASE_NAME }} | |
| repo_name: "newrelic/fluent-bit-package" | |
| schema: "custom" | |
| schema_url: "https://github.com/newrelic/fluent-bit-package/releases/download/${{ env.PRE_RELEASE_NAME }}/generated-linux-schema-staging.yaml" | |
| aws_access_key_id: ${{ secrets.OHAI_AWS_ACCESS_KEY_ID_STAGING }} | |
| aws_secret_access_key: ${{ secrets.OHAI_AWS_SECRET_ACCESS_KEY_STAGING }} | |
| aws_s3_bucket_name: "nr-downloads-ohai-staging" | |
| aws_s3_lock_bucket_name: "onhost-ci-lock-staging" | |
| access_point_host: "staging" | |
| run_id: ${{ github.run_id }} | |
| aws_region: "us-east-1" | |
| aws_role_session_name: ${{ secrets.OHAI_AWS_ROLE_SESSION_NAME_STAGING }} | |
| aws_role_arn: ${{ secrets.OHAI_AWS_ROLE_ARN_STAGING }} | |
| # used for signing package stuff | |
| gpg_passphrase: ${{ secrets.OHAI_GPG_PASSPHRASE }} | |
| gpg_private_key_base64: ${{ secrets.OHAI_GPG_PRIVATE_KEY_BASE64 }} # base64 encoded | |
| run_e2e_tests_staging: | |
| needs: [ setup_environment, run_e2e_tests_prerelease, publish_linux_to_staging ] | |
| if: | | |
| always() && !failure() && !cancelled() | |
| name: Run E2E tests for all supported packages installing NRIA+FB from staging | |
| uses: ./.github/workflows/run_e2e_tests.yml | |
| with: | |
| gh_release_name: ${{ needs.setup_environment.outputs.pre_release_name }} | |
| infra_agent_version: latest | |
| infra_agent_env: staging | |
| test_report_filename: test-report-staging.xml | |
| secrets: inherit |