[SPIKE] DT-909: Can we switch to using workload identity to auth as SAs across TDR GHAs? #7208
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Unit, Smoke, Connected and Integration tests | |
| on: | |
| workflow_dispatch: {} | |
| pull_request: | |
| schedule: | |
| - cron: '0 4 * * *' # run at 4 AM UTC, 12PM EST. | |
| # do not allow concurrent runs of this workflow on the same branch | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| test_unit: | |
| name: Unit tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| # Needed for integration with workload identity | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '17' | |
| cache: 'gradle' | |
| - name: 'Auth as jade-k8-sa' | |
| uses: 'google-github-actions/auth@v2' | |
| with: | |
| # Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting | |
| workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider' | |
| token_format: 'access_token' | |
| service_account: '[email protected]' | |
| access_token_scopes: 'profile, email, openid' | |
| - name: Run unit tests | |
| env: | |
| # required for sonarqube reports | |
| SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} | |
| # output plain logs instead of json | |
| TDR_LOG_APPENDER: 'Console-Standard' | |
| run: | | |
| # assemble code, run unit tests, and generate scan | |
| ./gradlew --scan assemble check jacocoTestReport sonar | |
| test_connected: | |
| name: Connected tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 180 | |
| # Needed for integration with workload identity | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '17' | |
| cache: 'gradle' | |
| - name: 'Auth as jade-k8-sa' | |
| uses: 'google-github-actions/auth@v2' | |
| with: | |
| # Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting | |
| workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider' | |
| service_account: '[email protected]' | |
| - name: Run connected tests | |
| env: | |
| # required for azure tests | |
| AZURE_CREDENTIALS_APPLICATIONID: 0e29ec36-04e8-44d5-ae7c-50dc15135571 | |
| AZURE_CREDENTIALS_HOMETENANTID: fad90753-2022-4456-9b0a-c7e5b934e408 | |
| AZURE_CREDENTIALS_SECRET: ${{ secrets.AZURE_CREDENTIALS_SECRET }} | |
| # required for synapse tests | |
| AZURE_SYNAPSE_SQLADMINPASSWORD: ${{ secrets.AZURE_SYNAPSE_SQLADMINPASSWORD }} | |
| AZURE_SYNAPSE_SQLADMINUSER: ${{ secrets.AZURE_SYNAPSE_SQLADMINUSER }} | |
| AZURE_SYNAPSE_WORKSPACENAME: tdr-snps-int-east-us-ondemand.sql.azuresynapse.net | |
| # required for testAzureBillingProfile | |
| # uses an azure marketplace app with this hardcoded deployment email | |
| JADE_USER_EMAIL: [email protected] | |
| # output plain logs instead of json | |
| TDR_LOG_APPENDER: 'Console-Standard' | |
| # required for integration tests | |
| RBS_INSTANCEURL: https://buffer.tools.integ.envs.broadinstitute.org | |
| RBS_POOLID: datarepo_v1 | |
| GOOGLE_CLOUD_PROJECT: broad-jade-dev | |
| run: | | |
| # run connected tests | |
| ./gradlew --scan --warn testConnected | |
| test_integration: | |
| name: Integration tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 300 | |
| # Needed for integration with workload identity | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| services: | |
| postgres: | |
| image: postgres:11 | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: postgres | |
| ports: | |
| - 5432:5432 | |
| options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '17' | |
| cache: 'gradle' | |
| - name: 'Auth as jade-k8-sa' | |
| uses: 'google-github-actions/auth@v2' | |
| with: | |
| # Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting | |
| workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider' | |
| service_account: '[email protected]' | |
| - name: Run integration tests | |
| env: | |
| # required for azure tests | |
| AZURE_CREDENTIALS_APPLICATIONID: 0e29ec36-04e8-44d5-ae7c-50dc15135571 | |
| AZURE_CREDENTIALS_HOMETENANTID: fad90753-2022-4456-9b0a-c7e5b934e408 | |
| AZURE_CREDENTIALS_SECRET: ${{ secrets.AZURE_CREDENTIALS_SECRET }} | |
| AZURE_SYNAPSE_SQLADMINUSER: ${{ secrets.AZURE_SYNAPSE_SQLADMINUSER }} | |
| AZURE_SYNAPSE_SQLADMINPASSWORD: ${{ secrets.AZURE_SYNAPSE_SQLADMINPASSWORD }} | |
| AZURE_SYNAPSE_WORKSPACENAME: tdr-snps-int-east-us-ondemand.sql.azuresynapse.net | |
| # required for integration tests | |
| IT_JADE_API_URL: http://localhost:8080 | |
| # postgres connection details | |
| PGHOST: 127.0.0.1 | |
| PGPASSWORD: postgres | |
| # required for integration tests | |
| RBS_INSTANCEURL: https://buffer.tools.integ.envs.broadinstitute.org | |
| RBS_POOLID: datarepo_v1 | |
| # output plain logs instead of json | |
| TDR_LOG_APPENDER: 'Console-Standard' | |
| run: | | |
| # wait for postgres to be ready | |
| pg_isready -h ${PGHOST} -t 30 | |
| # create the datarepo and stairway databases | |
| psql -U postgres -f ./scripts/init-db/postgres-init.sql | |
| # build code and run local api | |
| ./gradlew bootJar | |
| export DATA_REPO_JAR=$(find . -type f -name jade-data-repo-*-SNAPSHOT.jar) | |
| java -jar ${DATA_REPO_JAR} > local-api-output.log & | |
| # wait until api is ready | |
| timeout 30 bash -c 'until curl -s ${IT_JADE_API_URL}/status; do sleep 1; done' | |
| # run integration tests | |
| ./gradlew --scan --warn testIntegration | |
| - name: Upload API logs | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: integration-test-api-logs | |
| path: local-api-output.log | |
| retention-days: 7 | |
| # git_hash: | |
| # name: Extract git hash | |
| # runs-on: ubuntu-latest | |
| # outputs: | |
| # version: ${{ steps.config.outputs.GIT_HASH }} | |
| # steps: | |
| # - name: Checkout code | |
| # uses: actions/checkout@v4 | |
| # - name: Get the latest git hash | |
| # id: config | |
| # run: | | |
| # GIT_HASH=$(git rev-parse --short HEAD) | |
| # echo "GIT_HASH=${GIT_HASH}" >> $GITHUB_OUTPUT | |
| # echo "Latest git hash in branch is ${GIT_HASH}" | |
| # report-to-sherlock: | |
| # # only runs on pull requests and reports the appVersion even if tests fail | |
| # if: github.event_name == 'pull_request' | |
| # uses: broadinstitute/sherlock/.github/workflows/client-report-app-version.yaml@main | |
| # needs: git_hash | |
| # with: | |
| # new-version: ${{ needs.git_hash.outputs.version }} | |
| # chart-name: 'datarepo' | |
| # permissions: | |
| # contents: read | |
| # id-token: write | |
| # report-workflow: | |
| # if: github.ref == 'refs/heads/develop' | |
| # uses: broadinstitute/sherlock/.github/workflows/client-report-workflow.yaml@main | |
| # # dependency is not required but makes the action run page more readable | |
| # needs: git_hash | |
| # with: | |
| # relates-to-chart-releases: 'datarepo-dev' | |
| # notify-slack-channels-upon-workflow-failure: ${{ vars.SLACK_NOTIFICATION_CHANNELS }} | |
| # notify-slack-channels-upon-workflow-retry: ${{ vars.SLACK_NOTIFICATION_CHANNELS }} | |
| # permissions: | |
| # id-token: write |