-
Notifications
You must be signed in to change notification settings - Fork 5
200 lines (200 loc) · 8.06 KB
/
int-and-connected-test-run.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
name: Unit, Smoke, Connected and Integration tests
on:
workflow_dispatch: {}
pull_request:
schedule:
- cron: '0 4 * * *' # run at 4 AM UTC, 12PM EST.
# do not allow concurrent runs of this workflow on the same branch
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
test_unit:
name: Unit tests
runs-on: ubuntu-latest
timeout-minutes: 60
# Needed for integration with workload identity
permissions:
contents: 'read'
id-token: 'write'
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '17'
cache: 'gradle'
- name: 'Auth as jade-k8-sa'
uses: 'google-github-actions/auth@v2'
with:
# Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting
workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
token_format: 'access_token'
service_account: '[email protected]'
access_token_scopes: 'profile, email, openid'
- name: Run unit tests
env:
# required for sonarqube reports
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
# output plain logs instead of json
TDR_LOG_APPENDER: 'Console-Standard'
run: |
# assemble code, run unit tests, and generate scan
./gradlew --scan assemble check jacocoTestReport sonar
test_connected:
name: Connected tests
runs-on: ubuntu-latest
timeout-minutes: 180
# Needed for integration with workload identity
permissions:
contents: 'read'
id-token: 'write'
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '17'
cache: 'gradle'
- name: 'Auth as jade-k8-sa'
uses: 'google-github-actions/auth@v2'
with:
# Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting
workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
service_account: '[email protected]'
- name: Run connected tests
env:
# required for azure tests
AZURE_CREDENTIALS_APPLICATIONID: 0e29ec36-04e8-44d5-ae7c-50dc15135571
AZURE_CREDENTIALS_HOMETENANTID: fad90753-2022-4456-9b0a-c7e5b934e408
AZURE_CREDENTIALS_SECRET: ${{ secrets.AZURE_CREDENTIALS_SECRET }}
# required for synapse tests
AZURE_SYNAPSE_SQLADMINPASSWORD: ${{ secrets.AZURE_SYNAPSE_SQLADMINPASSWORD }}
AZURE_SYNAPSE_SQLADMINUSER: ${{ secrets.AZURE_SYNAPSE_SQLADMINUSER }}
AZURE_SYNAPSE_WORKSPACENAME: tdr-snps-int-east-us-ondemand.sql.azuresynapse.net
# required for testAzureBillingProfile
# uses an azure marketplace app with this hardcoded deployment email
JADE_USER_EMAIL: [email protected]
# output plain logs instead of json
TDR_LOG_APPENDER: 'Console-Standard'
# required for integration tests
RBS_INSTANCEURL: https://buffer.tools.integ.envs.broadinstitute.org
RBS_POOLID: datarepo_v1
GOOGLE_CLOUD_PROJECT: broad-jade-dev
run: |
# run connected tests
./gradlew --scan --warn testConnected
test_integration:
name: Integration tests
runs-on: ubuntu-latest
timeout-minutes: 300
# Needed for integration with workload identity
permissions:
contents: 'read'
id-token: 'write'
services:
postgres:
image: postgres:11
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: postgres
ports:
- 5432:5432
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '17'
cache: 'gradle'
- name: 'Auth as jade-k8-sa'
uses: 'google-github-actions/auth@v2'
with:
# Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting
workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
service_account: '[email protected]'
- name: Run integration tests
env:
# required for azure tests
AZURE_CREDENTIALS_APPLICATIONID: 0e29ec36-04e8-44d5-ae7c-50dc15135571
AZURE_CREDENTIALS_HOMETENANTID: fad90753-2022-4456-9b0a-c7e5b934e408
AZURE_CREDENTIALS_SECRET: ${{ secrets.AZURE_CREDENTIALS_SECRET }}
AZURE_SYNAPSE_SQLADMINUSER: ${{ secrets.AZURE_SYNAPSE_SQLADMINUSER }}
AZURE_SYNAPSE_SQLADMINPASSWORD: ${{ secrets.AZURE_SYNAPSE_SQLADMINPASSWORD }}
AZURE_SYNAPSE_WORKSPACENAME: tdr-snps-int-east-us-ondemand.sql.azuresynapse.net
# required for integration tests
IT_JADE_API_URL: http://localhost:8080
# postgres connection details
PGHOST: 127.0.0.1
PGPASSWORD: postgres
# required for integration tests
RBS_INSTANCEURL: https://buffer.tools.integ.envs.broadinstitute.org
RBS_POOLID: datarepo_v1
# output plain logs instead of json
TDR_LOG_APPENDER: 'Console-Standard'
run: |
# wait for postgres to be ready
pg_isready -h ${PGHOST} -t 30
# create the datarepo and stairway databases
psql -U postgres -f ./scripts/init-db/postgres-init.sql
# build code and run local api
./gradlew bootJar
export DATA_REPO_JAR=$(find . -type f -name jade-data-repo-*-SNAPSHOT.jar)
java -jar ${DATA_REPO_JAR} > local-api-output.log &
# wait until api is ready
timeout 30 bash -c 'until curl -s ${IT_JADE_API_URL}/status; do sleep 1; done'
# run integration tests
./gradlew --scan --warn testIntegration
- name: Upload API logs
if: always()
uses: actions/upload-artifact@v4
with:
name: integration-test-api-logs
path: local-api-output.log
retention-days: 7
# git_hash:
# name: Extract git hash
# runs-on: ubuntu-latest
# outputs:
# version: ${{ steps.config.outputs.GIT_HASH }}
# steps:
# - name: Checkout code
# uses: actions/checkout@v4
# - name: Get the latest git hash
# id: config
# run: |
# GIT_HASH=$(git rev-parse --short HEAD)
# echo "GIT_HASH=${GIT_HASH}" >> $GITHUB_OUTPUT
# echo "Latest git hash in branch is ${GIT_HASH}"
# report-to-sherlock:
# # only runs on pull requests and reports the appVersion even if tests fail
# if: github.event_name == 'pull_request'
# uses: broadinstitute/sherlock/.github/workflows/client-report-app-version.yaml@main
# needs: git_hash
# with:
# new-version: ${{ needs.git_hash.outputs.version }}
# chart-name: 'datarepo'
# permissions:
# contents: read
# id-token: write
# report-workflow:
# if: github.ref == 'refs/heads/develop'
# uses: broadinstitute/sherlock/.github/workflows/client-report-workflow.yaml@main
# # dependency is not required but makes the action run page more readable
# needs: git_hash
# with:
# relates-to-chart-releases: 'datarepo-dev'
# notify-slack-channels-upon-workflow-failure: ${{ vars.SLACK_NOTIFICATION_CHANNELS }}
# notify-slack-channels-upon-workflow-retry: ${{ vars.SLACK_NOTIFICATION_CHANNELS }}
# permissions:
# id-token: write