-
Notifications
You must be signed in to change notification settings - Fork 3
299 lines (253 loc) · 10.2 KB
/
system-tests-and-deploy.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
name: System tests and deploy
on:
push:
branches:
- master
- canary
pull_request:
branches:
- master
concurrency:
# New commits cancel previous builds only on pull requests. This is because `github.head_ref`` is only set on pull requests and `github.sha`` is unique for each commit.
group: system-tests-${{ github.head_ref || github.sha }}
cancel-in-progress: true
env:
CARGO_TERM_COLOR: always
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Check out repository code
uses: actions/checkout@v4
# This would cause all other tests to be skipped
- name: Make sure we have no focused system-tests
run: |
test "$(grep -r 'test.only' ./system-tests/src/ | wc -l)" -eq 0
- name: Free up disk space
run: |
echo "Space before:"
df -h
sudo rm -rf /usr/share/dotnet
sudo rm -rf /opt/ghc
sudo rm -rf "/usr/local/share/boost"
sudo rm -rf /usr/local/lib/android
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
echo "Space after:"
df -h
# If glcoud cli is installed skaffold spams the output because we're not logged in and in some cases might even fail the build.
- name: Hide gcloud cli from skaffold
run: sudo mv /usr/bin/gcloud /tmp/gcloud
- name: Install skaffold
uses: yokawasa/[email protected]
with:
skaffold: "2.13.1"
- name: Run npm ci so that shared module gets copied
run: npm ci
- name: Download tmc-langs
run: bin/download-tmc-langs
- name: Build
run: bin/skaffold-build-test-env-and-serialize-output
- name: Store skaffold output json
uses: actions/upload-artifact@v4
with:
name: skaffold-build-output
path: ./skaffold-build-output.json
- name: Export production images
run: bin/skaffold-export-production-images
- name: Store built images
uses: actions/upload-artifact@v4
with:
name: skaffold-images
path: ./skaffold-images
retention-days: 5
# If glcoud cli is installed skaffold spams the output because we're not logged in and in some cases might even fail the build.
- name: Unhide gcloud cli if master
run: sudo mv /tmp/gcloud /usr/bin/gcloud
if: ${{ github.ref == 'refs/heads/master' }}
- name: Authenticate to Google Cloud if master
env:
GCLOUD_SERVICE_KEY: ${{ secrets.GCLOUD_SERVICE_KEY }}
run: |
echo Authenticating to Google Cloud
echo "$GCLOUD_SERVICE_KEY" | python -m base64 -d > /tmp/key.json
gcloud auth activate-service-account --key-file=/tmp/key.json
if gcloud auth configure-docker -q; then
echo "Authenticated to Google Cloud..."
else
echo "Authentication to Google Cloud failed. Exiting..."
exit 1
fi
if: ${{ github.ref == 'refs/heads/master' }}
# These images are used for deployments and caching
- name: Push images if master
run: skaffold build -p push-images --filename ./skaffold.production.yaml
if: ${{ github.ref == 'refs/heads/master' }}
- name: Push latest images if master
run: skaffold build -p push-images,latest-tag --filename ./skaffold.production.yaml
if: ${{ github.ref == 'refs/heads/master' }}
- name: Push images tagged with sha if master
run: |
# shellcheck disable=SC2016
jq --raw-output '.builds[].tag' < skaffold-build-output.json | xargs -d $'\n' sh -c 'for arg do echo "> docker push $arg"; docker push "$arg"; done'
if: ${{ github.ref == 'refs/heads/master' }}
system-tests:
needs: build
runs-on: ubuntu-latest
strategy:
matrix:
shard: [1, 2]
steps:
- name: Free up disk space
run: |
echo "Space before:"
df -h
sudo rm -rf /usr/share/dotnet
sudo rm -rf /opt/ghc
sudo rm -rf "/usr/local/share/boost"
sudo rm -rf /usr/local/lib/android
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
echo "Space after:"
df -h
- name: Check out repository code
uses: actions/checkout@v4
# This would cause all other tests to be skipped
- name: Make sure we have no focused system-tests
run: |
test "$(grep -r 'test.only' ./system-tests/src/ | wc -l)" -eq 0
- name: Install kustomize, kubectl & skaffold
uses: yokawasa/[email protected]
with:
kubectl: "1.30.3"
kustomize: "5.4.1"
skaffold: "2.13.1"
- name: Workaround for kube-proxy set nf_conntrack_max permission denied
run: |
# Workarounds this issue when starting up kube-system/kube-proxy:
# I0719 17:18:25.711233 1 conntrack.go:100] Set sysctl 'net/netfilter/nf_conntrack_max' to 131072
# F0719 17:18:25.711251 1 server.go:489] open /proc/sys/net/netfilter/nf_conntrack_max: permission denied
sudo sysctl net/netfilter/nf_conntrack_max=131072
- name: Install, start & configure minikube
uses: manusa/[email protected]
with:
minikube version: "v1.33.1"
kubernetes version: "v1.30.0"
start args: "--addons ingress --memory 10g"
driver: "docker"
github token: ${{ secrets.GITHUB_TOKEN }}
- name: Enable snippet directives
run: |
kubectl patch configmap ingress-nginx-controller -p '{"data":{"allow-snippet-annotations":"true"}}' -n ingress-nginx --type merge
- name: Debug kube-system
run: |
kubectl get pods --namespace kube-system
PODS=$(kubectl get pods --namespace kube-system -o name | cut -d '/' -f 2)
for pod in $PODS
do
echo "------------------"
kubectl describe pod "$pod" --namespace kube-system
done
- name: Debug ingress
run: |
kubectl get pods --namespace ingress-nginx
PODS=$(kubectl get pods --namespace ingress-nginx -o name | cut -d '/' -f 2)
for pod in $PODS
do
echo "------------------"
kubectl describe pod "$pod" --namespace ingress-nginx
done
- name: Run npm ci for shared-modules
run: npm ci
- name: Download skaffold build output artifact
uses: actions/download-artifact@v4
with:
name: skaffold-build-output
path: skaffold-build-output
- name: Download built images
uses: actions/download-artifact@v4
with:
name: skaffold-images
path: skaffold-images
- name: Import downloaded images to minikube
run: bin/skaffold-import-images-to-minikube
- name: Deploy to local minikube
run: skaffold deploy --force --filename ./skaffold.production.yaml --build-artifacts=./skaffold-build-output/skaffold-build-output.json
- name: Configure ingress
run: echo "$(minikube ip) project-331.local" | sudo tee --append /etc/hosts
- name: Setup system test environment
run: npm ci
working-directory: ./system-tests/
- name: Run system tests
run: npm run test -- -- --shard="${{ matrix.shard }}/2"
working-directory: ./system-tests/
# To prevent accidentally introducing flaky tests in a pr. Skipped on master in order to not slow down deploys.
- name: Run system tests again to make sure they're stable (if not master)
run: npm run test -- -- --shard="${{ matrix.shard }}/2"
working-directory: ./system-tests/
if: ${{ github.ref != 'refs/heads/master' }}
- name: Headless lms logs
run: kubectl logs 'deployment/headless-lms'
if: failure()
- uses: actions/upload-artifact@v4
if: failure()
with:
name: test-results-${{ matrix.shard }}
path: ./system-tests/test-results/
if-no-files-found: ignore
- uses: actions/upload-artifact@v4
if: failure()
with:
name: playwright-html-report-${{ matrix.shard }}
path: ./system-tests/playwright-report/index.html
if-no-files-found: ignore
deploy:
needs: system-tests
if: ${{ github.ref == 'refs/heads/master' }}
runs-on: ubuntu-latest
environment:
name: production
url: https://courses.mooc.fi
steps:
- name: Check out repository code
uses: actions/checkout@v4
- name: Download skaffold build output artifact
uses: actions/download-artifact@v4
with:
name: skaffold-build-output
path: skaffold-build-output
- name: Install kustomize, kubectl & skaffold
uses: yokawasa/[email protected]
with:
kubectl: "1.30.3"
kustomize: "5.4.1"
skaffold: "2.13.1"
- uses: google-github-actions/setup-gcloud@94337306dda8180d967a56932ceb4ddcf01edae7
with:
service_account_key: ${{ secrets.GCLOUD_SERVICE_KEY_DEPLOY_TO_PRODUCTION }}
project_id: ${{ secrets.GKE_PROJECT }}
- run: |-
gcloud --quiet auth configure-docker
# Get the GKE credentials so we can deploy to the cluster
- uses: google-github-actions/get-gke-credentials@fb08709ba27618c31c09e014e1d8364b02e5042e
with:
cluster_name: ${{ secrets.GKE_CLUSTER }}
location: ${{ secrets.GKE_ZONE }}
credentials: ${{ secrets.GCLOUD_SERVICE_KEY_DEPLOY_TO_PRODUCTION }}
- name: Masking
run: |
IPS=$(kubectl config view | grep server | tr -s ' ' | cut -d ' ' -f 3 | xargs echo)
echo "::add-mask::$IPS"
- name: Deploy with Skaffold
run: skaffold deploy --force --filename ./skaffold.production.yaml -p production --build-artifacts=./skaffold-build-output/skaffold-build-output.json --namespace courses-moocfi
typecheck:
runs-on: ubuntu-latest
steps:
- name: Check out repository code
uses: actions/checkout@v4
- name: Run npm ci in repo root
run: npm ci
- name: Run npm ci system-tests
run: npm ci
working-directory: ./system-tests/
- name: Run tsc
run: npx tsc --noEmit
working-directory: ./system-tests/