forked from Azure/azure-service-operator
-
Notifications
You must be signed in to change notification settings - Fork 0
/
azure-pipelines.yml
346 lines (311 loc) · 14.6 KB
/
azure-pipelines.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
# By default, ADO runs jobs on pushes and on PRs. Since we're only tracking/after PRs, turn off
# tiggers on push.
trigger:
branches:
exclude:
- '*'
# PR Section describes what happens on PR
pr:
branches:
include:
- master
resources:
- repo: self
pool:
vmImage: 'ubuntu-latest'
variables:
tag: '$(Build.BuildId)'
MAJOR_VERSION: 0
MINOR_VERSION: 1
PATCH_VERSION: $(Build.BuildId)
IMAGE_NAME: 'candidate/k8s/azureserviceoperator'
ASO_HELM_CHART_NAME: 'candidate/k8s/asohelmchart'
KUBEBUILDER_ASSETS: $(go env GOPATH)/bin
# Set these values in Azure Devops variables
# AKS_CLUSTER_RG: Name of Resource group where AKS clusters used for testing are present
# OPERATOR_NAMESPACE: Namespace in AKS cluster that operator is deployed to (Default: azureoperator-system)
# ASO-DEVOPS-MI: Name of Managed Identity to use to deploy the Operator as in AKS
# POD-IDENTITY-CLIENTID: ClientID of Managed Identity to use to deploy the Operator as in AKS
# AZURE_OPERATOR_KEYVAULT: Keyvault passed to Operator deployment to use for secrets
# AZURE_CLIENT_ID: Client ID of Service Principal used to run integration tests on Kind Cluster, login to Helm registry ACR
# AZURE_CLIENT_SECRET: Secret for the above Client ID
# AZURE_SUBSCRIPTION_ID: Subscription ID that the pipeline runs against
# AZURE_TENANT_ID: Tenant ID that the pipeline runs against
# Pipeline ACR - Temporary ACR used to hold the image until AKS deploy testing is done.
# Release ACR - Only after the deploy test succeeds, the image and helm chart are pushed to the release ACR.
# The release pipeline exposes the images in Release ACR through mcr.microsoft.com
# CONTAINER_REGISTRY: Service Connection Name for Release ACR.
# CONTAINER_REGISTRY_NAME: Name of the Release ACR.
# PIPELINE_CONTAINER_REGISTRY: Service Connection Name for temporary ACR used in pipeline.
# PIPELINE_CONTAINER_REGISTRY_NAME: Name of the temporary ACR used in the pipeline.
# TEST_APIM_NAME: Used in integration tests - Name of the APIM service that is pre-configured.
# TEST_APIM_RG: Used in integration tests - Name of the resource group the pre-configured APIM service is in.
# Agent VM image name
vmImageName: 'ubuntu-latest'
steps:
- checkout: self # self represents the repo where the initial Pipelines YAML file was found
lfs: "true"
- task: Bash@3
name: "check_changes"
inputs:
targetType: 'filePath'
filePath: './scripts/check-changes.sh'
- script: |
make validate-copyright-headers
make validate-cainjection-files
condition: eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true')
displayName: 'Validate Source'
continueOnError: 'false'
- task: GoTool@0
displayName: Get Go 1.13.7
condition: eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true')
inputs:
version: '1.13.7'
- task: Go@0
displayName: Get Go dependencies
condition: eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true')
inputs:
command: 'get'
arguments: '-d'
workingDirectory: '$(System.DefaultWorkingDirectory)'
- task: Bash@3
displayName: Install kubebuilder, kustomize and test dependencies
condition: eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true')
inputs:
targetType: 'inline'
script: |
set -x
mkdir -p $(System.DefaultWorkingDirectory)/bin
go env
os=$(go env GOOS)
arch=$(go env GOARCH)
curl -sL https://go.kubebuilder.io/dl/2.3.1/${os}/${arch} | tar -xz -C $(go env GOPATH)/bin
export PATH=$PATH:$(go env path)/bin
echo '##vso[task.prependpath]$(go env path)/bin'
# download kustomize
curl -o $(System.DefaultWorkingDirectory)/bin/kustomize -sL https://go.kubebuilder.io/kustomize/${os}/${arch}
# set permission
chmod a+x $(System.DefaultWorkingDirectory)/bin/*
chmod a+x $(System.DefaultWorkingDirectory)/bin/kustomize
export PATH=$PATH:$(System.DefaultWorkingDirectory)/bin
echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/bin'
which kustomize
kustomize version
cp $(System.DefaultWorkingDirectory)/bin/kustomize $(System.DefaultWorkingDirectory)/kustomize
go mod download
make install-test-dependencies
workingDirectory: '$(System.DefaultWorkingDirectory)'
- script: |
set -e
export PATH=$PATH:$(go env GOPATH)/bin
GO111MODULE="on" go get sigs.k8s.io/[email protected]
kind create cluster
export KUBECONFIG=$(kind get kubeconfig-path --name="kind")
kubectl cluster-info
make install
export TEST_APIM_RG=$(TEST_APIM_RG)
export TEST_APIM_NAME=$(TEST_APIM_NAME)
make test-integration-controllers
displayName: Run tests on a Kind Cluster
condition: eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true')
continueOnError: 'false'
env:
GO111MODULE: on
AZURE_SUBSCRIPTION_ID: $(AZURE_SUBSCRIPTION_ID)
AZURE_TENANT_ID: $(AZURE_TENANT_ID)
AZURE_CLIENT_ID: $(AZURE_CLIENT_ID)
AZURE_CLIENT_SECRET: $(AZURE_CLIENT_SECRET)
REQUEUE_AFTER: $(REQUEUE_AFTER)
KUBEBUILDER_ASSETS: $(go env GOPATH)/bin
BUILD_ID: $(Build.BuildId)
workingDirectory: '$(System.DefaultWorkingDirectory)'
- bash: |
export PATH=$PATH:$(go env GOPATH)/bin
make test-process-coverage
displayName: Render Coverage Reports
condition: eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true')
continueOnError: true
workingDirectory: '$(System.DefaultWorkingDirectory)'
- task: PublishCodeCoverageResults@1
displayName: Publish Codecoverage results
condition: eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true')
inputs:
codeCoverageTool: Cobertura
summaryFileLocation: $(System.DefaultWorkingDirectory)/reports/merged-coverage.xml
- task: PublishBuildArtifacts@1
displayName: Publish test reports
condition: eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true')
inputs:
pathtoPublish: '$(System.DefaultWorkingDirectory)/reports'
artifactName: reports
- task: AzureCLI@2
displayName: Test - Cleanup Azure Resources
condition: always()
inputs:
azureSubscription: 'ASO Subscription'
scriptType: 'bash'
scriptLocation: 'inlineScript'
inlineScript: 'make test-cleanup-azure-resources'
- script: docker build -t $(IMAGE_NAME) .
displayName: Docker build
condition: eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true')
workingDirectory: '$(System.DefaultWorkingDirectory)'
- task: Docker@2
displayName: Login to temporary pipeline ACR
condition: eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true')
inputs:
containerRegistry: $(PIPELINE_CONTAINER_REGISTRY)
command: 'login'
- task: Docker@2
displayName: Build and Push Docker Image to temporary ACR for validation
condition: eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true')
inputs:
containerRegistry: $(PIPELINE_CONTAINER_REGISTRY)
repository: '$(IMAGE_NAME)'
command: 'buildAndPush'
Dockerfile: '$(System.DefaultWorkingDirectory)/Dockerfile'
buildContext: '$(System.DefaultWorkingDirectory)'
tags: '$(MAJOR_VERSION).$(MINOR_VERSION).$(PATCH_VERSION)'
- task: AzureCLI@2
inputs:
azureSubscription: 'ASO Subscription'
scriptType: 'bash'
scriptLocation: 'inlineScript'
inlineScript: |
az resource list --tag freeforpipeline=true --query '[].{Name:name}' -o table > clusters.txt
clustername=$(tail -n 1 clusters.txt)
if [ "$clustername" == "" ]; then
echo AKS clusters unavailable
exit 1
fi
echo $clustername
echo "##vso[task.setvariable variable=chosenclustername]$clustername"
echo 'az aks get-credentials -g $(AKS_CLUSTER_RG) -n $clustername'
az aks get-credentials -g $(AKS_CLUSTER_RG) -n $clustername
# Set tags to not available for the selected cluster so it doesn't get used in another run
az resource tag --tags 'freeforpipeline=false' -g $(AKS_CLUSTER_RG) -n $clustername --resource-type Microsoft.ContainerService/managedClusters
workingDirectory: '$(System.DefaultWorkingDirectory)'
failOnStandardError: true
displayName: Deploy to AKS - Find available AKS cluster and connect to it
condition: eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true')
- script: |
kubectl create namespace cert-manager
kubectl label namespace cert-manager cert-manager.io/disable-validation=true
kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/v0.12.0/cert-manager.yaml
displayName: Deploy to AKS - Install Cert Manager
condition: eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true')
- task: HelmInstaller@1
displayName: Deploy to AKS - Install latest Helm
condition: eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true')
inputs:
helmVersionToInstall: 'latest'
- script: |
# Replace image repository in values.yaml to the official image in ACR
img="$MAJOR_VERSION.$MINOR_VERSION.$PATCH_VERSION"
echo $img
sed -i -e 's@azureserviceoperator:latest@azureserviceoperator:'${img}'@' charts/azure-service-operator/values.yaml
displayName: Deploy to AKS - Replace image in values.yaml
condition: eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true')
- task: Bash@3
displayName: Deploy to AKS - Helm Deploy
condition: eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true')
inputs:
targetType: 'inline'
script: |
kubectl delete namespace $(OPERATOR_NAMESPACE)
imagename="$(PIPELINE_CONTAINER_REGISTRY_NAME)/$(IMAGE_NAME):$(MAJOR_VERSION).$(MINOR_VERSION).$(PATCH_VERSION)"
echo $imagename
# Apply CRDs as Helm 3 does not install CRDs for Helm upgrade
kubectl apply -f ./charts/azure-service-operator/crds/
# Deploy using Helm
helm upgrade --install aso charts/azure-service-operator-0.1.0.tgz -n $(OPERATOR_NAMESPACE) --create-namespace \
--set azureSubscriptionID=$(AZURE_SUBSCRIPTION_ID) \
--set azureTenantID=$(AZURE_TENANT_ID) \
--set azureClientID=$(AZURE_CLIENT_ID) \
--set azureClientSecret=$(AZURE_CLIENT_SECRET) \
--set azureUseMI=true \
--set installAadPodIdentity=true \
--set azureOperatorKeyvault=$(AZURE_OPERATOR_KEYVAULT) \
--set aad-pod-identity.azureIdentity.resourceID="/subscriptions/$(AZURE_SUBSCRIPTION_ID)/resourcegroups/resourcegroup-azure-operators/providers/Microsoft.ManagedIdentity/userAssignedIdentities/$(ASO-DEVOPS-MI)" \
--set aad-pod-identity.azureIdentity.clientID=$(POD-IDENTITY-CLIENTID) \
--set image.repository=$imagename
# Verify namespace and pods
kubectl get namespace
kubectl get pods -n $(OPERATOR_NAMESPACE)
kubectl describe pods -n $(OPERATOR_NAMESPACE)
kubectl rollout status deployment/azureoperator-controller-manager -n $(OPERATOR_NAMESPACE) --timeout 120s
kubectl logs -n $(OPERATOR_NAMESPACE) deployment/azureoperator-controller-manager -c manager
- task: AzureCLI@2
inputs:
azureSubscription: 'ASO Subscription'
scriptType: 'bash'
scriptLocation: 'inlineScript'
inlineScript: |
echo "Chosen AKS Cluster name"
echo $(chosenclustername)
# Delete CRDs to clean up cluster
kubectl delete -f ./charts/azure-service-operator/crds/
# Remove Helm deployment
helm list -n $(OPERATOR_NAMESPACE)
helm delete aso -n $(OPERATOR_NAMESPACE)
# Set tags to available for the selected cluster to put it back into the free pool
echo "Setting tags back to free"
az resource tag --tags 'freeforpipeline=true' -g $(AKS_CLUSTER_RG) -n $(chosenclustername) --resource-type Microsoft.ContainerService/managedClusters
workingDirectory: '$(System.DefaultWorkingDirectory)'
failOnStandardError: true
displayName: Deploy to AKS - Clean up deployment and release cluster back to free pool
condition: eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true')
- task: Docker@2
condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
displayName: Login to release ACR
inputs:
containerRegistry: $(CONTAINER_REGISTRY)
command: 'login'
- task: Docker@2
condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
displayName: Build and Push Docker Image $(IMAGE_NAME)
inputs:
containerRegistry: $(CONTAINER_REGISTRY)
repository: '$(IMAGE_NAME)'
command: 'buildAndPush'
Dockerfile: '$(System.DefaultWorkingDirectory)/Dockerfile'
buildContext: '$(System.DefaultWorkingDirectory)'
tags: |
latest
$(MAJOR_VERSION).$(MINOR_VERSION).$(PATCH_VERSION)
- script: |
export HELM_EXPERIMENTAL_OCI=1
cd charts/azure-service-operator
versiontag="$(CONTAINER_REGISTRY_NAME)/$(ASO_HELM_CHART_NAME):$(MAJOR_VERSION).$(MINOR_VERSION).$(PATCH_VERSION)"
latesttag="$(CONTAINER_REGISTRY_NAME)/$(ASO_HELM_CHART_NAME):latest"
echo $versiontag
echo $latesttag
echo $(AZURE_CLIENT_SECRET)|helm registry login $(CONTAINER_REGISTRY_NAME) --username $(AZURE_CLIENT_ID) --password-stdin
helm chart save . $versiontag
helm chart save . $latesttag
helm chart push $versiontag
helm chart push $latesttag
continueOnError: 'true'
displayName: Push Helm chart to MCR repo
workingDirectory: '$(System.DefaultWorkingDirectory)'
condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
- script: |
echo "updating kustomize image patch file for manager resource"
echo $IMAGE_NAME
img="$IMAGE_NAME:$MAJOR_VERSION.$MINOR_VERSION.$PATCH_VERSION"
echo $img
sed -i -e 's@controller:latest@'${img}'@g' ./config/default/manager_image_patch.yaml
which kustomize
kustomize build config/default > $(Build.ArtifactStagingDirectory)/setup.yaml
set -x
echo $img > $(Build.ArtifactStagingDirectory)/azure-service-operator.txt
cp charts/azure-service-operator-0.1.0.tgz $(Build.ArtifactStagingDirectory)
continueOnError: 'false'
displayName: Package artifacts (image, setup.yaml and helm charts) for release
workingDirectory: '$(System.DefaultWorkingDirectory)'
condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
- task: PublishBuildArtifacts@1
condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
inputs:
pathtoPublish: $(Build.ArtifactStagingDirectory)
artifactName: drop