forked from DataDog/datadog-agent
-
Notifications
You must be signed in to change notification settings - Fork 0
/
.gitlab-ci.yml
250 lines (223 loc) · 9.91 KB
/
.gitlab-ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
---
include:
- /.gitlab/maintenance_jobs.yml
- /.gitlab/deps_build.yml
- /.gitlab/deps_fetch.yml
- /.gitlab/source_test.yml
- /.gitlab/binary_build.yml
- /.gitlab/integration_test.yml
- /.gitlab/package_build.yml
- /.gitlab/internal_deploy.yml
- /.gitlab/kitchen_deploy.yml
- /.gitlab/kitchen_testing.yml
- /.gitlab/pkg_metrics.yml
- /.gitlab/image_build.yml
- /.gitlab/image_scan.yml
- /.gitlab/check_deploy.yml
- /.gitlab/image_deploy.yml
- /.gitlab/deploy_6.yml
- /.gitlab/deploy_7.yml
- /.gitlab/choco_build.yml
- /.gitlab/choco_deploy.yml
- /.gitlab/deploy_invalidate.yml
- /.gitlab/internal_image_deploy.yml
- /.gitlab/trigger_release.yml
- /.gitlab/e2e.yml
- /.gitlab/kitchen_cleanup.yml
- /.gitlab/functional_test.yml
- /.gitlab/functional_test_cleanup.yml
- /.gitlab/notify.yml
# FIXME: our current Gitlab version doesn't support importing a file more than once
# For now, the workaround is to include "common" files once in the top-level .gitlab-ci.yml file
# See: https://gitlab.com/gitlab-org/gitlab/-/issues/28987
- /.gitlab/kitchen_common/cleanup.yml
- /.gitlab/kitchen_common/testing.yml
- /.gitlab/docker_common/tag_job_templates.yml
default:
retry:
max: 2
when:
- runner_system_failure
- stuck_or_timeout_failure
- unknown_failure
- api_failure
workflow:
rules:
- if: $CI_COMMIT_TAG =~ /^dca-/ || $CI_COMMIT_TAG == null || $DEPLOY_AGENT == "true"
stages:
- maintenance_jobs
- deps_build
- deps_fetch
- source_test
- binary_build
- integration_test
- package_build
- internal_deploy
- kitchen_deploy
- kitchen_testing
- pkg_metrics
- image_build
- image_scan
- check_deploy
- image_deploy
- deploy6
- deploy7
- choco_build
- choco_deploy
- deploy_invalidate
- internal_image_deploy
- trigger_release
- e2e
- kitchen_cleanup
- functional_test
- functional_test_cleanup
- notify
variables:
# The SRC_PATH is in the GOPATH of the builders which
# currently is /go
SRC_PATH: /go/src/github.com/DataDog/datadog-agent
# Directory in which we execute the omnibus build.
# For an unknown reason, it does not go well with
# a ruby dependency if we build directly into $CI_PROJECT_DIR/.omnibus
OMNIBUS_BASE_DIR: /omnibus
# Directory in which we put the artifacts after the build
# Must be in $CI_PROJECT_DIR
OMNIBUS_PACKAGE_DIR: $CI_PROJECT_DIR/omnibus/pkg/
# Directory in which we put the SUSE artifacts after the SUSE build
# Must be in $CI_PROJECT_DIR
# RPM builds and SUSE RPM builds create artifacts with the same name.
# To differentiate them, we put them in different folders. That also
# avoids accidentally overwriting files when downloading artifacts from
# both RPM and SUSE rpm jobs.
OMNIBUS_PACKAGE_DIR_SUSE: $CI_PROJECT_DIR/omnibus/suse/pkg
DD_AGENT_TESTING_DIR: $CI_PROJECT_DIR/test/kitchen
STATIC_BINARIES_DIR: bin/static
DOGSTATSD_BINARIES_DIR: bin/dogstatsd
AGENT_BINARIES_DIR: bin/agent
CLUSTER_AGENT_BINARIES_DIR: bin/datadog-cluster-agent
CLUSTER_AGENT_CLOUDFOUNDRY_BINARIES_DIR: bin/datadog-cluster-agent-cloudfoundry
SYSTEM_PROBE_BINARIES_DIR: bin/system-probe
DEB_S3_BUCKET: apt.datad0g.com
RPM_S3_BUCKET: yum.datad0g.com
MACOS_S3_BUCKET: dd-agent-macostesting
WIN_S3_BUCKET: dd-agent-mstesting
PROCESS_S3_BUCKET: datad0g-process-agent
ANDROID_S3_BUCKET: dd-agent-androidtesting
DEB_RPM_BUCKET_BRANCH: nightly # branch of the DEB_S3_BUCKET and RPM_S3_BUCKET repos to release to, 'nightly' or 'beta'
DEB_TESTING_S3_BUCKET: apttesting.datad0g.com
RPM_TESTING_S3_BUCKET: yumtesting.datad0g.com
WINDOWS_TESTING_S3_BUCKET_A6: pipelines/A6/$CI_PIPELINE_ID
WINDOWS_TESTING_S3_BUCKET_A7: pipelines/A7/$CI_PIPELINE_ID
WINDOWS_BUILDS_S3_BUCKET: $WIN_S3_BUCKET/builds
ANDROID_BUILDS_S3_BUCKET: $ANDROID_S3_BUCKET/builds
DEB_RPM_TESTING_BUCKET_BRANCH: testing # branch of the DEB_TESTING_S3_BUCKET and RPM_TESTING_S3_BUCKET repos to release to, 'testing'
DD_REPO_BRANCH_NAME: $CI_COMMIT_REF_NAME
S3_CP_OPTIONS: --only-show-errors --region us-east-1 --sse AES256
S3_CP_CMD: aws s3 cp $S3_CP_OPTIONS
S3_ARTIFACTS_URI: s3://dd-ci-artefacts-build-stable/$CI_PROJECT_NAME/$CI_PIPELINE_ID
S3_PERMANENT_ARTIFACTS_URI: s3://dd-ci-persistent-artefacts-build-stable/$CI_PROJECT_NAME
## comment out both lines below (S3_OMNIBUS_CACHE_BUCKET and USE_S3_CACHING) to allow
## build to succeed with S3 caching disabled.
S3_OMNIBUS_CACHE_BUCKET: dd-ci-datadog-agent-omnibus-cache-build-stable
USE_S3_CACHING: --omnibus-s3-cache
S3_DSD6_URI: s3://dsd6-staging
RELEASE_VERSION_6: nightly
RELEASE_VERSION_7: nightly-a7
DATADOG_AGENT_BUILDIMAGES: v4452649-b69b5d5
DATADOG_AGENT_BUILDERS: v4529511-a0d9303
DATADOG_AGENT_WINBUILDIMAGES: v4319944-68f7e12
DATADOG_AGENT_ARMBUILDIMAGES: v4319944-68f7e12
DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v4319944-68f7e12
BCC_VERSION: v0.19.0
DATADOG_AGENT_EMBEDDED_PATH: /opt/datadog-agent/embedded
DEB_GPG_KEY_ID: 8387EEAF
DEB_GPG_KEY_NAME: "Datadog, Inc <[email protected]>"
DEB_GPG_KEY_SSM_NAME: ci.datadog-agent.deb_signing_private_key_${DEB_GPG_KEY_ID}
DEB_SIGNING_PASSPHRASE_SSM_NAME: ci.datadog-agent.deb_signing_key_passphrase_${DEB_GPG_KEY_ID}
RPM_GPG_KEY_ID: e09422b3
RPM_GPG_KEY_SSM_NAME: ci.datadog-agent.rpm_signing_private_key_e09422b3
RPM_SIGNING_PASSPHRASE_SSM_NAME: ci.datadog-agent.rpm_signing_key_passphrase_e09422b3
# docker.io authentication
DOCKER_REGISTRY_LOGIN_SSM_KEY: docker_hub_login
DOCKER_REGISTRY_PWD_SSM_KEY: docker_hub_pwd
DOCKER_REGISTRY_URL: docker.io
#
# Condition mixins for simplification of rules
#
# FIXME: currently, the only good way we have to compose rules is by using YAML anchors.
# However, as they are YAML features (and not Gitlab features), they have to be present
# in each file that needs them, which results in a lot of copy-pasting.
# In the future, we may be able to leverage YAML functions (https://gitlab.com/groups/gitlab-org/-/epics/4381)
# and cross-file references (https://gitlab.com/gitlab-org/gitlab/-/issues/266173) to remove this
# currently needed copy-paste.
#
# The list of all available rules is given below. To use a rule in a specific file, copy it.
#
.if_master_branch: &if_master_branch
if: $CI_COMMIT_BRANCH == "master"
.if_not_master_branch: &if_not_master_branch
if: $CI_COMMIT_BRANCH != "master"
.if_tagged_commit: &if_tagged_commit
if: $CI_COMMIT_TAG != null
.if_not_tagged_commit: &if_not_tagged_commit
if: $CI_COMMIT_TAG == null
# Rules to run all builds conditionally.
# By default:
# - on master and deploy pipelines, all builds are run
# - on branch pipelines, only a subset of build jobs are run (the ARM and MacOS jobs are not run).
# RUN_ALL_BUILDS can be set to true to force all build jobs to be run on a branch pipeline.
# RUN_ALL_BUILDS has no effect on master/deploy pipelines: they always run all builds (as some jobs
# on master and deploy pipelines depend on jobs that are only run if we run all builds).
.if_run_all_builds: &if_run_all_builds
if: $CI_COMMIT_BRANCH == "master" || $DEPLOY_AGENT == "true" || $RUN_ALL_BUILDS == "true"
.if_not_run_all_builds: &if_not_run_all_builds
if: $CI_COMMIT_BRANCH != "master" && $DEPLOY_AGENT != "true" && $RUN_ALL_BUILDS != "true"
# Run job only on deploy pipelines
.if_deploy: &if_deploy
if: $DEPLOY_AGENT == "true"
.if_deploy_6: &if_deploy_6
if: $DEPLOY_AGENT == "true" && $RELEASE_VERSION_6 != ""
.if_deploy_7: &if_deploy_7
if: $DEPLOY_AGENT == "true" && $RELEASE_VERSION_7 != ""
.if_not_deploy: &if_not_deploy
if: $DEPLOY_AGENT != "true"
# Rules to trigger test kitchen setup, run, and cleanup.
# By default:
# - on master and deploy pipelines, kitchen tests are run
# - on branch pipelines, kitchen tests are not run
# RUN_KITCHEN_TESTS can be set to true to force kitchen tests to be run on a branch pipeline.
# RUN_KITCHEN_TESTS can be set to false to force kithcen tests to not run on master/deploy pipelines.
.if_kitchen: &if_kitchen
if: ($CI_COMMIT_BRANCH == "master" || $DEPLOY_AGENT == "true" || $RUN_KITCHEN_TESTS == "true") && $RUN_KITCHEN_TESTS != "false"
.if_not_kitchen: &if_not_kitchen
if: ($CI_COMMIT_BRANCH != "master" && $DEPLOY_AGENT != "true" && $RUN_KITCHEN_TESTS != "true") || $RUN_KITCHEN_TESTS == "false"
# true only on deploys and when RELEASE_VERSION_X is NOT "nightly". In this setting
# we are building either a new tagged version of the agent (an RC for example).
# In both cases the artifacts should be uploaded to our staging repository.
.if_deploy_on_tag_6: &if_deploy_on_tag_6
# no RELEASE_VERSION means a nightly build for omnibus
if: $DEPLOY_AGENT == "true" && $RELEASE_VERSION_6 != "nightly" && $RELEASE_VERSION_6 != ""
.if_not_deploy_on_tag_6: &if_not_deploy_on_tag_6
# no RELEASE_VERSION means a nightly build for omnibus
if: $DEPLOY_AGENT != "true" || $RELEASE_VERSION_6 == "nightly" || $RELEASE_VERSION_6 == ""
.if_deploy_on_tag_7: &if_deploy_on_tag_7
# no RELEASE_VERSION means a nightly build for omnibus
if: $DEPLOY_AGENT == "true" && $RELEASE_VERSION_7 != "nightly-a7" && $RELEASE_VERSION_7 != ""
.if_not_deploy_on_tag_7: &if_not_deploy_on_tag_7
# no RELEASE_VERSION means a nightly build for omnibus
if: $DEPLOY_AGENT != "true" || $RELEASE_VERSION_7 == "nightly-a7" || $RELEASE_VERSION_7 == ""
# true only on deploys and when RELEASE_VERSION_X is "nightly". In this setting we
# build from master and update the nightly build for windows, linux and docker.
.if_deploy_on_nightly: &if_deploy_on_nightly
if: $DEPLOY_AGENT == "true" && $RELEASE_VERSION_6 == "nightly" && $RELEASE_VERSION_7 == "nightly-a7"
.if_not_deploy_on_nightly: &if_not_deploy_on_nightly
if: $DEPLOY_AGENT != "true" || $RELEASE_VERSION_6 != "nightly" || $RELEASE_VERSION_7 != "nightly-a7"
# true only when RELEASE_VERSION_X is not set
.if_version_6: &if_version_6
if: $RELEASE_VERSION_6 != ""
.if_not_version_6: &if_not_version_6
if: $RELEASE_VERSION_6 == ""
.if_version_7: &if_version_7
if: $RELEASE_VERSION_7 != ""
.if_not_version_7: &if_not_version_7
if: $RELEASE_VERSION_7 == ""