-
Notifications
You must be signed in to change notification settings - Fork 16
243 lines (213 loc) · 11.2 KB
/
pull_request.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
name: Pull Request Workflow
on: [pull_request]
# So that only one PR workflow will be run simultaneously. This ensures that there will not be multiple runners accessing
# the same Terraform state at the same time.
concurrency: pr-workflow-${{ github.event.pull_request.number }}
env:
GH_TOKEN: ${{ github.token }}
PRE_RELEASE_NAME: tmp-pr-${{ github.event.pull_request.number }}
# Used to re-sign Linux artifacts
GPG_MAIL: ${{ secrets.LOGGING_GPG_MAIL }}
GPG_PASSPHRASE: ${{ secrets.OHAI_GPG_PASSPHRASE }}
GPG_PRIVATE_KEY_BASE64: ${{ secrets.OHAI_GPG_PRIVATE_KEY_BASE64 }} # base64 encoded
jobs:
# Empties the GH pre-relase
# Generates strategy matrices that can be used by other jobs to run the build or testing of all supported packages
setup_environment:
runs-on: ubuntu-latest
outputs:
pre_release_name: ${{ steps.set_vars.outputs.pre_release_name }}
sles_matrix: ${{ steps.set-matrices.outputs.sles_matrix }}
linux_and_windows_matrix: ${{ steps.set-matrices.outputs.linux_and_windows_matrix }}
steps:
- name: Checkout code
uses: actions/checkout@v3
# This is a workaround required to pass environment variables to the run_e2e_tests workflow
# See: https://stackoverflow.com/a/73305536 and https://github.com/orgs/community/discussions/26671#discussioncomment-3252793
- name: Make dynamically-generated env variables available to be passed down to reusable workflows
id: set_vars
run: |
echo "pre_release_name=$PRE_RELEASE_NAME" >> $GITHUB_OUTPUT
- name: (Re)create pre-release
run: |
pre_release_exists=$(gh release view $PRE_RELEASE_NAME &>/dev/null && echo "true" || echo "false")
if [[ $pre_release_exists == "true" ]]; then
echo "Deleting existing pre-release"
gh release delete ${{ env.PRE_RELEASE_NAME }} -y --cleanup-tag
fi
pre_release_tag=$PRE_RELEASE_NAME
pre_release_title="Temporary release to build and test artifacts from PR#${{ github.event.pull_request.number }}"
pre_release_notes="Created by PR: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/pull/${{ github.event.pull_request.number }}"
# Releases created from a runner are always DRAFT
echo "Creating release: $pre_release_tag"
gh release create "$pre_release_tag" --title "$pre_release_title" --notes "$pre_release_notes"
# We need the pre-release to NOT be a draft, otherwise it won't be visible to download packages from the Ansible-managed hosts
echo "Updating release to be a pre-release"
gh release edit "$pre_release_tag" --draft=false --prerelease
- name: Install python
uses: actions/setup-python@v4
with:
python-version: '3.10'
- name: Compute and upload matrices
id: set-matrices
run: |
make versions/generateMatrices
echo "linux_and_windows_matrix=$( cat versions/linuxAndWindowsMatrix.json )" >> "$GITHUB_OUTPUT"
echo "sles_matrix=$( cat versions/slesMatrix.json )" >> "$GITHUB_OUTPUT"
gh release upload ${{ env.PRE_RELEASE_NAME }} versions/linuxAndWindowsMatrix.json --repo newrelic/fluent-bit-package
gh release upload ${{ env.PRE_RELEASE_NAME }} versions/slesMatrix.json --repo newrelic/fluent-bit-package
- name: Compute and upload schemas
run: |
make schemas/generateSchemas;
gh release upload ${{ env.PRE_RELEASE_NAME }} schemas/generated-linux-schema-staging.yaml --repo newrelic/fluent-bit-package
gh release upload ${{ env.PRE_RELEASE_NAME }} schemas/generated-linux-schema-production.yaml --repo newrelic/fluent-bit-package
# Downloads all Fluent Bit packages that are officially supported, preferably from the New Relic Infrastructure Agent
# repository (Linux packages, already re-signed by NR) or Logging's S3 bucket (Windows packages, already packaged for the NRIA).
# If these are not available, they are downloaded from the official Fluent Bit repository and repackaged to be used by the NRIA.
download_official_packages:
needs: [setup_environment]
runs-on: ubuntu-latest
if: ${{ needs.setup_environment.outputs.linux_and_windows_matrix != '[]' }}
strategy:
fail-fast: true
matrix:
include: ${{ fromJson(needs.setup_environment.outputs.linux_and_windows_matrix) }}
name: ${{ matrix.osDistro }}-${{ matrix.osVersion }}-${{ matrix.arch }}
steps:
- name: Check out repository
uses: actions/checkout@v3
- name: Attempt downloading package from New Relic repository
id: download_package_from_nr
run: |
mkdir -p packages
if wget --directory-prefix=packages "${{ matrix.nrPackageUrl }}"; then
echo "result=success" >> "$GITHUB_OUTPUT"
else
echo "result=failure" >> "$GITHUB_OUTPUT"
fi
- name: Download, rename and resign Linux package
if: ${{ steps.download_package_from_nr.outputs.result == 'failure' && matrix.osDistro != 'windows-server' }}
run: |
curl ${{ matrix.packageUrl }} -o packages/${{ matrix.targetPackageName }}
sudo apt-get install -y debsigs
bash ./scripts/sign.sh
- name: Download and re-zip Windows package
if: ${{ steps.download_package_from_nr.outputs.result == 'failure' && matrix.osDistro == 'windows-server' }}
run: |
wget ${{ matrix.packageUrl }}
unzip fluent-bit-${{ matrix.fbVersion }}-${{ matrix.arch }}.zip
zip -r -j packages/${{ matrix.targetPackageName }} \
fluent-bit-${{ matrix.fbVersion }}-${{ matrix.arch }}/bin/fluent-bit.exe \
fluent-bit-${{ matrix.fbVersion }}-${{ matrix.arch }}/bin/fluent-bit.dll
# gh release upload can have issues if multiple jobs attempt uploading the same file concurrently (it can happen
# for those distros using the same package, such as Windows). To avoid this, we first push all the files to a
# shared filesystem and let the "prepare_prerelease" step below upload them later, sequentially. This GH action
# ensures that if two jobs attempt pushing the same file, they get overwritten (last one prevails).
- uses: actions/upload-artifact@v3
with:
# Artifacts are pushed to *shared network folders* that have this name and that contain
# the artifact inside of them. Example: fluent-bit-2.1.8-386.exe/fluent-bit-2.1.8-386.exe
name: ${{ matrix.targetPackageName }}
path: packages/${{ matrix.targetPackageName }}
upload_official_packages_to_prerelease:
needs: [ download_official_packages ]
runs-on: ubuntu-latest
steps:
- name: Download all artifacts from shared filesystem
uses: actions/download-artifact@v3
with:
path: packages
- name: Push all artifacts to pre-release
run:
# To understand the need for /*/*, see comment in "upload artifacts" step above
gh release upload ${{ env.PRE_RELEASE_NAME }} packages/*/* --repo newrelic/fluent-bit-package
spin_up_suse:
needs: setup_environment
if: ${{ needs.setup_environment.outputs.sles_matrix != '[]' }}
uses: ./.github/workflows/run_task.yml
with:
container_make_target: "terraform/ec2-suse-builders/provision PRE_RELEASE_NAME=${{ needs.setup_environment.outputs.pre_release_name }}"
secrets: inherit
build_suse_packages:
needs: [setup_environment,spin_up_suse]
uses: ./.github/workflows/run_task.yml
with:
container_make_target: "ansible/build-fb-suse/run PRE_RELEASE_NAME=${{ needs.setup_environment.outputs.pre_release_name }}"
secrets: inherit
tear_down_suse:
needs: [setup_environment,build_suse_packages]
uses: ./.github/workflows/run_task.yml
with:
container_make_target: "terraform/ec2-suse-builders/clean PRE_RELEASE_NAME=${{ needs.setup_environment.outputs.pre_release_name }}"
secrets: inherit
sign_suse_packages:
needs: [setup_environment, build_suse_packages]
runs-on: ubuntu-latest
if: ${{ needs.setup_environment.outputs.sles_matrix != '[]' }}
strategy:
fail-fast: true
matrix:
include: ${{ fromJson(needs.setup_environment.outputs.sles_matrix) }}
name: ${{ matrix.osDistro }}-${{ matrix.osVersion }}-${{ matrix.arch }}
steps:
- name: Check out repository
uses: actions/checkout@v3
- name: Fetch package from GitHub pre-release
run: |
mkdir -p packages
gh release download ${{ env.PRE_RELEASE_NAME }} --pattern ${{ matrix.targetPackageName }} --dir packages
- name: Sign package
run: |
sudo apt-get install -y debsigs
bash ./scripts/sign.sh
- name: Upload signed asset
run:
gh release upload ${{ env.PRE_RELEASE_NAME }} packages/* --clobber
# Runs E2E tests using the packages available in the tmp-pr-#PR prerelease
run_e2e_tests_prerelease:
needs: [ setup_environment, sign_suse_packages, upload_official_packages_to_prerelease ]
name: Run E2E tests by installing NRIA from Production and installing Fluent Bit from the PR pre-release
uses: ./.github/workflows/run_e2e_tests.yml
with:
gh_release_name: ${{ needs.setup_environment.outputs.pre_release_name }}
infra_agent_version: latest
infra_agent_env: prerelease
test_report_filename: test-report-prerelease.xml
secrets: inherit
publish_linux_to_staging:
name: Publish linux packages to staging
needs: [ setup_environment, run_e2e_tests_prerelease ]
runs-on: ubuntu-latest
steps:
- name: Publish linux packages to staging
uses: newrelic/infrastructure-publish-action@v1
with:
app_name: fluent-bit
tag: ${{ env.PRE_RELEASE_NAME }}
repo_name: "newrelic/fluent-bit-package"
schema: "custom"
schema_url: "https://github.com/newrelic/fluent-bit-package/releases/download/${{ env.PRE_RELEASE_NAME }}/generated-linux-schema-staging.yaml"
aws_access_key_id: ${{ secrets.OHAI_AWS_ACCESS_KEY_ID_STAGING }}
aws_secret_access_key: ${{ secrets.OHAI_AWS_SECRET_ACCESS_KEY_STAGING }}
aws_s3_bucket_name: "nr-downloads-ohai-staging"
aws_s3_lock_bucket_name: "onhost-ci-lock-staging"
access_point_host: "staging"
run_id: ${{ github.run_id }}
aws_region: "us-east-1"
aws_role_session_name: ${{ secrets.OHAI_AWS_ROLE_SESSION_NAME_STAGING }}
aws_role_arn: ${{ secrets.OHAI_AWS_ROLE_ARN_STAGING }}
# used for signing package stuff
gpg_passphrase: ${{ secrets.OHAI_GPG_PASSPHRASE }}
gpg_private_key_base64: ${{ secrets.OHAI_GPG_PRIVATE_KEY_BASE64 }} # base64 encoded
run_e2e_tests_staging:
needs: [ setup_environment, run_e2e_tests_prerelease, publish_linux_to_staging ]
if: |
always() && !failure() && !cancelled()
name: Run E2E tests for all supported packages installing NRIA+FB from staging
uses: ./.github/workflows/run_e2e_tests.yml
with:
gh_release_name: ${{ needs.setup_environment.outputs.pre_release_name }}
infra_agent_version: latest
infra_agent_env: staging
test_report_filename: test-report-staging.xml
secrets: inherit