forked from elastic/elasticsearch
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add automation for building/uploading/updating lucene snapshots daily (…
- Loading branch information
1 parent
b155444
commit a7cb51c
Showing
10 changed files
with
353 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| steps: | ||
| - trigger: apache-lucene-build-snapshot | ||
| label: Trigger pipeline to build lucene snapshot | ||
| key: lucene-build | ||
| if: build.env("LUCENE_BUILD_ID") == null || build.env("LUCENE_BUILD_ID") == "" | ||
| - wait | ||
| - label: Upload and update lucene snapshot | ||
| command: .buildkite/scripts/lucene-snapshot/upload-snapshot.sh | ||
| timeout_in_minutes: 15 | ||
| env: | ||
| USE_LUCENE_SNAPSHOT_CREDS: "true" | ||
| UPDATE_ES_LUCENE_SNAPSHOT: "true" | ||
| agents: | ||
| provider: gcp | ||
| image: family/elasticsearch-ubuntu-2004 | ||
| machineType: custom-32-98304 | ||
| buildDirectory: /dev/shm/bk | ||
| - wait | ||
| - trigger: "elasticsearch-lucene-snapshot-tests" | ||
| async: true |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| steps: | ||
| - label: sanity-check | ||
| command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files precommit | ||
| timeout_in_minutes: 300 | ||
| agents: | ||
| provider: gcp | ||
| image: family/elasticsearch-ubuntu-2004 | ||
| machineType: custom-32-98304 | ||
| buildDirectory: /dev/shm/bk | ||
| - wait: null | ||
| - label: part1 | ||
| command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart1 | ||
| timeout_in_minutes: 300 | ||
| agents: | ||
| provider: gcp | ||
| image: family/elasticsearch-ubuntu-2004 | ||
| machineType: custom-32-98304 | ||
| buildDirectory: /dev/shm/bk | ||
| - label: part2 | ||
| command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2 | ||
| timeout_in_minutes: 300 | ||
| agents: | ||
| provider: gcp | ||
| image: family/elasticsearch-ubuntu-2004 | ||
| machineType: custom-32-98304 | ||
| buildDirectory: /dev/shm/bk | ||
| - label: part3 | ||
| command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart3 | ||
| timeout_in_minutes: 300 | ||
| agents: | ||
| provider: gcp | ||
| image: family/elasticsearch-ubuntu-2004 | ||
| machineType: custom-32-98304 | ||
| buildDirectory: /dev/shm/bk | ||
| - group: bwc-snapshots | ||
| steps: | ||
| - label: "{{matrix.BWC_VERSION}} / bwc-snapshots" | ||
| command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files v$$BWC_VERSION#bwcTest | ||
| timeout_in_minutes: 300 | ||
| matrix: | ||
| setup: | ||
| BWC_VERSION: | ||
| - 7.17.13 | ||
| - 8.9.1 | ||
| - 8.10.0 | ||
| agents: | ||
| provider: gcp | ||
| image: family/elasticsearch-ubuntu-2004 | ||
| machineType: custom-32-98304 | ||
| buildDirectory: /dev/shm/bk | ||
| env: | ||
| BWC_VERSION: "{{matrix.BWC_VERSION}}" | ||
| - label: rest-compat | ||
| command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkRestCompat | ||
| timeout_in_minutes: 300 | ||
| agents: | ||
| provider: gcp | ||
| image: family/elasticsearch-ubuntu-2004 | ||
| machineType: custom-32-98304 | ||
| buildDirectory: /dev/shm/bk |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| steps: | ||
| - label: Update lucene-snapshot branch | ||
| command: .buildkite/scripts/lucene-snapshot/update-branch.sh | ||
| timeout_in_minutes: 15 | ||
| - wait | ||
| - trigger: "elasticsearch-lucene-snapshot-tests" | ||
| async: true |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| #!/bin/bash | ||
|
|
||
| set -euo pipefail | ||
|
|
||
| # WARNING: this script will echo the credentials to the console. It is meant to be called from another script and captured in a variable. | ||
| # It should really only be used inside .buildkite/hooks/pre-command | ||
|
|
||
| VAULT_ROLE_ID=$(vault read -field=role-id secret/ci/elastic-elasticsearch/legacy-vault-credentials) | ||
| VAULT_SECRET_ID=$(vault read -field=secret-id secret/ci/elastic-elasticsearch/legacy-vault-credentials) | ||
| VAULT_ADDR=https://secrets.elastic.co:8200 | ||
|
|
||
| unset VAULT_TOKEN | ||
| VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id=$VAULT_ROLE_ID secret_id=$VAULT_SECRET_ID) | ||
| export VAULT_TOKEN | ||
|
|
||
| vault read -format=json aws-elastic/creds/lucene-snapshots |
13 changes: 13 additions & 0 deletions
13
.buildkite/scripts/lucene-snapshot/remove-verification-metadata.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| import re | ||
|
|
||
| regex = re.compile( | ||
| r'<component group="org.apache.lucene" name="lucene-.*?</component>\s*', | ||
| re.MULTILINE | re.DOTALL, | ||
| ) | ||
|
|
||
| with open("gradle/verification-metadata.xml", "r+") as f: | ||
| text = f.read() | ||
| text = regex.sub("", text) | ||
| f.seek(0) | ||
| f.truncate() | ||
| f.write(text) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| #!/bin/bash | ||
|
|
||
| set -euo pipefail | ||
|
|
||
| if [[ "$BUILDKITE_BRANCH" != "lucene_snapshot" ]]; then | ||
| echo "Error: This script should only be run on the lucene_snapshot branch" | ||
| exit 1 | ||
| fi | ||
|
|
||
| echo --- Updating lucene_snapshot branch with main | ||
|
|
||
| git config --global user.name elasticsearchmachine | ||
| git config --global user.email '[email protected]' | ||
|
|
||
| git checkout lucene_snapshot | ||
| git fetch origin main | ||
| git merge --no-edit origin/main | ||
| git push origin lucene_snapshot |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| #!/bin/bash | ||
|
|
||
| set -euo pipefail | ||
|
|
||
| echo --- Update Lucene snapshot in Elasticsearch | ||
|
|
||
| LUCENE_SNAPSHOT_VERSION=${LUCENE_SNAPSHOT_VERSION:-} | ||
|
|
||
| if [[ -z "$LUCENE_SNAPSHOT_VERSION" ]]; then | ||
| LUCENE_SNAPSHOT_VERSION=$(buildkite-agent meta-data get lucene-snapshot-version) | ||
| fi | ||
|
|
||
| echo "Lucene Snapshot Version: $LUCENE_SNAPSHOT_VERSION" | ||
|
|
||
| git checkout "$BUILDKITE_BRANCH" | ||
| git pull --ff-only origin "$BUILDKITE_BRANCH" | ||
|
|
||
| # Replace `lucene = <version>` string in version.properties and maintain the same indentation | ||
| sed -E "s/^(lucene *= *[^ ]* *).*\$/\1$LUCENE_SNAPSHOT_VERSION/" build-tools-internal/version.properties > new-version.properties | ||
| mv new-version.properties build-tools-internal/version.properties | ||
|
|
||
| # Remove stale verification metadata, because generating them just appends new ones | ||
| python3 .buildkite/scripts/lucene-snapshot/remove-verification-metadata.py | ||
| ./gradlew --write-verification-metadata sha256 | ||
|
|
||
| # 9.8.0-snapshot-1f25c68 -> 9.8.0 | ||
| VERSION=$(echo "$LUCENE_SNAPSHOT_VERSION" | cut -f 1 -d '-') | ||
| VERSION_SNAKE=$(echo "$VERSION" | sed -E 's/\./_/g') | ||
|
|
||
| sed -E "s/^(:lucene_version: *).*\$/\1$VERSION/" docs/Versions.asciidoc > docs/Versions.asciidoc.new | ||
| sed -E "s/^(:lucene_version_path: *).*\$/\1$VERSION_SNAKE/" docs/Versions.asciidoc.new > docs/Versions.asciidoc | ||
| rm -f docs/Versions.asciidoc.new | ||
|
|
||
| if git diff-index --quiet HEAD --; then | ||
| echo 'No changes to commit.' | ||
| else | ||
| git config --global user.name elasticsearchmachine | ||
| git config --global user.email '[email protected]' | ||
|
|
||
| git add build-tools-internal/version.properties | ||
| git add gradle/verification-metadata.xml | ||
| git add docs/Versions.asciidoc | ||
|
|
||
| git commit -m "[Automated] Update Lucene snapshot to $LUCENE_SNAPSHOT_VERSION" | ||
| git push origin "$BUILDKITE_BRANCH" | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| #!/bin/bash | ||
|
|
||
| set -euo pipefail | ||
|
|
||
| LUCENE_BUILD_ID=${LUCENE_BUILD_ID:-} | ||
|
|
||
| if [[ -z "$LUCENE_BUILD_ID" ]]; then | ||
| build_json=$(curl -sH "Authorization: Bearer $BUILDKITE_API_TOKEN" "https://api.buildkite.com/v2/organizations/elastic/pipelines/$BUILDKITE_PIPELINE_SLUG/builds/$BUILDKITE_BUILD_NUMBER") | ||
| LUCENE_BUILD_ID=$(jq -r '.jobs[] | select(.step_key == "lucene-build").triggered_build.id' <<< "$build_json") | ||
| fi | ||
|
|
||
| export LUCENE_BUILD_ID | ||
|
|
||
| LUCENE_SHA=$(buildkite-agent meta-data get --build "$LUCENE_BUILD_ID" lucene-snapshot-sha) | ||
| export LUCENE_SHA | ||
|
|
||
| LUCENE_SNAPSHOT_VERSION=$(buildkite-agent meta-data get --build "$LUCENE_BUILD_ID" lucene-snapshot-version) | ||
| export LUCENE_SNAPSHOT_VERSION | ||
|
|
||
| echo --- Downloading lucene snapshot | ||
|
|
||
| mkdir lucene-snapshot | ||
| cd lucene-snapshot | ||
| buildkite-agent artifact download --build "$LUCENE_BUILD_ID" lucene-snapshot.tar.gz . | ||
| tar -xvf lucene-snapshot.tar.gz | ||
| cd - | ||
|
|
||
| echo --- Upload lucene snapshot to S3 | ||
|
|
||
| if ! which aws; then | ||
| curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" | ||
| unzip awscliv2.zip | ||
| sudo ./aws/install | ||
| rm -rf awscliv2.zip aws | ||
| fi | ||
|
|
||
| aws s3 sync lucene-snapshot/ "s3://download.elasticsearch.org/lucenesnapshots/$LUCENE_SHA/" --acl public-read | ||
|
|
||
| if [[ "${UPDATE_ES_LUCENE_SNAPSHOT:-}" ]]; then | ||
| .buildkite/scripts/lucene-snapshot/update-es-snapshot.sh | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters