From e169722b893e6467f6cccb036097dba0675803b8 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 10 Jul 2024 18:25:57 +0200 Subject: [PATCH 1/7] use eval to run the compiled publish command --- .buildkite/scripts/steps/artifacts/publish.sh | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index 08c6ecc1e25ad..0d40ed2430eb3 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -2,6 +2,7 @@ set -euo pipefail +source env.env source .buildkite/scripts/common/util.sh source .buildkite/scripts/steps/artifacts/env.sh @@ -12,7 +13,7 @@ function download { download_artifact "$1" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" download_artifact "$1.sha512.txt" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" sha512sum -c "$1.sha512.txt" - rm "$1.sha512.txt" + rm -f "$1.sha512.txt" } mkdir -p target @@ -63,29 +64,29 @@ if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]]; then export BEATS_MANIFEST_URL=$(jq -r .manifest_url /tmp/beats_manifest.json) PUBLISH_CMD=$(cat << EOF - docker run --rm \ - --name release-manager \ - -e VAULT_ADDR \ - -e VAULT_ROLE_ID \ - -e VAULT_SECRET_ID \ - --mount type=bind,readonly=false,src="$PWD/target",target=/artifacts/target \ - docker.elastic.co/infra/release-manager:latest \ - cli collect \ - --project kibana \ - --branch "$KIBANA_BASE_BRANCH" \ - --commit "$GIT_COMMIT" \ - --workflow "$WORKFLOW" \ - --version "$BASE_VERSION" \ - --qualifier "$VERSION_QUALIFIER" \ - --dependency "beats:$BEATS_MANIFEST_URL" \ - --artifact-set main +docker run --rm \ + --name release-manager \ + -e VAULT_ADDR \ + -e VAULT_ROLE_ID \ + -e VAULT_SECRET_ID \ + --mount type=bind,readonly=false,src="$PWD/target",target=/artifacts/target \ + docker.elastic.co/infra/release-manager:latest \ + cli collect \ + --project kibana \ + --branch "$KIBANA_BASE_BRANCH" \ + --commit "$GIT_COMMIT" \ + --workflow "$WORKFLOW" \ + --version "$BASE_VERSION" \ + --qualifier "$VERSION_QUALIFIER" \ + --dependency "beats:$BEATS_MANIFEST_URL" \ + --artifact-set main EOF ) if [[ "${DRY_RUN:-}" =~ ^(1|true)$ ]]; then - PUBLISH_CMD+=(" --dry-run") + PUBLISH_CMD="$PUBLISH_CMD --dry-run" fi - "${PUBLISH_CMD[@]}" + eval "$PUBLISH_CMD" KIBANA_SUMMARY=$(curl -s "$KIBANA_MANIFEST_LATEST" | jq -re '.summary_url') From f64939d3f1bf52c71318f99cb0f7d8efd3e3c3bf Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 10 Jul 2024 18:28:28 +0200 Subject: [PATCH 2/7] allow 'Publish artifacts' step also if DRY_RUN is active, publish would be prevented by the dry_run flag --- .buildkite/scripts/steps/artifacts/publish.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index 0d40ed2430eb3..8735a3e8757ed 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -2,7 +2,6 @@ set -euo pipefail -source env.env source .buildkite/scripts/common/util.sh source .buildkite/scripts/steps/artifacts/env.sh @@ -55,7 +54,7 @@ echo "--- Pull latest Release Manager CLI" docker pull docker.elastic.co/infra/release-manager:latest echo "--- Publish artifacts" -if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]]; then +if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]] || [[ "${DRY_RUN:-}" =~ ^(1|true)$ ]]; then export VAULT_ROLE_ID="$(get_vault_role_id)" export VAULT_SECRET_ID="$(get_vault_secret_id)" export VAULT_ADDR="https://secrets.elastic.co:8200" From 9b1da2a11008970e968d0a211448fa53236464cb Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Mon, 15 Jul 2024 10:44:34 +0200 Subject: [PATCH 3/7] only collect release_manager_args instead of eval-ing the whole command --- .buildkite/scripts/steps/artifacts/publish.sh | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index 8735a3e8757ed..c988782246b63 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -62,30 +62,31 @@ if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]] || [[ "${DRY_RUN:-}" =~ ^( download_artifact beats_manifest.json /tmp --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" export BEATS_MANIFEST_URL=$(jq -r .manifest_url /tmp/beats_manifest.json) - PUBLISH_CMD=$(cat << EOF -docker run --rm \ - --name release-manager \ - -e VAULT_ADDR \ - -e VAULT_ROLE_ID \ - -e VAULT_SECRET_ID \ - --mount type=bind,readonly=false,src="$PWD/target",target=/artifacts/target \ - docker.elastic.co/infra/release-manager:latest \ - cli collect \ - --project kibana \ - --branch "$KIBANA_BASE_BRANCH" \ - --commit "$GIT_COMMIT" \ - --workflow "$WORKFLOW" \ - --version "$BASE_VERSION" \ - --qualifier "$VERSION_QUALIFIER" \ - --dependency "beats:$BEATS_MANIFEST_URL" \ - --artifact-set main + RELEASE_MANAGER_ARGS=$(cat << EOF + cli + collect + --project kibana + --branch "$KIBANA_BASE_BRANCH" + --commit "$GIT_COMMIT" + --workflow "$WORKFLOW" + --version "$BASE_VERSION" + --qualifier "$VERSION_QUALIFIER" + --dependency "beats:$BEATS_MANIFEST_URL" + --artifact-set main EOF ) - if [[ "${DRY_RUN:-}" =~ ^(1|true)$ ]]; then - PUBLISH_CMD="$PUBLISH_CMD --dry-run" + + if [[ "$DRY_RUN" =~ ^(1|true)$ ]]; then + RELEASE_MANAGER_ARGS="$RELEASE_MANAGER_ARGS --dry-run" fi - eval "$PUBLISH_CMD" + docker run --rm \ + --name release-manager \ + -e VAULT_ADDR \ + -e VAULT_ROLE_ID \ + -e VAULT_SECRET_ID \ + --mount type=bind,readonly=false,src="$PWD/target",target=/artifacts/target \ + docker.elastic.co/infra/release-manager:latest $RELEASE_MANAGER_ARGS KIBANA_SUMMARY=$(curl -s "$KIBANA_MANIFEST_LATEST" | jq -re '.summary_url') From 40bb6dda136abba10d904f75bc2112cdeed3a17b Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Mon, 15 Jul 2024 14:30:08 +0200 Subject: [PATCH 4/7] try without the quotes --- .buildkite/scripts/steps/artifacts/publish.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index c988782246b63..85ca4bf6791e8 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -68,7 +68,7 @@ if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]] || [[ "${DRY_RUN:-}" =~ ^( --project kibana --branch "$KIBANA_BASE_BRANCH" --commit "$GIT_COMMIT" - --workflow "$WORKFLOW" + --workflow $WORKFLOW --version "$BASE_VERSION" --qualifier "$VERSION_QUALIFIER" --dependency "beats:$BEATS_MANIFEST_URL" From de7d1e454b91c9e3d1cc6d3776a5643f8a128011 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Mon, 15 Jul 2024 17:19:05 +0200 Subject: [PATCH 5/7] remove double quotes from all params, to avoid double quoting --- .buildkite/scripts/steps/artifacts/publish.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index 85ca4bf6791e8..5da0471ff8627 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -66,12 +66,12 @@ if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]] || [[ "${DRY_RUN:-}" =~ ^( cli collect --project kibana - --branch "$KIBANA_BASE_BRANCH" - --commit "$GIT_COMMIT" + --branch $KIBANA_BASE_BRANCH + --commit $GIT_COMMIT --workflow $WORKFLOW - --version "$BASE_VERSION" - --qualifier "$VERSION_QUALIFIER" - --dependency "beats:$BEATS_MANIFEST_URL" + --version $BASE_VERSION + --qualifier $VERSION_QUALIFIER + --dependency beats:$BEATS_MANIFEST_URL --artifact-set main EOF ) From 979b44a95e263515ea02926293b4c8c9985f69a5 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Tue, 16 Jul 2024 07:55:36 +0200 Subject: [PATCH 6/7] use a plain if/else, nothing else seems to be working that well --- .buildkite/scripts/steps/artifacts/publish.sh | 57 +++++++++++-------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index 5da0471ff8627..a24ba072ef6aa 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -62,32 +62,43 @@ if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]] || [[ "${DRY_RUN:-}" =~ ^( download_artifact beats_manifest.json /tmp --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" export BEATS_MANIFEST_URL=$(jq -r .manifest_url /tmp/beats_manifest.json) - RELEASE_MANAGER_ARGS=$(cat << EOF - cli - collect - --project kibana - --branch $KIBANA_BASE_BRANCH - --commit $GIT_COMMIT - --workflow $WORKFLOW - --version $BASE_VERSION - --qualifier $VERSION_QUALIFIER - --dependency beats:$BEATS_MANIFEST_URL - --artifact-set main -EOF -) - if [[ "$DRY_RUN" =~ ^(1|true)$ ]]; then - RELEASE_MANAGER_ARGS="$RELEASE_MANAGER_ARGS --dry-run" + docker run --rm \ + --name release-manager \ + -e VAULT_ADDR \ + -e VAULT_ROLE_ID \ + -e VAULT_SECRET_ID \ + --mount type=bind,readonly=false,src="$PWD/target",target=/artifacts/target \ + docker.elastic.co/infra/release-manager:latest \ + cli collect \ + --project kibana \ + --branch "$KIBANA_BASE_BRANCH" \ + --commit "$GIT_COMMIT" \ + --workflow "$WORKFLOW" \ + --version "$BASE_VERSION" \ + --qualifier "$VERSION_QUALIFIER" \ + --dependency "beats:$BEATS_MANIFEST_URL" \ + --artifact-set main \ + --dry-run + else + docker run --rm \ + --name release-manager \ + -e VAULT_ADDR \ + -e VAULT_ROLE_ID \ + -e VAULT_SECRET_ID \ + --mount type=bind,readonly=false,src="$PWD/target",target=/artifacts/target \ + docker.elastic.co/infra/release-manager:latest \ + cli collect \ + --project kibana \ + --branch "$KIBANA_BASE_BRANCH" \ + --commit "$GIT_COMMIT" \ + --workflow "$WORKFLOW" \ + --version "$BASE_VERSION" \ + --qualifier "$VERSION_QUALIFIER" \ + --dependency "beats:$BEATS_MANIFEST_URL" \ + --artifact-set main fi - docker run --rm \ - --name release-manager \ - -e VAULT_ADDR \ - -e VAULT_ROLE_ID \ - -e VAULT_SECRET_ID \ - --mount type=bind,readonly=false,src="$PWD/target",target=/artifacts/target \ - docker.elastic.co/infra/release-manager:latest $RELEASE_MANAGER_ARGS - KIBANA_SUMMARY=$(curl -s "$KIBANA_MANIFEST_LATEST" | jq -re '.summary_url') cat << EOF | buildkite-agent annotate --style "info" --context artifacts-summary From e4272a3731fa2b2fdc8a815f0e20f8c0ebae13de Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Tue, 16 Jul 2024 08:00:28 +0200 Subject: [PATCH 7/7] remove unnecessary flag --- .buildkite/scripts/steps/artifacts/publish.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index a24ba072ef6aa..ddb1b15496163 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -12,7 +12,7 @@ function download { download_artifact "$1" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" download_artifact "$1.sha512.txt" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" sha512sum -c "$1.sha512.txt" - rm -f "$1.sha512.txt" + rm "$1.sha512.txt" } mkdir -p target