From 07d4d87f3e217284be97bbfb0f626accdb764d24 Mon Sep 17 00:00:00 2001 From: Massimiliano Giovagnoli Date: Tue, 6 Sep 2022 13:51:07 +0200 Subject: [PATCH 1/5] feat: configure in_repo_config for evolution repository Signed-off-by: Massimiliano Giovagnoli --- config/config.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/config/config.yaml b/config/config.yaml index ac43342b749..94b374454d2 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -49,6 +49,19 @@ sinker: max_pod_age: 24h terminated_pod_ttl: 2h +in_repo_config: + enabled: + # The key can be one of "*" for "globally", "org" or "org/repo". + # The narrowest match is used. Here the key is "kubernetes/kubernetes". + falcosecurity/evolution: true + + # Clusters must be allowed before they can be used. Here we allow the "default" + # cluster globally. This setting also allows using "*" for "globally", "org" or "org/repo" as key. + # All clusters that are allowed for the specific repo, its org or + # globally can be used. + allowed_clusters: + falcosecurity/evolution: ["default"] + branch-protection: enforce_admins: true # rules apply to admins too! restrictions: # restrict who can push From 665ef27a707aef3e928ced4ecad4a97ee9864881 Mon Sep 17 00:00:00 2001 From: Massimiliano Giovagnoli Date: Tue, 6 Sep 2022 13:52:36 +0200 Subject: [PATCH 2/5] feat: run prowjob check before prs submission in evolution repo Signed-off-by: Massimiliano Giovagnoli --- .../check-prow-config/check-prow-config.yaml | 51 ++++++++++++------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/config/jobs/check-prow-config/check-prow-config.yaml b/config/jobs/check-prow-config/check-prow-config.yaml index 255f5f3cf0a..da0091d09f7 100644 --- a/config/jobs/check-prow-config/check-prow-config.yaml +++ b/config/jobs/check-prow-config/check-prow-config.yaml @@ -1,22 +1,39 @@ presubmits: falcosecurity/test-infra: - - name: check-prow-config - branches: - - ^master$ - decorate: true - skip_report: false - always_run: true - spec: - containers: - - image: gcr.io/k8s-prow/checkconfig:v20220901-5db9cf5fa2 - command: - - checkconfig - args: - - --config-path=config/config.yaml - - --job-config-path=config/jobs - - --plugin-config=config/plugins.yaml - nodeSelector: - Archtype: "x86" + - name: check-prow-config + branches: + - ^master$ + decorate: true + skip_report: false + always_run: true + spec: + containers: + - image: gcr.io/k8s-prow/checkconfig:v20220901-5db9cf5fa2 + command: + - checkconfig + args: + - --config-path=config/config.yaml + - --job-config-path=config/jobs + - --plugin-config=config/plugins.yaml + nodeSelector: + Archtype: "x86" + falcosecurity/evolution: + - name: check-prow-config-evolution + always_run: true + decorate: true + extra_refs: + - org: falcosecurity + repo: test-infra + base_ref: master + spec: + containers: + - image: gcr.io/k8s-prow/checkconfig:v20220901-5db9cf5fa2 + command: + - checkconfig + args: + - --plugin-config=../test-infra/config/plugins.yaml + - --config-path=../test-infra/config/config.yaml + - --prow-yaml-repo-name=$(REPO_OWNER)/$(REPO_NAME) periodics: - name: check-prow-config-periodic interval: 1h From b12281728c8d9913f2174553e44f205dba7e8628 Mon Sep 17 00:00:00 2001 From: Massimiliano Giovagnoli Date: Thu, 8 Sep 2022 12:35:52 +0200 Subject: [PATCH 3/5] ci(config/jobs): run prow config check on evolution only with changes Signed-off-by: Massimiliano Giovagnoli --- config/jobs/check-prow-config/check-prow-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/jobs/check-prow-config/check-prow-config.yaml b/config/jobs/check-prow-config/check-prow-config.yaml index da0091d09f7..ac941f7b9fe 100644 --- a/config/jobs/check-prow-config/check-prow-config.yaml +++ b/config/jobs/check-prow-config/check-prow-config.yaml @@ -19,7 +19,7 @@ presubmits: Archtype: "x86" falcosecurity/evolution: - name: check-prow-config-evolution - always_run: true + run_if_changed: '^org/org.yaml$|^.prow/peribolos.yaml$' decorate: true extra_refs: - org: falcosecurity From 36768f80b2a06e45f2ff2c29fa7a5e7be4d33199 Mon Sep 17 00:00:00 2001 From: Massimiliano Giovagnoli Date: Fri, 9 Sep 2022 20:35:10 +0200 Subject: [PATCH 4/5] refactor(images/autobump): make bump of component and config optional Signed-off-by: Massimiliano Giovagnoli --- images/autobump/autobump.sh | 21 ++++++++++++------ images/autobump/bump.sh | 44 +++++++++++++++++++++++++++---------- 2 files changed, 47 insertions(+), 18 deletions(-) diff --git a/images/autobump/autobump.sh b/images/autobump/autobump.sh index 78376b7d059..707730dd69e 100644 --- a/images/autobump/autobump.sh +++ b/images/autobump/autobump.sh @@ -53,20 +53,27 @@ main() { echo "Bumping ${PROW_INSTANCE_NAME} to upstream (prow.k8s.io) version..." >&2 /bump.sh --upstream - cd "$(git rev-parse --show-toplevel)" - old_version=$(git show "HEAD:${PROW_CONTROLLER_MANAGER_FILE}" | extract-version) - version=$(cat "${PROW_CONTROLLER_MANAGER_FILE}" | extract-version) + title="Bump ${PROW_INSTANCE_NAME} configs" - if [[ -z "${version}" ]]; then + if -n "${PROW_CONTROLLER_MANAGER_FILE}"; then + cd "$(git rev-parse --show-toplevel)" + old_version=$(git show "HEAD:${PROW_CONTROLLER_MANAGER_FILE}" | extract-version) + version=$(cat "${PROW_CONTROLLER_MANAGER_FILE}" | extract-version) + + if [[ -z "${version}" ]]; then echo "Failed to fetch version from ${PROW_CONTROLLER_MANAGER_FILE}" exit 1 - fi - if [[ "${old_version}" == "${version}" ]]; then + fi + if [[ "${old_version}" == "${version}" ]]; then echo "Bump did not change the Prow version: it's still ${version}. Aborting no-op bump." >&2 return 0 + fi + + title+=" and components from ${old_version} to ${version}" fi + git add -u - title="Bump ${PROW_INSTANCE_NAME} from ${old_version} to ${version}" + comparison=$(extract-commit "${old_version}")...$(extract-commit "${version}") body="Included changes: https://github.com/kubernetes/test-infra/compare/${comparison}" diff --git a/images/autobump/bump.sh b/images/autobump/bump.sh index a50b570aea6..9a9dfe3e741 100644 --- a/images/autobump/bump.sh +++ b/images/autobump/bump.sh @@ -68,42 +68,62 @@ main() { fi echo -e "Bumping: 'gcr.io/k8s-prow/' images to $(color-version "${new_version}") ..." >&2 - local component_file_dir_array - IFS=, read -ra component_file_dir_array <<< "${COMPONENT_FILE_DIR}" bumpfiles=() - for c in "${component_file_dir_array[@]}"; do - # This expands wildcards into files if they exist - bumpfiles+=(${c}/*.yaml) - done - bumpfiles+=("${CONFIG_PATH}") + if test -n "${COMPONENT_FILE_DIR}"; then + local component_file_dir_array + IFS=, read -ra component_file_dir_array <<< "${COMPONENT_FILE_DIR}" + for c in "${component_file_dir_array[@]}"; do + # This expands wildcards into files if they exist + bumpfiles+=(${c}/*.yaml) + done + fi + + if test -n "${CONFIG_PATH}"; then + bumpfiles+=("${CONFIG_PATH}") + fi + if [[ -n "${JOB_CONFIG_PATH}" ]]; then bumpfiles+=($(grep -rl -e "gcr.io/k8s-prow/" "${JOB_CONFIG_PATH}"; true)) fi + echo "Found ${#bumpfiles[@]} files to bump." + + if (( ${#bumpfiles[@]} <= 0 )); then + echo "Nothing to do." + exit 0 + fi + echo "Attempting to bump the following files:" >&2 for bf in "${bumpfiles[@]}"; do echo -e "$bf" done + local token="$(gcloud auth print-access-token)" + # Update image tags in the identified files. This supports both normal image and -arm64 images local matcher="gcr.io\/k8s-prow\/\([[:alnum:]_-]\+\):v[a-f0-9-]\+\(-arm64\)\{0,1\}" local replacer="s/${matcher}/gcr.io\/k8s-prow\/\1:${new_version}\2/I" + for file in "${bumpfiles[@]}"; do ${SED} -i "${replacer}" "${file}" local images="$(grep -o "${matcher}" "${file}")" local arr=(${images//\\n/}) + # image is in the format of gcr.io/k8s-prow/[image_name]:[tag] for image in ${arr[@]+"${arr[@]}"}; do echo "Checking the existence of ${image}" + # Use the Docker Registry v2 API to query the image manifest to check if the given image tag exists or not. # The manifest_url is in the format of https://gcr.io/v2/k8s-prow/[image_name]/manifests/[tag] # Check more details from https://stackoverflow.com/a/55344819/13578870 local manifest_url=$(echo "$image" | ${SED} "s/:/\/manifests\//" | ${SED} "s/gcr.io/https:\/\/gcr.io\/v2/") if ! curl --fail -L -H "Authorization: Bearer $token" -o /dev/null -s "${manifest_url}"; then echo "The image ${image} does not exist, please double check." >&2 + # Revert the changes for this file. git checkout -- "${file}" + return 1 fi done @@ -114,13 +134,15 @@ main() { check-args() { if [[ -z "${COMPONENT_FILE_DIR}" ]]; then - echo "ERROR: COMPONENT_FILE_DIR must be specified as an env var." >&2 - return 1 + echo "info: COMPONENT_FILE_DIR is empty." >&2 fi if [[ -z "${CONFIG_PATH}" ]]; then - echo "ERROR: CONFIG_PATH must be specified as an env var." >&2 - return 1 + echo "info: CONFIG_PATH is empty." >&2 + fi + if [[ -z "${JOB_CONFIG_PATH}" ]]; then + echo "info: JOB_CONFIG_PATH is empty." >&2 fi + return } check-requirements() { From ba53927cf76ebbf9d497f6eb707e76a7d915a738 Mon Sep 17 00:00:00 2001 From: Massimiliano Giovagnoli Date: Fri, 9 Sep 2022 20:52:02 +0200 Subject: [PATCH 5/5] ci(autobump): add autobump for evolution jobs Signed-off-by: Massimiliano Giovagnoli --- config/jobs/autobump/evolution.yaml | 64 +++++++++++++++++++ .../{autobump.yaml => test-infra.yaml} | 0 images/autobump/autobump.sh | 28 ++++---- 3 files changed, 78 insertions(+), 14 deletions(-) create mode 100644 config/jobs/autobump/evolution.yaml rename config/jobs/autobump/{autobump.yaml => test-infra.yaml} (100%) diff --git a/config/jobs/autobump/evolution.yaml b/config/jobs/autobump/evolution.yaml new file mode 100644 index 00000000000..ca7061e9025 --- /dev/null +++ b/config/jobs/autobump/evolution.yaml @@ -0,0 +1,64 @@ +periodics: +- name: ci-prow-autobump-evolution + # interval: 15m + cron: "05 15 * * 1" # Run at 15:05 PST (15:05 UTC) Mon + decorate: true + extra_refs: + # Check out the repo containing the config and deployment files for your Prow instance. + - org: falcosecurity + repo: evolution + base_ref: main + spec: + containers: + - image: 292999226676.dkr.ecr.eu-west-1.amazonaws.com/test-infra/autobump # see images/autobump + imagePullPolicy: Always + command: + - /autobump.sh + args: + - /etc/github-token/oauth + # Make the bot name and email match the user data of the provided token's user. + - "Falco Automation" + - 51138685+poiana@users.noreply.github.com + - /root/gpg-signing-key/poiana.asc + - "EC9875C7B990D55F3B44D6E45F284448FF941C8F" + volumeMounts: + - name: service + mountPath: /etc/service-account + readOnly: true + - name: github + mountPath: /etc/github-token + readOnly: true + - name: gpg-signing-key + mountPath: /root/gpg-signing-key/ + readOnly: true + env: + # autobump.sh args + - name: GH_PROXY + value: https://api.github.com + # GitHub org containing the repo where the Prow config and component files live. + - name: GH_ORG + value: falcosecurity + # GitHub repo where the Prow config and component deployment files live. + - name: GH_REPO + value: evolution + # bump.sh args + # Repo relative path of the ProwJob config file or directory. + # Omit this if ProwJobs are only defined in config.yaml (or are not configured at all). + - name: JOB_CONFIG_PATH + value: .prow/ + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /etc/service-account/service-account.json + volumes: + - name: service + secret: + secretName: gcloud-credentials + - name: github + secret: + # Secret containing a GitHub user access token with `repo` scope for creating PRs. + secretName: oauth-token + - name: gpg-signing-key + secret: + secretName: poiana-gpg-signing-key + defaultMode: 0400 + nodeSelector: + Archtype: "x86" diff --git a/config/jobs/autobump/autobump.yaml b/config/jobs/autobump/test-infra.yaml similarity index 100% rename from config/jobs/autobump/autobump.yaml rename to config/jobs/autobump/test-infra.yaml diff --git a/images/autobump/autobump.sh b/images/autobump/autobump.sh index 707730dd69e..a15bfe9defc 100644 --- a/images/autobump/autobump.sh +++ b/images/autobump/autobump.sh @@ -56,20 +56,20 @@ main() { title="Bump ${PROW_INSTANCE_NAME} configs" if -n "${PROW_CONTROLLER_MANAGER_FILE}"; then - cd "$(git rev-parse --show-toplevel)" - old_version=$(git show "HEAD:${PROW_CONTROLLER_MANAGER_FILE}" | extract-version) - version=$(cat "${PROW_CONTROLLER_MANAGER_FILE}" | extract-version) - - if [[ -z "${version}" ]]; then - echo "Failed to fetch version from ${PROW_CONTROLLER_MANAGER_FILE}" - exit 1 - fi - if [[ "${old_version}" == "${version}" ]]; then - echo "Bump did not change the Prow version: it's still ${version}. Aborting no-op bump." >&2 - return 0 - fi - - title+=" and components from ${old_version} to ${version}" + cd "$(git rev-parse --show-toplevel)" + old_version=$(git show "HEAD:${PROW_CONTROLLER_MANAGER_FILE}" | extract-version) + version=$(cat "${PROW_CONTROLLER_MANAGER_FILE}" | extract-version) + + if [[ -z "${version}" ]]; then + echo "Failed to fetch version from ${PROW_CONTROLLER_MANAGER_FILE}" + exit 1 + fi + if [[ "${old_version}" == "${version}" ]]; then + echo "Bump did not change the Prow version: it's still ${version}. Aborting no-op bump." >&2 + return 0 + fi + + title+=" and components from ${old_version} to ${version}" fi git add -u