From 871e1ef6e4698748aabb769db8c40f9278ee72fa Mon Sep 17 00:00:00 2001 From: Max Jonas Werner Date: Wed, 2 Aug 2023 17:31:35 +0200 Subject: [PATCH] Make import-flux2-assets.sh aware of the desired Flux version Since the website will be built for the 2.0.x minor release, the import script needs to fetch the API docs for the controller versions of the latest 2.0.x Flux release. This change makes the script aware of the desired version (derived from the `BRANCH` environment variable that is set by Netlify). This change is a subset of the changes implemented in #1577. We cannot simply backport 1577 since the changes in there required some changes to the controller API docs which haven't been released as part of a Flux 2.0.x release, yet. Signed-off-by: Max Jonas Werner --- .gitignore | 3 ++ Makefile | 23 ++++++++-- hack/import-flux2-assets.sh | 86 +++++++++++++++++++++---------------- 3 files changed, 71 insertions(+), 41 deletions(-) diff --git a/.gitignore b/.gitignore index 6b8416a2e..0d556ceba 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ static/install.sh # local GH token, used for some scripts .gh-token + +# build directories +/bin diff --git a/Makefile b/Makefile index 8d217a0e6..1e2fc9e34 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,12 @@ HUGO_BIND_ADDRESS ?= 127.0.0.1 BUILDER_CLI := docker # BUILDER_CLI := okteto LYCHEE_IMAGE_NAME ?= lycheeverse/lychee:202105190720247e4977 +YQ_VERSION ?= v4.34.2 +BRANCH ?= main + +REPO_ROOT := $(shell git rev-parse --show-toplevel) +OS := $(shell uname -s | tr '[:upper:]' '[:lower:]') +BIN_DIR := $(REPO_ROOT)/bin help: ## Display this help menu. @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) @@ -20,11 +26,20 @@ help: ## Display this help menu. prereqs: npm install + python3 -m pip install -r requirements.txt + +.PHONY: yq +yq: $(BIN_DIR)/yq-$(YQ_VERSION) -gen-content: ## Generates content from external sources. +$(BIN_DIR)/yq-$(YQ_VERSION): + mkdir -p $(BIN_DIR) && wget https://github.com/mikefarah/yq/releases/download/$(YQ_VERSION)/yq_$(OS)_$(shell go env GOARCH) -O $(BIN_DIR)/yq-$(YQ_VERSION) && \ + chmod +x $(BIN_DIR)/yq-$(YQ_VERSION) && \ + cp $(BIN_DIR)/yq-$(YQ_VERSION) $(BIN_DIR)/yq + +gen-content: yq ## Generates content from external sources. hack/gen-content.py hack/import-calendar.py - hack/import-flux2-assets.sh + PATH=$(BIN_DIR):$(PATH) BRANCH=$(BRANCH) hack/import-flux2-assets.sh serve: gen-content prereqs ## Spawns a development server. hugo server \ @@ -51,7 +66,7 @@ preview-build: gen-content prereqs ## Builds a preview build (for e.g. a pull re branch-build: gen-content prereqs ## Builds a Git branch (for e.g. development branches). hugo \ --baseURL $(DEPLOY_PRIME_URL) \ - --environment branch \ + --buildDrafts \ --buildFuture \ --gc \ --minify \ @@ -63,7 +78,7 @@ docker-preview: docker-serve .PHONY: docker-serve docker-serve: docker run -v $(shell pwd):/site -p 1313:1313 -it $(SUPPORT_IMAGE_NAME) \ - make \"MAKEFLAGS=$(MAKEFLAGS)\" serve HUGO_BIND_ADDRESS=0.0.0.0 + make \"MAKEFLAGS=$(MAKEFLAGS)\" serve HUGO_BIND_ADDRESS=0.0.0.0 GITHUB_TOKEN=$(GITHUB_TOKEN) BRANCH=$(BRANCH) .PHONY: docker-push docker-push-hugo docker-push-support docker-push: docker-push-hugo docker-push-support diff --git a/hack/import-flux2-assets.sh b/hack/import-flux2-assets.sh index e937c9a35..3cac6cff4 100755 --- a/hack/import-flux2-assets.sh +++ b/hack/import-flux2-assets.sh @@ -1,9 +1,11 @@ -#!/bin/sh -x +#!/usr/bin/env bash + +set -euxo pipefail COMPONENTS_DIR="content/en/flux/components" FLUX_DIR="content/en/flux/cmd" -if [ -z "${GITHUB_USER}" ]; then +if [ -z "${GITHUB_USER:-}" ]; then GITHUB_USER=fluxcdbot fi @@ -19,7 +21,7 @@ fatal() { # Set os, fatal if operating system not supported setup_verify_os() { - if [ -z "${OS}" ]; then + if [ -z "${OS:-}" ]; then OS=$(uname) fi case ${OS} in @@ -36,7 +38,7 @@ setup_verify_os() { # Set arch, fatal if architecture not supported setup_verify_arch() { - if [ -z "${ARCH}" ]; then + if [ -z "${ARCH:-}" ]; then ARCH=$(uname -m) fi case ${ARCH} in @@ -59,21 +61,14 @@ setup_verify_arch() { controller_version() { - url=https://api.github.com/repos/fluxcd/$1/releases - out=$(curl -w "%{http_code}" -o /tmp/releases -u "$GITHUB_USER:$GITHUB_TOKEN" -s "$url") - if [ "${out}" -ne "200" ] ; then - echo "error fetching releases from $url: $out" - cat /tmp/releases - exit 1 - fi - - jq -r '.[] | .tag_name' < /tmp/releases | sed 's/.*\///' | awk '{ if ($1 ~ /-/) print; else print $0"_" ; }' | sort -rV | sed 's/_$//' | head -n1 + ks_url=$(curl -sL "https://raw.githubusercontent.com/fluxcd/flux2/v${VERSION_FLUX}/manifests/bases/${1}/kustomization.yaml" | yq '.resources[]|select(. == "*crds.yaml*")') + echo "${ks_url}" | cut -d/ -f8 } gen_crd_doc() { URL="$1" DEST="$2" - HUGETABLE="$3" + HUGETABLE="${3:-}" TMP="$(mktemp)" curl -u "$GITHUB_USER:$GITHUB_TOKEN" -# -Lf "$URL" > "$TMP" @@ -85,7 +80,7 @@ gen_crd_doc() { # 3. While others go # Here is the title you're looking for... # - FIRST_LINE="$(grep -vE "^$' "$TMP" | cut -d' ' -f2|cut -d':' -f2)" + WEIGHT="$(grep -E '^$' "$TMP" | cut -d' ' -f2|cut -d':' -f2 || true)" if [ -z "${WEIGHT}" ] ; then WEIGHT=0 fi @@ -119,6 +114,44 @@ gen_crd_doc() { fi } +{ + # get flux cmd docs + setup_verify_os + setup_verify_arch + + TMP="$(mktemp -d)" + TMP_METADATA="$TMP/flux.json" + TMP_BIN="$TMP/flux.tar.gz" + + if [[ "${BRANCH}" =~ ^pull/[[:digit:]]*/head$ ]] ; then + BRANCH=$(curl -sSfL "https://api.github.com/repos/fluxcd/website/pulls/$(echo ${BRANCH}|cut -d/ -f2)" | jq .base.ref -r) + fi + + page=1 + while [ -z "${VERSION_FLUX:-}" ] ; do + tags=$(curl -u "${GITHUB_USER}:${GITHUB_TOKEN}" -sSfL "https://api.github.com/repos/fluxcd/flux2/releases?page=${page}" | jq .[].tag_name -r) || false + for tag in $tags ; do + if [ "${BRANCH}" = "main" ] ; then + VERSION_FLUX="${tag#v}" + break + fi + if [ "${tag%.*}" = "${BRANCH/-/.}" ] ; then + VERSION_FLUX="${tag#v}" + break + fi + done + ((page = page + 1)) + done + + curl -u "$GITHUB_USER:$GITHUB_TOKEN" -o "${TMP_BIN}" --retry 3 -sSfL "https://github.com/fluxcd/flux2/releases/download/v${VERSION_FLUX}/flux_${VERSION_FLUX}_${OS}_${ARCH}.tar.gz" + tar xfz "${TMP_BIN}" -C "${TMP}" + + rm -rf "${FLUX_DIR:?}/*" + "${TMP}/flux" docgen --path "${FLUX_DIR}" + + rm -rf "$TMP" +} + { # source-controller CRDs SOURCE_VER="$(controller_version source-controller)" @@ -170,27 +203,6 @@ gen_crd_doc() { gen_crd_doc "https://raw.githubusercontent.com/fluxcd/image-automation-controller/$IMG_AUTO_VER/docs/spec/v1beta1/imageupdateautomations.md" "$COMPONENTS_DIR/image/imageupdateautomations.md" } -{ - # get flux cmd docs - setup_verify_os - setup_verify_arch - - TMP="$(mktemp -d)" - TMP_METADATA="$TMP/flux.json" - TMP_BIN="$TMP/flux.tar.gz" - - curl -u "$GITHUB_USER:$GITHUB_TOKEN" -o "${TMP_METADATA}" --retry 3 -sSfL "https://api.github.com/repos/fluxcd/flux2/releases/latest" - VERSION_FLUX=$(grep '"tag_name":' "${TMP_METADATA}" | sed -E 's/.*"([^"]+)".*/\1/' | cut -c 2-) - - curl -u "$GITHUB_USER:$GITHUB_TOKEN" -o "${TMP_BIN}" --retry 3 -sSfL "https://github.com/fluxcd/flux2/releases/download/v${VERSION_FLUX}/flux_${VERSION_FLUX}_${OS}_${ARCH}.tar.gz" - tar xfz "${TMP_BIN}" -C "${TMP}" - - rm -rf "${FLUX_DIR:?}/*" - "${TMP}/flux" docgen --path "${FLUX_DIR}" - - rm -rf "$TMP" -} - { # provide Flux install script if [ ! -d static ]; then