Make import-flux2-assets.sh aware of the desired Flux version

Since the website will be built for the 2.0.x minor release, the import script needs to fetch the API docs for the controller versions of the latest 2.0.x Flux release. This change makes the script aware of the desired version (derived from the `BRANCH` environment variable that is set by Netlify). This change is a subset of the changes implemented in #1577. We cannot simply backport 1577 since the changes in there required some changes to the controller API docs which haven't been released as part of a Flux 2.0.x release, yet. Signed-off-by: Max Jonas Werner <[email protected]>
fluxcd · Aug 3, 2023 · 871e1ef · 871e1ef
1 parent cb2bea2
commit 871e1ef
Show file tree

Hide file tree

Showing 3 changed files with 71 additions and 41 deletions.
diff --git a/.gitignore b/.gitignore
@@ -15,3 +15,6 @@ static/install.sh
 
 # local GH token, used for some scripts
 .gh-token
+
+# build directories
+/bin
diff --git a/Makefile b/Makefile
@@ -13,18 +13,33 @@ HUGO_BIND_ADDRESS          ?= 127.0.0.1
 BUILDER_CLI                := docker
 # BUILDER_CLI                := okteto
 LYCHEE_IMAGE_NAME          ?= lycheeverse/lychee:202105190720247e4977
+YQ_VERSION                 ?= v4.34.2
+BRANCH                     ?= main
+
+REPO_ROOT := $(shell git rev-parse --show-toplevel)
+OS := $(shell uname -s | tr '[:upper:]' '[:lower:]')
+BIN_DIR := $(REPO_ROOT)/bin
 
 help:  ## Display this help menu.
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 .PHONY: help
 
 prereqs:
 	npm install
+	python3 -m pip install -r requirements.txt
+
+.PHONY: yq
+yq: $(BIN_DIR)/yq-$(YQ_VERSION)
 
-gen-content: ## Generates content from external sources.
+$(BIN_DIR)/yq-$(YQ_VERSION):
+	mkdir -p $(BIN_DIR) && wget https://github.com/mikefarah/yq/releases/download/$(YQ_VERSION)/yq_$(OS)_$(shell go env GOARCH) -O $(BIN_DIR)/yq-$(YQ_VERSION) && \
+	    chmod +x $(BIN_DIR)/yq-$(YQ_VERSION) && \
+	    cp $(BIN_DIR)/yq-$(YQ_VERSION) $(BIN_DIR)/yq
+
+gen-content: yq ## Generates content from external sources.
 	hack/gen-content.py
 	hack/import-calendar.py
-	hack/import-flux2-assets.sh
+	PATH=$(BIN_DIR):$(PATH) BRANCH=$(BRANCH) hack/import-flux2-assets.sh
 
 serve: gen-content prereqs ## Spawns a development server.
 	hugo server \
@@ -51,7 +66,7 @@ preview-build: gen-content prereqs ## Builds a preview build (for e.g. a pull re
 branch-build: gen-content prereqs ## Builds a Git branch (for e.g. development branches).
 	hugo \
 		--baseURL $(DEPLOY_PRIME_URL) \
-		--environment branch \
+		--buildDrafts \
 		--buildFuture \
 		--gc \
 		--minify \
@@ -63,7 +78,7 @@ docker-preview: docker-serve
 .PHONY: docker-serve
 docker-serve:
 	docker run -v $(shell pwd):/site -p 1313:1313 -it $(SUPPORT_IMAGE_NAME) \
-		make \"MAKEFLAGS=$(MAKEFLAGS)\" serve HUGO_BIND_ADDRESS=0.0.0.0
+		make \"MAKEFLAGS=$(MAKEFLAGS)\" serve HUGO_BIND_ADDRESS=0.0.0.0 GITHUB_TOKEN=$(GITHUB_TOKEN) BRANCH=$(BRANCH)
 
 .PHONY: docker-push docker-push-hugo docker-push-support
 docker-push: docker-push-hugo docker-push-support

diff --git a/hack/import-flux2-assets.sh b/hack/import-flux2-assets.sh
@@ -1,9 +1,11 @@
-#!/bin/sh -x
+#!/usr/bin/env bash
+
+set -euxo pipefail
 
 COMPONENTS_DIR="content/en/flux/components"
 FLUX_DIR="content/en/flux/cmd"
 
-if [ -z "${GITHUB_USER}" ]; then
+if [ -z "${GITHUB_USER:-}" ]; then
     GITHUB_USER=fluxcdbot
 fi
 
@@ -19,7 +21,7 @@ fatal() {
 
 # Set os, fatal if operating system not supported
 setup_verify_os() {
-    if [ -z "${OS}" ]; then
+    if [ -z "${OS:-}" ]; then
         OS=$(uname)
     fi
     case ${OS} in
@@ -36,7 +38,7 @@ setup_verify_os() {
 
 # Set arch, fatal if architecture not supported
 setup_verify_arch() {
-    if [ -z "${ARCH}" ]; then
+    if [ -z "${ARCH:-}" ]; then
         ARCH=$(uname -m)
     fi
     case ${ARCH} in
@@ -59,21 +61,14 @@ setup_verify_arch() {
 
 
 controller_version() {
-  url=https://api.github.com/repos/fluxcd/$1/releases
-  out=$(curl -w "%{http_code}" -o /tmp/releases -u "$GITHUB_USER:$GITHUB_TOKEN" -s "$url")
-  if [ "${out}" -ne "200" ] ; then
-      echo "error fetching releases from $url: $out"
-      cat /tmp/releases
-      exit 1
-  fi
-
-  jq -r '.[] | .tag_name' < /tmp/releases | sed 's/.*\///' | awk '{ if ($1 ~ /-/) print; else print $0"_" ; }' | sort -rV | sed 's/_$//' | head -n1
+  ks_url=$(curl -sL "https://raw.githubusercontent.com/fluxcd/flux2/v${VERSION_FLUX}/manifests/bases/${1}/kustomization.yaml" | yq '.resources[]|select(. == "*crds.yaml*")')
+  echo "${ks_url}" | cut -d/ -f8
 }
 
 gen_crd_doc() {
   URL="$1"
   DEST="$2"
-  HUGETABLE="$3"
+  HUGETABLE="${3:-}"
 
   TMP="$(mktemp)"
   curl -u "$GITHUB_USER:$GITHUB_TOKEN" -# -Lf "$URL" > "$TMP"
@@ -85,7 +80,7 @@ gen_crd_doc() {
   # 3. While others go     # Here is the title you're looking for...
   #
 
-  FIRST_LINE="$(grep -vE "^<!--" "$TMP" | head -n1)"
+  FIRST_LINE="$(grep -vEm1 "^<!--" "$TMP")"
   if echo "$FIRST_LINE" | grep -q "<h1>" ; then
     TITLE="$(echo "$FIRST_LINE" | cut -d'<' -f2 | cut -d'>' -f2 | sed 's/^\#\ //')"
   elif echo "$FIRST_LINE" | grep -E "^# "; then
@@ -95,7 +90,7 @@ gen_crd_doc() {
     exit 1
   fi
 
-  WEIGHT="$(grep -E '^<!-- menuweight:[[:digit:]]+ -->$' "$TMP" | cut -d' ' -f2|cut -d':' -f2)"
+  WEIGHT="$(grep -E '^<!-- menuweight:[[:digit:]]+ -->$' "$TMP" | cut -d' ' -f2|cut -d':' -f2 || true)"
   if [ -z "${WEIGHT}" ] ; then
     WEIGHT=0
   fi
@@ -119,6 +114,44 @@ gen_crd_doc() {
   fi
 }
 
+{
+  # get flux cmd docs
+  setup_verify_os
+  setup_verify_arch
+
+  TMP="$(mktemp -d)"
+  TMP_METADATA="$TMP/flux.json"
+  TMP_BIN="$TMP/flux.tar.gz"
+
+  if [[ "${BRANCH}" =~ ^pull/[[:digit:]]*/head$ ]] ; then
+    BRANCH=$(curl -sSfL "https://api.github.com/repos/fluxcd/website/pulls/$(echo ${BRANCH}|cut -d/ -f2)" | jq .base.ref -r)
+  fi
+
+  page=1
+  while [ -z "${VERSION_FLUX:-}" ] ; do
+    tags=$(curl -u "${GITHUB_USER}:${GITHUB_TOKEN}" -sSfL "https://api.github.com/repos/fluxcd/flux2/releases?page=${page}" | jq .[].tag_name -r) || false
+    for tag in $tags ; do
+      if [ "${BRANCH}" = "main" ] ; then
+        VERSION_FLUX="${tag#v}"
+        break
+      fi
+      if [ "${tag%.*}" = "${BRANCH/-/.}" ] ; then
+        VERSION_FLUX="${tag#v}"
+        break
+      fi
+    done
+    ((page = page + 1))
+  done
+
+  curl -u "$GITHUB_USER:$GITHUB_TOKEN" -o "${TMP_BIN}" --retry 3 -sSfL "https://github.com/fluxcd/flux2/releases/download/v${VERSION_FLUX}/flux_${VERSION_FLUX}_${OS}_${ARCH}.tar.gz"
+  tar xfz "${TMP_BIN}" -C "${TMP}"
+
+  rm -rf "${FLUX_DIR:?}/*"
+  "${TMP}/flux" docgen --path "${FLUX_DIR}"
+
+  rm -rf "$TMP"
+}
+
 {
   # source-controller CRDs
   SOURCE_VER="$(controller_version source-controller)"
@@ -170,27 +203,6 @@ gen_crd_doc() {
   gen_crd_doc "https://raw.githubusercontent.com/fluxcd/image-automation-controller/$IMG_AUTO_VER/docs/spec/v1beta1/imageupdateautomations.md" "$COMPONENTS_DIR/image/imageupdateautomations.md"
 }
 
-{
-  # get flux cmd docs
-  setup_verify_os
-  setup_verify_arch
-
-  TMP="$(mktemp -d)"
-  TMP_METADATA="$TMP/flux.json"
-  TMP_BIN="$TMP/flux.tar.gz"
-
-  curl -u "$GITHUB_USER:$GITHUB_TOKEN" -o "${TMP_METADATA}" --retry 3 -sSfL "https://api.github.com/repos/fluxcd/flux2/releases/latest"
-  VERSION_FLUX=$(grep '"tag_name":' "${TMP_METADATA}" | sed -E 's/.*"([^"]+)".*/\1/' | cut -c 2-)
-
-  curl -u "$GITHUB_USER:$GITHUB_TOKEN" -o "${TMP_BIN}" --retry 3 -sSfL "https://github.com/fluxcd/flux2/releases/download/v${VERSION_FLUX}/flux_${VERSION_FLUX}_${OS}_${ARCH}.tar.gz"
-  tar xfz "${TMP_BIN}" -C "${TMP}"
-
-  rm -rf "${FLUX_DIR:?}/*"
-  "${TMP}/flux" docgen --path "${FLUX_DIR}"
-
-  rm -rf "$TMP"
-}
-
 {
   # provide Flux install script
   if [ ! -d static ]; then