[CI] Various CI improvements

.dockerignore

@@ -1,3 +1,4 @@
 ./.dapper
 ./.cache
 ./dist
+./build

.github/workflows/e2e/scripts/create-project-namespace.sh

@@ -5,8 +5,13 @@ source $(dirname $0)/entry
 
 cd $(dirname $0)/../../../..
 
-kubectl create namespace e2e-hpo || true
-kubectl label namespace e2e-hpo field.cattle.io/projectId=p-example --overwrite
+USE_RANCHER=${USE_RANCHER:-"false"}
+if [ "$USE_RANCHER" = "true" ]; then
+  kubectl apply -f ./examples/helm-project-operator/ci/project.yaml
+fi
+
+kubectl apply -f ./examples/helm-project-operator/ci/namespace.yaml
+
 sleep "${DEFAULT_SLEEP_TIMEOUT_SECONDS}"
 if ! kubectl get namespace cattle-project-p-example; then
     echo "ERROR: Expected cattle-project-p-example namespace to exist after ${DEFAULT_SLEEP_TIMEOUT_SECONDS} seconds, not found"

.github/workflows/e2e/scripts/create-projecthelmchart.sh

@@ -5,7 +5,7 @@ source $(dirname $0)/entry
 
 cd $(dirname $0)/../../../..
 
-kubectl apply -f ./examples/helm-project-operator/ci-example.yaml
+kubectl apply -f ./examples/helm-project-operator/ci/project-helm-chart.yaml
 sleep ${DEFAULT_SLEEP_TIMEOUT_SECONDS};
 
 if ! kubectl get -n cattle-helm-system job/helm-install-project-operator-example-chart-dummy; then

.github/workflows/e2e/scripts/delete-projecthelmchart.sh

@@ -5,7 +5,7 @@ source $(dirname $0)/entry
 
 cd $(dirname $0)/../../../..
 
-kubectl delete -f ./examples/helm-project-operator/ci-example.yaml
+kubectl delete -f ./examples/helm-project-operator/ci/project-helm-chart.yaml
 if kubectl get -n cattle-helm-system job/helm-delete-project-operator-example-chart-dummy --ignore-not-found; then
     if ! kubectl wait --for=condition=complete --timeout="${KUBECTL_WAIT_TIMEOUT}" -n cattle-helm-system job/helm-delete-project-operator-example-chart-dummy; then
         echo "ERROR: Helm Uninstall Job for Example Chart never completed after ${KUBECTL_WAIT_TIMEOUT}"

.github/workflows/hl-ci.yaml

@@ -1,6 +1,7 @@
 name: "[helm-locker] CI"
 
 on:
+  workflow_call:
   pull_request:
   push:
     branches:

.github/workflows/hl-e2e.yaml

@@ -14,7 +14,7 @@ on:
       - 'pkg/helm-locker/**'
 
 env:
-  CLUSTER_NAME : test-cluster
+  CLUSTER_NAME : e2e-ci-helm-locker
   K3S_VERSION : v1.27.9-k3s1
 
 jobs:
@@ -32,12 +32,12 @@ jobs:
         run : |
           curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
           sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
-      - name: Set up k3d
-        run : ./.github/workflows/e2e/scripts/install-k3d.sh
       - name: build
         run: BUILD_TARGET=helm-locker make build
-      - name : Setup cluster
-        run :  CLUSTER_NAME=${{ env.CLUSTER_NAME }} K3S_VERSION=${{ env.K3S_VERSION }} ./scripts/setup-cluster.sh
+      - name : Install k3d
+        run : ./.github/workflows/e2e/scripts/install-k3d.sh
+      - name : Setup k3d cluster
+        run : ./.github/workflows/e2e/scripts/setup-cluster.sh
       # temporary hack to run the helm-locker controller in the k3d cluster
       - name : run helm-locker
         run : |

.github/workflows/hpo-ci.yaml

@@ -2,11 +2,20 @@ name: "[helm-project-operator] ci"
 
 env:
   CGO_ENABLED: 0
-  YQ_VERSION: v4.25.1
+  YQ_VERSION: v4.44.3
+
+on:
+  pull_request:
+  workflow_call:
+    inputs:
+      skip_build:
+        default: 'false'
+        required: false
+        type: string
 
-on: [ pull_request ]
 permissions:
   contents: read
+
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -30,6 +39,7 @@ jobs:
       - name: Perform CI
         run : make ci
   build-images:
+    if: ${{ inputs.skip_build != 'true' }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

.github/workflows/hpo-e2e-ci.yaml

@@ -26,11 +26,12 @@ on:
       - 'Makefile'
 
 env:
+  CLUSTER_NAME: e2e-ci-helm-project-operator
   CGO_ENABLED: 0
-  YQ_VERSION: v4.25.1
+  YQ_VERSION: v4.44.3
   E2E_CI: true
   REPO: rancher
-  TAG: dev
+  TAG: v0.0.0-dev.1
   APISERVER_PORT: 8001
   DEFAULT_SLEEP_TIMEOUT_SECONDS: 10
   KUBECTL_WAIT_TIMEOUT: 120s
@@ -47,6 +48,8 @@ jobs:
         k3s_version:
           # k3d version list k3s | sed 's/+/-/' | sort -h
           - ${{ github.event.inputs.k3s_version || 'v1.20.15-k3s1' }}
+    env:
+      K3S_VERSION: ${{ matrix.k3s_version }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -62,21 +65,16 @@ jobs:
         run: |
           REPO=${REPO} TAG=${TAG} ./scripts/build;
           REPO=${REPO} TAG=${TAG} ./scripts/package;
-      - name: Provision k3d Cluster
-        uses: AbsaOSS/k3d-action@v2
-        # k3d will automatically create a network named k3d-test-cluster-1 with the range 172.18.0.0/16
-        with:
-          cluster-name: "e2e-ci-helm-project-operator"
-          args: >-
-            --agents 1
-            --network "nw01"
-            --image docker.io/rancher/k3s:${{matrix.k3s_version}}
+      - name : Install k3d
+        run : ./.github/workflows/e2e/scripts/install-k3d.sh
+      - name : Setup k3d cluster
+        run : ./.github/workflows/e2e/scripts/setup-cluster.sh
       - name: Import Images Into k3d
         run: |
-          k3d image import ${REPO}/helm-project-operator:${TAG} -c e2e-ci-helm-project-operator;
+          k3d image import ${REPO}/helm-project-operator:${TAG} -c "$CLUSTER_NAME";
       - name: Setup kubectl context
         run: |
-          kubectl config use-context k3d-e2e-ci-helm-project-operator;
+          kubectl config use-context "k3d-$CLUSTER_NAME";
       - name: Set Up Tmate Debug Session
         if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.enable_tmate == 'true' }}
         uses: mxschmitt/action-tmate@v3
@@ -103,4 +101,4 @@ jobs:
 
       - name: Delete k3d cluster
         if: always()
-        run: k3d cluster delete e2e-ci-helm-project-operator
+        run: k3d cluster delete "$CLUSTER_NAME";

.github/workflows/publish.yaml

@@ -8,11 +8,65 @@ on:
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
+  YQ_VERSION: v4.44.3
 
 jobs:
+  helm-locker-ci:
+    uses: ./.github/workflows/hl-ci.yaml
+  helm-project-operator-ci:
+    uses: ./.github/workflows/hpo-ci.yaml
+    with:
+      skip_build: 'true'
+  goreleaser:
+    name: Build go binaries and helm chart
+    runs-on : ubuntu-latest
+    needs: [
+      helm-locker-ci,
+      helm-project-operator-ci,
+    ]
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      GORELEASER_CURRENT_TAG: ${{ github.ref_name }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - run: git fetch --force --tags
+      - name : setup Go
+        uses : actions/setup-go@v5
+        with:
+          go-version: 1.22
+      - name: Install mikefarah/yq
+        run: |
+          sudo wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq;
+      - name : Install helm
+        run : |
+          curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
+          chmod 700 get_helm.sh
+          ./get_helm.sh
+          helm version
+      - name: Package release helm charts
+        run: make package-helm
+      - run: mkdir -p ./build/artifacts/ && mv -v ./dist/artifacts/ ./build/
+      - name: Prepare embedded helm chart
+        run: make build-chart
+      - uses: goreleaser/goreleaser-action@v6
+        with:
+          distribution: goreleaser
+          version: latest
+          args: release --clean
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GORELEASER_CURRENT_TAG: ${{ github.ref_name }}
   push:
+    needs: [
+      helm-locker-ci,
+      helm-project-operator-ci,
+    ]
     name : Build and push helm-locker & Helm-Project-Operator images
     runs-on : ubuntu-latest
+    env:
+      TAG: ${{ needs.prebuild-env.outputs.branch_static_tag }}
     permissions:
       contents: read
       packages: write
@@ -23,18 +77,12 @@ jobs:
         uses: actions/checkout@v4
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
-      - name : Setup go
-        uses: actions/setup-go@v5
-        with:
-          go-version: 1.22
       - name: Log in to the Container registry
         uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      - name : Build, test & validate
-        run : BUILD_TARGET=helm-locker make ci
       - name: Extract metadata (tags, labels) for helm-locker image
         id: meta-locker
         uses: docker/metadata-action@v5

.gitignore

@@ -1,10 +1,12 @@
 /.dapper
 /.cache
 /bin
+/build
 !/cmd/helm-project-operator/fs/.gitkeep
 /cmd/helm-project-operator/fs/*
 /dist
 *.swp
 .idea
 /helm-project-operator
 /.vscode
+get_helm.sh

.goreleaser.yaml

@@ -0,0 +1,63 @@
+# Make sure to check the documentation at https://goreleaser.com
+version: 2 # the goreleaser config version
+before:
+  hooks:
+    - go mod tidy
+builds:
+  - id: helm-locker
+    main: ./cmd/helm-locker/main.go
+    goos:
+      - linux
+    goarch:
+      - amd64
+      - arm64
+    binary: helm-locker
+    ldflags:
+      - -extldflags
+      - -static
+      - -s
+      - -X main.Version={{.Version}} -X main.GitCommit={{.Commit}}
+    flags:
+      - -trimpath
+    env:
+      - CGO_ENABLED=0
+  - id: helm-project-operator
+    main: ./cmd/helm-project-operator/main.go
+    goos:
+      - linux
+    goarch:
+      - amd64
+      - arm64
+    binary: helm-project-operator
+    ldflags:
+      - -extldflags
+      - -static
+      - -s
+      - -X main.Version={{.Version}} -X main.GitCommit={{.Commit}}
+    flags:
+      - -trimpath
+    env:
+      - CGO_ENABLED=0
+archives:
+  - id: helm-project-operator
+    builds:
+      - helm-project-operator
+    name_template: '{{ .Binary }}-{{ .Arch }}'
+  - id: helm-locker
+    builds:
+      - helm-locker
+    name_template: '{{ .Binary }}-{{ .Arch }}'
+release:
+  prerelease: auto
+  extra_files:
+    - glob : ./build/artifacts/*.tgz
+checksum:
+  name_template: 'checksums.txt'
+snapshot:
+  version_template: "{{ incpatch .Version }}-next"
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - '^docs:'
+      - '^test:'

Dockerfile.dapper

@@ -4,6 +4,8 @@ RUN git -C / clone --branch release-v3.9.0 --depth=1 https://github.com/rancher/
 RUN make -C /helm
 
 FROM registry.suse.com/bci/golang:1.22
+# Add buildx plugin
+COPY --from=docker.io/docker/buildx-bin:0.17.1 /buildx /usr/libexec/docker/cli-plugins/docker-buildx
 
 ARG DAPPER_HOST_ARCH
 ENV ARCH $DAPPER_HOST_ARCH
@@ -18,12 +20,20 @@ ENV YQ_VERSION=v4.30.8
 RUN wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${ARCH} -O /usr/bin/yq && chmod +x /usr/bin/yq;
 
 COPY --from=helm ./helm/bin/helm /usr/local/bin/
+RUN if [ "${ARCH}" != "s390x" ]; then \
+        helm plugin install https://github.com/quintush/helm-unittest; \
+    fi
 
-ENV DAPPER_ENV REPO TAG DRONE_TAG CROSS
+ENV DAPPER_ENV REPO TAG HELM_IMAGE_TAG HELM_CHART_VERSION HELM_CHART_VERSION_DEV CROSS CROSS_ARCH USE_DOCKER_BUILDX TARGET_ARCH
 ENV DAPPER_SOURCE /go/src/github.com/rancher/helm-project-operator/
 ENV DAPPER_OUTPUT ./bin ./dist
-ENV DAPPER_DOCKER_SOCKET true
+ENV GOCACHE /root/.cache/go-build
 ENV GOPATH /go
+ENV DAPPER_DOCKER_SOCKET true
+ENV DAPPER_RUN_ARGS --privileged
+VOLUME /var/lib/rancher/k3s
+VOLUME /var/lib/cni
+VOLUME /var/log
 ENV HOME ${DAPPER_SOURCE}
 WORKDIR ${DAPPER_SOURCE}
 

Makefile

@@ -1,8 +1,25 @@
-TARGETS := $(shell ls scripts)
+TARGETS := $(shell ls scripts|grep -ve "^util-\|entry")
+LOCAL_TARGETS := $(addprefix local-,$(TARGETS))
 
+.dapper:
+	@echo Downloading dapper
+	@curl -sL https://releases.rancher.com/dapper/latest/dapper-$$(uname -s)-$$(uname -m) > .dapper.tmp
+	@@chmod +x .dapper.tmp
+	@./.dapper.tmp -v
+	@mv .dapper.tmp .dapper
+
+# Default behavior for targets without dapper
 $(TARGETS):
-	./scripts/$@
+	@scripts/$@
+
+# Behavior for targets prefixed with "local-" using dapper
+$(LOCAL_TARGETS): local-%: .dapper
+	./.dapper $(@:local-%=%)
 
 .DEFAULT_GOAL := default
+.PHONY: $(TARGETS) $(LOCAL_TARGETS) list
 
-.PHONY: $(TARGETS)
+list:
+	@LC_ALL=C $(MAKE) -pRrq -f $(firstword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/(^|\n)# Files(\n|$$)/,/(^|\n)# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | grep -E -v -e '^[^[:alnum:]]' -e '^$@$$'
+# IMPORTANT: The line above must be indented by (at least one)
+#            *actual TAB character* - *spaces* do *not* work.