feat: Add integration test with dagger and docs

Signed-off-by: Ross Fairbanks <[email protected]>

.github/workflows/benchmark-pipeline-test.yml

@@ -0,0 +1,24 @@
+name: Benchmark Pipeline Test
+
+on:
+  push:
+
+concurrency:
+  group: benchmark
+
+jobs:
+  pipeline:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dagger/dagger-for-github@v7
+        with:
+          module: "."
+          version: "0.15.1"
+          args: benchmark-pipeline-test
+              --source='.'
+              --cncf-project='falco'
+              --config='modern-ebpf'
+              --version='0.39.2'
+              --benchmark-job-url='https://raw.githubusercontent.com/falcosecurity/cncf-green-review-testing/e93136094735c1a52cbbef3d7e362839f26f4944/benchmark-tests/falco-benchmark-tests.yaml'
+              --benchmark-job-duration-mins=2

.gitignore

@@ -6,3 +6,4 @@ infrastructure/equinix-metal/.terraform.lock.hcl
 /internal/dagger
 /internal/querybuilder
 /internal/telemetry
+kind-in-cluster

Dockerfile

@@ -1,3 +1,6 @@
 FROM alpine:3.21
 
-RUN apk add ca-certificates kubectl --no-cache
+RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
+    apk update
+
+RUN apk add ca-certificates flux kubectl --no-cache

clusters/base/kepler.yaml

@@ -28,7 +28,7 @@ spec:
   chart:
     spec:
       chart: kepler
-      version: '0.5.3'
+      version: '0.5.12'
       sourceRef:
         kind: HelmRepository
         name: kepler

dagger.json

@@ -2,5 +2,12 @@
   "name": "green-reviews-tooling",
   "engineVersion": "v0.15.1",
   "sdk": "go",
+  "dependencies": [
+    {
+      "name": "k3s",
+      "source": "github.com/marcosnils/daggerverse/k3s@k3s/v0.1.7",
+      "pin": "833ec36632b2457862f6e3bf1f7107ad65e3e515"
+    }
+  ],
   "source": "."
 }

docs/dagger/README.md

@@ -0,0 +1,68 @@
+# Dagger
+
+Run benchmark pipeline locally using [dagger](https://docs.dagger.io/).
+
+Docs use `kind` other tooling like minikube or k3d should also work but is untested.
+
+## Tools
+
+These CLIs are needed
+
+- `dagger` https://docs.dagger.io/install
+- `helm` https://helm.sh/docs/helm/helm_install/
+- `kind` https://kind.sigs.k8s.io/docs/user/quick-start/
+- `yq` https://github.com/mikefarah/yq/#install
+
+## Setup
+
+- Create kind cluster
+
+```sh
+kind create cluster
+kind get kubeconfig | yq e '.clusters[0].cluster.server = "https://kubernetes.default"' - > kind-in-cluster
+```
+
+- Install Dagger engine and configure CLI https://docs.dagger.io/integrations/kubernetes
+
+```sh
+helm upgrade --install --namespace=dagger --create-namespace \
+    dagger oci://registry.dagger.io/dagger-helm
+
+kubectl wait --for condition=Ready --timeout=60s pod \
+    --selector=name=dagger-dagger-helm-engine --namespace=dagger
+
+DAGGER_ENGINE_POD_NAME="$(kubectl get pod \
+    --selector=name=dagger-dagger-helm-engine --namespace=dagger \
+    --output=jsonpath='{.items[0].metadata.name}')"
+export DAGGER_ENGINE_POD_NAME
+
+_EXPERIMENTAL_DAGGER_RUNNER_HOST="kube-pod://$DAGGER_ENGINE_POD_NAME?namespace=dagger"
+export _EXPERIMENTAL_DAGGER_RUNNER_HOST
+```
+
+## Run pipeline
+
+- Bootstrap flux and install manifests from [/clusters/base/](/clusters/base/)
+
+```sh
+dagger call setup-cluster --source=. --kubeconfig=/src/kind-in-cluster
+```
+
+- Run the pipeline and execute tests on completion
+
+```sh
+dagger call benchmark-pipeline-test --source=. --kubeconfig=/src/kind-in-cluster
+   --cncf-project='falco'
+   --config='modern-ebpf'
+   --version='0.39.2'
+   --benchmark-job-url='https://raw.githubusercontent.com/falcosecurity/cncf-green-review-testing/e93136094735c1a52cbbef3d7e362839f26f4944/benchmark-tests/falco-benchmark-tests.yaml'
+   --benchmark-job-duration-mins=2
+```
+
+## Debugging
+
+- Get an interactive terminal for trouble shooting
+
+```sh
+dagger call terminal --source=. --kubeconfig=/src/kind-in-cluster
+```

main.go

@@ -10,6 +10,10 @@ import (
 	"github.com/cncf-tags/green-reviews-tooling/pkg/pipeline"
 )
 
+const (
+	clusterName = "green-reviews-test"
+)
+
 type GreenReviewsTooling struct{}
 
 // BenchmarkPipeline measures the sustainability footprint of CNCF projects.
@@ -22,22 +26,98 @@ func (m *GreenReviewsTooling) BenchmarkPipeline(ctx context.Context,
 	benchmarkJobURL,
 	kubeconfig string,
 	benchmarkJobDurationMins int) (*dagger.Container, error) {
-	p, err := newPipeline(source, kubeconfig)
+	p, err := newPipeline(ctx, source, kubeconfig)
 	if err != nil {
 		return nil, err
 	}
 
 	return p.Benchmark(ctx, cncfProject, config, version, benchmarkJobURL, benchmarkJobDurationMins)
 }
 
-func newPipeline(source *dagger.Directory, kubeconfig string) (*pipeline.Pipeline, error) {
+// BenchmarkPipelineTest tests the pipeline.
+func (m *GreenReviewsTooling) BenchmarkPipelineTest(ctx context.Context,
+	source *dagger.Directory,
+	// +optional
+	// +default="falco"
+	cncfProject,
+	// +optional
+	// +default="modern-ebpf"
+	config,
+	// +optional
+	// +default="0.39.2"
+	version,
+	// +optional
+	// +default="https://raw.githubusercontent.com/falcosecurity/cncf-green-review-testing/e93136094735c1a52cbbef3d7e362839f26f4944/benchmark-tests/falco-benchmark-tests.yaml"
+	benchmarkJobURL,
+	// +optional
+	kubeconfig string,
+	// +optional
+	// +default=2
+	benchmarkJobDurationMins int) (*dagger.Container, error) {
+	p, err := newPipeline(ctx, source, kubeconfig)
+	if err != nil {
+		return nil, err
+	}
+
+	if kubeconfig == "" {
+		_, err = p.SetupCluster(ctx)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return p.Benchmark(ctx,
+		cncfProject,
+		config,
+		version,
+		benchmarkJobURL,
+		benchmarkJobDurationMins)
+}
+
+// SetupCluster installs cluster components in an empty cluster for CI/CD and
+// local development.
+func (m *GreenReviewsTooling) SetupCluster(ctx context.Context,
+	source *dagger.Directory,
+	// +optional
+	kubeconfig string) (*dagger.Container, error) {
+	p, err := newPipeline(ctx, source, kubeconfig)
+	if err != nil {
+		return nil, err
+	}
+
+	return p.SetupCluster(ctx)
+}
+
+// Terminal returns dagger interactive terminal configured with kubeconfig
+// for trouble shooting.
+func (m *GreenReviewsTooling) Terminal(ctx context.Context,
+	source *dagger.Directory,
+	// +optional
+	kubeconfig string) (*dagger.Container, error) {
+	p, err := newPipeline(ctx, source, kubeconfig)
+	if err != nil {
+		return nil, err
+	}
+
+	return p.Terminal(ctx)
+}
+
+func newPipeline(ctx context.Context, source *dagger.Directory, kubeconfig string) (*pipeline.Pipeline, error) {
 	var configFile *dagger.File
 	var err error
 
 	container := build(source)
-	configFile, err = getKubeconfig(kubeconfig)
-	if err != nil {
-		return nil, err
+
+	if kubeconfig == "" {
+		configFile, err = startK3sCluster(ctx)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		configFile, err = getKubeconfig(kubeconfig)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	return pipeline.New(container, source, configFile)
@@ -62,3 +142,14 @@ func getKubeconfig(configFilePath string) (*dagger.File, error) {
 	dir := dag.Directory().WithNewFile(filePath, string(contents))
 	return dir.File(filePath), nil
 }
+
+func startK3sCluster(ctx context.Context) (*dagger.File, error) {
+	k3s := dag.K3S(clusterName)
+	kServer := k3s.Server()
+
+	kServer, err := kServer.Start(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return k3s.Config(), nil
+}

pkg/cmd/cmd.go

@@ -1,5 +1,7 @@
 package cmd
 
+import "fmt"
+
 func Apply(manifest string) []string {
 	return []string{
 		"kubectl",
@@ -19,6 +21,58 @@ func Delete(manifest string) []string {
 	}
 }
 
+func FluxInstall() []string {
+	return []string{
+		"flux",
+		"install",
+	}
+}
+
+func FluxReconcile(resource, name string) []string {
+	return []string{
+		"flux",
+		"reconcile",
+		resource,
+		name,
+	}
+}
+
+func GetNodeNames() []string {
+	return []string{
+		"kubectl",
+		"get",
+		"node",
+		"-o",
+		"name",
+	}
+}
+
+func LabelNode(nodeName string, labels map[string]string) []string {
+	args := []string{
+		"kubectl",
+		"label",
+		nodeName,
+	}
+	for k, v := range labels {
+		args = append(args, fmt.Sprintf("%s=%s", k, v))
+	}
+	return args
+}
+
+func Patch(resource, name, namespace, path, value string) []string {
+	return []string{
+		"kubectl",
+		"patch",
+		resource,
+		name,
+		"-n",
+		namespace,
+		"--type=json",
+		"-p",
+		fmt.Sprintf(`[{"op": "add", "path": "%s", "value": %s}]`, path, value),
+	}
+}
+
 func WaitForNamespace(namespace string) []string {
 	return []string{
 		"kubectl",

pkg/pipeline/benchmark_test.go

@@ -0,0 +1,24 @@
+package pipeline
+
+import (
+	"context"
+	"log"
+
+	"github.com/cncf-tags/green-reviews-tooling/internal/dagger"
+)
+
+// BenchmarkTest runs the pipeline and executes tests on completion.
+func (p *Pipeline) BenchmarkTest(ctx context.Context,
+	cncfProject,
+	config,
+	version,
+	benchmarkJobURL string,
+	benchmarkJobDurationMins int) (*dagger.Container, error) {
+	_, err := p.Benchmark(ctx, cncfProject, config, version, benchmarkJobURL, benchmarkJobDurationMins)
+	if err != nil {
+		log.Printf("benchmark failed: %v", err)
+	}
+
+	// TODO Add tests.
+	return p.container, nil
+}