diff --git a/.github/workflows/benchmark-pipeline-test.yml b/.github/workflows/benchmark-pipeline-test.yml new file mode 100644 index 0000000..b7b89e8 --- /dev/null +++ b/.github/workflows/benchmark-pipeline-test.yml @@ -0,0 +1,24 @@ +name: Benchmark Pipeline Test + +on: + push: + +concurrency: + group: benchmark + +jobs: + pipeline: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: dagger/dagger-for-github@v7 + with: + module: "." + version: "0.15.1" + args: benchmark-pipeline-test + --source='.' + --cncf-project='falco' + --config='modern-ebpf' + --version='0.39.2' + --benchmark-job-url='https://raw.githubusercontent.com/falcosecurity/cncf-green-review-testing/e93136094735c1a52cbbef3d7e362839f26f4944/benchmark-tests/falco-benchmark-tests.yaml' + --benchmark-job-duration-mins=2 diff --git a/.gitignore b/.gitignore index d8b6cc0..891c1fd 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ infrastructure/equinix-metal/.terraform.lock.hcl /internal/dagger /internal/querybuilder /internal/telemetry +kind-in-cluster diff --git a/Dockerfile b/Dockerfile index b25e626..71f207c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,6 @@ FROM alpine:3.21 -RUN apk add ca-certificates kubectl --no-cache +RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \ + apk update + +RUN apk add ca-certificates flux kubectl --no-cache diff --git a/clusters/base/kepler.yaml b/clusters/base/kepler.yaml index 13eb5f6..3a1a9b6 100644 --- a/clusters/base/kepler.yaml +++ b/clusters/base/kepler.yaml @@ -28,7 +28,7 @@ spec: chart: spec: chart: kepler - version: '0.5.3' + version: '0.5.12' sourceRef: kind: HelmRepository name: kepler diff --git a/dagger.json b/dagger.json index 60e8246..897814f 100644 --- a/dagger.json +++ b/dagger.json @@ -2,5 +2,12 @@ "name": "green-reviews-tooling", "engineVersion": "v0.15.1", "sdk": "go", + "dependencies": [ + { + "name": "k3s", + "source": "github.com/marcosnils/daggerverse/k3s@k3s/v0.1.7", + "pin": "833ec36632b2457862f6e3bf1f7107ad65e3e515" + } + ], "source": "." } diff --git a/docs/dagger/README.md b/docs/dagger/README.md new file mode 100644 index 0000000..4ee109d --- /dev/null +++ b/docs/dagger/README.md @@ -0,0 +1,68 @@ +# Dagger + +Run benchmark pipeline locally using [dagger](https://docs.dagger.io/). + +Docs use `kind` other tooling like minikube or k3d should also work but is untested. + +## Tools + +These CLIs are needed + +- `dagger` https://docs.dagger.io/install +- `helm` https://helm.sh/docs/helm/helm_install/ +- `kind` https://kind.sigs.k8s.io/docs/user/quick-start/ +- `yq` https://github.com/mikefarah/yq/#install + +## Setup + +- Create kind cluster + +```sh +kind create cluster +kind get kubeconfig | yq e '.clusters[0].cluster.server = "https://kubernetes.default"' - > kind-in-cluster +``` + +- Install Dagger engine and configure CLI https://docs.dagger.io/integrations/kubernetes + +```sh +helm upgrade --install --namespace=dagger --create-namespace \ + dagger oci://registry.dagger.io/dagger-helm + +kubectl wait --for condition=Ready --timeout=60s pod \ + --selector=name=dagger-dagger-helm-engine --namespace=dagger + +DAGGER_ENGINE_POD_NAME="$(kubectl get pod \ + --selector=name=dagger-dagger-helm-engine --namespace=dagger \ + --output=jsonpath='{.items[0].metadata.name}')" +export DAGGER_ENGINE_POD_NAME + +_EXPERIMENTAL_DAGGER_RUNNER_HOST="kube-pod://$DAGGER_ENGINE_POD_NAME?namespace=dagger" +export _EXPERIMENTAL_DAGGER_RUNNER_HOST +``` + +## Run pipeline + +- Bootstrap flux and install manifests from [/clusters/base/](/clusters/base/) + +```sh +dagger call setup-cluster --source=. --kubeconfig=/src/kind-in-cluster +``` + +- Run the pipeline and execute tests on completion + +```sh +dagger call benchmark-pipeline-test --source=. --kubeconfig=/src/kind-in-cluster + --cncf-project='falco' + --config='modern-ebpf' + --version='0.39.2' + --benchmark-job-url='https://raw.githubusercontent.com/falcosecurity/cncf-green-review-testing/e93136094735c1a52cbbef3d7e362839f26f4944/benchmark-tests/falco-benchmark-tests.yaml' + --benchmark-job-duration-mins=2 +``` + +## Debugging + +- Get an interactive terminal for trouble shooting + +```sh +dagger call terminal --source=. --kubeconfig=/src/kind-in-cluster +``` diff --git a/main.go b/main.go index 50d0812..038c2d5 100644 --- a/main.go +++ b/main.go @@ -10,6 +10,10 @@ import ( "github.com/cncf-tags/green-reviews-tooling/pkg/pipeline" ) +const ( + clusterName = "green-reviews-test" +) + type GreenReviewsTooling struct{} // BenchmarkPipeline measures the sustainability footprint of CNCF projects. @@ -22,7 +26,7 @@ func (m *GreenReviewsTooling) BenchmarkPipeline(ctx context.Context, benchmarkJobURL, kubeconfig string, benchmarkJobDurationMins int) (*dagger.Container, error) { - p, err := newPipeline(source, kubeconfig) + p, err := newPipeline(ctx, source, kubeconfig) if err != nil { return nil, err } @@ -30,14 +34,90 @@ func (m *GreenReviewsTooling) BenchmarkPipeline(ctx context.Context, return p.Benchmark(ctx, cncfProject, config, version, benchmarkJobURL, benchmarkJobDurationMins) } -func newPipeline(source *dagger.Directory, kubeconfig string) (*pipeline.Pipeline, error) { +// BenchmarkPipelineTest tests the pipeline. +func (m *GreenReviewsTooling) BenchmarkPipelineTest(ctx context.Context, + source *dagger.Directory, + // +optional + // +default="falco" + cncfProject, + // +optional + // +default="modern-ebpf" + config, + // +optional + // +default="0.39.2" + version, + // +optional + // +default="https://raw.githubusercontent.com/falcosecurity/cncf-green-review-testing/e93136094735c1a52cbbef3d7e362839f26f4944/benchmark-tests/falco-benchmark-tests.yaml" + benchmarkJobURL, + // +optional + kubeconfig string, + // +optional + // +default=2 + benchmarkJobDurationMins int) (*dagger.Container, error) { + p, err := newPipeline(ctx, source, kubeconfig) + if err != nil { + return nil, err + } + + if kubeconfig == "" { + _, err = p.SetupCluster(ctx) + if err != nil { + return nil, err + } + } + + return p.Benchmark(ctx, + cncfProject, + config, + version, + benchmarkJobURL, + benchmarkJobDurationMins) +} + +// SetupCluster installs cluster components in an empty cluster for CI/CD and +// local development. +func (m *GreenReviewsTooling) SetupCluster(ctx context.Context, + source *dagger.Directory, + // +optional + kubeconfig string) (*dagger.Container, error) { + p, err := newPipeline(ctx, source, kubeconfig) + if err != nil { + return nil, err + } + + return p.SetupCluster(ctx) +} + +// Terminal returns dagger interactive terminal configured with kubeconfig +// for trouble shooting. +func (m *GreenReviewsTooling) Terminal(ctx context.Context, + source *dagger.Directory, + // +optional + kubeconfig string) (*dagger.Container, error) { + p, err := newPipeline(ctx, source, kubeconfig) + if err != nil { + return nil, err + } + + return p.Terminal(ctx) +} + +func newPipeline(ctx context.Context, source *dagger.Directory, kubeconfig string) (*pipeline.Pipeline, error) { var configFile *dagger.File var err error container := build(source) - configFile, err = getKubeconfig(kubeconfig) - if err != nil { - return nil, err + + if kubeconfig == "" { + configFile, err = startK3sCluster(ctx) + if err != nil { + return nil, err + } + } else { + configFile, err = getKubeconfig(kubeconfig) + if err != nil { + return nil, err + } } return pipeline.New(container, source, configFile) @@ -62,3 +142,14 @@ func getKubeconfig(configFilePath string) (*dagger.File, error) { dir := dag.Directory().WithNewFile(filePath, string(contents)) return dir.File(filePath), nil } + +func startK3sCluster(ctx context.Context) (*dagger.File, error) { + k3s := dag.K3S(clusterName) + kServer := k3s.Server() + + kServer, err := kServer.Start(ctx) + if err != nil { + return nil, err + } + return k3s.Config(), nil +} diff --git a/pkg/cmd/cmd.go b/pkg/cmd/cmd.go index c216fe2..6310db3 100644 --- a/pkg/cmd/cmd.go +++ b/pkg/cmd/cmd.go @@ -1,5 +1,7 @@ package cmd +import "fmt" + func Apply(manifest string) []string { return []string{ "kubectl", @@ -19,6 +21,58 @@ func Delete(manifest string) []string { } } +func FluxInstall() []string { + return []string{ + "flux", + "install", + } +} + +func FluxReconcile(resource, name string) []string { + return []string{ + "flux", + "reconcile", + resource, + name, + } +} + +func GetNodeNames() []string { + return []string{ + "kubectl", + "get", + "node", + "-o", + "name", + } +} + +func LabelNode(nodeName string, labels map[string]string) []string { + args := []string{ + "kubectl", + "label", + nodeName, + } + for k, v := range labels { + args = append(args, fmt.Sprintf("%s=%s", k, v)) + } + return args +} + +func Patch(resource, name, namespace, path, value string) []string { + return []string{ + "kubectl", + "patch", + resource, + name, + "-n", + namespace, + "--type=json", + "-p", + fmt.Sprintf(`[{"op": "add", "path": "%s", "value": %s}]`, path, value), + } +} + func WaitForNamespace(namespace string) []string { return []string{ "kubectl", diff --git a/pkg/pipeline/benchmark_test.go b/pkg/pipeline/benchmark_test.go new file mode 100644 index 0000000..6d3bef8 --- /dev/null +++ b/pkg/pipeline/benchmark_test.go @@ -0,0 +1,24 @@ +package pipeline + +import ( + "context" + "log" + + "github.com/cncf-tags/green-reviews-tooling/internal/dagger" +) + +// BenchmarkTest runs the pipeline and executes tests on completion. +func (p *Pipeline) BenchmarkTest(ctx context.Context, + cncfProject, + config, + version, + benchmarkJobURL string, + benchmarkJobDurationMins int) (*dagger.Container, error) { + _, err := p.Benchmark(ctx, cncfProject, config, version, benchmarkJobURL, benchmarkJobDurationMins) + if err != nil { + log.Printf("benchmark failed: %v", err) + } + + // TODO Add tests. + return p.container, nil +} diff --git a/pkg/pipeline/setup_cluster.go b/pkg/pipeline/setup_cluster.go new file mode 100644 index 0000000..084ce7e --- /dev/null +++ b/pkg/pipeline/setup_cluster.go @@ -0,0 +1,114 @@ +package pipeline + +import ( + "context" + "fmt" + "strings" + + "github.com/cncf-tags/green-reviews-tooling/internal/dagger" + "github.com/cncf-tags/green-reviews-tooling/pkg/cmd" +) + +const ( + monitoringNamespace = "monitoring" +) + +// SetupCluster bootstraps flux and installs manifests from /clusters/base/ for +// CI/CD and local development. +func (p *Pipeline) SetupCluster(ctx context.Context) (*dagger.Container, error) { + // Install flux. + _, err := p.exec(ctx, cmd.FluxInstall()) + if err != nil { + return nil, err + } + + // Add node labels. + nodeName, err := p.getNodeName(ctx) + if err != nil { + return nil, err + } + _, err = p.exec(ctx, cmd.LabelNode(nodeName, nodeLabels())) + if err != nil { + return nil, err + } + + // Apply cluster manifests. + for _, manifest := range clusterManifests() { + _, err = p.execWithDir(ctx, manifest, cmd.Apply(manifest)) + if err != nil { + return nil, err + } + } + + // Patch helmrelease values to ensure all pods will start. + for _, patch := range localSetupPatches() { + _, err = p.exec(ctx, patch) + if err != nil { + return nil, err + } + } + + // Kepler depends on kube-prometheus-stack. + _, err = p.exec(ctx, cmd.FluxReconcile("helmrelease", "kepler")) + if err != nil { + return nil, err + } + + // Wait until all pods are ready. + _, err = p.exec(ctx, cmd.WaitForNamespace(monitoringNamespace)) + if err != nil { + return nil, err + } + + // Enable terminal to debug. + // return p.Terminal(ctx) + + return p.container, nil +} + +func (p *Pipeline) getNodeName(ctx context.Context) (string, error) { + stdout, err := p.exec(ctx, cmd.GetNodeNames()) + if err != nil { + return "", err + } + + parts := strings.Split(stdout, "\n") + if len(parts) == 0 { + return "", fmt.Errorf("failed to get node name from %s", stdout) + } + + return parts[0], nil +} + +func clusterManifests() []string { + return []string{ + // Namespace must be created first for dependencies. + "/clusters/base/monitoring-namespace.yaml", + "/clusters/base", + } +} + +// localSetupPatches patch flux manifests to disable resources not available +// in the k3s container like host mounts. +func localSetupPatches() [][]string { + return [][]string{ + cmd.Patch("helmrelease", + "kube-prometheus-stack", + "flux-system", + "/spec/values/prometheus-node-exporter", + `{"hostRootFsMount": {"enabled": false}}`), + cmd.Patch("helmrelease", + "kepler", + "flux-system", + "/spec/values/canMount", + `{"usrSrc": false}`), + } +} + +func nodeLabels() map[string]string { + return map[string]string{ + "cncf-project": "wg-green-reviews", + "cncf-project-sub": "internal", + "node-role.kubernetes.io/benchmark": "true", + } +}