register a secret in the harness cluster (#206)

currently the `k3s` harness wires in all the necessary auth at the
`containerd` level.

this means the cluster can transparently pull private images without the
need for cluster namespace/sa scoped secrets in the traditional sense.

this has been working great, but is limited in that only `containerd`
can leverage the creds when pulling the image(s). since not all
applications can leverage this store, this PR also uses those same creds
but stores them as a secret scoped to the cluster.

the secret created is by default benign:
`imagetest-registry-auth/kube-system`. everything will still use the
default containerd creds to pull images. but creating this secret allows
for things like `post_hooks` to copy the secret around to the various
applications that may need to leverage these pull creds.

the best example for this is something like crossplane, which hijacks
the method of using `kubernetes.io/dockerconfigjson` type secrets to
handle application level credentials.

internal/harness/k3s/k3s.go

@@ -3,6 +3,7 @@ package k3s
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"html/template"
 	"io"
@@ -11,12 +12,16 @@ import (
 
 	"github.com/chainguard-dev/terraform-provider-imagetest/internal/docker"
 	"github.com/chainguard-dev/terraform-provider-imagetest/internal/harness"
+	"github.com/docker/cli/cli/config/configfile"
+	dtypes "github.com/docker/cli/cli/config/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/mount"
 	"github.com/docker/go-connections/nat"
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/google/uuid"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
@@ -215,15 +220,6 @@ rules:
 		return nil, fmt.Errorf("adding k3s service teardown to stack: %w", err)
 	}
 
-	// Run the post start hooks
-	for _, hook := range h.Hooks.PostStart {
-		if err := resp.Run(ctx, harness.Command{
-			Args: hook,
-		}); err != nil {
-			return nil, fmt.Errorf("running post start hook: %w", err)
-		}
-	}
-
 	kcfg, err := h.kubeconfig(ctx, resp, func(cfg *api.Config) error {
 		if resp.NetworkSettings == nil && resp.NetworkSettings.Networks == nil {
 			return fmt.Errorf("no network settings found")
@@ -257,6 +253,20 @@ rules:
 		return nil, fmt.Errorf("creating kubernetes client: %w", err)
 	}
 
+	// Add the registries auth as a secret to the cluster
+	if err := h.registrySecret(ctx); err != nil {
+		return nil, fmt.Errorf("adding registry secret: %w", err)
+	}
+
+	// Run the post start hooks after we're all done with the cluster setup
+	for _, hook := range h.Hooks.PostStart {
+		if err := resp.Run(ctx, harness.Command{
+			Args: hook,
+		}); err != nil {
+			return nil, fmt.Errorf("running post start hook: %w", err)
+		}
+	}
+
 	return resp, nil
 }
 
@@ -412,6 +422,43 @@ func (h *k3s) kubeconfig(ctx context.Context, resp *docker.Response, config func
 	return clientcmd.Write(*kcfg)
 }
 
+func (h *k3s) registrySecret(ctx context.Context) error {
+	dockerconfig := configfile.ConfigFile{
+		AuthConfigs: make(map[string]dtypes.AuthConfig),
+	}
+
+	for name, reg := range h.Service.Registries {
+		dockerconfig.AuthConfigs[name] = dtypes.AuthConfig{
+			Username: reg.Auth.Username,
+			Password: reg.Auth.Password,
+			Auth:     reg.Auth.Auth,
+		}
+	}
+
+	dockerConfigJSON, err := json.Marshal(dockerconfig)
+	if err != nil {
+		return fmt.Errorf("marshaling docker config: %w", err)
+	}
+
+	ns := "kube-system"
+
+	_, err = h.kcli.CoreV1().Secrets(ns).Create(ctx, &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "imagetest-registry-auth",
+			Namespace: ns,
+		},
+		Type: corev1.SecretTypeDockerConfigJson,
+		Data: map[string][]byte{
+			".dockerconfigjson": dockerConfigJSON,
+		},
+	}, metav1.CreateOptions{})
+	if err != nil {
+		return fmt.Errorf("creating registry secret: %w", err)
+	}
+
+	return nil
+}
+
 func tmpl(tpl string, data interface{}) (string, error) {
 	t, err := template.New("config").Parse(tpl)
 	if err != nil {

internal/provider/harness_k3s_resource_test.go

@@ -245,6 +245,15 @@ resource "imagetest_harness_k3s" "test" {
   name = "test"
   inventory = data.imagetest_inventory.this
 
+  registries = {
+    "foo" = {
+      auth = {
+        username = "testuser"
+        password = "testpass"
+      }
+    }
+  }
+
   hooks = {
     post_start = [
       "echo 'post' > /tmp/hi",
@@ -254,6 +263,7 @@ resource "imagetest_harness_k3s" "test" {
   provisioner "local-exec" {
     command = <<EOF
 docker exec ${self.id} sh -c "cat /tmp/hi"
+docker exec ${self.id} sh -c "kubectl get secret -n kube-system imagetest-registry-auth -o json"
       EOF
   }
 }