diff --git a/examples/whoami/whoami.yaml b/examples/whoami/whoami.yaml index 6b4c261..aab15f1 100644 --- a/examples/whoami/whoami.yaml +++ b/examples/whoami/whoami.yaml @@ -19,8 +19,8 @@ spec: spec: containers: - name: whoami - image: "traefik/whoami:latest" - imagePullPolicy: Never + image: traefik/whoami:latest + imagePullPolicy: IfNotPresent ports: - name: http containerPort: 80 diff --git a/pkg/build/builder.go b/pkg/build/builder.go index 52ccebd..b344726 100644 --- a/pkg/build/builder.go +++ b/pkg/build/builder.go @@ -58,12 +58,17 @@ func (b *builder) Build(opts *types.BuildOptions) error { log.Info("Latest k3s version is", opts.K3sVersion) } + imageFormat := types.ImageBundleTar + if opts.CreateRegistry { + imageFormat = types.ImageBundleRegistry + } packageMeta := types.PackageMeta{ - MetaVersion: "v1", - Name: opts.Name, - Version: opts.BuildVersion, - K3sVersion: opts.K3sVersion, - Arch: opts.Arch, + MetaVersion: "v1", + Name: opts.Name, + Version: opts.BuildVersion, + K3sVersion: opts.K3sVersion, + Arch: opts.Arch, + ImageBundleFormat: imageFormat, } if opts.ConfigFile != "" { @@ -186,33 +191,29 @@ func (b *builder) bundleImages(opts *types.BuildOptions, parser types.ManifestPa log.Info("Detected the following images to bundle with the package:", imageNames) + downloader := images.NewImageDownloader() + + var imgRdr io.ReadCloser if opts.CreateRegistry { log.Info("Building private image registry to bundle with the package") - artifacts, err := images.NewImageDownloader().BuildRegistry(&types.BuildRegistryOptions{ + imgRdr, err = downloader.BuildRegistry(&types.BuildRegistryOptions{ Name: opts.Name, AppVersion: opts.BuildVersion, Arch: opts.Arch, Images: imageNames, PullPolicy: opts.PullPolicy, - // TODO: Make more configurable }) - if err != nil { - return err - } - for _, artifact := range artifacts { - if err := b.writer.Put(artifact); err != nil { - return err - } - } - return nil + } else { + log.Info("Exporting images to tar archives to bundle with the package") + // TODO: Switch to opts here as well + imgRdr, err = downloader.SaveImages(imageNames, opts.Arch, opts.PullPolicy) } - - rdr, err := images.NewImageDownloader().SaveImages(imageNames, opts.Arch, opts.PullPolicy) if err != nil { return err } + log.Info("Adding container images to package") - images, err := util.ArtifactFromReader(types.ArtifactImages, types.ManifestUserImagesFile, rdr) + images, err := util.ArtifactFromReader(types.ArtifactImages, types.ManifestUserImagesFile, imgRdr) if err != nil { return err } diff --git a/pkg/cmd/install.go b/pkg/cmd/install.go index 6836a81..b9f1d9e 100644 --- a/pkg/cmd/install.go +++ b/pkg/cmd/install.go @@ -39,6 +39,8 @@ var ( installDockerOpts types.DockerClusterOptions ) +// TODO: Add flags for user to supply registry certs, port, and/or password + func init() { var currentUser *user.User diff --git a/pkg/images/build_registry.go b/pkg/images/build_registry.go index 2bac6ee..04f882a 100644 --- a/pkg/images/build_registry.go +++ b/pkg/images/build_registry.go @@ -1,39 +1,27 @@ package images import ( - "bytes" "context" "errors" "fmt" - "io/ioutil" + "io" "time" dockertypes "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" - "golang.org/x/crypto/bcrypt" + "github.com/tinyzimmer/k3p/pkg/images/registry" "github.com/tinyzimmer/k3p/pkg/log" "github.com/tinyzimmer/k3p/pkg/types" - "github.com/tinyzimmer/k3p/pkg/util" ) -const kubenabImage = "docker.bintray.io/kubenab:0.3.4" - -var requiredRegistryImages = []string{"registry:2", "busybox", kubenabImage} +var requiredRegistryImages = []string{"registry:2", "busybox", registry.KubenabImage} func setOptDefaults(opts *types.BuildRegistryOptions) *types.BuildRegistryOptions { - if opts.RegistrySecret == "" { - opts.RegistrySecret = util.GenerateToken(16) - } - if opts.AppVersion == "" { opts.AppVersion = types.VersionLatest } - if opts.RegistryNodePort == "" { - opts.RegistryNodePort = "30100" - } - if opts.PullPolicy == "" { opts.PullPolicy = types.PullPolicyAlways } @@ -41,7 +29,7 @@ func setOptDefaults(opts *types.BuildRegistryOptions) *types.BuildRegistryOption return opts } -func (d *dockerImageDownloader) BuildRegistry(opts *types.BuildRegistryOptions) ([]*types.Artifact, error) { +func (d *dockerImageDownloader) BuildRegistry(opts *types.BuildRegistryOptions) (io.ReadCloser, error) { opts = setOptDefaults(opts) cli, err := getDockerClient() @@ -50,85 +38,6 @@ func (d *dockerImageDownloader) BuildRegistry(opts *types.BuildRegistryOptions) } defer cli.Close() - regDataImgName := fmt.Sprintf("%s-private-registry-data:%s", opts.Name, opts.AppVersion) - - // Generate certificates for the registry - // TODO: Allow user to supply certificates - log.Info("Generating PKI for registry TLS") - caCert, caPriv, err := generateCACertificate(opts.Name) - if err != nil { - return nil, err - } - registryCert, registryPriv, err := generateRegistryCertificate(caCert, caPriv, opts.Name) - if err != nil { - return nil, err - } - caCertPem, _, err := encodeToPEM(caCert, caPriv) - if err != nil { - return nil, err - } - registryCertPEM, registryKeyPEM, err := encodeToPEM(registryCert, registryPriv) - if err != nil { - return nil, err - } - - caCertificate := &types.Artifact{ - Type: types.ArtifactEtc, - Name: "registry-ca.crt", - Body: ioutil.NopCloser(bytes.NewReader(caCertPem)), - Size: int64(len(caCertPem)), - } - - // Generate htpasswd file for the registry - log.Info("Generating secrets for registry authentication") - passwordBytes, err := bcrypt.GenerateFromPassword([]byte(opts.RegistrySecret), bcrypt.DefaultCost) - if err != nil { - return nil, err - } - htpasswd := append([]byte("registry:"), passwordBytes...) - htpasswd = append(htpasswd, []byte("\n")...) - - // Create a manifest for the registry - log.Info("Generating kubernetes manifests for the private registry") - var buf bytes.Buffer - err = registryTmpl.Execute(&buf, map[string]string{ - "TLSCertificate": string(registryCertPEM), - "TLSPrivateKey": string(registryKeyPEM), - "TLSCACertificate": string(caCertPem), - "RegistryAuthHtpasswd": string(htpasswd), - "KubenabImage": kubenabImage, - "RegistryDataImage": regDataImgName, - "RegistryNodePort": opts.RegistryNodePort, - }) - if err != nil { - return nil, err - } - body := buf.Bytes() - deploymentManifest := &types.Artifact{ - Type: types.ArtifactManifest, - Name: fmt.Sprintf("%s-private-registry-deployment.yaml", opts.Name), - Body: ioutil.NopCloser(bytes.NewReader(body)), - Size: int64(len(body)), - } - - // Generate a registries.yaml - var yamlBuf bytes.Buffer - err = registriesYamlTmpl.Execute(&yamlBuf, map[string]string{ - "Username": "registry", - "Password": opts.RegistrySecret, - "RegistryNodePort": opts.RegistryNodePort, - }) - if err != nil { - return nil, err - } - registriesBody := yamlBuf.Bytes() - registriesYamlArtifact := &types.Artifact{ - Type: types.ArtifactEtc, - Name: "registries.yaml", - Body: ioutil.NopCloser(bytes.NewReader(registriesBody)), - Size: int64(len(registriesBody)), - } - // Ensure all needed images are present userImages := sanitizeImageNameSlice(opts.Images) for _, img := range append(requiredRegistryImages, userImages...) { @@ -228,22 +137,11 @@ func (d *dockerImageDownloader) BuildRegistry(opts *types.BuildRegistryOptions) } // Commit the registry volume container to an image - _, err = cli.ContainerCommit(context.TODO(), volContainerID, dockertypes.ContainerCommitOptions{Reference: regDataImgName}) + _, err = cli.ContainerCommit(context.TODO(), volContainerID, dockertypes.ContainerCommitOptions{Reference: opts.RegistryImageName()}) if err != nil { return nil, err } // Save all images for the registry - rdr, err := cli.ImageSave(context.TODO(), append(requiredRegistryImages, regDataImgName)) - if err != nil { - return nil, err - } - - // Create artifact for registry images - registryArtifact, err := util.ArtifactFromReader(types.ArtifactImages, "private-registry.tar", rdr) - if err != nil { - return nil, err - } - - return []*types.Artifact{caCertificate, registriesYamlArtifact, deploymentManifest, registryArtifact}, nil + return cli.ImageSave(context.TODO(), append(requiredRegistryImages, opts.RegistryImageName())) } diff --git a/pkg/images/registry/generate.go b/pkg/images/registry/generate.go new file mode 100644 index 0000000..96be803 --- /dev/null +++ b/pkg/images/registry/generate.go @@ -0,0 +1,205 @@ +package registry + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "strconv" + "strings" + "time" + + "golang.org/x/crypto/bcrypt" + + "github.com/tinyzimmer/k3p/pkg/log" +) + +// These should be moved to types (or eventual apis) package +const ( + RegistryUser = "registry" + + RegistryNamespace = "kube-system" + RegistryTLSSecret = "registry-tls" + RegistryAuthSecret = "registry-htpasswd" + KubenabTLSSecret = "kubenab-tls" + + RegistryK8sAppName = "private-registry" + KubenabK8sAppName = "kubenab" + + RegistryCAPath = "/etc/rancher/k3s/registry-ca.crt" + + KubenabImage = "docker.bintray.io/kubenab:0.3.4" +) + +// GenerateRegistryAuthSecret will create a kubernetes secret cotaining an htpasswd file +// for registry basic auth. +func GenerateRegistryAuthSecret(secret string) ([]byte, error) { + // Generate htpasswd file for the registry + log.Info("Generating secrets for registry authentication") + passwordBytes, err := bcrypt.GenerateFromPassword([]byte(secret), bcrypt.DefaultCost) + if err != nil { + return nil, err + } + htpasswd := append([]byte(RegistryUser), []byte(":")...) + htpasswd = append(htpasswd, passwordBytes...) + htpasswd = append(htpasswd, []byte("\n")...) + + return executeTemplate(registryAuthSecretTmpl, map[string]interface{}{ + "RegistryAuthSecret": RegistryAuthSecret, + "RegistryNamespace": RegistryNamespace, + "RegistryK8sAppName": RegistryK8sAppName, + "RegistryAuthHtpasswd": string(htpasswd), + }) +} + +// GenerateRegistryDeployments will generate Deployments objects for the registry. +func GenerateRegistryDeployments(dataImageName string) ([]byte, error) { + return executeTemplate(registryDeploymentsTmpl, map[string]interface{}{ + "KubenabImage": KubenabImage, + "KubenabTLSSecret": KubenabTLSSecret, + "KubenabK8sAppName": KubenabK8sAppName, + "RegistryK8sAppName": RegistryK8sAppName, + "RegistryNamespace": RegistryNamespace, + "RegistryAuthSecret": RegistryAuthSecret, + "RegistryTLSSecret": RegistryTLSSecret, + "RegistryDataImage": dataImageName, + }) +} + +// GenerateRegistryServices will generate Service objects for the registry. +func GenerateRegistryServices(port int) ([]byte, error) { + return executeTemplate(registryServicesTmpl, map[string]interface{}{ + "KubenabK8sAppName": KubenabK8sAppName, + "RegistryK8sAppName": RegistryK8sAppName, + "RegistryNamespace": RegistryNamespace, + "RegistryNodePort": strconv.Itoa(port), + }) +} + +// GenerateRegistriesYaml will generate the registries.yaml used to configure containerd. +func GenerateRegistriesYaml(secret string, port int) ([]byte, error) { + return executeTemplate(registriesYamlTmpl, map[string]interface{}{ + "RegistryNodePort": strconv.Itoa(port), + "Username": RegistryUser, + "Password": secret, + "RegistryCAPath": RegistryCAPath, + }) +} + +// GenerateRegistryTLSSecrets will generate secrets and configurations for registry TLS. +func GenerateRegistryTLSSecrets(name string) (caCertBytes, k8sManifests []byte, err error) { + // Generate certificates for the registry + // TODO: Allow user to supply certificates + caCert, caPriv, err := generateCACertificate(name) + if err != nil { + return nil, nil, err + } + registryCert, registryPriv, err := generateRegistryCertificate(caCert, caPriv, name) + if err != nil { + return nil, nil, err + } + caCertPEM, _, err := encodeToPEM(caCert, caPriv) + if err != nil { + return nil, nil, err + } + registryCertPEM, registryKeyPEM, err := encodeToPEM(registryCert, registryPriv) + if err != nil { + return nil, nil, err + } + manifests, err := executeTemplate(registryTLSTmpl, map[string]interface{}{ + "KubenabTLSSecret": KubenabTLSSecret, + "KubenabK8sAppName": KubenabK8sAppName, + "RegistryTLSSecret": RegistryTLSSecret, + "RegistryK8sAppName": RegistryK8sAppName, + "RegistryNamespace": RegistryNamespace, + "TLSCertificate": string(registryCertPEM), + "TLSPrivateKey": string(registryKeyPEM), + "TLSCACertificate": string(caCertPEM), + }) + return caCertPEM, manifests, err +} + +func generateCACertificate(name string) (*x509.Certificate, *rsa.PrivateKey, error) { + // Generate a 4096-bit RSA private key + caPriv, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return nil, nil, err + } + fixName := strings.Replace(name, "_", "-", -1) + caCert := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + CommonName: fmt.Sprintf("%s-registry-ca", fixName), + Organization: []string{fmt.Sprintf("%s-private-registry", fixName)}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().Add(time.Hour * 24 * 365 * 10), // 10 years - obviously needs to be handled better + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + IsCA: true, + } + caDerBytes, err := x509.CreateCertificate(rand.Reader, caCert, caCert, caPriv.Public(), caPriv) + if err != nil { + return nil, nil, err + } + caCertSigned, err := x509.ParseCertificate(caDerBytes) + if err != nil { + return nil, nil, err + } + return caCertSigned, caPriv, nil +} + +func generateRegistryCertificate(caCert *x509.Certificate, caKey *rsa.PrivateKey, name string) (*x509.Certificate, *rsa.PrivateKey, error) { + priv, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return nil, nil, err + } + fixName := strings.Replace(name, "_", "-", -1) + cert := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + CommonName: fmt.Sprintf("%s-private-registry", fixName), + Organization: []string{fmt.Sprintf("%s-private-registry", fixName)}, + }, + DNSNames: []string{"localhost", "kubenab.kube-system.svc"}, + NotBefore: time.Now(), + NotAfter: time.Now().Add(time.Hour * 24 * 365 * 10), // 10 years - obviously needs to be handled better + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + derBytes, err := x509.CreateCertificate(rand.Reader, cert, caCert, priv.Public(), caKey) + if err != nil { + return nil, nil, err + } + certSigned, err := x509.ParseCertificate(derBytes) + if err != nil { + return nil, nil, err + } + return certSigned, priv, nil +} + +func encodeToPEM(rawCert *x509.Certificate, rawKey *rsa.PrivateKey) (cert, key []byte, err error) { + var certout bytes.Buffer + + // encode the certificate + if err := pem.Encode(&certout, &pem.Block{Type: "CERTIFICATE", Bytes: rawCert.Raw}); err != nil { + return nil, nil, err + } + certBytes := certout.Bytes() + + var keyout bytes.Buffer + + // encode the private key + if err := pem.Encode(&keyout, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(rawKey)}); err != nil { + return nil, nil, err + } + keyBytes := keyout.Bytes() + + return certBytes, keyBytes, nil +} diff --git a/pkg/images/registry_templates.go b/pkg/images/registry/templates.go similarity index 65% rename from pkg/images/registry_templates.go rename to pkg/images/registry/templates.go index de58a08..1a9f6e8 100644 --- a/pkg/images/registry_templates.go +++ b/pkg/images/registry/templates.go @@ -1,11 +1,55 @@ -package images +package registry import ( + "bytes" "text/template" "github.com/Masterminds/sprig" ) +func executeTemplate(tmpl *template.Template, vars map[string]interface{}) ([]byte, error) { + var buf bytes.Buffer + if err := tmpl.Execute(&buf, vars); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +var registryServicesTmpl = template.Must(template.New("").Funcs(sprig.TxtFuncMap()).Parse(`--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .RegistryK8sAppName }} + namespace: {{ .RegistryNamespace }} + labels: + k8s-app: {{ .RegistryK8sAppName }} +spec: + type: NodePort + selector: + k8s-app: {{ .RegistryK8sAppName }} + ports: + - port: 5000 + protocol: TCP + targetPort: 5000 + nodePort: {{ .RegistryNodePort }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .KubenabK8sAppName }} + namespace: {{ .RegistryNamespace }} + labels: + k8s-app: {{ .KubenabK8sAppName }} +spec: + selector: + k8s-app: {{ .KubenabK8sAppName }} + type: ClusterIP + ports: + - port: 443 + protocol: "TCP" + name: https +`)) + var registriesYamlTmpl = template.Must(template.New("").Funcs(sprig.TxtFuncMap()).Parse(` mirrors: registry.private: @@ -18,30 +62,30 @@ configs: username: {{ .Username }} password: {{ .Password }} tls: - ca_file: /etc/rancher/k3s/registry-ca.crt + ca_file: {{ .RegistryCAPath }} `)) -var registryTmpl = template.Must(template.New("").Funcs(sprig.TxtFuncMap()).Parse(`--- +var registryAuthSecretTmpl = template.Must(template.New("").Funcs(sprig.TxtFuncMap()).Parse(` apiVersion: v1 kind: Secret metadata: - name: registry-tls - namespace: kube-system + name: {{ .RegistryAuthSecret }} + namespace: {{ .RegistryNamespace }} labels: - k8s-app: private-registry -type: kubernetes.io/tls + k8s-app: {{ .RegistryK8sAppName }} +type: Opaque data: - tls.crt: {{ .TLSCertificate | b64enc }} - tls.key: {{ .TLSPrivateKey | b64enc }} - ca.crt: {{ .TLSCACertificate | b64enc }} ---- + htpasswd: {{ .RegistryAuthHtpasswd | b64enc }} +`)) + +var registryTLSTmpl = template.Must(template.New("").Funcs(sprig.TxtFuncMap()).Parse(`--- apiVersion: v1 kind: Secret metadata: - name: kubenab-tls - namespace: kube-system + name: {{ .RegistryTLSSecret }} + namespace: {{ .RegistryNamespace }} labels: - k8s-app: kubenab + k8s-app: {{ .RegistryK8sAppName }} type: kubernetes.io/tls data: tls.crt: {{ .TLSCertificate | b64enc }} @@ -51,41 +95,69 @@ data: apiVersion: v1 kind: Secret metadata: - name: registry-htpasswd - namespace: kube-system + name: {{ .KubenabTLSSecret }} + namespace: {{ .RegistryNamespace }} labels: - k8s-app: private-registry -type: Opaque + k8s-app: {{ .KubenabK8sAppName }} +type: kubernetes.io/tls data: - htpasswd: {{ .RegistryAuthHtpasswd | b64enc }} + tls.crt: {{ .TLSCertificate | b64enc }} + tls.key: {{ .TLSPrivateKey | b64enc }} + ca.crt: {{ .TLSCACertificate | b64enc }} --- +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: MutatingWebhookConfiguration +metadata: + name: kubenab-mutate +webhooks: +- name: kubenab-mutate.kubenab.com + objectSelector: + matchExpressions: + - key: k8s-app + operator: NotIn + values: ["kube-dns", "kubenab", "private-registry", "metrics-server"] + rules: + - operations: [ "CREATE", "UPDATE" ] + apiGroups: [""] + apiVersions: ["v1"] + resources: ["pods"] + failurePolicy: Fail + clientConfig: + service: + name: kubenab + namespace: kube-system + path: "/mutate" + caBundle: {{ .TLSCACertificate | b64enc }} +`)) + +var registryDeploymentsTmpl = template.Must(template.New("").Funcs(sprig.TxtFuncMap()).Parse(`--- apiVersion: apps/v1 kind: Deployment metadata: - name: private-registry - namespace: kube-system + name: {{ .RegistryK8sAppName }} + namespace: {{ .RegistryNamespace }} labels: - k8s-app: private-registry + k8s-app: {{ .RegistryK8sAppName }} spec: replicas: 1 selector: matchLabels: - k8s-app: private-registry + k8s-app: {{ .RegistryK8sAppName }} template: metadata: labels: - k8s-app: private-registry + k8s-app: {{ .RegistryK8sAppName }} spec: priorityClassName: system-cluster-critical volumes: - name: registry-data emptyDir: {} - - name: registry-tls + - name: {{ .RegistryTLSSecret }} secret: - secretName: registry-tls - - name: registry-htpasswd + secretName: {{ .RegistryTLSSecret }} + - name: {{ .RegistryAuthSecret }} secret: - secretName: registry-htpasswd + secretName: {{ .RegistryAuthSecret }} initContainers: - name: data-extractor image: {{ .RegistryDataImage }} @@ -95,7 +167,7 @@ spec: - name: registry-data mountPath: /var/lib/registry containers: - - name: private-registry + - name: {{ .RegistryK8sAppName }} image: registry:2 imagePullPolicy: Never env: @@ -114,33 +186,33 @@ spec: volumeMounts: - name: registry-data mountPath: /var/lib/registry - - name: registry-tls + - name: {{ .RegistryTLSSecret }} mountPath: /etc/tls/certs readOnly: true - - name: registry-htpasswd + - name: {{ .RegistryAuthSecret }} mountPath: /etc/auth readOnly: true --- apiVersion: apps/v1 kind: Deployment metadata: - name: kubenab - namespace: kube-system + name: {{ .KubenabK8sAppName }} + namespace: {{ .RegistryNamespace }} labels: - k8s-app: kubenab + k8s-app: {{ .KubenabK8sAppName }} spec: selector: matchLabels: - k8s-app: kubenab + k8s-app: {{ .KubenabK8sAppName }} replicas: 1 template: metadata: labels: - k8s-app: kubenab + k8s-app: {{ .KubenabK8sAppName }} spec: priorityClassName: system-cluster-critical containers: - - name: kubenab + - name: {{ .KubenabK8sAppName }} image: {{ .KubenabImage }} imagePullPolicy: Never env: @@ -156,67 +228,10 @@ spec: - containerPort: 443 name: https volumeMounts: - - name: tls + - name: {{ .KubenabTLSSecret }} mountPath: /etc/admission-controller/tls volumes: - - name: tls + - name: {{ .KubenabTLSSecret }} secret: - secretName: kubenab-tls ---- -apiVersion: v1 -kind: Service -metadata: - name: private-registry - namespace: kube-system - labels: - k8s-app: private-registry -spec: - type: NodePort - selector: - k8s-app: private-registry - ports: - - port: 5000 - protocol: TCP - targetPort: 5000 - nodePort: {{ .RegistryNodePort }} ---- -apiVersion: v1 -kind: Service -metadata: - name: kubenab - namespace: kube-system - labels: - k8s-app: kubenab -spec: - selector: - k8s-app: kubenab - type: ClusterIP - ports: - - port: 443 - protocol: "TCP" - name: https ---- -apiVersion: admissionregistration.k8s.io/v1beta1 -kind: MutatingWebhookConfiguration -metadata: - name: kubenab-mutate -webhooks: -- name: kubenab-mutate.kubenab.com - objectSelector: - matchExpressions: - - key: k8s-app - operator: NotIn - values: ["kube-dns", "kubenab", "private-registry", "metrics-server"] - rules: - - operations: [ "CREATE", "UPDATE" ] - apiGroups: [""] - apiVersions: ["v1"] - resources: ["pods"] - failurePolicy: Fail - clientConfig: - service: - name: kubenab - namespace: kube-system - path: "/mutate" - caBundle: {{ .TLSCACertificate | b64enc }} + secretName: {{ .KubenabTLSSecret }} `)) diff --git a/pkg/images/util.go b/pkg/images/util.go index 3639805..c65f160 100644 --- a/pkg/images/util.go +++ b/pkg/images/util.go @@ -1,16 +1,9 @@ package images import ( - "bytes" "context" - "crypto/rand" - "crypto/rsa" - "crypto/x509" - "crypto/x509/pkix" - "encoding/pem" "errors" "fmt" - "math/big" "net/http" "strings" "time" @@ -164,87 +157,6 @@ func getHostPortForContainer(cli *client.Client, containerID string, portProto s return localPort, nil } -func generateCACertificate(name string) (*x509.Certificate, *rsa.PrivateKey, error) { - // Generate a 4096-bit RSA private key - caPriv, err := rsa.GenerateKey(rand.Reader, 4096) - if err != nil { - return nil, nil, err - } - fixName := strings.Replace(name, "_", "-", -1) - caCert := &x509.Certificate{ - SerialNumber: big.NewInt(1), - Subject: pkix.Name{ - CommonName: fmt.Sprintf("%s-registry-ca", fixName), - Organization: []string{fmt.Sprintf("%s-private-registry", fixName)}, - }, - NotBefore: time.Now(), - NotAfter: time.Now().Add(time.Hour * 24 * 365 * 10), // 10 years - obviously needs to be handled better - KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - BasicConstraintsValid: true, - IsCA: true, - } - caDerBytes, err := x509.CreateCertificate(rand.Reader, caCert, caCert, caPriv.Public(), caPriv) - if err != nil { - return nil, nil, err - } - caCertSigned, err := x509.ParseCertificate(caDerBytes) - if err != nil { - return nil, nil, err - } - return caCertSigned, caPriv, nil -} - -func generateRegistryCertificate(caCert *x509.Certificate, caKey *rsa.PrivateKey, name string) (*x509.Certificate, *rsa.PrivateKey, error) { - priv, err := rsa.GenerateKey(rand.Reader, 4096) - if err != nil { - return nil, nil, err - } - fixName := strings.Replace(name, "_", "-", -1) - cert := &x509.Certificate{ - SerialNumber: big.NewInt(1), - Subject: pkix.Name{ - CommonName: fmt.Sprintf("%s-private-registry", fixName), - Organization: []string{fmt.Sprintf("%s-private-registry", fixName)}, - }, - DNSNames: []string{"localhost", "kubenab.kube-system.svc"}, - NotBefore: time.Now(), - NotAfter: time.Now().Add(time.Hour * 24 * 365 * 10), // 10 years - obviously needs to be handled better - KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - BasicConstraintsValid: true, - } - derBytes, err := x509.CreateCertificate(rand.Reader, cert, caCert, priv.Public(), caKey) - if err != nil { - return nil, nil, err - } - certSigned, err := x509.ParseCertificate(derBytes) - if err != nil { - return nil, nil, err - } - return certSigned, priv, nil -} - -func encodeToPEM(rawCert *x509.Certificate, rawKey *rsa.PrivateKey) (cert, key []byte, err error) { - var certout bytes.Buffer - - // encode the certificate - if err := pem.Encode(&certout, &pem.Block{Type: "CERTIFICATE", Bytes: rawCert.Raw}); err != nil { - return nil, nil, err - } - certBytes := certout.Bytes() - - var keyout bytes.Buffer - - // encode the private key - if err := pem.Encode(&keyout, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(rawKey)}); err != nil { - return nil, nil, err - } - keyBytes := keyout.Bytes() - - return certBytes, keyBytes, nil -} - func waitForLocalRegistry(port string, timeout time.Duration) error { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() diff --git a/pkg/install/installer.go b/pkg/install/installer.go index 9b1ac4f..38e8793 100644 --- a/pkg/install/installer.go +++ b/pkg/install/installer.go @@ -2,14 +2,18 @@ package install import ( "bufio" + "bytes" "errors" "fmt" + "io" "io/ioutil" "os" "os/exec" + "path" "strings" "time" + "github.com/tinyzimmer/k3p/pkg/images/registry" "github.com/tinyzimmer/k3p/pkg/log" "github.com/tinyzimmer/k3p/pkg/types" "github.com/tinyzimmer/k3p/pkg/util" @@ -75,6 +79,13 @@ func (i *installer) Install(target types.Node, pkg types.Package, opts *types.In } } + if meta.ImageBundleFormat == types.ImageBundleRegistry { + log.Info("Package was generated with private registry") + if err := setupPrivateRegistry(target, meta, opts); err != nil { + return err + } + } + installedConfig := &types.InstallConfig{InstallOptions: opts} log.Debugf("Built installation config %+v\n", installedConfig) @@ -121,3 +132,63 @@ func promptEULA(eula *types.Artifact, autoAccept bool) error { } } } + +func setupPrivateRegistry(target types.Node, meta *types.PackageMeta, opts *types.InstallOptions) error { + registryManifestPath := path.Join(types.K3sManifestsDir, "private-registry") + + log.Info("Generating PKI for registry TLS") + caCert, secrets, err := registry.GenerateRegistryTLSSecrets(meta.GetName()) + if err != nil { + return err + } + if err := target.WriteFile(nopCloser(caCert), registry.RegistryCAPath, "0644", size(caCert)); err != nil { + return err + } + if err := target.WriteFile(nopCloser(secrets), path.Join(registryManifestPath, "registry-tls-secrets.yaml"), "0644", size(secrets)); err != nil { + return err + } + + log.Info("Writing secrets for registry authentication") + if opts.RegistrySecret == "" { + log.Info("Generating password for registry authentication") + opts.RegistrySecret = util.GenerateToken(16) + } + authSecret, err := registry.GenerateRegistryAuthSecret(opts.RegistrySecret) + if err != nil { + return err + } + if err := target.WriteFile(nopCloser(authSecret), path.Join(registryManifestPath, "registry-auth-secret.yaml"), "0644", size(authSecret)); err != nil { + return err + } + + log.Info("Writing deployments and services for the private registry") + svcs, err := registry.GenerateRegistryServices(opts.GetRegistryNodePort()) + if err != nil { + return err + } + deployments, err := registry.GenerateRegistryDeployments(meta.GetRegistryImageName()) + if err != nil { + return err + } + if err := target.WriteFile(nopCloser(svcs), path.Join(registryManifestPath, "registry-services.yaml"), "0644", size(svcs)); err != nil { + return err + } + if err := target.WriteFile(nopCloser(deployments), path.Join(registryManifestPath, "registry-deployments.yaml"), "0644", size(deployments)); err != nil { + return err + } + + log.Info("Writing containerd configuration for the private registry") + registryConf, err := registry.GenerateRegistriesYaml(opts.RegistrySecret, opts.GetRegistryNodePort()) + if err != nil { + return err + } + if err := target.WriteFile(nopCloser(registryConf), types.K3sRegistriesYamlPath, "0644", size(registryConf)); err != nil { + return err + } + + return nil +} + +func size(b []byte) int64 { return int64(len(b)) } + +func nopCloser(b []byte) io.ReadCloser { return ioutil.NopCloser(bytes.NewReader(b)) } diff --git a/pkg/types/constants.go b/pkg/types/constants.go index 2b77c73..46d5450 100644 --- a/pkg/types/constants.go +++ b/pkg/types/constants.go @@ -48,6 +48,9 @@ const K3sBinDir = "/usr/local/bin" // K3sEtcDir is the directory where configuration files are stored for k3s. const K3sEtcDir = "/etc/rancher/k3s" +// K3sRegistriesYamlPath is the path where the k3s containerd configuration is stored. +const K3sRegistriesYamlPath = "/etc/rancher/k3s/registries.yaml" + // K3sKubeconfig is the path where the admin kubeconfig is stored on the system. const K3sKubeconfig = "/etc/rancher/k3s/k3s.yaml" @@ -66,6 +69,9 @@ const K3pDockerNodeNameLabel = "k3p.io/node-name" // K3pDockerNodeRoleLabel is the label where the node role is placed. const K3pDockerNodeRoleLabel = "k3p.io/node-role" +// DefaultRegistryPort is the default node port used when a package includes a private registry. +const DefaultRegistryPort = 30100 + // K3sRole represents the different roles a machine can take in the cluster type K3sRole string @@ -97,3 +103,14 @@ const ( // ArtifactEtc is an artifact to be placed in /etc/rancher/k3s. ArtifactEtc ArtifactType = "etc" ) + +// ImageBundleFormat declares how the images were bundled in a package. Currently +// either via raw tar balls, or a pre-loaded private registry. +type ImageBundleFormat string + +const ( + // ImageBundleTar represents raw image tarballs. + ImageBundleTar ImageBundleFormat = "raw" + // ImageBundleRegistry represents a pre-loaded private registry. + ImageBundleRegistry ImageBundleFormat = "registry" +) diff --git a/pkg/types/image_downloader.go b/pkg/types/image_downloader.go index 25ddff9..774a2a0 100644 --- a/pkg/types/image_downloader.go +++ b/pkg/types/image_downloader.go @@ -6,11 +6,11 @@ import "io" // them to tar archives or deployable registries. It can be implemented by different // runtimes such as docker, containerd, podman, etc. type ImageDownloader interface { - // SaveImages should return a reader containing the contents of the exported + // SaveImages will return a reader containing the contents of the exported // images provided as arguments. SaveImages(images []string, arch string, pullPolicy PullPolicy) (io.ReadCloser, error) - // BuildRegistry should build a container registry with the given images and return a - // slice of artifacts to be bundled in a package. The artifacts should usually contain - // a container image and manifest for launching it. - BuildRegistry(*BuildRegistryOptions) ([]*Artifact, error) + // BuildRegistry will build a container registry with the given images and return a + // a reader to a container image holding the backed up contents. It will be unpacked into + // a running registry with auto-generated TLS at installation time. + BuildRegistry(*BuildRegistryOptions) (io.ReadCloser, error) } diff --git a/pkg/types/installer.go b/pkg/types/installer.go index 35360da..1687d8a 100644 --- a/pkg/types/installer.go +++ b/pkg/types/installer.go @@ -29,10 +29,12 @@ type InstallOptions struct { KubeconfigMode string // The port that the k3s API server should listen on APIListenPort int - // Extra arguments to pass to the k3s server or agent process that are not included - // in the package. + // Extra arguments to pass to the k3s server process that are not included + // in the package. This includes arguments to the agent running on a server. K3sServerArgs []string - K3sAgentArgs []string + // Extra arguments to pass to the k3s agent process that are not included + // in the package. + K3sAgentArgs []string // Whether to run with --cluster-init InitHA bool // Whether to run as a server or agent @@ -40,23 +42,39 @@ type InstallOptions struct { // Variables contain substitutions to perform on manifests before // installing them to the system. Variables map[string]string + // The password to use for authentication to the registry, if this is blank one will + // be generated. + RegistrySecret string + // The node port that the private registry will listen on when installed. Defaults to + // 30100. + RegistryNodePort int +} + +// GetRegistryNodePort returns the node port to use for a private-registry. +func (opts *InstallOptions) GetRegistryNodePort() int { + if opts.RegistryNodePort != 0 { + return opts.RegistryNodePort + } + return DefaultRegistryPort } // DeepCopy creates a copy of these installation options. func (opts *InstallOptions) DeepCopy() *InstallOptions { newOpts := &InstallOptions{ - NodeName: opts.NodeName, - AcceptEULA: opts.AcceptEULA, - ServerURL: opts.ServerURL, - NodeToken: opts.NodeToken, - ResolvConf: opts.ResolvConf, - KubeconfigMode: opts.KubeconfigMode, - APIListenPort: opts.APIListenPort, - K3sServerArgs: make([]string, len(opts.K3sServerArgs)), - K3sAgentArgs: make([]string, len(opts.K3sAgentArgs)), - InitHA: opts.InitHA, - K3sRole: opts.K3sRole, - Variables: make(map[string]string), + NodeName: opts.NodeName, + AcceptEULA: opts.AcceptEULA, + ServerURL: opts.ServerURL, + NodeToken: opts.NodeToken, + ResolvConf: opts.ResolvConf, + KubeconfigMode: opts.KubeconfigMode, + APIListenPort: opts.APIListenPort, + K3sServerArgs: make([]string, len(opts.K3sServerArgs)), + K3sAgentArgs: make([]string, len(opts.K3sAgentArgs)), + InitHA: opts.InitHA, + K3sRole: opts.K3sRole, + Variables: make(map[string]string), + RegistrySecret: opts.RegistrySecret, + RegistryNodePort: opts.RegistryNodePort, } copy(newOpts.K3sServerArgs, opts.K3sServerArgs) copy(newOpts.K3sAgentArgs, opts.K3sAgentArgs) diff --git a/pkg/types/package_meta.go b/pkg/types/package_meta.go index af4c7ef..3b4acd6 100644 --- a/pkg/types/package_meta.go +++ b/pkg/types/package_meta.go @@ -17,6 +17,8 @@ type PackageMeta struct { K3sVersion string `json:"k3sVersion,omitempty"` // The architecture the package was built for Arch string `json:"arch,omitempty"` + // The format with which images were bundles in the archive. + ImageBundleFormat ImageBundleFormat `json:"imageBundleFormat,omitempty"` // A listing of the contents of the package Manifest *Manifest `json:"manifest,omitempty"` // A configuration containing installation variables @@ -29,12 +31,13 @@ type PackageMeta struct { // TODO: DeepCopy functions need to be generated func (p *PackageMeta) DeepCopy() *PackageMeta { meta := &PackageMeta{ - MetaVersion: p.MetaVersion, - Name: p.Name, - Version: p.Version, - K3sVersion: p.K3sVersion, - Arch: p.Arch, - PackageConfigRaw: make([]byte, len(p.PackageConfigRaw)), + MetaVersion: p.MetaVersion, + Name: p.Name, + Version: p.Version, + K3sVersion: p.K3sVersion, + Arch: p.Arch, + ImageBundleFormat: p.ImageBundleFormat, + PackageConfigRaw: make([]byte, len(p.PackageConfigRaw)), } copy(meta.PackageConfigRaw, p.PackageConfigRaw) if p.Manifest != nil { @@ -106,6 +109,13 @@ func (p *PackageMeta) GetManifest() *Manifest { return p.Manifest } // GetPackageConfig returns the package config if of the package or nil if there is none. func (p *PackageMeta) GetPackageConfig() *PackageConfig { return p.PackageConfig } +// GetRegistryImageName returns the name that would have been used for a container image +// containing the registry contents. +// TODO: Needing to keep this logic here and BuildRegistryOptions is not a good design probably. +func (p *PackageMeta) GetRegistryImageName() string { + return fmt.Sprintf("%s-private-registry-data:%s", p.Name, p.Version) +} + // NewEmptyMeta returns a new empty PackageMeta instance. func NewEmptyMeta() *PackageMeta { return &PackageMeta{Manifest: NewEmptyManifest()} diff --git a/pkg/types/registry_options.go b/pkg/types/registry_options.go index 3726c91..9e5e7b9 100644 --- a/pkg/types/registry_options.go +++ b/pkg/types/registry_options.go @@ -1,5 +1,7 @@ package types +import "fmt" + // BuildRegistryOptions are options for configuring an in-cluster private // container registry. type BuildRegistryOptions struct { @@ -14,10 +16,9 @@ type BuildRegistryOptions struct { Arch string // Pull policy to use while building the registry PullPolicy PullPolicy - // The password to use for authentication to the registry, if this is blank one will - // be generated. - RegistrySecret string - // The node port that the private registry will listen on when installed. Defaults to - // 30100. - RegistryNodePort string +} + +// RegistryImageName returns the name to use for the image containing the registry contents. +func (opts *BuildRegistryOptions) RegistryImageName() string { + return fmt.Sprintf("%s-private-registry-data:%s", opts.Name, opts.AppVersion) }