feat(private-containers): private registries

cluster/kube/apply.go

@@ -86,6 +86,29 @@ func applyNetPolicies(ctx context.Context, kc kubernetes.Interface, b builder.Ne
 // 	return err
 // }
 
+func applyServiceCredentials(ctx context.Context, kc kubernetes.Interface, b builder.ServiceCredentials) error {
+	obj, err := kc.CoreV1().Secrets(b.NS()).Get(ctx, b.Name(), metav1.GetOptions{})
+	metricsutils.IncCounterVecWithLabelValuesFiltered(kubeCallsCounter, "secrets-get", err, errors.IsNotFound)
+
+	switch {
+	case err == nil:
+		obj, err = b.Update(obj)
+		if err == nil {
+			_, err = kc.CoreV1().Secrets(b.NS()).Update(ctx, obj, metav1.UpdateOptions{})
+			metricsutils.IncCounterVecWithLabelValues(kubeCallsCounter, "secrets-get", err)
+
+		}
+	case errors.IsNotFound(err):
+		obj, err = b.Create()
+		if err == nil {
+			_, err = kc.CoreV1().Secrets(b.NS()).Create(ctx, obj, metav1.CreateOptions{})
+			metricsutils.IncCounterVecWithLabelValues(kubeCallsCounter, "secrets-create", err)
+		}
+	}
+	return err
+
+}
+
 func applyDeployment(ctx context.Context, kc kubernetes.Interface, b builder.Deployment) error {
 	obj, err := kc.AppsV1().Deployments(b.NS()).Get(ctx, b.Name(), metav1.GetOptions{})
 	metricsutils.IncCounterVecWithLabelValuesFiltered(kubeCallsCounter, "deployments-get", err, errors.IsNotFound)

cluster/kube/builder/service_credentials.go

@@ -0,0 +1,111 @@
+package builder
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	mani "github.com/akash-network/akash-api/go/manifest/v2beta2"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+type ServiceCredentials interface {
+	NS() string
+	Name() string
+	Create() (*corev1.Secret, error)
+	Update(obj *corev1.Secret) (*corev1.Secret, error)
+}
+
+type serviceCredentials struct {
+	ns          string
+	serviceName string
+	credentials *mani.ServiceImageCredentials
+	// TODO: labels for deleting
+	// labels      map[string]string
+}
+
+func NewServiceCredentials(ns string, serviceName string, credentials *mani.ServiceImageCredentials) ServiceCredentials {
+	return serviceCredentials{
+		ns:          ns,
+		serviceName: serviceName,
+		credentials: credentials,
+	}
+}
+
+func (b serviceCredentials) NS() string {
+	return b.ns
+}
+
+func (b serviceCredentials) Name() string {
+	return fmt.Sprintf("docker-creds-%v", b.serviceName)
+}
+
+func (b serviceCredentials) Create() (*corev1.Secret, error) {
+	// see https://github.com/kubernetes/kubectl/blob/master/pkg/cmd/create/create_secret_docker.go#L280-L298
+
+	data, err := b.encodeSecret()
+	if err != nil {
+		return nil, err
+	}
+
+	obj := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: b.ns,
+			Name:      b.Name(),
+		},
+		Data: map[string][]byte{
+			corev1.DockerConfigJsonKey: data,
+		},
+		Type: corev1.SecretTypeDockerConfigJson,
+	}
+
+	return obj, nil
+}
+
+func (b serviceCredentials) Update(obj *corev1.Secret) (*corev1.Secret, error) {
+	// see https://github.com/kubernetes/kubectl/blob/master/pkg/cmd/create/create_secret_docker.go#L280-L298
+
+	data, err := b.encodeSecret()
+	if err != nil {
+		return nil, err
+	}
+
+	obj.Data = map[string][]byte{
+		corev1.DockerConfigJsonKey: data,
+	}
+	obj.Type = corev1.SecretTypeDockerConfigJson
+	return obj, nil
+}
+
+type dockerCredentialsEntry struct {
+	Username string `json:"username,omitempty"`
+	Password string `json:"password,omitempty"`
+	Email    string `json:"email,omitempty"`
+	Auth     string `json:"auth,omitempty"`
+}
+
+type dockerCredentials struct {
+	Auths map[string]dockerCredentialsEntry `json:"auths"`
+}
+
+func (b serviceCredentials) encodeSecret() ([]byte, error) {
+	entry := dockerCredentialsEntry{
+		Username: strings.TrimSpace(b.credentials.Username),
+		Password: strings.TrimSpace(b.credentials.Password),
+		Email:    strings.TrimSpace(b.credentials.Email),
+		Auth:     encodeAuth(strings.TrimSpace(b.credentials.Username), strings.TrimSpace(b.credentials.Password)),
+	}
+	creds := dockerCredentials{
+		Auths: map[string]dockerCredentialsEntry{
+			b.credentials.Host: entry,
+		},
+	}
+	return json.Marshal(creds)
+}
+
+func encodeAuth(username, password string) string {
+	value := username + ":" + password
+	return base64.StdEncoding.EncodeToString([]byte(value))
+}

cluster/kube/builder/workload.go

@@ -26,6 +26,7 @@
 type workloadBase interface {
 	builderBase
 	Name() string
+	NS() string
 }
 
 type Workload struct {
@@ -54,6 +55,10 @@
 	return b.deployment.ManifestGroup().Services[b.serviceIdx].Name
 }
 
+func (b *Workload) NS() string {
+	return LidNS(b.deployment.LeaseID())
+}
+
 func (b *Workload) container() corev1.Container {
 	falseValue := false
 
@@ -125,7 +130,7 @@
 
 	// fixme: ram is never expected to be nil
 	if mem := service.Resources.Memory; mem != nil {
 		requestedRam := sdlutil.ComputeCommittedResources(b.settings.MemoryCommitLevel, mem.Quantity)
 		kcontainer.Resources.Requests[corev1.ResourceMemory] = resource.NewQuantity(int64(requestedRam.Value()), resource.DecimalSI).DeepCopy()
 		kcontainer.Resources.Limits[corev1.ResourceMemory] = resource.NewQuantity(int64(mem.Quantity.Value()+requestedMem), resource.DecimalSI).DeepCopy()
 	}
@@ -362,11 +367,19 @@
 }
 
 func (b *Workload) imagePullSecrets() []corev1.LocalObjectReference {
-	if b.settings.DockerImagePullSecretsName == "" {
+
+	sname := b.settings.DockerImagePullSecretsName
+
+	service := &b.deployment.ManifestGroup().Services[b.serviceIdx]
+	if service.Credentials != nil {
+		sname = NewServiceCredentials(b.NS(), b.Name(), service.Credentials).Name()
+	}
+
+	if sname == "" {
 		return nil
 	}
 
-	return []corev1.LocalObjectReference{{Name: b.settings.DockerImagePullSecretsName}}
+	return []corev1.LocalObjectReference{{Name: sname}}
 }
 
 func (b *Workload) addEnvVarsForDeployment(envVarsAlreadyAdded map[string]int, env []corev1.EnvVar) []corev1.EnvVar {

cluster/kube/client.go

@@ -190,6 +190,7 @@ type deploymentService struct {
 	statefulSet   builder.StatefulSet
 	localService  builder.Service
 	globalService builder.Service
+	credentials   builder.ServiceCredentials
 }
 
 type deploymentApplies struct {
@@ -268,6 +269,10 @@ func (c *client) Deploy(ctx context.Context, deployment ctypes.IDeployment) (err
 
 		svc := &deploymentService{}
 
+		if service.Credentials != nil {
+			svc.credentials = builder.NewServiceCredentials(workload.NS(), workload.Name(), service.Credentials)
+		}
+
 		persistent := false
 		for i := range service.Resources.Storage {
 			attrVal := service.Resources.Storage[i].Attributes.Find(sdl.StorageAttributePersistent)
@@ -291,6 +296,7 @@ func (c *client) Deploy(ctx context.Context, deployment ctypes.IDeployment) (err
 
 		svc.localService = builder.BuildService(workload, false)
 		svc.globalService = builder.BuildService(workload, true)
+
 	}
 
 	if err := applyNS(ctx, c.kc, applies.ns); err != nil {
@@ -320,6 +326,13 @@ func (c *client) Deploy(ctx context.Context, deployment ctypes.IDeployment) (err
 		applyObjs := applies.services[svcIdx]
 		service := &group.Services[svcIdx]
 
+		if applyObjs.credentials != nil {
+			if err = applyServiceCredentials(ctx, c.kc, applyObjs.credentials); err != nil {
+				c.log.Error("applying credentials", "err", err, "lease", lid, "service", service.Name)
+				return err
+			}
+		}
+
 		if applyObjs.statefulSet != nil {
 			if err = applyStatefulSet(ctx, c.kc, applyObjs.statefulSet); err != nil {
 				c.log.Error("applying statefulSet", "err", err, "lease", lid, "service", service.Name)

go.mod

@@ -3,7 +3,7 @@ module github.com/akash-network/provider
 go 1.21
 
 require (
-	github.com/akash-network/akash-api v0.0.61
+	github.com/akash-network/akash-api v0.0.65
 	github.com/akash-network/node v0.32.3
 	github.com/avast/retry-go/v4 v4.5.0
 	github.com/blang/semver/v4 v4.0.0

go.sum

@@ -197,8 +197,8 @@ github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia
 github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
 github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
-github.com/akash-network/akash-api v0.0.61 h1:Hj/IBr9cFMsFs4VjymLZCoX/5dNfZSFd4iFwUSqhBtQ=
-github.com/akash-network/akash-api v0.0.61/go.mod h1:pNr61L4+0sheol7ZK0HjgK3rxpIAbYBGq1w1oH4B0+M=
+github.com/akash-network/akash-api v0.0.65 h1:Jadkbu9rvE5UIrd2t0FRbEGMeco/KUEzD/8w6zqxPlI=
+github.com/akash-network/akash-api v0.0.65/go.mod h1:pNr61L4+0sheol7ZK0HjgK3rxpIAbYBGq1w1oH4B0+M=
 github.com/akash-network/cometbft v0.34.27-akash h1:V1dApDOr8Ee7BJzYyQ7Z9VBtrAul4+baMeA6C49dje0=
 github.com/akash-network/cometbft v0.34.27-akash/go.mod h1:BcCbhKv7ieM0KEddnYXvQZR+pZykTKReJJYf7YC7qhw=
 github.com/akash-network/ledger-go v0.14.3 h1:LCEFkTfgGA2xFMN2CtiKvXKE7dh0QSM77PJHCpSkaAo=