feat: addon secret

cmd/terway-controlplane/terway-controlplane.go

@@ -44,6 +44,7 @@ import (
 	"github.com/AliyunContainerService/terway/pkg/controller/webhook"
 	"github.com/AliyunContainerService/terway/pkg/metric"
 	"github.com/AliyunContainerService/terway/pkg/utils"
+	"github.com/AliyunContainerService/terway/pkg/utils/k8sclient"
 	"github.com/AliyunContainerService/terway/pkg/version"
 	"github.com/AliyunContainerService/terway/types/controlplane"
 )
@@ -89,7 +90,7 @@ func main() {
 	restConfig.QPS = cfg.KubeClientQPS
 	restConfig.Burst = cfg.KubeClientBurst
 	restConfig.UserAgent = version.UA
-	utils.RegisterClients(restConfig)
+	k8sclient.RegisterClients(restConfig)
 
 	log.Info("using config", "config", cfg)
 

pkg/aliyun/credential/sts.go

@@ -18,6 +18,7 @@
 
 	"github.com/AliyunContainerService/terway/pkg/backoff"
 	"github.com/AliyunContainerService/terway/pkg/utils"
+	"github.com/AliyunContainerService/terway/pkg/utils/k8sclient"
 )
 
 type EncryptedCredentialInfo struct {
@@ -74,7 +75,7 @@
 			}
 			return false
 		}, func() error {
-			secret, err = utils.K8sClient.CoreV1().Secrets(e.secretNamespace).Get(context.Background(), e.secretName, metav1.GetOptions{})
+			secret, err = k8sclient.K8sClient.CoreV1().Secrets(e.secretNamespace).Get(context.Background(), e.secretName, metav1.GetOptions{})
 			if err != nil {
 				return err
 			}

pkg/apis/crds/register.go

@@ -7,7 +7,7 @@
 
 	"k8s.io/apimachinery/pkg/api/errors"
 
-	"github.com/AliyunContainerService/terway/pkg/utils"
+	"github.com/AliyunContainerService/terway/pkg/utils/k8sclient"
 
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
@@ -110,7 +110,7 @@
 func RegisterCRDs() error {
 	crds := []string{CRDPodENI, CRDPodNetworking}
 	for _, crd := range crds {
-		err := createOrUpdateCRD(utils.APIExtensionsClient, crd)
+		err := createOrUpdateCRD(k8sclient.APIExtensionsClient, crd)
 		if err != nil {
 			return err
 		}
@@ -121,7 +121,7 @@
 
 func RegisterCRD(crds []string) error {
 	for _, crd := range crds {
-		err := createCRD(utils.APIExtensionsClient, crd)
+		err := createCRD(k8sclient.APIExtensionsClient, crd)
 		if err != nil {
 			return err
 		}

pkg/cert/webhook.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/AliyunContainerService/terway/pkg/logger"
 	"github.com/AliyunContainerService/terway/pkg/utils"
+	"github.com/AliyunContainerService/terway/pkg/utils/k8sclient"
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -37,7 +38,7 @@ const (
 // SyncCert sync cert for webhook
 func SyncCert(ns, name, domain, certDir string) error {
 	secretName := fmt.Sprintf("%s-webhook-cert", name)
-	cs := utils.K8sClient
+	cs := k8sclient.K8sClient
 	// check secret
 	var serverCertBytes, serverKeyBytes, caCertBytes []byte
 

pkg/controller/endpoint/endpoint.go

@@ -8,7 +8,8 @@ import (
 	"time"
 
 	register "github.com/AliyunContainerService/terway/pkg/controller"
-	"github.com/AliyunContainerService/terway/pkg/utils"
+	"github.com/AliyunContainerService/terway/pkg/utils/k8sclient"
+
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -86,12 +87,12 @@ func (m *Endpoint) RegisterEndpoints() error {
 			},
 		}}
 	ctx := context.Background()
-	oldEP, err := utils.K8sClient.CoreV1().Endpoints(m.Namespace).Get(ctx, m.Name, metav1.GetOptions{})
+	oldEP, err := k8sclient.K8sClient.CoreV1().Endpoints(m.Namespace).Get(ctx, m.Name, metav1.GetOptions{})
 	if err != nil {
 		if !errors.IsNotFound(err) {
 			return err
 		}
-		_, err = utils.K8sClient.CoreV1().Endpoints(m.Namespace).Create(ctx, &v1.Endpoints{
+		_, err = k8sclient.K8sClient.CoreV1().Endpoints(m.Namespace).Create(ctx, &v1.Endpoints{
 			ObjectMeta: metav1.ObjectMeta{
 				Namespace: m.Namespace,
 				Name:      m.Name,
@@ -107,7 +108,7 @@ func (m *Endpoint) RegisterEndpoints() error {
 	}
 	copyEP := oldEP.DeepCopy()
 	copyEP.Subsets = newEPSubnet
-	_, err = utils.K8sClient.CoreV1().Endpoints(m.Namespace).Update(ctx, copyEP, metav1.UpdateOptions{})
+	_, err = k8sclient.K8sClient.CoreV1().Endpoints(m.Namespace).Update(ctx, copyEP, metav1.UpdateOptions{})
 	log.Info("register endpoint", "ip", m.PodIP)
 	return err
 }

pkg/k8s/k8s.go

@@ -34,6 +34,7 @@
 	"github.com/AliyunContainerService/terway/pkg/storage"
 	"github.com/AliyunContainerService/terway/pkg/tracing"
 	"github.com/AliyunContainerService/terway/pkg/utils"
+	"github.com/AliyunContainerService/terway/pkg/utils/k8sclient"
 	"github.com/AliyunContainerService/terway/pkg/version"
 	"github.com/AliyunContainerService/terway/types"
 	"github.com/AliyunContainerService/terway/types/daemon"
@@ -116,7 +117,7 @@
 	if err != nil {
 		return nil, err
 	}
-	utils.RegisterClients(restConfig)
+	k8sclient.RegisterClients(restConfig)
 
 	nodeName := os.Getenv("NODE_NAME")
 	if nodeName == "" {
@@ -137,7 +138,7 @@
 	var nodeCidr *types.IPNetSet
 	if daemonMode == daemon.ModeVPC {
 		// vpc mode not support ipv6
-		nodeCidr, err = nodeCidrFromAPIServer(utils.K8sClient, nodeName)
+		nodeCidr, err = nodeCidrFromAPIServer(k8sclient.K8sClient, nodeName)
 		if err != nil {
 			return nil, fmt.Errorf("error retrieving node cidr for '%s': %w", nodeName, err)
 		}
@@ -153,7 +154,7 @@
 	recorder := broadcaster.NewRecorder(scheme.Scheme, source)
 
 	sink := &typedv1.EventSinkImpl{
-		Interface: typedv1.New(utils.K8sClient.CoreV1().RESTClient()).Events(""),
+		Interface: typedv1.New(k8sclient.K8sClient.CoreV1().RESTClient()).Events(""),
 	}
 	broadcaster.StartRecordingToSink(sink)
 

pkg/utils/k8s.go

@@ -5,30 +5,9 @@ import (
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
-	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
-
-	networkingclientset "github.com/AliyunContainerService/terway/pkg/generated/clientset/versioned"
 )
 
-// K8sClient k8s client set
-var K8sClient kubernetes.Interface
-
-// APIExtensionsClient k8s client set
-var APIExtensionsClient apiextensionsclient.Interface
-
-// NetworkClient network client set
-var NetworkClient networkingclientset.Interface
-
-// RegisterClients create all k8s clients
-func RegisterClients(restConfig *rest.Config) {
-	K8sClient = kubernetes.NewForConfigOrDie(restConfig)
-	APIExtensionsClient = apiextensionsclient.NewForConfigOrDie(restConfig)
-	NetworkClient = networkingclientset.NewForConfigOrDie(restConfig)
-}
-
 var stsKinds = []string{"StatefulSet"}
 
 // SetStsKinds set custom sts workload kinds

pkg/utils/k8sclient/client.go

@@ -0,0 +1,25 @@
+package k8sclient
+
+import (
+	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+
+	networkingclientset "github.com/AliyunContainerService/terway/pkg/generated/clientset/versioned"
+)
+
+// K8sClient k8s client set
+var K8sClient kubernetes.Interface
+
+// APIExtensionsClient k8s client set
+var APIExtensionsClient apiextensionsclient.Interface
+
+// NetworkClient network client set
+var NetworkClient networkingclientset.Interface
+
+// RegisterClients create all k8s clients
+func RegisterClients(restConfig *rest.Config) {
+	K8sClient = kubernetes.NewForConfigOrDie(restConfig)
+	APIExtensionsClient = apiextensionsclient.NewForConfigOrDie(restConfig)
+	NetworkClient = networkingclientset.NewForConfigOrDie(restConfig)
+}

tests/connection_test.go

@@ -20,7 +20,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 
 	"github.com/AliyunContainerService/terway/pkg/generated/clientset/versioned"
-	"github.com/AliyunContainerService/terway/pkg/utils"
+	"github.com/AliyunContainerService/terway/pkg/utils/k8sclient"
 )
 
 func TestConnectionTestSuite(t *testing.T) {
@@ -40,8 +40,8 @@ type ConnectionTestSuite struct {
 
 func (s *ConnectionTestSuite) SetupSuite() {
 	s.RestConf = ctrl.GetConfigOrDie()
-	utils.RegisterClients(s.RestConf)
-	s.ClientSet = utils.K8sClient
+	k8sclient.RegisterClients(s.RestConf)
+	s.ClientSet = k8sclient.K8sClient
 	s.PodNetworkingClientSet, _ = versioned.NewForConfig(s.RestConf)
 
 	ctx := context.Background()

tests/trunk/perf_test.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1"
-	"github.com/AliyunContainerService/terway/pkg/utils"
+	"github.com/AliyunContainerService/terway/pkg/utils/k8sclient"
 	terwayTypes "github.com/AliyunContainerService/terway/types"
 
 	"github.com/google/uuid"
@@ -20,20 +20,20 @@ import (
 func Test_10KPod(t *testing.T) {
 	restConf := ctrl.GetConfigOrDie()
 	ns := "perf"
-	utils.RegisterClients(restConf)
+	k8sclient.RegisterClients(restConf)
 	ctx := context.Background()
 
-	_, _ = utils.K8sClient.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{
+	_, _ = k8sclient.K8sClient.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{
 		ObjectMeta: metav1.ObjectMeta{Name: ns, Labels: map[string]string{"trunk": "trunk", "perf": "perf"}},
 	}, metav1.CreateOptions{})
 
 	pn := newPodNetworking("pn", nil, nil, nil,
 		&metav1.LabelSelector{MatchLabels: map[string]string{"trunk": "trunk"}})
-	_, _ = utils.NetworkClient.NetworkV1beta1().PodNetworkings().Create(ctx, pn, metav1.CreateOptions{})
+	_, _ = k8sclient.NetworkClient.NetworkV1beta1().PodNetworkings().Create(ctx, pn, metav1.CreateOptions{})
 
 	for i := 0; i < 10000; i++ {
 		name := uuid.NewString()
-		peni, err := utils.NetworkClient.NetworkV1beta1().PodENIs(ns).Create(ctx, &v1beta1.PodENI{
+		peni, err := k8sclient.NetworkClient.NetworkV1beta1().PodENIs(ns).Create(ctx, &v1beta1.PodENI{
 			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: map[string]string{
 				terwayTypes.PodNetworking: "pn",
 				terwayTypes.PodENI:        "true",
@@ -68,9 +68,9 @@ func Test_10KPod(t *testing.T) {
 		update := peni.DeepCopy()
 		update.Status.Phase = v1beta1.ENIPhaseBind
 		update.Status.PodLastSeen = metav1.NewTime(time.Now().Add(time.Hour))
-		_, _ = utils.NetworkClient.NetworkV1beta1().PodENIs(update.Namespace).UpdateStatus(ctx, update, metav1.UpdateOptions{})
+		_, _ = k8sclient.NetworkClient.NetworkV1beta1().PodENIs(update.Namespace).UpdateStatus(ctx, update, metav1.UpdateOptions{})
 
-		_, _ = utils.K8sClient.CoreV1().Pods(ns).Create(ctx, &corev1.Pod{
+		_, _ = k8sclient.K8sClient.CoreV1().Pods(ns).Create(ctx, &corev1.Pod{
 			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
 			Spec: corev1.PodSpec{
 				Containers: []corev1.Container{

types/daemon/config.go

@@ -3,6 +3,7 @@
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 
 	"github.com/AliyunContainerService/terway/pkg/aliyun/instance"
 	"github.com/AliyunContainerService/terway/types/secret"
@@ -16,6 +17,14 @@
 	"github.com/AliyunContainerService/terway/types/route"
 )
 
+const (
+	addonSecretPath      = "/var/addon/alibaba-addon-secret"
+	addonSecretKeyID     = "access-key-id"
+	addonSecretKeySecret = "access-key-secret"
+)
+
+var addonSecretRootPath = addonSecretPath
+
 // Config configuration of terway daemon
 type Config struct {
 	Version                string              `yaml:"version" json:"version"`
@@ -126,7 +135,21 @@
 		return nil, err
 	}
 
-	return MergeConfigAndUnmarshal(cfg, data)
+	config, err := MergeConfigAndUnmarshal(cfg, data)
+	if err != nil {
+		return nil, err
+	}
+
+	ak, sk, err := GetAddonSecret()
+	if err != nil {
+		return nil, err
+	}
+	if ak != "" && sk != "" {
+		config.AccessID = secret.Secret(ak)
+		config.AccessSecret = secret.Secret(sk)
+	}
+
+	return config, nil
 }
 
 func MergeConfigAndUnmarshal(topCfg, baseCfg []byte) (*Config, error) {
@@ -147,3 +170,22 @@
 
 	return config, err
 }
+
+// GetAddonSecret return ak/sk from file, return nil if not present.
+func GetAddonSecret() (string, string, error) {
+	keyID, err := os.ReadFile(filepath.Join(addonSecretRootPath, addonSecretKeyID))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return "", "", nil
+		}
+		return "", "", err
+	}
+	keySecret, err := os.ReadFile(filepath.Join(addonSecretRootPath, addonSecretKeySecret))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return "", "", nil
+		}
+		return "", "", err
+	}
+	return string(keyID), string(keySecret), nil
+}

types/daemon/config_test.go

@@ -1,6 +1,8 @@
 package daemon
 
 import (
+	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -87,3 +89,19 @@ func Test_MergeConfigAndUnmarshal(t *testing.T) {
 	assert.Equal(t, "ordered", cfg.VSwitchSelectionPolicy)
 	t.Logf("%+v", cfg)
 }
+
+func TestGetAddonSecret(t *testing.T) {
+	dir, err := os.MkdirTemp("", "")
+	assert.NoError(t, err)
+
+	defer os.RemoveAll(dir)
+
+	_ = os.WriteFile(filepath.Join(dir, addonSecretKeyID), []byte("key"), 0700)
+	_ = os.WriteFile(filepath.Join(dir, addonSecretKeySecret), []byte("secret"), 0700)
+	addonSecretRootPath = dir
+
+	ak, sk, err := GetAddonSecret()
+	assert.NoError(t, err)
+	assert.Equal(t, "key", ak)
+	assert.Equal(t, "secret", sk)
+}