policy,init: do not start policy in exclusive eni mode

Signed-off-by: l1b0k <[email protected]>

charts/terway/templates/daemonset.yaml

@@ -106,6 +106,12 @@ spec:
           command:
             - /bin/init.sh
           env:
+          - name: TERWAY_DAEMON_MODE
+            value: "{{.Values.daemonMode}}"
+          - name: K8S_NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
           - name: DISABLE_POLICY
             valueFrom:
               configMapKeyRef:

cmd/terway-cli/cni.go

@@ -8,7 +8,6 @@ import (
 	"strings"
 
 	"github.com/Jeffail/gabs/v2"
-	"github.com/docker/docker/pkg/parsers/kernel"
 	"github.com/spf13/cobra"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	cliflag "k8s.io/component-base/cli/flag"
@@ -71,7 +70,7 @@ var cniCmd = &cobra.Command{
 func processCNIConfig(cmd *cobra.Command, args []string) error {
 	flag.Parse()
 
-	_checkKernelVersion = kernel.CheckKernelVersion
+	_checkKernelVersion = checkKernelVersion
 
 	_switchDataPathV2 = switchDataPathV2
 

cmd/terway-cli/cni_linux.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/docker/docker/pkg/parsers/kernel"
 	"github.com/vishvananda/netlink"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 
@@ -25,3 +26,7 @@ func switchDataPathV2() bool {
 	_, err := netlink.LinkByName("cilium_net")
 	return errors.As(err, &netlink.LinkNotFoundError{})
 }
+
+func checkKernelVersion(k, major, minor int) bool {
+	return kernel.CheckKernelVersion(k, major, minor)
+}

cmd/terway-cli/cni_unsupport.go

@@ -5,3 +5,7 @@ package main
 func switchDataPathV2() bool {
 	return true
 }
+
+func checkKernelVersion(k, major, minor int) bool {
+	return false
+}

cmd/terway-cli/main.go

@@ -94,7 +94,7 @@ var (
 )
 
 func init() {
-	rootCmd.AddCommand(listCmd, showCmd, mappingCmd, executeCmd, metadataCmd, cniCmd)
+	rootCmd.AddCommand(listCmd, showCmd, mappingCmd, executeCmd, metadataCmd, cniCmd, nodeconfigCmd)
 }
 
 func main() {

cmd/terway-cli/node.go

@@ -0,0 +1,173 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/spf13/cobra"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	k8sClient "sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/AliyunContainerService/terway/pkg/utils/nodecap"
+	"github.com/AliyunContainerService/terway/pkg/version"
+	"github.com/AliyunContainerService/terway/types"
+	"github.com/AliyunContainerService/terway/types/daemon"
+)
+
+const eniOnlyCNI = `{
+  "cniVersion": "0.4.0",
+  "name": "terway-chainer",
+  "plugins": [
+    {
+      "capabilities": {
+        "bandwidth": true
+      },
+      "host_stack_cidrs": [
+        "169.254.20.10/32"
+      ],
+      "type": "terway"
+    }
+  ]
+}`
+
+const cniFilePath = "/etc/cni/net.d/10-terway.conflist"
+const nodeCapabilitiesFile = "/var/run/eni/node_capabilities"
+
+type Task struct {
+	Name string
+	Func func(cmd *cobra.Command, args []string) error
+}
+
+var tasks = []Task{
+	{
+		Name: "get eni config",
+		Func: getENIConfig,
+	},
+	{
+		Name: "eniOnly",
+		Func: overrideCNI,
+	},
+	{
+		Name: "dual stack",
+		Func: dualStack,
+	},
+}
+
+var nodeconfigCmd = &cobra.Command{
+	Use:          "nodeconfig",
+	SilenceUsage: true,
+	Run: func(cmd *cobra.Command, args []string) {
+		for _, task := range tasks {
+			err := task.Func(cmd, args)
+			if err != nil {
+				_, _ = fmt.Fprintf(os.Stderr, "task: %serror: %v\n", task.Name, err)
+				os.Exit(1)
+			}
+		}
+	},
+}
+
+var eniCfg *daemon.Config
+
+func getENIConfig(cmd *cobra.Command, args []string) error {
+	var err error
+	eniCfg, err = daemon.GetConfigFromFileWithMerge("/etc/eni/eni.json", nil)
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("eni config: %+v\n", eniCfg)
+	return nil
+}
+
+func overrideCNI(cmd *cobra.Command, args []string) error {
+	restConfig := ctrl.GetConfigOrDie()
+	restConfig.UserAgent = version.UA
+	c, err := k8sClient.New(restConfig, k8sClient.Options{
+		Scheme: types.Scheme,
+		Mapper: types.NewRESTMapper(),
+	})
+	if err != nil {
+		return err
+	}
+
+	node := &corev1.Node{}
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
+	defer cancel()
+
+	nodeName := os.Getenv("K8S_NODE_NAME")
+
+	err = c.Get(ctx, k8sClient.ObjectKey{
+		Name: nodeName,
+	}, node, &k8sClient.GetOptions{
+		Raw: &metav1.GetOptions{
+			ResourceVersion: "0",
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("get node %s error: %w", nodeName, err)
+	}
+
+	store := nodecap.NewFileNodeCapabilities(nodeCapabilitiesFile)
+	return setExclusiveMode(store, node.Labels, cniFilePath)
+}
+
+func setExclusiveMode(store nodecap.NodeCapabilitiesStore, labels map[string]string, cniPath string) error {
+	err := store.Load()
+	if err != nil {
+		return err
+	}
+	now := types.NodeExclusiveENIMode(labels)
+
+	switch store.Get(nodecap.NodeCapabilityExclusiveENI) {
+	case string(types.ExclusiveENIOnly):
+		if now != types.ExclusiveENIOnly {
+			return fmt.Errorf("exclusive eni mode changed")
+		}
+	case string(types.ExclusiveDefault):
+		if now != types.ExclusiveDefault {
+			return fmt.Errorf("exclusive eni mode changed")
+		}
+	case "":
+		// empty for new node, or rebooted
+	}
+
+	store.Set(nodecap.NodeCapabilityExclusiveENI, string(now))
+	err = store.Save()
+	if err != nil {
+		return err
+	}
+
+	// write cni config
+	if now == types.ExclusiveENIOnly {
+		err = os.WriteFile(cniPath, []byte(eniOnlyCNI), 0644)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func dualStack(cmd *cobra.Command, args []string) error {
+	store := nodecap.NewFileNodeCapabilities(nodeCapabilitiesFile)
+
+	val := ""
+	switch eniCfg.IPStack {
+	case "dual", "ipv6":
+		val = "true"
+	default:
+		val = "false"
+	}
+
+	err := store.Load()
+	if err != nil {
+		return err
+	}
+
+	store.Set(nodecap.NodeCapabilityIPv6, val)
+	return store.Save()
+}

cmd/terway-cli/node_test.go

@@ -0,0 +1,94 @@
+package main
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/AliyunContainerService/terway/pkg/utils/nodecap"
+)
+
+func TestExclusiveModeNewNode(t *testing.T) {
+	tempFile, err := os.CreateTemp("", "test_node_capabilities")
+	assert.NoError(t, err)
+	defer os.Remove(tempFile.Name())
+
+	store := nodecap.NewFileNodeCapabilities(tempFile.Name())
+	labels := map[string]string{"k8s.aliyun.com/exclusive-mode-eni-type": "eniOnly"}
+	cniPath := tempFile.Name() + "_cni_config"
+
+	err = setExclusiveMode(store, labels, cniPath)
+	assert.NoError(t, err)
+	assert.Equal(t, "eniOnly", store.Get(nodecap.NodeCapabilityExclusiveENI))
+}
+
+func TestExclusiveModeDoesNotChangeWhenAlreadySet(t *testing.T) {
+	tempFile, err := os.CreateTemp("", "test_node_capabilities")
+	assert.NoError(t, err)
+	defer os.Remove(tempFile.Name())
+
+	err = os.WriteFile(tempFile.Name(), []byte("cni_exclusive_eni = eniOnly"), 0644)
+	assert.NoError(t, err)
+
+	store := nodecap.NewFileNodeCapabilities(tempFile.Name())
+	labels := map[string]string{"k8s.aliyun.com/exclusive-mode-eni-type": "eniOnly"}
+	cniPath := tempFile.Name() + "_cni_config"
+
+	err = setExclusiveMode(store, labels, cniPath)
+	assert.NoError(t, err)
+	assert.Equal(t, "eniOnly", store.Get(nodecap.NodeCapabilityExclusiveENI))
+}
+
+func TestExclusiveModeFailsWhenChangedFromExclusive(t *testing.T) {
+	tempFile, err := os.CreateTemp("", "test_node_capabilities")
+	assert.NoError(t, err)
+	defer os.Remove(tempFile.Name())
+
+	err = os.WriteFile(tempFile.Name(), []byte("cni_exclusive_eni = eniOnly"), 0644)
+	assert.NoError(t, err)
+
+	store := nodecap.NewFileNodeCapabilities(tempFile.Name())
+	labels := map[string]string{"k8s.aliyun.com/exclusive-mode-eni-type": "default"}
+	cniPath := tempFile.Name() + "_cni_config"
+
+	err = setExclusiveMode(store, labels, cniPath)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "exclusive eni mode changed")
+}
+
+func TestExclusiveModeWritesCNIConfigWhenSet(t *testing.T) {
+	tempFile, err := os.CreateTemp("", "test_node_capabilities")
+	assert.NoError(t, err)
+	defer os.Remove(tempFile.Name())
+
+	store := nodecap.NewFileNodeCapabilities(tempFile.Name())
+	labels := map[string]string{"k8s.aliyun.com/exclusive-mode-eni-type": "eniOnly"}
+	cniPath := tempFile.Name() + "_cni_config"
+	defer os.Remove(cniPath)
+
+	err = setExclusiveMode(store, labels, cniPath)
+	assert.NoError(t, err)
+
+	content, err := os.ReadFile(cniPath)
+	assert.NoError(t, err)
+	assert.Contains(t, string(content), `"type": "terway"`)
+}
+
+func TestExclusiveModeDoesNotWriteCNIConfigWhenNotSet(t *testing.T) {
+	tempFile, err := os.CreateTemp("", "test_node_capabilities")
+	assert.NoError(t, err)
+	defer os.Remove(tempFile.Name())
+
+	store := nodecap.NewFileNodeCapabilities(tempFile.Name())
+	labels := map[string]string{"k8s.aliyun.com/exclusive-mode-eni-type": "default"}
+	cniPath := tempFile.Name() + "_cni_config"
+	defer os.Remove(cniPath)
+
+	err = setExclusiveMode(store, labels, cniPath)
+	assert.NoError(t, err)
+
+	_, err = os.ReadFile(cniPath)
+	assert.Error(t, err)
+	assert.True(t, os.IsNotExist(err))
+}

daemon/builder.go

@@ -108,6 +108,11 @@ func (b *NetworkServiceBuilder) InitK8S() *NetworkServiceBuilder {
 		b.err = fmt.Errorf("error init k8s: %w", err)
 		return b
 	}
+
+	if types.NodeExclusiveENIMode(b.service.k8s.Node().Labels) == types.ExclusiveENIOnly {
+		b.service.daemonMode = daemon.ModeENIOnly
+	}
+
 	return b
 }
 

daemon/server.go

@@ -68,7 +68,11 @@ func stackTriger() {
 
 // Run terway daemon
 func Run(ctx context.Context, socketFilePath, debugSocketListen, configFilePath, daemonMode string) error {
-	// init.sh already create the /var/run/eni
+	err := os.MkdirAll(filepath.Dir(lockFile), 0700)
+	if err != nil {
+		return err
+	}
+
 	lock, err := filemutex.New(lockFile)
 	if err != nil {
 		return fmt.Errorf("error create lock file: %s, %w", lockFile, err)

init.sh

@@ -5,14 +5,18 @@ set -o nounset
 
 # install CNIs
 cp -f /usr/bin/terway /opt/cni/bin/
-cp -f /usr/bin/cilium-cni /opt/cni/bin/
 chmod +x /opt/cni/bin/terway
-chmod +x /opt/cni/bin/cilium-cni
+
+if [ "$TERWAY_DAEMON_MODE" != "VPC" ]; then
+  cp -f /usr/bin/cilium-cni /opt/cni/bin/
+  chmod +x /opt/cni/bin/cilium-cni
+fi
 
 # init cni config
 cp /tmp/eni/eni_conf /etc/eni/eni.json
 
 terway-cli cni /tmp/eni/10-terway.conflist /tmp/eni/10-terway.conf --output /etc/cni/net.d/10-terway.conflist
+terway-cli nodeconfig
 
 node_capabilities=/var/run/eni/node_capabilities
 if [ ! -f "$node_capabilities" ]; then
@@ -42,4 +46,7 @@ cp $node_capabilities /var-run-eni/node_capabilities
 
 sysctl -w net.ipv4.conf.eth0.rp_filter=0
 modprobe sch_htb || true
-chroot /host sh -c "systemctl disable eni.service; rm -f /etc/udev/rules.d/75-persistent-net-generator.rules /lib/udev/rules.d/60-net.rules /lib/udev/rules.d/61-eni.rules /lib/udev/write_net_rules"
+
+if [ "$TERWAY_DAEMON_MODE" != "VPC" ]; then
+  chroot /host sh -c "systemctl disable eni.service; rm -f /etc/udev/rules.d/75-persistent-net-generator.rules /lib/udev/rules.d/60-net.rules /lib/udev/rules.d/61-eni.rules /lib/udev/write_net_rules"
+fi