feat: --network-dataplane flag (#318)

* feat: --network-dataplane flag

* commit from codespace

pkg/operator/options/options.go

@@ -62,14 +62,14 @@ type Options struct {
 	ClusterEndpoint                string // => APIServerName in bootstrap, except needs to be w/o https/port
 	VMMemoryOverheadPercent        float64
 	ClusterID                      string
-	KubeletClientTLSBootstrapToken string   // => TLSBootstrapToken in bootstrap (may need to be per node/nodepool)
-	SSHPublicKey                   string   // ssh.publicKeys.keyData => VM SSH public key // TODO: move to v1alpha2.AKSNodeClass?
-	NetworkPlugin                  string   // => NetworkPlugin in bootstrap
-	NetworkPolicy                  string   // => NetworkPolicy in bootstrap
+	KubeletClientTLSBootstrapToken string // => TLSBootstrapToken in bootstrap (may need to be per node/nodepool)
+	SSHPublicKey                   string // ssh.publicKeys.keyData => VM SSH public key // TODO: move to v1alpha2.AKSNodeClass?
+	NetworkPlugin                  string // => NetworkPlugin in bootstrap
+	NetworkPolicy                  string // => NetworkPolicy in bootstrap
+	NetworkDataplane               string
 	NodeIdentities                 []string // => Applied onto each VM
 
 	SubnetID string // => VnetSubnetID to use (for nodes in Azure CNI Overlay and Azure CNI + pod subnet; for for nodes and pods in Azure CNI), unless overridden via AKSNodeClass
-
 	setFlags map[string]bool
 }
 
@@ -81,6 +81,7 @@ func (o *Options) AddFlags(fs *coreoptions.FlagSet) {
 	fs.StringVar(&o.SSHPublicKey, "ssh-public-key", env.WithDefaultString("SSH_PUBLIC_KEY", ""), "[REQUIRED] VM SSH public key.")
 	fs.StringVar(&o.NetworkPlugin, "network-plugin", env.WithDefaultString("NETWORK_PLUGIN", "azure"), "The network plugin used by the cluster.")
 	fs.StringVar(&o.NetworkPolicy, "network-policy", env.WithDefaultString("NETWORK_POLICY", ""), "The network policy used by the cluster.")
+	fs.StringVar(&o.NetworkDataplane, "network-dataplane", env.WithDefaultString("NETWORK_DATAPLANE", "cilium"), "The network dataplane used by the cluster.")
 	fs.StringVar(&o.SubnetID, "vnet-subnet-id", env.WithDefaultString("VNET_SUBNET_ID", ""), "The default subnet ID to use for new nodes. This must be a valid ARM resource ID for subnet that does not overlap with the service CIDR or the pod CIDR")
 	fs.Var(newNodeIdentitiesValue(env.WithDefaultString("NODE_IDENTITIES", ""), &o.NodeIdentities), "node-identities", "User assigned identities for nodes.")
 }

pkg/operator/options/options_validation.go

@@ -31,6 +31,7 @@ func (o Options) Validate() error {
 		o.validateRequiredFields(),
 		o.validateEndpoint(),
 		o.validateVMMemoryOverheadPercent(),
+		o.validateNetworkDataplane(),
 		o.validateVnetSubnetID(),
 		validate.Struct(o),
 	)
@@ -44,6 +45,12 @@ func (o Options) validateVnetSubnetID() error {
 	return nil
 }
 
+func (o Options) validateNetworkDataplane() error {
+	if o.NetworkDataplane != "azure" && o.NetworkDataplane != "cilium" {
+		return fmt.Errorf("network dataplane %s is not a valid network dataplane, valid dataplanes are ('azure', 'cilium')", o.NetworkDataplane)
+	}
+	return nil
+}
 func (o Options) validateEndpoint() error {
 	if o.ClusterEndpoint == "" {
 		return nil

pkg/operator/options/suite_test.go

@@ -113,8 +113,18 @@ var _ = Describe("Options", func() {
 			}))
 		})
 	})
-
 	Context("Validation", func() {
+		It("should fail validation when networkDataplane is not invalid", func() {
+			err := opts.Parse(
+				fs,
+				"--cluster-endpoint", "https://karpenter-000000000000.hcp.westus2.staging.azmk8s.io",
+				"--kubelet-bootstrap-token", "flag-bootstrap-token",
+				"--ssh-public-key", "flag-ssh-public-key",
+				"--network-dataplane", "ciluum",
+			)
+			Expect(err).To(MatchError(ContainSubstring("network dataplane ciluum is not a valid network dataplane, valid dataplanes are ('azure', 'cilium')")))
+		})
+
 		It("should fail validation when clusterName not included", func() {
 			err := opts.Parse(
 				fs,

pkg/providers/launchtemplate/launchtemplate.go

@@ -121,13 +121,16 @@ func (p *Provider) getStaticParameters(ctx context.Context, instanceType *cloudp
 	labels = lo.Assign(labels, vnetLabels)
 
 	// TODO: Make conditional on epbf dataplane
-	// This label is required for the cilium agent daemonset because
-	// we select the nodes for the daemonset based on this label
-	//              - key: kubernetes.azure.com/ebpf-dataplane
-	//            operator: In
-	//            values:
-	//              - cilium
-	labels[vnetDataPlaneLabel] = networkDataplaneCilium
+	if options.FromContext(ctx).NetworkDataplane == networkDataplaneCilium {
+		// This label is required for the cilium agent daemonset because
+		// we select the nodes for the daemonset based on this label
+		//              - key: kubernetes.azure.com/ebpf-dataplane
+		//            operator: In
+		//            values:
+		//              - cilium
+
+		labels[vnetDataPlaneLabel] = networkDataplaneCilium
+	}
 
 	return &parameters.StaticParameters{
 		ClusterName:                    options.FromContext(ctx).ClusterName,

pkg/test/options.go

@@ -33,6 +33,7 @@ type OptionsFields struct {
 	SSHPublicKey                   *string
 	NetworkPlugin                  *string
 	NetworkPolicy                  *string
+	NetworkDataplane               *string
 	VMMemoryOverheadPercent        *float64
 	NodeIdentities                 []string
 	SubnetID                       *string
@@ -53,6 +54,7 @@ func Options(overrides ...OptionsFields) *azoptions.Options {
 		SSHPublicKey:                   lo.FromPtrOr(options.SSHPublicKey, "test-ssh-public-key"),
 		NetworkPlugin:                  lo.FromPtrOr(options.NetworkPlugin, "azure"),
 		NetworkPolicy:                  lo.FromPtrOr(options.NetworkPolicy, "cilium"),
+		NetworkDataplane:               lo.FromPtrOr(options.NetworkDataplane, "cilium"),
 		VMMemoryOverheadPercent:        lo.FromPtrOr(options.VMMemoryOverheadPercent, 0.075),
 		NodeIdentities:                 options.NodeIdentities,
 		SubnetID:                       lo.FromPtrOr(options.SubnetID, "/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/sillygeese/providers/Microsoft.Network/virtualNetworks/karpentervnet/subnets/karpentersub"),