From c852a71c3d565f81cf526e47c289e022787ada71 Mon Sep 17 00:00:00 2001 From: Serge Logvinov Date: Sun, 4 Feb 2024 18:39:01 +0200 Subject: [PATCH] feat: support CloudDualStackNodeIPs Talos CCM now supports the `CloudDualStackNodeIPs` feature gate. This feature allows the user(cloud) to specify a list of IPv4 and IPv6 addresses for each node in the cluster. https://github.com/kubernetes/kubernetes/pull/120275 Signed-off-by: Serge Logvinov --- pkg/talos/helper.go | 25 +++++++++++++-------- pkg/talos/helper_test.go | 25 +++++++++++++++++---- pkg/talos/instances.go | 32 ++++++++++++++++++++++----- pkg/utils/net/net.go | 41 ++++++++++++++++++++++++++++++++++ pkg/utils/net/net_test.go | 46 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 150 insertions(+), 19 deletions(-) diff --git a/pkg/talos/helper.go b/pkg/talos/helper.go index 6bdddc1..0d21b49 100644 --- a/pkg/talos/helper.go +++ b/pkg/talos/helper.go @@ -4,6 +4,7 @@ import ( "context" "crypto/x509" "fmt" + "strings" utilsnet "github.com/siderolabs/talos-cloud-controller-manager/pkg/utils/net" "github.com/siderolabs/talos/pkg/machinery/resources/network" @@ -17,11 +18,11 @@ import ( "k8s.io/utils/strings/slices" ) -func getNodeAddresses(config *cloudConfig, platform, nodeIP string, ifaces []network.AddressStatusSpec) []v1.NodeAddress { +func getNodeAddresses(config *cloudConfig, platform string, nodeIPs []string, ifaces []network.AddressStatusSpec) []v1.NodeAddress { var publicIPv4s, publicIPv6s, publicIPs []string switch platform { - case "nocloud", "metal": + case "nocloud", "metal", "openstack": for _, iface := range ifaces { if iface.LinkName == "kubespan" { continue @@ -29,6 +30,10 @@ func getNodeAddresses(config *cloudConfig, platform, nodeIP string, ifaces []net ip := iface.Address.Addr() if ip.IsGlobalUnicast() && !ip.IsPrivate() { + if slices.Contains(nodeIPs, ip.String()) { + continue + } + if ip.Is6() { publicIPv6s = append(publicIPv6s, ip.String()) } else { @@ -41,6 +46,10 @@ func getNodeAddresses(config *cloudConfig, platform, nodeIP string, ifaces []net if iface.LinkName == "external" { ip := iface.Address.Addr() + if slices.Contains(nodeIPs, ip.String()) { + continue + } + if ip.Is6() { publicIPv6s = append(publicIPv6s, ip.String()) } else { @@ -50,14 +59,12 @@ func getNodeAddresses(config *cloudConfig, platform, nodeIP string, ifaces []net } } - addresses := []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: nodeIP}} - - if config.Global.PreferIPv6 { - publicIPs = utilsnet.SortedNodeIPs(nodeIP, publicIPv6s, publicIPv4s) - } else { - publicIPs = utilsnet.SortedNodeIPs(nodeIP, publicIPv4s, publicIPv6s) + addresses := []v1.NodeAddress{} + for _, ip := range utilsnet.PreferedDualStackNodeIPs(config.Global.PreferIPv6, nodeIPs) { + addresses = append(addresses, v1.NodeAddress{Type: v1.NodeInternalIP, Address: ip}) } + publicIPs = utilsnet.PreferedDualStackNodeIPs(config.Global.PreferIPv6, append(publicIPv4s, publicIPv6s...)) for _, ip := range publicIPs { addresses = append(addresses, v1.NodeAddress{Type: v1.NodeExternalIP, Address: ip}) } @@ -105,7 +112,7 @@ func csrNodeChecks(ctx context.Context, kclient clientkubernetes.Interface, x509 if node != nil { if providedIP, ok := node.ObjectMeta.Annotations[cloudproviderapi.AnnotationAlphaProvidedIPAddr]; ok { - nodeAddrs = append(nodeAddrs, providedIP) + nodeAddrs = append(nodeAddrs, strings.Split(providedIP, ",")...) } for _, ip := range node.Status.Addresses { diff --git a/pkg/talos/helper_test.go b/pkg/talos/helper_test.go index 22f3e68..81861ba 100644 --- a/pkg/talos/helper_test.go +++ b/pkg/talos/helper_test.go @@ -6,6 +6,7 @@ import ( "fmt" "net" "net/netip" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -45,6 +46,21 @@ func TestGetNodeAddresses(t *testing.T) { {Type: v1.NodeInternalIP, Address: "192.168.0.1"}, }, }, + { + name: "nocloud has dualstack", + cfg: cfg, + platform: "nocloud", + providedIP: "192.168.0.1,fd00:192:168:0::1", + ifaces: []network.AddressStatusSpec{ + {Address: netip.MustParsePrefix("192.168.0.1/24")}, + {Address: netip.MustParsePrefix("fe80::e0b5:71ff:fe24:7e60/64")}, + {Address: netip.MustParsePrefix("fd00:192:168:0::1/64")}, + }, + expected: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "192.168.0.1"}, + {Type: v1.NodeInternalIP, Address: "fd00:192:168::1"}, + }, + }, { name: "nocloud has many PublicIPs", cfg: cfg, @@ -62,14 +78,14 @@ func TestGetNodeAddresses(t *testing.T) { expected: []v1.NodeAddress{ {Type: v1.NodeInternalIP, Address: "192.168.0.1"}, {Type: v1.NodeExternalIP, Address: "1.2.3.4"}, - {Type: v1.NodeExternalIP, Address: "2001:1234:4321::32"}, + {Type: v1.NodeExternalIP, Address: "2001:1234::1"}, }, }, { name: "nocloud has many PublicIPs (IPv6 preferred)", cfg: cloudConfig{Global: cloudConfigGlobal{PreferIPv6: true}}, platform: "nocloud", - providedIP: "192.168.0.1", + providedIP: "192.168.0.1,fd15:1:2::192:168:0:1", ifaces: []network.AddressStatusSpec{ {Address: netip.MustParsePrefix("192.168.0.1/24")}, {Address: netip.MustParsePrefix("fe80::e0b5:71ff:fe24:7e60/64")}, @@ -80,8 +96,9 @@ func TestGetNodeAddresses(t *testing.T) { {Address: netip.MustParsePrefix("2001:1234:4321::32/64")}, }, expected: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "fd15:1:2:0:192:168:0:1"}, {Type: v1.NodeInternalIP, Address: "192.168.0.1"}, - {Type: v1.NodeExternalIP, Address: "2001:1234:4321::32"}, + {Type: v1.NodeExternalIP, Address: "2001:1234::1"}, {Type: v1.NodeExternalIP, Address: "1.2.3.4"}, }, }, @@ -124,7 +141,7 @@ func TestGetNodeAddresses(t *testing.T) { }, } { t.Run(tt.name, func(t *testing.T) { - addresses := getNodeAddresses(&tt.cfg, tt.platform, tt.providedIP, tt.ifaces) + addresses := getNodeAddresses(&tt.cfg, tt.platform, strings.Split(tt.providedIP, ","), tt.ifaces) assert.Equal(t, tt.expected, addresses) }) diff --git a/pkg/talos/instances.go b/pkg/talos/instances.go index a405bd3..09f3282 100644 --- a/pkg/talos/instances.go +++ b/pkg/talos/instances.go @@ -5,7 +5,9 @@ import ( "fmt" "strings" + "github.com/siderolabs/talos-cloud-controller-manager/pkg/utils/net" "github.com/siderolabs/talos-cloud-controller-manager/pkg/utils/platform" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" v1 "k8s.io/api/core/v1" cloudprovider "k8s.io/cloud-provider" @@ -46,16 +48,34 @@ func (i *instances) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloud klog.V(4).Info("instances.InstanceMetadata() called, node: ", node.Name) if providedIP, ok := node.ObjectMeta.Annotations[cloudproviderapi.AnnotationAlphaProvidedIPAddr]; ok { - meta, err := i.c.getNodeMetadata(ctx, providedIP) - if err != nil { - return nil, fmt.Errorf("error getting metadata from the node %s: %w", node.Name, err) + nodeIPs := net.PreferedDualStackNodeIPs(i.c.config.Global.PreferIPv6, strings.Split(providedIP, ",")) + + var ( + meta *runtime.PlatformMetadataSpec + err error + nodeIP string + ) + + for _, ip := range nodeIPs { + meta, err = i.c.getNodeMetadata(ctx, ip) + if err == nil { + nodeIP = ip + + break + } + + klog.Errorf("error getting metadata from the node %s: %v", node.Name, err) + } + + if meta == nil { + return nil, fmt.Errorf("error getting metadata from the node %s", node.Name) } klog.V(5).Infof("instances.InstanceMetadata() resource: %+v", meta) providerID := meta.ProviderID if providerID == "" { - providerID = fmt.Sprintf("%s://%s/%s", ProviderName, meta.Platform, providedIP) + providerID = fmt.Sprintf("%s://%s/%s", ProviderName, meta.Platform, nodeIP) } // Fix for Azure, resource group name must be lower case. @@ -66,12 +86,12 @@ func (i *instances) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloud } } - ifaces, err := i.c.getNodeIfaces(ctx, providedIP) + ifaces, err := i.c.getNodeIfaces(ctx, nodeIP) if err != nil { return nil, fmt.Errorf("error getting interfaces list from the node %s: %w", node.Name, err) } - addresses := getNodeAddresses(i.c.config, meta.Platform, providedIP, ifaces) + addresses := getNodeAddresses(i.c.config, meta.Platform, nodeIPs, ifaces) addresses = append(addresses, v1.NodeAddress{Type: v1.NodeHostName, Address: node.Name}) diff --git a/pkg/utils/net/net.go b/pkg/utils/net/net.go index c2ba9be..3426cf3 100644 --- a/pkg/utils/net/net.go +++ b/pkg/utils/net/net.go @@ -38,3 +38,44 @@ func SortedNodeIPs(nodeIP string, first, second []string) (res []string) { return res } + +// PreferedDualStackNodeIPs returns the first IPv4 and IPv6 addresses from the list of IPs. +func PreferedDualStackNodeIPs(preferIPv6 bool, ips []string) []string { + var ipv6, ipv4 string + + for _, ip := range ips { + if nip, err := netip.ParseAddr(ip); err == nil { + if nip.Is6() { + if ipv6 == "" { + ipv6 = nip.String() + } + } else { + if ipv4 == "" { + ipv4 = nip.String() + } + } + } + } + + res := []string{} + + if preferIPv6 { + if ipv6 != "" { + res = append(res, ipv6) + } + + if ipv4 != "" { + res = append(res, ipv4) + } + } else { + if ipv4 != "" { + res = append(res, ipv4) + } + + if ipv6 != "" { + res = append(res, ipv6) + } + } + + return res +} diff --git a/pkg/utils/net/net_test.go b/pkg/utils/net/net_test.go index e635fc0..e2bb92f 100644 --- a/pkg/utils/net/net_test.go +++ b/pkg/utils/net/net_test.go @@ -106,3 +106,49 @@ func TestSortedNodeIPs(t *testing.T) { }) } } + +func TestPreferedDualStackNodeIPs(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { //nolint:govet + name string + preferIPv6 bool + nodeIPs []string + expected []string + }{ + { + name: "one nodeIP", + preferIPv6: false, + nodeIPs: []string{"192.168.0.1"}, + expected: []string{"192.168.0.1"}, + }, + { + name: "dualstack nodeIP", + preferIPv6: false, + nodeIPs: []string{"192.168.0.1", "fd00::1"}, + expected: []string{"192.168.0.1", "fd00::1"}, + }, + { + name: "dualstack nodeIP preferIPv6", + preferIPv6: true, + nodeIPs: []string{"192.168.0.1", "fd00::1"}, + expected: []string{"fd00::1", "192.168.0.1"}, + }, + { + name: "dualstack nodeIP preferIPv6 with external IPs", + preferIPv6: true, + nodeIPs: []string{"192.168.0.1", "fd00::1", "1.1.1.1", "1111:2222:3333::1", "2000:123:123::9b87:57a7:38bf:6c71"}, + expected: []string{"fd00::1", "192.168.0.1"}, + }, + } { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + result := utilnet.PreferedDualStackNodeIPs(tt.preferIPv6, tt.nodeIPs) + + assert.Equal(t, fmt.Sprintf("%v", tt.expected), fmt.Sprintf("%v", result)) + }) + } +}