Skip to content

Commit

Permalink
feat: support CloudDualStackNodeIPs
Browse files Browse the repository at this point in the history 
Talos CCM now supports the `CloudDualStackNodeIPs` feature gate. This feature
allows the user(cloud) to specify a list of IPv4 and IPv6 addresses for each node in
the cluster. kubernetes/kubernetes#120275

Signed-off-by: Serge Logvinov <[email protected]>
  • Loading branch information
@sergelogvinov
sergelogvinov committed Feb 5, 2024
1 parent 33faa60 commit 81668b7
Show file tree
Hide file tree
Showing 5 changed files with 152 additions and 19 deletions.
25 changes: 16 additions & 9 deletions pkg/talos/helper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"crypto/x509"
"fmt"
"strings"

utilsnet "github.com/siderolabs/talos-cloud-controller-manager/pkg/utils/net"
"github.com/siderolabs/talos/pkg/machinery/resources/network"
Expand All @@ -17,18 +18,22 @@ import (
"k8s.io/utils/strings/slices"
)

func getNodeAddresses(config *cloudConfig, platform, nodeIP string, ifaces []network.AddressStatusSpec) []v1.NodeAddress {
func getNodeAddresses(config *cloudConfig, platform string, nodeIPs []string, ifaces []network.AddressStatusSpec) []v1.NodeAddress {
var publicIPv4s, publicIPv6s, publicIPs []string

switch platform {
case "nocloud", "metal":
case "nocloud", "metal", "openstack":
for _, iface := range ifaces {
if iface.LinkName == "kubespan" {
continue
}

ip := iface.Address.Addr()
if ip.IsGlobalUnicast() && !ip.IsPrivate() {
if slices.Contains(nodeIPs, ip.String()) {
continue
}

if ip.Is6() {
publicIPv6s = append(publicIPv6s, ip.String())
} else {
Expand All @@ -41,6 +46,10 @@ func getNodeAddresses(config *cloudConfig, platform, nodeIP string, ifaces []net
if iface.LinkName == "external" {
ip := iface.Address.Addr()

if slices.Contains(nodeIPs, ip.String()) {
continue
}

if ip.Is6() {
publicIPv6s = append(publicIPv6s, ip.String())
} else {
Expand All @@ -50,14 +59,12 @@ func getNodeAddresses(config *cloudConfig, platform, nodeIP string, ifaces []net
}
}

addresses := []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: nodeIP}}

if config.Global.PreferIPv6 {
publicIPs = utilsnet.SortedNodeIPs(nodeIP, publicIPv6s, publicIPv4s)
} else {
publicIPs = utilsnet.SortedNodeIPs(nodeIP, publicIPv4s, publicIPv6s)
addresses := []v1.NodeAddress{}
for _, ip := range utilsnet.PreferedDualStackNodeIPs(config.Global.PreferIPv6, nodeIPs) {
addresses = append(addresses, v1.NodeAddress{Type: v1.NodeInternalIP, Address: ip})
}

publicIPs = utilsnet.PreferedDualStackNodeIPs(config.Global.PreferIPv6, append(publicIPv4s, publicIPv6s...))
for _, ip := range publicIPs {
addresses = append(addresses, v1.NodeAddress{Type: v1.NodeExternalIP, Address: ip})
}
Expand Down Expand Up @@ -105,7 +112,7 @@ func csrNodeChecks(ctx context.Context, kclient clientkubernetes.Interface, x509

if node != nil {
if providedIP, ok := node.ObjectMeta.Annotations[cloudproviderapi.AnnotationAlphaProvidedIPAddr]; ok {
nodeAddrs = append(nodeAddrs, providedIP)
nodeAddrs = append(nodeAddrs, strings.Split(providedIP, ",")...)
}

for _, ip := range node.Status.Addresses {
Expand Down
25 changes: 21 additions & 4 deletions pkg/talos/helper_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"fmt"
"net"
"net/netip"
"strings"
"testing"

"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -45,6 +46,21 @@ func TestGetNodeAddresses(t *testing.T) {
{Type: v1.NodeInternalIP, Address: "192.168.0.1"},
},
},
{
name: "nocloud has dualstack",
cfg: cfg,
platform: "nocloud",
providedIP: "192.168.0.1,fd00:192:168:0::1",
ifaces: []network.AddressStatusSpec{
{Address: netip.MustParsePrefix("192.168.0.1/24")},
{Address: netip.MustParsePrefix("fe80::e0b5:71ff:fe24:7e60/64")},
{Address: netip.MustParsePrefix("fd00:192:168:0::1/64")},
},
expected: []v1.NodeAddress{
{Type: v1.NodeInternalIP, Address: "192.168.0.1"},
{Type: v1.NodeInternalIP, Address: "fd00:192:168::1"},
},
},
{
name: "nocloud has many PublicIPs",
cfg: cfg,
Expand All @@ -62,14 +78,14 @@ func TestGetNodeAddresses(t *testing.T) {
expected: []v1.NodeAddress{
{Type: v1.NodeInternalIP, Address: "192.168.0.1"},
{Type: v1.NodeExternalIP, Address: "1.2.3.4"},
{Type: v1.NodeExternalIP, Address: "2001:1234:4321::32"},
{Type: v1.NodeExternalIP, Address: "2001:1234::1"},
},
},
{
name: "nocloud has many PublicIPs (IPv6 preferred)",
cfg: cloudConfig{Global: cloudConfigGlobal{PreferIPv6: true}},
platform: "nocloud",
providedIP: "192.168.0.1",
providedIP: "192.168.0.1,fd15:1:2::192:168:0:1",
ifaces: []network.AddressStatusSpec{
{Address: netip.MustParsePrefix("192.168.0.1/24")},
{Address: netip.MustParsePrefix("fe80::e0b5:71ff:fe24:7e60/64")},
Expand All @@ -80,8 +96,9 @@ func TestGetNodeAddresses(t *testing.T) {
{Address: netip.MustParsePrefix("2001:1234:4321::32/64")},
},
expected: []v1.NodeAddress{
{Type: v1.NodeInternalIP, Address: "fd15:1:2:0:192:168:0:1"},
{Type: v1.NodeInternalIP, Address: "192.168.0.1"},
{Type: v1.NodeExternalIP, Address: "2001:1234:4321::32"},
{Type: v1.NodeExternalIP, Address: "2001:1234::1"},
{Type: v1.NodeExternalIP, Address: "1.2.3.4"},
},
},
Expand Down Expand Up @@ -124,7 +141,7 @@ func TestGetNodeAddresses(t *testing.T) {
},
} {
t.Run(tt.name, func(t *testing.T) {
addresses := getNodeAddresses(&tt.cfg, tt.platform, tt.providedIP, tt.ifaces)
addresses := getNodeAddresses(&tt.cfg, tt.platform, strings.Split(tt.providedIP, ","), tt.ifaces)

assert.Equal(t, tt.expected, addresses)
})
Expand Down
34 changes: 28 additions & 6 deletions pkg/talos/instances.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@ import (
"fmt"
"strings"

"github.com/siderolabs/talos-cloud-controller-manager/pkg/utils/net"
"github.com/siderolabs/talos-cloud-controller-manager/pkg/utils/platform"
"github.com/siderolabs/talos/pkg/machinery/resources/runtime"

v1 "k8s.io/api/core/v1"
cloudprovider "k8s.io/cloud-provider"
Expand Down Expand Up @@ -46,16 +48,34 @@ func (i *instances) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloud
klog.V(4).Info("instances.InstanceMetadata() called, node: ", node.Name)

if providedIP, ok := node.ObjectMeta.Annotations[cloudproviderapi.AnnotationAlphaProvidedIPAddr]; ok {
meta, err := i.c.getNodeMetadata(ctx, providedIP)
if err != nil {
return nil, fmt.Errorf("error getting metadata from the node %s: %w", node.Name, err)
nodeIPs := net.PreferedDualStackNodeIPs(i.c.config.Global.PreferIPv6, strings.Split(providedIP, ","))

var (
meta *runtime.PlatformMetadataSpec
err error
nodeIP string
)

for _, ip := range nodeIPs {
meta, err = i.c.getNodeMetadata(ctx, ip)
if err == nil {
nodeIP = ip

break
}

klog.Errorf("error getting metadata from the node %s: %v", node.Name, err)
}

if meta == nil {
return nil, fmt.Errorf("error getting metadata from the node %s", node.Name)
}

klog.V(5).Infof("instances.InstanceMetadata() resource: %+v", meta)

providerID := meta.ProviderID
if providerID == "" {
providerID = fmt.Sprintf("%s://%s/%s", ProviderName, meta.Platform, providedIP)
providerID = fmt.Sprintf("%s://%s/%s", ProviderName, meta.Platform, nodeIP)
}

// Fix for Azure, resource group name must be lower case.
Expand All @@ -66,19 +86,21 @@ func (i *instances) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloud
}
}

ifaces, err := i.c.getNodeIfaces(ctx, providedIP)
ifaces, err := i.c.getNodeIfaces(ctx, nodeIP)
if err != nil {
return nil, fmt.Errorf("error getting interfaces list from the node %s: %w", node.Name, err)
}

addresses := getNodeAddresses(i.c.config, meta.Platform, providedIP, ifaces)
addresses := getNodeAddresses(i.c.config, meta.Platform, nodeIPs, ifaces)

addresses = append(addresses, v1.NodeAddress{Type: v1.NodeHostName, Address: node.Name})

if meta.Hostname != "" && strings.IndexByte(meta.Hostname, '.') > 0 {
addresses = append(addresses, v1.NodeAddress{Type: v1.NodeInternalDNS, Address: meta.Hostname})
}

klog.V(5).Infof("instances.InstanceMetadata() addresses: %+v", addresses)

// Foreign node, update network only.
if i.c.config.Global.SkipForeignNode && !strings.HasPrefix(node.Spec.ProviderID, ProviderName) {
klog.V(4).Infof("instances.InstanceMetadata() node %s has foreign providerID: %s, skipped", node.Name, node.Spec.ProviderID)
Expand Down
41 changes: 41 additions & 0 deletions pkg/utils/net/net.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,3 +38,44 @@ func SortedNodeIPs(nodeIP string, first, second []string) (res []string) {

return res
}

// PreferedDualStackNodeIPs returns the first IPv4 and IPv6 addresses from the list of IPs.
func PreferedDualStackNodeIPs(preferIPv6 bool, ips []string) []string {
var ipv6, ipv4 string

for _, ip := range ips {
if nip, err := netip.ParseAddr(ip); err == nil {
if nip.Is6() {
if ipv6 == "" {
ipv6 = nip.String()
}
} else {
if ipv4 == "" {
ipv4 = nip.String()
}
}
}
}

res := []string{}

if preferIPv6 {
if ipv6 != "" {
res = append(res, ipv6)
}

if ipv4 != "" {
res = append(res, ipv4)
}
} else {
if ipv4 != "" {
res = append(res, ipv4)
}

if ipv6 != "" {
res = append(res, ipv6)
}
}

return res
}
46 changes: 46 additions & 0 deletions pkg/utils/net/net_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,3 +106,49 @@ func TestSortedNodeIPs(t *testing.T) {
})
}
}

func TestPreferedDualStackNodeIPs(t *testing.T) {
t.Parallel()

for _, tt := range []struct { //nolint:govet
name string
preferIPv6 bool
nodeIPs []string
expected []string
}{
{
name: "one nodeIP",
preferIPv6: false,
nodeIPs: []string{"192.168.0.1"},
expected: []string{"192.168.0.1"},
},
{
name: "dualstack nodeIP",
preferIPv6: false,
nodeIPs: []string{"192.168.0.1", "fd00::1"},
expected: []string{"192.168.0.1", "fd00::1"},
},
{
name: "dualstack nodeIP preferIPv6",
preferIPv6: true,
nodeIPs: []string{"192.168.0.1", "fd00::1"},
expected: []string{"fd00::1", "192.168.0.1"},
},
{
name: "dualstack nodeIP preferIPv6 with external IPs",
preferIPv6: true,
nodeIPs: []string{"192.168.0.1", "fd00::1", "1.1.1.1", "1111:2222:3333::1", "2000:123:123::9b87:57a7:38bf:6c71"},
expected: []string{"fd00::1", "192.168.0.1"},
},
} {
tt := tt

t.Run(tt.name, func(t *testing.T) {
t.Parallel()

result := utilnet.PreferedDualStackNodeIPs(tt.preferIPv6, tt.nodeIPs)

assert.Equal(t, fmt.Sprintf("%v", tt.expected), fmt.Sprintf("%v", result))
})
}
}

0 comments on commit 81668b7

Please sign in to comment.