Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adds basic support for advertising both v4 and v6 VIPs in a dual-stack cluster #1120

Closed
wants to merge 1 commit into from
Closed

Adds basic support for advertising both v4 and v6 VIPs in a dual-stack cluster #1120

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
186 changes: 105 additions & 81 deletions pkg/controllers/routing/bgp_policies.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"net"
"reflect"
"strconv"
"strings"
Expand Down Expand Up @@ -94,75 +95,95 @@ func (nrc *NetworkRoutingController) addPodCidrDefinedSet() error {

// create a defined set to represent all the advertisable IP associated with the services
func (nrc *NetworkRoutingController) addServiceVIPsDefinedSet() error {
var currentDefinedSet *gobgpapi.DefinedSet
err := nrc.bgpServer.ListDefinedSet(context.Background(),
&gobgpapi.ListDefinedSetRequest{DefinedType: gobgpapi.DefinedType_PREFIX, Name: "servicevipsdefinedset"},
func(ds *gobgpapi.DefinedSet) {
currentDefinedSet = ds
})
if err != nil {
return err
definedSets := map[string][]*gobgpapi.Prefix{
"servicevipsdefinedset4": {},
"servicevipsdefinedset6": {},
}
advIPPrefixList := make([]*gobgpapi.Prefix, 0)

advIps, _, _ := nrc.getAllVIPs()
for _, ip := range advIps {
advIPPrefixList = append(advIPPrefixList, &gobgpapi.Prefix{IpPrefix: ip + "/32", MaskLengthMin: 32, MaskLengthMax: 32})
}
if currentDefinedSet == nil {
clusterIPPrefixSet := &gobgpapi.DefinedSet{
DefinedType: gobgpapi.DefinedType_PREFIX,
Name: "servicevipsdefinedset",
Prefixes: advIPPrefixList,
if net.ParseIP(ip).To4() != nil {
definedSets["servicevipsdefinedset4"] = append(definedSets["servicevipsdefinedset4"],
&gobgpapi.Prefix{IpPrefix: ip + "/32", MaskLengthMin: 32, MaskLengthMax: 32})
} else if net.ParseIP(ip).To16() != nil {
definedSets["servicevipsdefinedset6"] = append(definedSets["servicevipsdefinedset6"],
&gobgpapi.Prefix{IpPrefix: ip + "/128", MaskLengthMin: 128, MaskLengthMax: 128})
} else {
return errors.New("found invalid address: " + ip)
}
}

for dsn, advIPPrefixList := range definedSets {
var currentDefinedSet *gobgpapi.DefinedSet
err := nrc.bgpServer.ListDefinedSet(context.Background(),
&gobgpapi.ListDefinedSetRequest{DefinedType: gobgpapi.DefinedType_PREFIX, Name: dsn},
func(ds *gobgpapi.DefinedSet) {
currentDefinedSet = ds
})
if err != nil {
return err
}
return nrc.bgpServer.AddDefinedSet(context.Background(), &gobgpapi.AddDefinedSetRequest{DefinedSet: clusterIPPrefixSet})
}

if reflect.DeepEqual(advIPPrefixList, currentDefinedSet.Prefixes) {
return nil
}
toAdd := make([]*gobgpapi.Prefix, 0)
toDelete := make([]*gobgpapi.Prefix, 0)
for _, prefix := range advIPPrefixList {
add := true
for _, currentPrefix := range currentDefinedSet.Prefixes {
if currentPrefix.IpPrefix == prefix.IpPrefix {
add = false
if currentDefinedSet == nil {
clusterIPPrefixSet := &gobgpapi.DefinedSet{
DefinedType: gobgpapi.DefinedType_PREFIX,
Name: dsn,
Prefixes: advIPPrefixList,
}
err := nrc.bgpServer.AddDefinedSet(context.Background(), &gobgpapi.AddDefinedSetRequest{DefinedSet: clusterIPPrefixSet})
if err != nil {
return err
} else {
continue
}
}
if add {
toAdd = append(toAdd, prefix)

if reflect.DeepEqual(advIPPrefixList, currentDefinedSet.Prefixes) {
continue
}
}
for _, currentPrefix := range currentDefinedSet.Prefixes {
shouldDelete := true
toAdd := make([]*gobgpapi.Prefix, 0)
toDelete := make([]*gobgpapi.Prefix, 0)
for _, prefix := range advIPPrefixList {
if currentPrefix.IpPrefix == prefix.IpPrefix {
shouldDelete = false
add := true
for _, currentPrefix := range currentDefinedSet.Prefixes {
if currentPrefix.IpPrefix == prefix.IpPrefix {
add = false
}
}
if add {
toAdd = append(toAdd, prefix)
}
}
if shouldDelete {
toDelete = append(toDelete, currentPrefix)
for _, currentPrefix := range currentDefinedSet.Prefixes {
shouldDelete := true
for _, prefix := range advIPPrefixList {
if currentPrefix.IpPrefix == prefix.IpPrefix {
shouldDelete = false
}
}
if shouldDelete {
toDelete = append(toDelete, currentPrefix)
}
}
clusterIPPrefixSet := &gobgpapi.DefinedSet{
DefinedType: gobgpapi.DefinedType_PREFIX,
Name: dsn,
Prefixes: toAdd,
}
err = nrc.bgpServer.AddDefinedSet(context.Background(), &gobgpapi.AddDefinedSetRequest{DefinedSet: clusterIPPrefixSet})
if err != nil {
return err
}
clusterIPPrefixSet = &gobgpapi.DefinedSet{
DefinedType: gobgpapi.DefinedType_PREFIX,
Name: dsn,
Prefixes: toDelete,
}
err = nrc.bgpServer.DeleteDefinedSet(context.Background(), &gobgpapi.DeleteDefinedSetRequest{DefinedSet: clusterIPPrefixSet, All: false})
if err != nil {
return err
}
}
clusterIPPrefixSet := &gobgpapi.DefinedSet{
DefinedType: gobgpapi.DefinedType_PREFIX,
Name: "servicevipsdefinedset",
Prefixes: toAdd,
}
err = nrc.bgpServer.AddDefinedSet(context.Background(), &gobgpapi.AddDefinedSetRequest{DefinedSet: clusterIPPrefixSet})
if err != nil {
return err
}
clusterIPPrefixSet = &gobgpapi.DefinedSet{
DefinedType: gobgpapi.DefinedType_PREFIX,
Name: "servicevipsdefinedset",
Prefixes: toDelete,
}
err = nrc.bgpServer.DeleteDefinedSet(context.Background(), &gobgpapi.DeleteDefinedSetRequest{DefinedSet: clusterIPPrefixSet, All: false})
if err != nil {
return err
}

return nil
}

Expand Down Expand Up @@ -429,19 +450,21 @@ func (nrc *NetworkRoutingController) addExportPolicies() error {

// statement to represent the export policy to permit advertising cluster IP's
// only to the global BGP peer or node specific BGP peer
statements = append(statements, &gobgpapi.Statement{
Conditions: &gobgpapi.Conditions{
PrefixSet: &gobgpapi.MatchSet{
MatchType: gobgpapi.MatchType_ANY,
Name: "servicevipsdefinedset",
},
NeighborSet: &gobgpapi.MatchSet{
MatchType: gobgpapi.MatchType_ANY,
Name: "externalpeerset",
for _, dsn := range []string{"servicevipsdefinedset4", "servicevipsdefinedset6"} {
statements = append(statements, &gobgpapi.Statement{
Conditions: &gobgpapi.Conditions{
PrefixSet: &gobgpapi.MatchSet{
MatchType: gobgpapi.MatchType_ANY,
Name: dsn,
},
NeighborSet: &gobgpapi.MatchSet{
MatchType: gobgpapi.MatchType_ANY,
Name: "externalpeerset",
},
},
},
Actions: &bgpActions,
})
Actions: &bgpActions,
})
}

if nrc.advertisePodCidr {
actions := gobgpapi.Actions{
Expand Down Expand Up @@ -534,20 +557,21 @@ func (nrc *NetworkRoutingController) addImportPolicies() error {
actions := gobgpapi.Actions{
RouteAction: gobgpapi.RouteAction_REJECT,
}
statements = append(statements, &gobgpapi.Statement{
Conditions: &gobgpapi.Conditions{
PrefixSet: &gobgpapi.MatchSet{
MatchType: gobgpapi.MatchType_ANY,
Name: "servicevipsdefinedset",
},
NeighborSet: &gobgpapi.MatchSet{
MatchType: gobgpapi.MatchType_ANY,
Name: "allpeerset",
for _, dsn := range []string{"servicevipsdefinedset4", "servicevipsdefinedset6"} {
statements = append(statements, &gobgpapi.Statement{
Conditions: &gobgpapi.Conditions{
PrefixSet: &gobgpapi.MatchSet{
MatchType: gobgpapi.MatchType_ANY,
Name: dsn,
},
NeighborSet: &gobgpapi.MatchSet{
MatchType: gobgpapi.MatchType_ANY,
Name: "allpeerset",
},
},
},
Actions: &actions,
})

Actions: &actions,
})
}
definition := gobgpapi.Policy{
Name: "kube_router_import",
Statements: statements,
Expand Down
35 changes: 27 additions & 8 deletions pkg/controllers/routing/ecmp_vip.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"net"
"strconv"

"github.com/cloudnativelabs/kube-router/pkg/utils"
Expand All @@ -21,22 +22,24 @@ import (
// bgpAdvertiseVIP advertises the service vip (cluster ip or load balancer ip or external IP) the configured peers
func (nrc *NetworkRoutingController) bgpAdvertiseVIP(vip string) error {

klog.V(2).Infof("Advertising route: '%s/%s via %s' to peers", vip, strconv.Itoa(32), nrc.nodeIP.String())
prefixLen, nh, afi := nrc.getVIPRoutingAttr(vip)

klog.V(2).Infof("Advertising route: '%s/%s via %s' to peers", vip, strconv.Itoa(prefixLen), nh)

a1, _ := ptypes.MarshalAny(&gobgpapi.OriginAttribute{
Origin: 0,
})
a2, _ := ptypes.MarshalAny(&gobgpapi.NextHopAttribute{
NextHop: nrc.nodeIP.String(),
NextHop: nh,
})
attrs := []*any.Any{a1, a2}
nlri1, _ := ptypes.MarshalAny(&gobgpapi.IPAddressPrefix{
Prefix: vip,
PrefixLen: 32,
PrefixLen: uint32(prefixLen),
})
_, err := nrc.bgpServer.AddPath(context.Background(), &gobgpapi.AddPathRequest{
Path: &gobgpapi.Path{
Family: &gobgpapi.Family{Afi: gobgpapi.Family_AFI_IP, Safi: gobgpapi.Family_SAFI_UNICAST},
Family: &gobgpapi.Family{Afi: afi, Safi: gobgpapi.Family_SAFI_UNICAST},
Nlri: nlri1,
Pattrs: attrs,
},
Expand All @@ -47,21 +50,22 @@ func (nrc *NetworkRoutingController) bgpAdvertiseVIP(vip string) error {

// bgpWithdrawVIP unadvertises the service vip
func (nrc *NetworkRoutingController) bgpWithdrawVIP(vip string) error {
klog.V(2).Infof("Withdrawing route: '%s/%s via %s' to peers", vip, strconv.Itoa(32), nrc.nodeIP.String())
prefixLen, nh, afi := nrc.getVIPRoutingAttr(vip)
klog.V(2).Infof("Withdrawing route: '%s/%s via %s' to peers", vip, strconv.Itoa(prefixLen), nh)

a1, _ := ptypes.MarshalAny(&gobgpapi.OriginAttribute{
Origin: 0,
})
a2, _ := ptypes.MarshalAny(&gobgpapi.NextHopAttribute{
NextHop: nrc.nodeIP.String(),
NextHop: nh,
})
attrs := []*any.Any{a1, a2}
nlri, _ := ptypes.MarshalAny(&gobgpapi.IPAddressPrefix{
Prefix: vip,
PrefixLen: 32,
PrefixLen: uint32(prefixLen),
})
path := gobgpapi.Path{
Family: &gobgpapi.Family{Afi: gobgpapi.Family_AFI_IP, Safi: gobgpapi.Family_SAFI_UNICAST},
Family: &gobgpapi.Family{Afi: afi, Safi: gobgpapi.Family_SAFI_UNICAST},
Nlri: nlri,
Pattrs: attrs,
}
Expand All @@ -73,6 +77,21 @@ func (nrc *NetworkRoutingController) bgpWithdrawVIP(vip string) error {
return err
}

func (nrc *NetworkRoutingController) getVIPRoutingAttr(vip string) (int, string, gobgpapi.Family_Afi) {
prefixLen := 32
afi := gobgpapi.Family_AFI_IP
nh := nrc.nodeIP.String()
if nrc.isDualStack && utils.MatchAddressFamily(net.ParseIP(vip), nrc.dualStackNextNodeIP) {
nh = nrc.dualStackNextNodeIP.String()
}
if net.ParseIP(vip).To4() == nil && net.ParseIP(vip).To16() != nil {
prefixLen = 128
afi = gobgpapi.Family_AFI_IP6
}

return prefixLen, nh, afi
}

func (nrc *NetworkRoutingController) advertiseVIPs(vips []string) {
for _, vip := range vips {
err := nrc.bgpAdvertiseVIP(vip)
Expand Down
11 changes: 11 additions & 0 deletions pkg/controllers/routing/network_routes_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ type NetworkRoutingController struct {
nodeInterface string
routerID string
isIpv6 bool
isDualStack bool
dualStackNextNodeIP net.IP
activeNodes map[string]bool
mu sync.Mutex
clientset kubernetes.Interface
Expand Down Expand Up @@ -1085,7 +1087,16 @@ func NewNetworkRoutingController(clientset kubernetes.Interface,
}
nrc.nodeIP = nodeIP
nrc.isIpv6 = nodeIP.To4() == nil
nrc.isDualStack = utils.IsNodeDualStack(node)

if nrc.isDualStack {
dualStackNextNodeIP, err := utils.GetNextFamilyNodeIP(node, nodeIP)
if err != nil {
return nil, errors.New("failed to get suitable node addresses for dual-stack: " + err.Error())
}
nrc.dualStackNextNodeIP = dualStackNextNodeIP

}
if kubeRouterConfig.RouterID != "" {
nrc.routerID = kubeRouterConfig.RouterID
} else {
Expand Down
46 changes: 46 additions & 0 deletions pkg/utils/node.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,3 +85,49 @@ func GetMTUFromNodeIP(nodeIP net.IP, overlayEnabled bool) (int, error) {
}
return 0, errors.New("failed to find interface with specified node IP")
}

// GetNextFamilyNodeIP is used in the dual stack case and returns the Node IP from
// the address family that is not the same as that returned from GetNodeIP
func GetNextFamilyNodeIP(node *apiv1.Node, firstNodeIP net.IP) (net.IP, error) {
addresses := node.Status.Addresses
addressMap := make(map[apiv1.NodeAddressType][]apiv1.NodeAddress)
for i := range addresses {
addressMap[addresses[i].Type] = append(addressMap[addresses[i].Type], addresses[i])
}
for _, address := range addressMap[apiv1.NodeInternalIP] {
if !MatchAddressFamily(net.ParseIP(address.Address), firstNodeIP) {
return net.ParseIP(address.Address), nil
}
}

for _, address := range addressMap[apiv1.NodeExternalIP] {
if !MatchAddressFamily(net.ParseIP(address.Address), firstNodeIP) {
return net.ParseIP(address.Address), nil
}
}

return nil, errors.New("Unable to find suitable next IP for dual-stack")
}

// IsNodeDualStack returns true after finding both an IPv4 and IPv6 address in either
// NodeInternalIP or NodeExternalIP, else returns false. Consider changing to look at
// IPv6DualStack feature gate when promoted to core
func IsNodeDualStack(node *apiv1.Node) bool {
hasv4 := false
hasv6 := false

addresses := node.Status.Addresses
for i := range addresses {
if addresses[i].Type == apiv1.NodeInternalIP || addresses[i].Type == apiv1.NodeExternalIP {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nitpic, but usually for stuff like this where there isn't an else, I like to invert the condition and have it continue so that the rest of the logic can be outdented and IMO it makes it a bit more readable.

Something like:

if addresses[i].Type != apiv1.NodeInternalIP && addresses[i].Type != apiv1.NodeExternalIP {
  continue
}

if net.ParseIP(addresses[i].Address).To4() != nil {
hasv4 = true
} else if net.ParseIP(addresses[i].Address).To16() != nil {
hasv6 = true
}
if hasv4 && hasv6 {
return true
}
}
}
return false
}
Loading