simplify punt fake gateway

This patch simplifies the way we handle the punt routes when routing for-me traffic VPP cannot handle to the host. Previously, we were using different fake gateways for ip4 & ip6. We now use the ip4 indirection in both cases. We also generate different hardware addresses using the uplink order in the configuration. Finally, we also make the isMain parameter public in the config. Signed-off-by: Nathan Skrzypczak <[email protected]>
projectcalico · Oct 5, 2023 · 45a7dae · 45a7dae
1 parent 79c8115
commit 45a7dae
Show file tree

Hide file tree

Showing 7 changed files with 80 additions and 52 deletions.
diff --git a/calico-vpp-agent/cni/cni_server.go b/calico-vpp-agent/cni/cni_server.go
@@ -38,7 +38,6 @@ import (
 	"github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common"
 	"github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers"
 	"github.com/projectcalico/vpp-dataplane/v3/config"
-	"github.com/projectcalico/vpp-dataplane/v3/vpp-manager/utils"
 	"github.com/projectcalico/vpp-dataplane/v3/vpplink"
 	"github.com/projectcalico/vpp-dataplane/v3/vpplink/types"
 )
@@ -479,15 +478,13 @@ forloop:
 	return nil
 }
 
-func (s *Server) getMainTap0Info() (tapSwIfIndex uint32, address net.IP) {
+func (s *Server) getMainInterface() *config.UplinkStatus {
 	for _, i := range common.VppManagerInfo.UplinkStatuses {
 		if i.IsMain {
-			tapSwIfIndex = i.TapSwIfIndex
-			break
+			return &i
 		}
 	}
-	address = utils.FakeVppNextHopIP4
-	return
+	return nil
 }
 
 func (s *Server) createRedirectToHostRules() (uint32, error) {
@@ -506,12 +503,15 @@ func (s *Server) createRedirectToHostRules() (uint32, error) {
 	if err != nil {
 		return types.InvalidID, err
 	}
-	tap0swifindex, tap0nexthop := s.getMainTap0Info()
+	mainInterface := s.getMainInterface()
+	if mainInterface == nil {
+		return types.InvalidID, fmt.Errorf("No main interface found")
+	}
 	for _, rule := range config.GetCalicoVppInitialConfig().RedirectToHostRules {
 		err = s.vpp.AddSessionRedirect(&types.SessionRedirect{
 			FiveTuple:  types.NewDst3Tuple(rule.Proto, net.ParseIP(rule.Ip), rule.Port),
 			TableIndex: index,
-		}, &types.RoutePath{Gw: tap0nexthop, SwIfIndex: tap0swifindex})
+		}, &types.RoutePath{Gw: config.VppHostPuntFakeGatewayAddress, SwIfIndex: mainInterface.TapSwIfIndex})
 		if err != nil {
 			return types.InvalidID, err
 		}

diff --git a/config/config.go b/config/config.go
@@ -58,6 +58,11 @@ const (
 	VppSigKillTimeout = 2
 	DefaultEncapSize  = 60 // Used to lower the MTU of the routes to the cluster
 
+	DefaultPhysicalNetworkName = ""
+
+	// BaseVppSideHardwareAddress is the base hardware address of VPP side of the HostPunt
+	// tap interface. It is used to generate hardware addresses for each uplink interface.
+	BaseVppSideHardwareAddress = "02:ca:11:c0:fd:00"
 )
 
 var (
@@ -130,6 +135,10 @@ var (
 	}
 
 	Info = &VppManagerInfo{}
+
+	// VppHostPuntFakeGatewayAddress is the fake gateway we use with a static neighbor
+	// in the punt table to route punted packets to the host
+	VppHostPuntFakeGatewayAddress = net.ParseIP("169.254.0.1")
 )
 
 func RunHook(hookScript *string, hookName string, params *VppManagerParams, log *logrus.Logger) {
@@ -222,7 +231,7 @@ func (i *InterfaceSpec) Validate(maxIfSpec *InterfaceSpec) error {
 
 type UplinkInterfaceSpec struct {
 	InterfaceSpec
-	IsMain              *bool             `json:"-"`
+	IsMain              bool              `json:"isMain"`
 	PhysicalNetworkName string            `json:"physicalNetworkName"`
 	InterfaceName       string            `json:"interfaceName"`
 	VppDriver           string            `json:"vppDriver"`
@@ -231,17 +240,26 @@ type UplinkInterfaceSpec struct {
 	// Mtu is the User specified MTU for uplink & the tap
 	Mtu       int    `json:"mtu"`
 	SwIfIndex uint32 `json:"-"`
+
+	// uplinkInterfaceIndex is the index of the uplinkInterface in the list
+	uplinkInterfaceIndex int `json:"-"`
 }
 
-func (u *UplinkInterfaceSpec) GetIsMain() bool {
-	if u.IsMain == nil {
-		return false
+func (u *UplinkInterfaceSpec) GetVppSideHardwareAddress() net.HardwareAddr {
+	mac, _ := net.ParseMAC(BaseVppSideHardwareAddress)
+	mac[len(mac)-1] = byte(u.uplinkInterfaceIndex)
+	if u.uplinkInterfaceIndex > 255 {
+		panic("too many uplinkinteraces")
 	}
-	return *u.IsMain
+	return mac
+}
+
+func (u *UplinkInterfaceSpec) SetUplinkInterfaceIndex(uplinkInterfaceIndex int) {
+	u.uplinkInterfaceIndex = uplinkInterfaceIndex
 }
 
-func (u *UplinkInterfaceSpec) Validate(maxIfSpec *InterfaceSpec, isMain bool) (err error) {
-	if !isMain && u.VppDriver == "" {
+func (u *UplinkInterfaceSpec) Validate(maxIfSpec *InterfaceSpec) (err error) {
+	if !u.IsMain && u.VppDriver == "" {
 		return errors.Errorf("vpp driver should be specified for secondary uplink interfaces")
 	}
 	return u.InterfaceSpec.Validate(maxIfSpec)
@@ -504,7 +522,11 @@ type UplinkStatus struct {
 	Mtu                 int
 	PhysicalNetworkName string
 
+	// FakeNextHopIP4 is the computed next hop for v4 routes added
+	// in linux to (ServiceCIDR, podCIDR, etc...) towards this interface
 	FakeNextHopIP4 net.IP
+	// FakeNextHopIP6 is the computed next hop for v6 routes added
+	// in linux to (ServiceCIDR, podCIDR, etc...) towards this interface
 	FakeNextHopIP6 net.IP
 }
 

diff --git a/vpp-manager/startup/startup.go b/vpp-manager/startup/startup.go
@@ -40,14 +40,30 @@ func NewVppManagerParams() *config.VppManagerParams {
 	}
 
 	/* uplinks configuration */
-	for index, uplink := range config.GetCalicoVppInterfaces().UplinkInterfaces {
-		_ = uplink.Validate(nil, index == 0)
+	isMainCount := 0
+	for _, uplink := range config.GetCalicoVppInterfaces().UplinkInterfaces {
 		params.UplinksSpecs = append(params.UplinksSpecs, uplink)
+		if uplink.IsMain {
+			isMainCount++
+		}
 	}
 	if len(params.UplinksSpecs) == 0 {
 		log.Panicf("No interface specified. Specify an interface through the environment variable")
 	}
-	params.UplinksSpecs[0].IsMain = &config.True
+	if isMainCount == 0 {
+		// By default the first interface is main
+		params.UplinksSpecs[0].IsMain = true
+	} else if isMainCount > 1 {
+		log.Panicf("Too many interfaces tagged Main")
+	}
+
+	for index, uplink := range params.UplinksSpecs {
+		uplink.SetUplinkInterfaceIndex(index)
+		err := uplink.Validate(nil)
+		if err != nil {
+			log.Panicf("error validating uplink %s %s", uplink.String(), err)
+		}
+	}
 
 	/* Drivers */
 	params.LoadedDrivers = make(map[string]bool)

diff --git a/vpp-manager/uplink/default.go b/vpp-manager/uplink/default.go
@@ -86,7 +86,7 @@ func (d *DefaultDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int,
 		log.Infof("Moved interface %s to VPP netns", d.spec.InterfaceName)
 	}
 	// refusing to run on secondary interfaces as we have no way to figure out the sw_if_index
-	if !d.spec.GetIsMain() {
+	if !d.spec.IsMain {
 		return fmt.Errorf("%s driver not supported for secondary interfaces", d.name)
 	}
 	swIfIndex, err := vpp.SearchInterfaceWithTag("main-" + d.spec.InterfaceName)

diff --git a/vpp-manager/uplink/dpdk.go b/vpp-manager/uplink/dpdk.go
@@ -177,7 +177,7 @@ func (d *DPDKDriver) RestoreLinux(allInterfacesPhysical bool) {
 func (d *DPDKDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) {
 	// Nothing to do VPP autocreates on startup
 	// refusing to run on secondary interfaces as we have no way to figure out the sw_if_index
-	if !d.spec.GetIsMain() {
+	if !d.spec.IsMain {
 		return fmt.Errorf("%s driver not supported for secondary interfaces", d.name)
 	}
 	swIfIndex, err := vpp.SearchInterfaceWithTag("main-" + d.spec.InterfaceName)

diff --git a/vpp-manager/utils/utils.go b/vpp-manager/utils/utils.go
@@ -47,12 +47,6 @@ import (
 	"github.com/projectcalico/vpp-dataplane/v3/vpplink"
 )
 
-var (
-	FakeVppNextHopIP4 = net.ParseIP("169.254.0.1")
-	FakeVppNextHopIP6 = net.ParseIP("fc00:ffff:ffff:ffff:ca11:c000:fd10:fffe")
-	VppSideMac, _     = net.ParseMAC("02:ca:11:c0:fd:10")
-)
-
 func IsDriverLoaded(driver string) (bool, error) {
 	_, err := os.Stat("/sys/bus/pci/drivers/" + driver)
 	if err == nil {

diff --git a/vpp-manager/vpp_runner.go b/vpp-manager/vpp_runner.go
@@ -43,8 +43,6 @@ import (
 	"github.com/projectcalico/vpp-dataplane/v3/vpplink/types"
 )
 
-const DefaultPhysicalNetworkName = ""
-
 type VppRunner struct {
 	params       *config.VppManagerParams
 	conf         []*config.LinuxInterfaceState
@@ -141,28 +139,26 @@ func (v *VppRunner) configureGlobalPunt() (err error) {
 }
 
 func (v *VppRunner) configurePunt(tapSwIfIndex uint32, ifState config.LinuxInterfaceState) (err error) {
-	for _, neigh := range []net.IP{utils.FakeVppNextHopIP4, utils.FakeVppNextHopIP6} {
-		err = v.vpp.AddNeighbor(&types.Neighbor{
-			SwIfIndex:    tapSwIfIndex,
-			IP:           neigh,
-			HardwareAddr: ifState.HardwareAddr,
+	err = v.vpp.AddNeighbor(&types.Neighbor{
+		SwIfIndex:    tapSwIfIndex,
+		IP:           config.VppHostPuntFakeGatewayAddress,
+		HardwareAddr: ifState.HardwareAddr,
+	})
+	if err != nil {
+		return errors.Wrapf(err, "Error adding neighbor %s to tap", config.VppHostPuntFakeGatewayAddress)
+	}
+	/* In the punt table (where all punted traffics ends), route to the tap */
+	for _, address := range ifState.Addresses {
+		err = v.vpp.RouteAdd(&types.Route{
+			Dst:   address.IPNet,
+			Table: common.PuntTableId,
+			Paths: []types.RoutePath{{
+				Gw:        config.VppHostPuntFakeGatewayAddress,
+				SwIfIndex: tapSwIfIndex,
+			}},
 		})
 		if err != nil {
-			return errors.Wrapf(err, "Error adding neighbor %s to tap", neigh)
-		}
-		/* In the punt table (where all punted traffics ends), route to the tap */
-		for _, address := range ifState.Addresses {
-			err = v.vpp.RouteAdd(&types.Route{
-				Dst:   address.IPNet,
-				Table: common.PuntTableId,
-				Paths: []types.RoutePath{{
-					Gw:        neigh,
-					SwIfIndex: tapSwIfIndex,
-				}},
-			})
-			if err != nil {
-				return errors.Wrapf(err, "error adding vpp side routes for interface")
-			}
+			return errors.Wrapf(err, "error adding vpp side routes for interface")
 		}
 	}
 
@@ -524,7 +520,7 @@ func (v *VppRunner) configureVppUplinkInterface(
 		}
 	}
 
-	if ifSpec.GetIsMain() {
+	if ifSpec.IsMain {
 		if config.GetCalicoVppInitialConfig().ExtraAddrCount > 0 {
 			err = v.addExtraAddresses(ifState.Addresses, config.GetCalicoVppInitialConfig().ExtraAddrCount, ifSpec.SwIfIndex)
 			if err != nil {
@@ -544,7 +540,7 @@ func (v *VppRunner) configureVppUplinkInterface(
 			HostInterfaceName: ifSpec.InterfaceName,
 			RxQueueSize:       config.GetCalicoVppInterfaces().VppHostTapSpec.RxQueueSize,
 			TxQueueSize:       config.GetCalicoVppInterfaces().VppHostTapSpec.TxQueueSize,
-			HardwareAddr:      utils.VppSideMac,
+			HardwareAddr:      ifSpec.GetVppSideHardwareAddress(),
 		},
 		HostNamespace:  "pid:1", // create tap in root netns
 		Tag:            "host-" + ifSpec.InterfaceName,
@@ -648,7 +644,7 @@ func (v *VppRunner) configureVppUplinkInterface(
 			PhysicalNetworkName: ifSpec.PhysicalNetworkName,
 			LinkIndex:           link.Attrs().Index,
 			Name:                link.Attrs().Name,
-			IsMain:              ifSpec.GetIsMain(),
+			IsMain:              ifSpec.IsMain,
 			FakeNextHopIP4:      fakeNextHopIP4,
 			FakeNextHopIP6:      fakeNextHopIP6,
 		}
@@ -866,7 +862,7 @@ func (v *VppRunner) runVpp() (err error) {
 	}
 
 	// add main network that has the default VRF
-	config.Info.PhysicalNets[DefaultPhysicalNetworkName] = config.PhysicalNetwork{VrfId: common.DefaultVRFIndex, PodVrfId: common.PodVRFIndex}
+	config.Info.PhysicalNets[config.DefaultPhysicalNetworkName] = config.PhysicalNetwork{VrfId: common.DefaultVRFIndex, PodVrfId: common.PodVRFIndex}
 
 	err = v.configureGlobalPunt()
 	if err != nil {