simplify punt fake gateway

calico-vpp-agent/cni/cni_server.go

@@ -38,7 +38,6 @@ import (
 	"github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common"
 	"github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers"
 	"github.com/projectcalico/vpp-dataplane/v3/config"
-	"github.com/projectcalico/vpp-dataplane/v3/vpp-manager/utils"
 	"github.com/projectcalico/vpp-dataplane/v3/vpplink"
 	"github.com/projectcalico/vpp-dataplane/v3/vpplink/types"
 )
@@ -479,15 +478,13 @@ forloop:
 	return nil
 }
 
-func (s *Server) getMainTap0Info() (tapSwIfIndex uint32, address net.IP) {
+func (s *Server) getMainInterface() *config.UplinkStatus {
 	for _, i := range common.VppManagerInfo.UplinkStatuses {
 		if i.IsMain {
-			tapSwIfIndex = i.TapSwIfIndex
-			break
+			return &i
 		}
 	}
-	address = utils.FakeVppNextHopIP4
-	return
+	return nil
 }
 
 func (s *Server) createRedirectToHostRules() (uint32, error) {
@@ -506,12 +503,15 @@ func (s *Server) createRedirectToHostRules() (uint32, error) {
 	if err != nil {
 		return types.InvalidID, err
 	}
-	tap0swifindex, tap0nexthop := s.getMainTap0Info()
+	mainInterface := s.getMainInterface()
+	if mainInterface == nil {
+		return types.InvalidID, fmt.Errorf("No main interface found")
+	}
 	for _, rule := range config.GetCalicoVppInitialConfig().RedirectToHostRules {
 		err = s.vpp.AddSessionRedirect(&types.SessionRedirect{
 			FiveTuple:  types.NewDst3Tuple(rule.Proto, net.ParseIP(rule.Ip), rule.Port),
 			TableIndex: index,
-		}, &types.RoutePath{Gw: tap0nexthop, SwIfIndex: tap0swifindex})
+		}, &types.RoutePath{Gw: config.VppHostPuntFakeGatewayAddress, SwIfIndex: mainInterface.TapSwIfIndex})
 		if err != nil {
 			return types.InvalidID, err
 		}

config/config.go

@@ -58,6 +58,11 @@ const (
 	VppSigKillTimeout = 2
 	DefaultEncapSize  = 60 // Used to lower the MTU of the routes to the cluster
 
+	DefaultPhysicalNetworkName = ""
+
+	// BaseVppSideHardwareAddress is the base hardware address of VPP side of the HostPunt
+	// tap interface. It is used to generate hardware addresses for each uplink interface.
+	BaseVppSideHardwareAddress = "02:ca:11:c0:fd:00"
 )
 
 var (
@@ -130,6 +135,10 @@ var (
 	}
 
 	Info = &VppManagerInfo{}
+
+	// VppHostPuntFakeGatewayAddress is the fake gateway we use with a static neighbor
+	// in the punt table to route punted packets to the host
+	VppHostPuntFakeGatewayAddress = net.ParseIP("169.254.0.1")
 )
 
 func RunHook(hookScript *string, hookName string, params *VppManagerParams, log *logrus.Logger) {
@@ -222,7 +231,7 @@ func (i *InterfaceSpec) Validate(maxIfSpec *InterfaceSpec) error {
 
 type UplinkInterfaceSpec struct {
 	InterfaceSpec
-	IsMain              *bool             `json:"-"`
+	IsMain              bool              `json:"isMain"`
 	PhysicalNetworkName string            `json:"physicalNetworkName"`
 	InterfaceName       string            `json:"interfaceName"`
 	VppDriver           string            `json:"vppDriver"`
@@ -231,17 +240,26 @@ type UplinkInterfaceSpec struct {
 	// Mtu is the User specified MTU for uplink & the tap
 	Mtu       int    `json:"mtu"`
 	SwIfIndex uint32 `json:"-"`
+
+	// uplinkInterfaceIndex is the index of the uplinkInterface in the list
+	uplinkInterfaceIndex int `json:"-"`
 }
 
-func (u *UplinkInterfaceSpec) GetIsMain() bool {
-	if u.IsMain == nil {
-		return false
+func (u *UplinkInterfaceSpec) GetVppSideHardwareAddress() net.HardwareAddr {
+	mac, _ := net.ParseMAC(BaseVppSideHardwareAddress)
+	mac[len(mac)-1] = byte(u.uplinkInterfaceIndex)
+	if u.uplinkInterfaceIndex > 255 {
+		panic("too many uplinkinteraces")
 	}
-	return *u.IsMain
+	return mac
+}
+
+func (u *UplinkInterfaceSpec) SetUplinkInterfaceIndex(uplinkInterfaceIndex int) {
+	u.uplinkInterfaceIndex = uplinkInterfaceIndex
 }
 
-func (u *UplinkInterfaceSpec) Validate(maxIfSpec *InterfaceSpec, isMain bool) (err error) {
-	if !isMain && u.VppDriver == "" {
+func (u *UplinkInterfaceSpec) Validate(maxIfSpec *InterfaceSpec) (err error) {
+	if !u.IsMain && u.VppDriver == "" {
 		return errors.Errorf("vpp driver should be specified for secondary uplink interfaces")
 	}
 	return u.InterfaceSpec.Validate(maxIfSpec)
@@ -504,7 +522,11 @@ type UplinkStatus struct {
 	Mtu                 int
 	PhysicalNetworkName string
 
+	// FakeNextHopIP4 is the computed next hop for v4 routes added
+	// in linux to (ServiceCIDR, podCIDR, etc...) towards this interface
 	FakeNextHopIP4 net.IP
+	// FakeNextHopIP6 is the computed next hop for v6 routes added
+	// in linux to (ServiceCIDR, podCIDR, etc...) towards this interface
 	FakeNextHopIP6 net.IP
 }
 

vpp-manager/startup/startup.go

@@ -40,14 +40,30 @@ func NewVppManagerParams() *config.VppManagerParams {
 	}
 
 	/* uplinks configuration */
-	for index, uplink := range config.GetCalicoVppInterfaces().UplinkInterfaces {
-		_ = uplink.Validate(nil, index == 0)
+	isMainCount := 0
+	for _, uplink := range config.GetCalicoVppInterfaces().UplinkInterfaces {
 		params.UplinksSpecs = append(params.UplinksSpecs, uplink)
+		if uplink.IsMain {
+			isMainCount++
+		}
 	}
 	if len(params.UplinksSpecs) == 0 {
 		log.Panicf("No interface specified. Specify an interface through the environment variable")
 	}
-	params.UplinksSpecs[0].IsMain = &config.True
+	if isMainCount == 0 {
+		// By default the first interface is main
+		params.UplinksSpecs[0].IsMain = true
+	} else if isMainCount > 1 {
+		log.Panicf("Too many interfaces tagged Main")
+	}
+
+	for index, uplink := range params.UplinksSpecs {
+		uplink.SetUplinkInterfaceIndex(index)
+		err := uplink.Validate(nil)
+		if err != nil {
+			log.Panicf("error validating uplink %s %s", uplink.String(), err)
+		}
+	}
 
 	/* Drivers */
 	params.LoadedDrivers = make(map[string]bool)

vpp-manager/uplink/default.go

@@ -86,7 +86,7 @@ func (d *DefaultDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int,
 		log.Infof("Moved interface %s to VPP netns", d.spec.InterfaceName)
 	}
 	// refusing to run on secondary interfaces as we have no way to figure out the sw_if_index
-	if !d.spec.GetIsMain() {
+	if !d.spec.IsMain {
 		return fmt.Errorf("%s driver not supported for secondary interfaces", d.name)
 	}
 	swIfIndex, err := vpp.SearchInterfaceWithTag("main-" + d.spec.InterfaceName)

vpp-manager/uplink/dpdk.go

@@ -177,7 +177,7 @@ func (d *DPDKDriver) RestoreLinux(allInterfacesPhysical bool) {
 func (d *DPDKDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) {
 	// Nothing to do VPP autocreates on startup
 	// refusing to run on secondary interfaces as we have no way to figure out the sw_if_index
-	if !d.spec.GetIsMain() {
+	if !d.spec.IsMain {
 		return fmt.Errorf("%s driver not supported for secondary interfaces", d.name)
 	}
 	swIfIndex, err := vpp.SearchInterfaceWithTag("main-" + d.spec.InterfaceName)

vpp-manager/utils/utils.go

@@ -47,12 +47,6 @@ import (
 	"github.com/projectcalico/vpp-dataplane/v3/vpplink"
 )
 
-var (
-	FakeVppNextHopIP4 = net.ParseIP("169.254.0.1")
-	FakeVppNextHopIP6 = net.ParseIP("fc00:ffff:ffff:ffff:ca11:c000:fd10:fffe")
-	VppSideMac, _     = net.ParseMAC("02:ca:11:c0:fd:10")
-)
-
 func IsDriverLoaded(driver string) (bool, error) {
 	_, err := os.Stat("/sys/bus/pci/drivers/" + driver)
 	if err == nil {

vpp-manager/vpp_runner.go

@@ -43,8 +43,6 @@ import (
 	"github.com/projectcalico/vpp-dataplane/v3/vpplink/types"
 )
 
-const DefaultPhysicalNetworkName = ""
-
 type VppRunner struct {
 	params       *config.VppManagerParams
 	conf         []*config.LinuxInterfaceState
@@ -141,28 +139,26 @@ func (v *VppRunner) configureGlobalPunt() (err error) {
 }
 
 func (v *VppRunner) configurePunt(tapSwIfIndex uint32, ifState config.LinuxInterfaceState) (err error) {
-	for _, neigh := range []net.IP{utils.FakeVppNextHopIP4, utils.FakeVppNextHopIP6} {
-		err = v.vpp.AddNeighbor(&types.Neighbor{
-			SwIfIndex:    tapSwIfIndex,
-			IP:           neigh,
-			HardwareAddr: ifState.HardwareAddr,
+	err = v.vpp.AddNeighbor(&types.Neighbor{
+		SwIfIndex:    tapSwIfIndex,
+		IP:           config.VppHostPuntFakeGatewayAddress,
+		HardwareAddr: ifState.HardwareAddr,
+	})
+	if err != nil {
+		return errors.Wrapf(err, "Error adding neighbor %s to tap", config.VppHostPuntFakeGatewayAddress)
+	}
+	/* In the punt table (where all punted traffics ends), route to the tap */
+	for _, address := range ifState.Addresses {
+		err = v.vpp.RouteAdd(&types.Route{
+			Dst:   address.IPNet,
+			Table: common.PuntTableId,
+			Paths: []types.RoutePath{{
+				Gw:        config.VppHostPuntFakeGatewayAddress,
+				SwIfIndex: tapSwIfIndex,
+			}},
 		})
 		if err != nil {
-			return errors.Wrapf(err, "Error adding neighbor %s to tap", neigh)
-		}
-		/* In the punt table (where all punted traffics ends), route to the tap */
-		for _, address := range ifState.Addresses {
-			err = v.vpp.RouteAdd(&types.Route{
-				Dst:   address.IPNet,
-				Table: common.PuntTableId,
-				Paths: []types.RoutePath{{
-					Gw:        neigh,
-					SwIfIndex: tapSwIfIndex,
-				}},
-			})
-			if err != nil {
-				return errors.Wrapf(err, "error adding vpp side routes for interface")
-			}
+			return errors.Wrapf(err, "error adding vpp side routes for interface")
 		}
 	}
 
@@ -524,7 +520,7 @@ func (v *VppRunner) configureVppUplinkInterface(
 		}
 	}
 
-	if ifSpec.GetIsMain() {
+	if ifSpec.IsMain {
 		if config.GetCalicoVppInitialConfig().ExtraAddrCount > 0 {
 			err = v.addExtraAddresses(ifState.Addresses, config.GetCalicoVppInitialConfig().ExtraAddrCount, ifSpec.SwIfIndex)
 			if err != nil {
@@ -544,7 +540,7 @@ func (v *VppRunner) configureVppUplinkInterface(
 			HostInterfaceName: ifSpec.InterfaceName,
 			RxQueueSize:       config.GetCalicoVppInterfaces().VppHostTapSpec.RxQueueSize,
 			TxQueueSize:       config.GetCalicoVppInterfaces().VppHostTapSpec.TxQueueSize,
-			HardwareAddr:      utils.VppSideMac,
+			HardwareAddr:      ifSpec.GetVppSideHardwareAddress(),
 		},
 		HostNamespace:  "pid:1", // create tap in root netns
 		Tag:            "host-" + ifSpec.InterfaceName,
@@ -648,7 +644,7 @@ func (v *VppRunner) configureVppUplinkInterface(
 			PhysicalNetworkName: ifSpec.PhysicalNetworkName,
 			LinkIndex:           link.Attrs().Index,
 			Name:                link.Attrs().Name,
-			IsMain:              ifSpec.GetIsMain(),
+			IsMain:              ifSpec.IsMain,
 			FakeNextHopIP4:      fakeNextHopIP4,
 			FakeNextHopIP6:      fakeNextHopIP6,
 		}
@@ -866,7 +862,7 @@ func (v *VppRunner) runVpp() (err error) {
 	}
 
 	// add main network that has the default VRF
-	config.Info.PhysicalNets[DefaultPhysicalNetworkName] = config.PhysicalNetwork{VrfId: common.DefaultVRFIndex, PodVrfId: common.PodVRFIndex}
+	config.Info.PhysicalNets[config.DefaultPhysicalNetworkName] = config.PhysicalNetwork{VrfId: common.DefaultVRFIndex, PodVrfId: common.PodVRFIndex}
 
 	err = v.configureGlobalPunt()
 	if err != nil {