From 45a7dae363b85bbd6277ee42d4b2f6960ea45dd6 Mon Sep 17 00:00:00 2001 From: Nathan Skrzypczak Date: Fri, 11 Aug 2023 18:52:25 +0200 Subject: [PATCH] simplify punt fake gateway This patch simplifies the way we handle the punt routes when routing for-me traffic VPP cannot handle to the host. Previously, we were using different fake gateways for ip4 & ip6. We now use the ip4 indirection in both cases. We also generate different hardware addresses using the uplink order in the configuration. Finally, we also make the isMain parameter public in the config. Signed-off-by: Nathan Skrzypczak --- calico-vpp-agent/cni/cni_server.go | 16 +++++----- config/config.go | 36 +++++++++++++++++----- vpp-manager/startup/startup.go | 22 ++++++++++++-- vpp-manager/uplink/default.go | 2 +- vpp-manager/uplink/dpdk.go | 2 +- vpp-manager/utils/utils.go | 6 ---- vpp-manager/vpp_runner.go | 48 ++++++++++++++---------------- 7 files changed, 80 insertions(+), 52 deletions(-) diff --git a/calico-vpp-agent/cni/cni_server.go b/calico-vpp-agent/cni/cni_server.go index 416f9df6..02facdb2 100644 --- a/calico-vpp-agent/cni/cni_server.go +++ b/calico-vpp-agent/cni/cni_server.go @@ -38,7 +38,6 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" "github.com/projectcalico/vpp-dataplane/v3/config" - "github.com/projectcalico/vpp-dataplane/v3/vpp-manager/utils" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) @@ -479,15 +478,13 @@ forloop: return nil } -func (s *Server) getMainTap0Info() (tapSwIfIndex uint32, address net.IP) { +func (s *Server) getMainInterface() *config.UplinkStatus { for _, i := range common.VppManagerInfo.UplinkStatuses { if i.IsMain { - tapSwIfIndex = i.TapSwIfIndex - break + return &i } } - address = utils.FakeVppNextHopIP4 - return + return nil } func (s *Server) createRedirectToHostRules() (uint32, error) { @@ -506,12 +503,15 @@ func (s *Server) createRedirectToHostRules() (uint32, error) { if err != nil { return types.InvalidID, err } - tap0swifindex, tap0nexthop := s.getMainTap0Info() + mainInterface := s.getMainInterface() + if mainInterface == nil { + return types.InvalidID, fmt.Errorf("No main interface found") + } for _, rule := range config.GetCalicoVppInitialConfig().RedirectToHostRules { err = s.vpp.AddSessionRedirect(&types.SessionRedirect{ FiveTuple: types.NewDst3Tuple(rule.Proto, net.ParseIP(rule.Ip), rule.Port), TableIndex: index, - }, &types.RoutePath{Gw: tap0nexthop, SwIfIndex: tap0swifindex}) + }, &types.RoutePath{Gw: config.VppHostPuntFakeGatewayAddress, SwIfIndex: mainInterface.TapSwIfIndex}) if err != nil { return types.InvalidID, err } diff --git a/config/config.go b/config/config.go index a73b61c2..d169911e 100644 --- a/config/config.go +++ b/config/config.go @@ -58,6 +58,11 @@ const ( VppSigKillTimeout = 2 DefaultEncapSize = 60 // Used to lower the MTU of the routes to the cluster + DefaultPhysicalNetworkName = "" + + // BaseVppSideHardwareAddress is the base hardware address of VPP side of the HostPunt + // tap interface. It is used to generate hardware addresses for each uplink interface. + BaseVppSideHardwareAddress = "02:ca:11:c0:fd:00" ) var ( @@ -130,6 +135,10 @@ var ( } Info = &VppManagerInfo{} + + // VppHostPuntFakeGatewayAddress is the fake gateway we use with a static neighbor + // in the punt table to route punted packets to the host + VppHostPuntFakeGatewayAddress = net.ParseIP("169.254.0.1") ) func RunHook(hookScript *string, hookName string, params *VppManagerParams, log *logrus.Logger) { @@ -222,7 +231,7 @@ func (i *InterfaceSpec) Validate(maxIfSpec *InterfaceSpec) error { type UplinkInterfaceSpec struct { InterfaceSpec - IsMain *bool `json:"-"` + IsMain bool `json:"isMain"` PhysicalNetworkName string `json:"physicalNetworkName"` InterfaceName string `json:"interfaceName"` VppDriver string `json:"vppDriver"` @@ -231,17 +240,26 @@ type UplinkInterfaceSpec struct { // Mtu is the User specified MTU for uplink & the tap Mtu int `json:"mtu"` SwIfIndex uint32 `json:"-"` + + // uplinkInterfaceIndex is the index of the uplinkInterface in the list + uplinkInterfaceIndex int `json:"-"` } -func (u *UplinkInterfaceSpec) GetIsMain() bool { - if u.IsMain == nil { - return false +func (u *UplinkInterfaceSpec) GetVppSideHardwareAddress() net.HardwareAddr { + mac, _ := net.ParseMAC(BaseVppSideHardwareAddress) + mac[len(mac)-1] = byte(u.uplinkInterfaceIndex) + if u.uplinkInterfaceIndex > 255 { + panic("too many uplinkinteraces") } - return *u.IsMain + return mac +} + +func (u *UplinkInterfaceSpec) SetUplinkInterfaceIndex(uplinkInterfaceIndex int) { + u.uplinkInterfaceIndex = uplinkInterfaceIndex } -func (u *UplinkInterfaceSpec) Validate(maxIfSpec *InterfaceSpec, isMain bool) (err error) { - if !isMain && u.VppDriver == "" { +func (u *UplinkInterfaceSpec) Validate(maxIfSpec *InterfaceSpec) (err error) { + if !u.IsMain && u.VppDriver == "" { return errors.Errorf("vpp driver should be specified for secondary uplink interfaces") } return u.InterfaceSpec.Validate(maxIfSpec) @@ -504,7 +522,11 @@ type UplinkStatus struct { Mtu int PhysicalNetworkName string + // FakeNextHopIP4 is the computed next hop for v4 routes added + // in linux to (ServiceCIDR, podCIDR, etc...) towards this interface FakeNextHopIP4 net.IP + // FakeNextHopIP6 is the computed next hop for v6 routes added + // in linux to (ServiceCIDR, podCIDR, etc...) towards this interface FakeNextHopIP6 net.IP } diff --git a/vpp-manager/startup/startup.go b/vpp-manager/startup/startup.go index 8a896729..8cab913b 100644 --- a/vpp-manager/startup/startup.go +++ b/vpp-manager/startup/startup.go @@ -40,14 +40,30 @@ func NewVppManagerParams() *config.VppManagerParams { } /* uplinks configuration */ - for index, uplink := range config.GetCalicoVppInterfaces().UplinkInterfaces { - _ = uplink.Validate(nil, index == 0) + isMainCount := 0 + for _, uplink := range config.GetCalicoVppInterfaces().UplinkInterfaces { params.UplinksSpecs = append(params.UplinksSpecs, uplink) + if uplink.IsMain { + isMainCount++ + } } if len(params.UplinksSpecs) == 0 { log.Panicf("No interface specified. Specify an interface through the environment variable") } - params.UplinksSpecs[0].IsMain = &config.True + if isMainCount == 0 { + // By default the first interface is main + params.UplinksSpecs[0].IsMain = true + } else if isMainCount > 1 { + log.Panicf("Too many interfaces tagged Main") + } + + for index, uplink := range params.UplinksSpecs { + uplink.SetUplinkInterfaceIndex(index) + err := uplink.Validate(nil) + if err != nil { + log.Panicf("error validating uplink %s %s", uplink.String(), err) + } + } /* Drivers */ params.LoadedDrivers = make(map[string]bool) diff --git a/vpp-manager/uplink/default.go b/vpp-manager/uplink/default.go index 61e8be99..d5636516 100644 --- a/vpp-manager/uplink/default.go +++ b/vpp-manager/uplink/default.go @@ -86,7 +86,7 @@ func (d *DefaultDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, log.Infof("Moved interface %s to VPP netns", d.spec.InterfaceName) } // refusing to run on secondary interfaces as we have no way to figure out the sw_if_index - if !d.spec.GetIsMain() { + if !d.spec.IsMain { return fmt.Errorf("%s driver not supported for secondary interfaces", d.name) } swIfIndex, err := vpp.SearchInterfaceWithTag("main-" + d.spec.InterfaceName) diff --git a/vpp-manager/uplink/dpdk.go b/vpp-manager/uplink/dpdk.go index 6e0103a9..398409b8 100644 --- a/vpp-manager/uplink/dpdk.go +++ b/vpp-manager/uplink/dpdk.go @@ -177,7 +177,7 @@ func (d *DPDKDriver) RestoreLinux(allInterfacesPhysical bool) { func (d *DPDKDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) { // Nothing to do VPP autocreates on startup // refusing to run on secondary interfaces as we have no way to figure out the sw_if_index - if !d.spec.GetIsMain() { + if !d.spec.IsMain { return fmt.Errorf("%s driver not supported for secondary interfaces", d.name) } swIfIndex, err := vpp.SearchInterfaceWithTag("main-" + d.spec.InterfaceName) diff --git a/vpp-manager/utils/utils.go b/vpp-manager/utils/utils.go index bf210a02..902c30fa 100644 --- a/vpp-manager/utils/utils.go +++ b/vpp-manager/utils/utils.go @@ -47,12 +47,6 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/vpplink" ) -var ( - FakeVppNextHopIP4 = net.ParseIP("169.254.0.1") - FakeVppNextHopIP6 = net.ParseIP("fc00:ffff:ffff:ffff:ca11:c000:fd10:fffe") - VppSideMac, _ = net.ParseMAC("02:ca:11:c0:fd:10") -) - func IsDriverLoaded(driver string) (bool, error) { _, err := os.Stat("/sys/bus/pci/drivers/" + driver) if err == nil { diff --git a/vpp-manager/vpp_runner.go b/vpp-manager/vpp_runner.go index 62f131ca..642917c3 100644 --- a/vpp-manager/vpp_runner.go +++ b/vpp-manager/vpp_runner.go @@ -43,8 +43,6 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) -const DefaultPhysicalNetworkName = "" - type VppRunner struct { params *config.VppManagerParams conf []*config.LinuxInterfaceState @@ -141,28 +139,26 @@ func (v *VppRunner) configureGlobalPunt() (err error) { } func (v *VppRunner) configurePunt(tapSwIfIndex uint32, ifState config.LinuxInterfaceState) (err error) { - for _, neigh := range []net.IP{utils.FakeVppNextHopIP4, utils.FakeVppNextHopIP6} { - err = v.vpp.AddNeighbor(&types.Neighbor{ - SwIfIndex: tapSwIfIndex, - IP: neigh, - HardwareAddr: ifState.HardwareAddr, + err = v.vpp.AddNeighbor(&types.Neighbor{ + SwIfIndex: tapSwIfIndex, + IP: config.VppHostPuntFakeGatewayAddress, + HardwareAddr: ifState.HardwareAddr, + }) + if err != nil { + return errors.Wrapf(err, "Error adding neighbor %s to tap", config.VppHostPuntFakeGatewayAddress) + } + /* In the punt table (where all punted traffics ends), route to the tap */ + for _, address := range ifState.Addresses { + err = v.vpp.RouteAdd(&types.Route{ + Dst: address.IPNet, + Table: common.PuntTableId, + Paths: []types.RoutePath{{ + Gw: config.VppHostPuntFakeGatewayAddress, + SwIfIndex: tapSwIfIndex, + }}, }) if err != nil { - return errors.Wrapf(err, "Error adding neighbor %s to tap", neigh) - } - /* In the punt table (where all punted traffics ends), route to the tap */ - for _, address := range ifState.Addresses { - err = v.vpp.RouteAdd(&types.Route{ - Dst: address.IPNet, - Table: common.PuntTableId, - Paths: []types.RoutePath{{ - Gw: neigh, - SwIfIndex: tapSwIfIndex, - }}, - }) - if err != nil { - return errors.Wrapf(err, "error adding vpp side routes for interface") - } + return errors.Wrapf(err, "error adding vpp side routes for interface") } } @@ -524,7 +520,7 @@ func (v *VppRunner) configureVppUplinkInterface( } } - if ifSpec.GetIsMain() { + if ifSpec.IsMain { if config.GetCalicoVppInitialConfig().ExtraAddrCount > 0 { err = v.addExtraAddresses(ifState.Addresses, config.GetCalicoVppInitialConfig().ExtraAddrCount, ifSpec.SwIfIndex) if err != nil { @@ -544,7 +540,7 @@ func (v *VppRunner) configureVppUplinkInterface( HostInterfaceName: ifSpec.InterfaceName, RxQueueSize: config.GetCalicoVppInterfaces().VppHostTapSpec.RxQueueSize, TxQueueSize: config.GetCalicoVppInterfaces().VppHostTapSpec.TxQueueSize, - HardwareAddr: utils.VppSideMac, + HardwareAddr: ifSpec.GetVppSideHardwareAddress(), }, HostNamespace: "pid:1", // create tap in root netns Tag: "host-" + ifSpec.InterfaceName, @@ -648,7 +644,7 @@ func (v *VppRunner) configureVppUplinkInterface( PhysicalNetworkName: ifSpec.PhysicalNetworkName, LinkIndex: link.Attrs().Index, Name: link.Attrs().Name, - IsMain: ifSpec.GetIsMain(), + IsMain: ifSpec.IsMain, FakeNextHopIP4: fakeNextHopIP4, FakeNextHopIP6: fakeNextHopIP6, } @@ -866,7 +862,7 @@ func (v *VppRunner) runVpp() (err error) { } // add main network that has the default VRF - config.Info.PhysicalNets[DefaultPhysicalNetworkName] = config.PhysicalNetwork{VrfId: common.DefaultVRFIndex, PodVrfId: common.PodVRFIndex} + config.Info.PhysicalNets[config.DefaultPhysicalNetworkName] = config.PhysicalNetwork{VrfId: common.DefaultVRFIndex, PodVrfId: common.PodVRFIndex} err = v.configureGlobalPunt() if err != nil {