diff --git a/calico-vpp-agent/cni/cni_server.go b/calico-vpp-agent/cni/cni_server.go index 416f9df6..02facdb2 100644 --- a/calico-vpp-agent/cni/cni_server.go +++ b/calico-vpp-agent/cni/cni_server.go @@ -38,7 +38,6 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" "github.com/projectcalico/vpp-dataplane/v3/config" - "github.com/projectcalico/vpp-dataplane/v3/vpp-manager/utils" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) @@ -479,15 +478,13 @@ forloop: return nil } -func (s *Server) getMainTap0Info() (tapSwIfIndex uint32, address net.IP) { +func (s *Server) getMainInterface() *config.UplinkStatus { for _, i := range common.VppManagerInfo.UplinkStatuses { if i.IsMain { - tapSwIfIndex = i.TapSwIfIndex - break + return &i } } - address = utils.FakeVppNextHopIP4 - return + return nil } func (s *Server) createRedirectToHostRules() (uint32, error) { @@ -506,12 +503,15 @@ func (s *Server) createRedirectToHostRules() (uint32, error) { if err != nil { return types.InvalidID, err } - tap0swifindex, tap0nexthop := s.getMainTap0Info() + mainInterface := s.getMainInterface() + if mainInterface == nil { + return types.InvalidID, fmt.Errorf("No main interface found") + } for _, rule := range config.GetCalicoVppInitialConfig().RedirectToHostRules { err = s.vpp.AddSessionRedirect(&types.SessionRedirect{ FiveTuple: types.NewDst3Tuple(rule.Proto, net.ParseIP(rule.Ip), rule.Port), TableIndex: index, - }, &types.RoutePath{Gw: tap0nexthop, SwIfIndex: tap0swifindex}) + }, &types.RoutePath{Gw: config.VppHostPuntFakeGatewayAddress, SwIfIndex: mainInterface.TapSwIfIndex}) if err != nil { return types.InvalidID, err } diff --git a/config/config.go b/config/config.go index a73b61c2..d169911e 100644 --- a/config/config.go +++ b/config/config.go @@ -58,6 +58,11 @@ const ( VppSigKillTimeout = 2 DefaultEncapSize = 60 // Used to lower the MTU of the routes to the cluster + DefaultPhysicalNetworkName = "" + + // BaseVppSideHardwareAddress is the base hardware address of VPP side of the HostPunt + // tap interface. It is used to generate hardware addresses for each uplink interface. + BaseVppSideHardwareAddress = "02:ca:11:c0:fd:00" ) var ( @@ -130,6 +135,10 @@ var ( } Info = &VppManagerInfo{} + + // VppHostPuntFakeGatewayAddress is the fake gateway we use with a static neighbor + // in the punt table to route punted packets to the host + VppHostPuntFakeGatewayAddress = net.ParseIP("169.254.0.1") ) func RunHook(hookScript *string, hookName string, params *VppManagerParams, log *logrus.Logger) { @@ -222,7 +231,7 @@ func (i *InterfaceSpec) Validate(maxIfSpec *InterfaceSpec) error { type UplinkInterfaceSpec struct { InterfaceSpec - IsMain *bool `json:"-"` + IsMain bool `json:"isMain"` PhysicalNetworkName string `json:"physicalNetworkName"` InterfaceName string `json:"interfaceName"` VppDriver string `json:"vppDriver"` @@ -231,17 +240,26 @@ type UplinkInterfaceSpec struct { // Mtu is the User specified MTU for uplink & the tap Mtu int `json:"mtu"` SwIfIndex uint32 `json:"-"` + + // uplinkInterfaceIndex is the index of the uplinkInterface in the list + uplinkInterfaceIndex int `json:"-"` } -func (u *UplinkInterfaceSpec) GetIsMain() bool { - if u.IsMain == nil { - return false +func (u *UplinkInterfaceSpec) GetVppSideHardwareAddress() net.HardwareAddr { + mac, _ := net.ParseMAC(BaseVppSideHardwareAddress) + mac[len(mac)-1] = byte(u.uplinkInterfaceIndex) + if u.uplinkInterfaceIndex > 255 { + panic("too many uplinkinteraces") } - return *u.IsMain + return mac +} + +func (u *UplinkInterfaceSpec) SetUplinkInterfaceIndex(uplinkInterfaceIndex int) { + u.uplinkInterfaceIndex = uplinkInterfaceIndex } -func (u *UplinkInterfaceSpec) Validate(maxIfSpec *InterfaceSpec, isMain bool) (err error) { - if !isMain && u.VppDriver == "" { +func (u *UplinkInterfaceSpec) Validate(maxIfSpec *InterfaceSpec) (err error) { + if !u.IsMain && u.VppDriver == "" { return errors.Errorf("vpp driver should be specified for secondary uplink interfaces") } return u.InterfaceSpec.Validate(maxIfSpec) @@ -504,7 +522,11 @@ type UplinkStatus struct { Mtu int PhysicalNetworkName string + // FakeNextHopIP4 is the computed next hop for v4 routes added + // in linux to (ServiceCIDR, podCIDR, etc...) towards this interface FakeNextHopIP4 net.IP + // FakeNextHopIP6 is the computed next hop for v6 routes added + // in linux to (ServiceCIDR, podCIDR, etc...) towards this interface FakeNextHopIP6 net.IP } diff --git a/vpp-manager/startup/startup.go b/vpp-manager/startup/startup.go index 8a896729..8cab913b 100644 --- a/vpp-manager/startup/startup.go +++ b/vpp-manager/startup/startup.go @@ -40,14 +40,30 @@ func NewVppManagerParams() *config.VppManagerParams { } /* uplinks configuration */ - for index, uplink := range config.GetCalicoVppInterfaces().UplinkInterfaces { - _ = uplink.Validate(nil, index == 0) + isMainCount := 0 + for _, uplink := range config.GetCalicoVppInterfaces().UplinkInterfaces { params.UplinksSpecs = append(params.UplinksSpecs, uplink) + if uplink.IsMain { + isMainCount++ + } } if len(params.UplinksSpecs) == 0 { log.Panicf("No interface specified. Specify an interface through the environment variable") } - params.UplinksSpecs[0].IsMain = &config.True + if isMainCount == 0 { + // By default the first interface is main + params.UplinksSpecs[0].IsMain = true + } else if isMainCount > 1 { + log.Panicf("Too many interfaces tagged Main") + } + + for index, uplink := range params.UplinksSpecs { + uplink.SetUplinkInterfaceIndex(index) + err := uplink.Validate(nil) + if err != nil { + log.Panicf("error validating uplink %s %s", uplink.String(), err) + } + } /* Drivers */ params.LoadedDrivers = make(map[string]bool) diff --git a/vpp-manager/uplink/default.go b/vpp-manager/uplink/default.go index 61e8be99..d5636516 100644 --- a/vpp-manager/uplink/default.go +++ b/vpp-manager/uplink/default.go @@ -86,7 +86,7 @@ func (d *DefaultDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, log.Infof("Moved interface %s to VPP netns", d.spec.InterfaceName) } // refusing to run on secondary interfaces as we have no way to figure out the sw_if_index - if !d.spec.GetIsMain() { + if !d.spec.IsMain { return fmt.Errorf("%s driver not supported for secondary interfaces", d.name) } swIfIndex, err := vpp.SearchInterfaceWithTag("main-" + d.spec.InterfaceName) diff --git a/vpp-manager/uplink/dpdk.go b/vpp-manager/uplink/dpdk.go index 6e0103a9..398409b8 100644 --- a/vpp-manager/uplink/dpdk.go +++ b/vpp-manager/uplink/dpdk.go @@ -177,7 +177,7 @@ func (d *DPDKDriver) RestoreLinux(allInterfacesPhysical bool) { func (d *DPDKDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) { // Nothing to do VPP autocreates on startup // refusing to run on secondary interfaces as we have no way to figure out the sw_if_index - if !d.spec.GetIsMain() { + if !d.spec.IsMain { return fmt.Errorf("%s driver not supported for secondary interfaces", d.name) } swIfIndex, err := vpp.SearchInterfaceWithTag("main-" + d.spec.InterfaceName) diff --git a/vpp-manager/utils/utils.go b/vpp-manager/utils/utils.go index bf210a02..902c30fa 100644 --- a/vpp-manager/utils/utils.go +++ b/vpp-manager/utils/utils.go @@ -47,12 +47,6 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/vpplink" ) -var ( - FakeVppNextHopIP4 = net.ParseIP("169.254.0.1") - FakeVppNextHopIP6 = net.ParseIP("fc00:ffff:ffff:ffff:ca11:c000:fd10:fffe") - VppSideMac, _ = net.ParseMAC("02:ca:11:c0:fd:10") -) - func IsDriverLoaded(driver string) (bool, error) { _, err := os.Stat("/sys/bus/pci/drivers/" + driver) if err == nil { diff --git a/vpp-manager/vpp_runner.go b/vpp-manager/vpp_runner.go index 62f131ca..642917c3 100644 --- a/vpp-manager/vpp_runner.go +++ b/vpp-manager/vpp_runner.go @@ -43,8 +43,6 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) -const DefaultPhysicalNetworkName = "" - type VppRunner struct { params *config.VppManagerParams conf []*config.LinuxInterfaceState @@ -141,28 +139,26 @@ func (v *VppRunner) configureGlobalPunt() (err error) { } func (v *VppRunner) configurePunt(tapSwIfIndex uint32, ifState config.LinuxInterfaceState) (err error) { - for _, neigh := range []net.IP{utils.FakeVppNextHopIP4, utils.FakeVppNextHopIP6} { - err = v.vpp.AddNeighbor(&types.Neighbor{ - SwIfIndex: tapSwIfIndex, - IP: neigh, - HardwareAddr: ifState.HardwareAddr, + err = v.vpp.AddNeighbor(&types.Neighbor{ + SwIfIndex: tapSwIfIndex, + IP: config.VppHostPuntFakeGatewayAddress, + HardwareAddr: ifState.HardwareAddr, + }) + if err != nil { + return errors.Wrapf(err, "Error adding neighbor %s to tap", config.VppHostPuntFakeGatewayAddress) + } + /* In the punt table (where all punted traffics ends), route to the tap */ + for _, address := range ifState.Addresses { + err = v.vpp.RouteAdd(&types.Route{ + Dst: address.IPNet, + Table: common.PuntTableId, + Paths: []types.RoutePath{{ + Gw: config.VppHostPuntFakeGatewayAddress, + SwIfIndex: tapSwIfIndex, + }}, }) if err != nil { - return errors.Wrapf(err, "Error adding neighbor %s to tap", neigh) - } - /* In the punt table (where all punted traffics ends), route to the tap */ - for _, address := range ifState.Addresses { - err = v.vpp.RouteAdd(&types.Route{ - Dst: address.IPNet, - Table: common.PuntTableId, - Paths: []types.RoutePath{{ - Gw: neigh, - SwIfIndex: tapSwIfIndex, - }}, - }) - if err != nil { - return errors.Wrapf(err, "error adding vpp side routes for interface") - } + return errors.Wrapf(err, "error adding vpp side routes for interface") } } @@ -524,7 +520,7 @@ func (v *VppRunner) configureVppUplinkInterface( } } - if ifSpec.GetIsMain() { + if ifSpec.IsMain { if config.GetCalicoVppInitialConfig().ExtraAddrCount > 0 { err = v.addExtraAddresses(ifState.Addresses, config.GetCalicoVppInitialConfig().ExtraAddrCount, ifSpec.SwIfIndex) if err != nil { @@ -544,7 +540,7 @@ func (v *VppRunner) configureVppUplinkInterface( HostInterfaceName: ifSpec.InterfaceName, RxQueueSize: config.GetCalicoVppInterfaces().VppHostTapSpec.RxQueueSize, TxQueueSize: config.GetCalicoVppInterfaces().VppHostTapSpec.TxQueueSize, - HardwareAddr: utils.VppSideMac, + HardwareAddr: ifSpec.GetVppSideHardwareAddress(), }, HostNamespace: "pid:1", // create tap in root netns Tag: "host-" + ifSpec.InterfaceName, @@ -648,7 +644,7 @@ func (v *VppRunner) configureVppUplinkInterface( PhysicalNetworkName: ifSpec.PhysicalNetworkName, LinkIndex: link.Attrs().Index, Name: link.Attrs().Name, - IsMain: ifSpec.GetIsMain(), + IsMain: ifSpec.IsMain, FakeNextHopIP4: fakeNextHopIP4, FakeNextHopIP6: fakeNextHopIP6, } @@ -866,7 +862,7 @@ func (v *VppRunner) runVpp() (err error) { } // add main network that has the default VRF - config.Info.PhysicalNets[DefaultPhysicalNetworkName] = config.PhysicalNetwork{VrfId: common.DefaultVRFIndex, PodVrfId: common.PodVRFIndex} + config.Info.PhysicalNets[config.DefaultPhysicalNetworkName] = config.PhysicalNetwork{VrfId: common.DefaultVRFIndex, PodVrfId: common.PodVRFIndex} err = v.configureGlobalPunt() if err != nil {