From e882e15c6c1adfd3ba1aa8be8c830eb8eb74c33c Mon Sep 17 00:00:00 2001 From: hedi bouattour Date: Fri, 10 Feb 2023 16:53:11 +0000 Subject: [PATCH 1/2] agent: refactor integration tests to include fv tests --- Makefile | 10 +- calico-vpp-agent/cmd/calico_vpp_dataplane.go | 5 +- calico-vpp-agent/cni/cni_node_test.go | 342 ++---------- calico-vpp-agent/cni/cni_pod_test.go | 300 ++--------- calico-vpp-agent/common_tests/common_tests.go | 507 ++++++++++++++++++ calico-vpp-agent/policy/host_endpoint.go | 1 - calico-vpp-agent/policy/policy_server.go | 10 +- .../{cmd => watch_dog}/watch_dog.go | 2 +- go.mod | 2 + go.sum | 2 + test/integration-tests/run-tests.sh | 6 +- 11 files changed, 621 insertions(+), 566 deletions(-) create mode 100644 calico-vpp-agent/common_tests/common_tests.go rename calico-vpp-agent/{cmd => watch_dog}/watch_dog.go (98%) diff --git a/Makefile b/Makefile index ef54fd17..ec0bc575 100644 --- a/Makefile +++ b/Makefile @@ -191,9 +191,13 @@ release: check-TAG check-CALICO_TAG @echo "***IMPORTANT***IMPORTANT***IMPORTANT***IMPORTANT***" @echo "Please update \"vppbranch\" in https://github.com/projectcalico/calico/blob/${CALICO_TAG}/calico/_config.yml to ${TAG} otherwise the install docs get broken." -.PHONY: run-integration-tests -run-integration-tests: - cd test/integration-tests;./run-tests.sh +.PHONY: run-integration-tests-cni +run-integration-tests-cni: + cd test/integration-tests;./run-tests.sh cni + +.PHONY: run-integration-tests-policy +run-integration-tests-policy: + cd test/integration-tests;./run-tests.sh policy .PHONY: test test: go-lint diff --git a/calico-vpp-agent/cmd/calico_vpp_dataplane.go b/calico-vpp-agent/cmd/calico_vpp_dataplane.go index d9c01591..3093aa24 100644 --- a/calico-vpp-agent/cmd/calico_vpp_dataplane.go +++ b/calico-vpp-agent/cmd/calico_vpp_dataplane.go @@ -41,6 +41,7 @@ import ( "github.com/projectcalico/vpp-dataplane/config" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/watchers" + "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/watch_dog" ) /* @@ -136,7 +137,7 @@ func main() { serviceServer := services.NewServiceServer(vpp, k8sclient, log.WithFields(logrus.Fields{"component": "services"})) prometheusServer := prometheus.NewPrometheusServer(vpp, log.WithFields(logrus.Fields{"component": "prometheus"})) localSIDWatcher := watchers.NewLocalSIDWatcher(vpp, clientv3, log.WithFields(logrus.Fields{"subcomponent": "localsid-watcher"})) - policyServer, err := policy.NewPolicyServer(vpp, log.WithFields(logrus.Fields{"component": "policy"})) + policyServer, err := policy.NewPolicyServer(vpp, log.WithFields(logrus.Fields{"component": "policy"}), true) if err != nil { log.Fatalf("Failed to create policy server %s", err) } @@ -154,7 +155,7 @@ func main() { routingServer.SetBGPConf(bgpConf) serviceServer.SetBGPConf(bgpConf) - watchDog := NewWatchDog(log.WithFields(logrus.Fields{"component": "watchDog"}), &t) + watchDog := watchdog.NewWatchDog(log.WithFields(logrus.Fields{"component": "watchDog"}), &t) Go(policyServer.ServePolicy) felixConfig := watchDog.Wait(policyServer.FelixConfigChan, "Waiting for FelixConfig to be provided by the calico pod") ourBGPSpec := watchDog.Wait(policyServer.GotOurNodeBGPchan, "Waiting for bgp spec to be provided on node add") diff --git a/calico-vpp-agent/cni/cni_node_test.go b/calico-vpp-agent/cni/cni_node_test.go index b170d34e..48a8dd41 100644 --- a/calico-vpp-agent/cni/cni_node_test.go +++ b/calico-vpp-agent/cni/cni_node_test.go @@ -19,31 +19,25 @@ import ( "fmt" "net" "os" - "os/exec" - "strconv" "strings" "testing" - "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" gs "github.com/onsi/gomega/gstruct" gobgpapi "github.com/osrg/gobgp/v3/api" - "github.com/pkg/errors" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/calico/felix/config" "github.com/projectcalico/calico/felix/proto" oldv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" "github.com/projectcalico/calico/libcalico-go/lib/backend/encap" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/common" + test "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/common_tests" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/connectivity" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/tests/mocks" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/tests/mocks/calico" agentConf "github.com/projectcalico/vpp-dataplane/config" "github.com/projectcalico/vpp-dataplane/vpplink" - "github.com/projectcalico/vpp-dataplane/vpplink/binapi/vppapi/interface_types" - "github.com/projectcalico/vpp-dataplane/vpplink/binapi/vppapi/ip_types" "github.com/projectcalico/vpp-dataplane/vpplink/types" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -57,15 +51,6 @@ const ( VppContainerExtraArgsName = "VPP_CONTAINER_EXTRA_ARGS" ) -var ( - // vppImage is the name of docker image containing VPP binary - vppImage string - // vppBinary is the full path to VPP binary inside docker image - vppBinary string - // vppContainerExtraArgs is a list of additionnal cli parameters for the VPP `docker run ...` - vppContainerExtraArgs []string = []string{} -) - // TestCniIntegration runs all the ginkgo integration test inside CNI package func TestCniIntegration(t *testing.T) { // skip test if test run is not integration test run (prevent accidental run of integration tests using go test ./...) @@ -82,20 +67,20 @@ func TestCniIntegration(t *testing.T) { var _ = BeforeSuite(func() { // extract common input for CNI integration tests var found bool - vppImage, found = os.LookupEnv(VppImageArgName) + test.VppImage, found = os.LookupEnv(VppImageArgName) if !found { - Expect(vppImage).ToNot(BeEmpty(), fmt.Sprintf("Please specify docker image containing "+ + Expect(test.VppImage).ToNot(BeEmpty(), fmt.Sprintf("Please specify docker image containing "+ "VPP binary using %s environment variable.", VppImageArgName)) } - vppBinary, found = os.LookupEnv(VppBinaryArgName) + test.VppBinary, found = os.LookupEnv(VppBinaryArgName) if !found { - Expect(vppBinary).ToNot(BeEmpty(), fmt.Sprintf("Please specify VPP binary (full path) "+ - "inside docker image %s using %s environment variable.", vppImage, VppBinaryArgName)) + Expect(test.VppBinary).ToNot(BeEmpty(), fmt.Sprintf("Please specify VPP binary (full path) "+ + "inside docker image %s using %s environment variable.", test.VppImage, VppBinaryArgName)) } vppContainerExtraArgsList, found := os.LookupEnv(VppContainerExtraArgsName) if found { - vppContainerExtraArgs = append(vppContainerExtraArgs, strings.Split(vppContainerExtraArgsList, ",")...) + test.VppContainerExtraArgs = append(test.VppContainerExtraArgs, strings.Split(vppContainerExtraArgsList, ",")...) } }) @@ -138,8 +123,8 @@ var _ = Describe("Node-related functionality of CNI", func() { }) JustBeforeEach(func() { - startVPP() - vpp, uplinkSwIfIndex = configureVPP(log) + test.StartVPP() + vpp, uplinkSwIfIndex = test.ConfigureVPP(log) // setup connectivity server (functionality target of tests) if ipamStub == nil { @@ -160,7 +145,7 @@ var _ = Describe("Node-related functionality of CNI", func() { It("should only configure correct routes in VPP", func() { By("Adding node") err := connectivityServer.UpdateIPConnectivity(&common.NodeConnectivity{ - Dst: *ipNet(AddedNodeIP + "/24"), + Dst: *test.IpNet(AddedNodeIP + "/24"), NextHop: net.ParseIP(GatewayIP), ResolvedProvider: connectivity.FLAT, Custom: nil, @@ -173,14 +158,14 @@ var _ = Describe("Node-related functionality of CNI", func() { Expect(routes).To(ContainElements( // route to destination going via gateway gs.MatchFields(gs.IgnoreExtras, gs.Fields{ - "Dst": gs.PointTo(Equal(*ipNet("10.0.200.0/24"))), + "Dst": gs.PointTo(Equal(*test.IpNet("10.0.200.0/24"))), "Paths": ContainElements(gs.MatchFields(gs.IgnoreExtras, gs.Fields{ "Gw": Equal(net.ParseIP(GatewayIP).To4()), })), }), // using gateway means using our uplink interface gs.MatchFields(gs.IgnoreExtras, gs.Fields{ - "Dst": gs.PointTo(Equal(*ipNet(GatewayIP + "/32"))), + "Dst": gs.PointTo(Equal(*test.IpNet(GatewayIP + "/32"))), "Paths": ContainElements(gs.MatchFields(gs.IgnoreExtras, gs.Fields{ "SwIfIndex": Equal(uplinkSwIfIndex), })), @@ -192,7 +177,7 @@ var _ = Describe("Node-related functionality of CNI", func() { BeforeEach(func() { // add node pool for IPSec (uses IPIP tunnels) ipamStub = mocks.NewIpamCacheStub() - ipamStub.AddPrefixIPPool(ipNet(AddedNodeIP+"/24"), &proto.IPAMPoolUpdate{ + ipamStub.AddPrefixIPPool(test.IpNet(AddedNodeIP+"/24"), &proto.IPAMPoolUpdate{ Id: fmt.Sprintf("custom-test-pool-for-ipsec-%s", AddedNodeIP+"/24"), Pool: &proto.IPAMPool{ Cidr: AddedNodeIP + "/24", @@ -227,7 +212,7 @@ var _ = Describe("Node-related functionality of CNI", func() { //Note: not testing setting of IPsecAsyncMode and threads dedicated to IPSec (CryptoWorkers) // inside RescanState() function call By("Adding node") - configureBGPNodeIPAddresses(connectivityServer) + test.ConfigureBGPNodeIPAddresses(connectivityServer) // FIXME The concept of Destination and NextHop in common.NodeConnectivity is not well defined // (is the Destination the IP of added node, or it subnet or totally unrelated network? Is // the nexthop the IP of added node or could it be IP of some intermediate router that is @@ -236,7 +221,7 @@ var _ = Describe("Node-related functionality of CNI", func() { // in connectivity provider implementation and check each test for semantics used in given // connectivity provider err := connectivityServer.UpdateIPConnectivity(&common.NodeConnectivity{ - Dst: *ipNet(AddedNodeIP + "/24"), + Dst: *test.IpNet(AddedNodeIP + "/24"), NextHop: net.ParseIP(AddedNodeIP), // next hop == other node IP (for IPSec impl) ResolvedProvider: connectivity.IPSEC, Custom: nil, @@ -265,10 +250,10 @@ var _ = Describe("Node-related functionality of CNI", func() { })) By("checking IPSec's IPIP tunnel interface attributes (Unnumbered)") - assertUnnumberedInterface(ipipSwIfIndex, "IPSec's IPIP tunnel interface", vpp) + test.AssertUnnumberedInterface(ipipSwIfIndex, "IPSec's IPIP tunnel interface", vpp) By("checking IPSec's IPIP tunnel interface attributes (GSO+CNAT)") - assertInterfaceGSOCNat(ipipSwIfIndex, "IPSec's IPIP tunnel interface", vpp) + test.AssertInterfaceGSOCNat(ipipSwIfIndex, "IPSec's IPIP tunnel interface", vpp) By("checking route for IPSec's IPIP tunnel from pod VRF") routes, err := vpp.GetRoutes(common.PodVRFIndex, false) @@ -276,7 +261,7 @@ var _ = Describe("Node-related functionality of CNI", func() { Expect(routes).To(ContainElements( // when IPIP is created it makes steering route with for NextHop/ from Pod VRF gs.MatchFields(gs.IgnoreExtras, gs.Fields{ - "Dst": gs.PointTo(Equal(*ipNet(AddedNodeIP + "/32"))), + "Dst": gs.PointTo(Equal(*test.IpNet(AddedNodeIP + "/32"))), "Paths": ContainElements(gs.MatchFields(gs.IgnoreExtras, gs.Fields{ "SwIfIndex": Equal(ipipSwIfIndex), })), @@ -334,7 +319,7 @@ var _ = Describe("Node-related functionality of CNI", func() { BeforeEach(func() { // add node pool for VXLAN ipamStub = mocks.NewIpamCacheStub() - ipamStub.AddPrefixIPPool(ipNet(AddedNodeIP+"/24"), &proto.IPAMPoolUpdate{ + ipamStub.AddPrefixIPPool(test.IpNet(AddedNodeIP+"/24"), &proto.IPAMPoolUpdate{ Id: fmt.Sprintf("custom-test-pool-for-vxlan-%s", AddedNodeIP+"/24"), Pool: &proto.IPAMPool{ Cidr: AddedNodeIP + "/24", @@ -358,13 +343,13 @@ var _ = Describe("Node-related functionality of CNI", func() { "can't properly create ???") By("Checking VPP's node graph modifications for VXLAN") - ipv4DecapNextIndex := assertNextNodeLink("vxlan4-input", "ip4-input", vpp) - assertNextNodeLink("vxlan6-input", "ip6-input", vpp) + ipv4DecapNextIndex := test.AssertNextNodeLink("vxlan4-input", "ip4-input", vpp) + test.AssertNextNodeLink("vxlan6-input", "ip6-input", vpp) By("Adding node") - configureBGPNodeIPAddresses(connectivityServer) + test.ConfigureBGPNodeIPAddresses(connectivityServer) err = connectivityServer.UpdateIPConnectivity(&common.NodeConnectivity{ - Dst: *ipNet(AddedNodeIP + "/24"), + Dst: *test.IpNet(AddedNodeIP + "/24"), NextHop: net.ParseIP(GatewayIP), ResolvedProvider: connectivity.VXLAN, Custom: nil, @@ -387,10 +372,10 @@ var _ = Describe("Node-related functionality of CNI", func() { })) By("checking VXLAN tunnel interface attributes (Unnumbered)") - assertUnnumberedInterface(vxlanSwIfIndex, "VXLAN tunnel interface", vpp) + test.AssertUnnumberedInterface(vxlanSwIfIndex, "VXLAN tunnel interface", vpp) By("checking VXLAN tunnel interface attributes (GSO+CNAT)") - assertInterfaceGSOCNat(vxlanSwIfIndex, "VXLAN tunnel interface", vpp) + test.AssertInterfaceGSOCNat(vxlanSwIfIndex, "VXLAN tunnel interface", vpp) By("checking VXLAN tunnel interface attributes (Up state)") interfaceDetails, err := vpp.GetInterfaceDetails(vxlanSwIfIndex) @@ -403,7 +388,7 @@ var _ = Describe("Node-related functionality of CNI", func() { Expect(routes).To(ContainElements( // when VXLAN is created it makes steering route with for NextHop/ from Pod VRF gs.MatchFields(gs.IgnoreExtras, gs.Fields{ - "Dst": gs.PointTo(Equal(*ipNet(GatewayIP + "/32"))), + "Dst": gs.PointTo(Equal(*test.IpNet(GatewayIP + "/32"))), "Paths": ContainElements(gs.MatchFields(gs.IgnoreExtras, gs.Fields{ "SwIfIndex": Equal(vxlanSwIfIndex), })), @@ -413,7 +398,7 @@ var _ = Describe("Node-related functionality of CNI", func() { Expect(routes).To(ContainElements( // steering route for NodeConnectivity.Dst using vxlan that is leading to the added node gs.MatchFields(gs.IgnoreExtras, gs.Fields{ - "Dst": gs.PointTo(Equal(*ipNet(AddedNodeIP + "/24"))), // NodeConnectivity.Dst + "Dst": gs.PointTo(Equal(*test.IpNet(AddedNodeIP + "/24"))), // NodeConnectivity.Dst "Paths": ContainElements(gs.MatchFields(gs.IgnoreExtras, gs.Fields{ "SwIfIndex": Equal(vxlanSwIfIndex), })), @@ -439,7 +424,7 @@ var _ = Describe("Node-related functionality of CNI", func() { BeforeEach(func() { // add node pool for IPIP ipamStub = mocks.NewIpamCacheStub() - ipamStub.AddPrefixIPPool(ipNet(AddedNodeIP+"/24"), &proto.IPAMPoolUpdate{ + ipamStub.AddPrefixIPPool(test.IpNet(AddedNodeIP+"/24"), &proto.IPAMPoolUpdate{ Id: fmt.Sprintf("custom-test-pool-for-ipip-%s", AddedNodeIP+"/24"), Pool: &proto.IPAMPool{ Cidr: AddedNodeIP + "/24", @@ -458,9 +443,9 @@ var _ = Describe("Node-related functionality of CNI", func() { It("should have IP-IP tunnel and route forwarding to it", func() { By("Adding node") - configureBGPNodeIPAddresses(connectivityServer) + test.ConfigureBGPNodeIPAddresses(connectivityServer) err := connectivityServer.UpdateIPConnectivity(&common.NodeConnectivity{ - Dst: *ipNet(AddedNodeIP + "/24"), + Dst: *test.IpNet(AddedNodeIP + "/24"), NextHop: net.ParseIP(GatewayIP), ResolvedProvider: connectivity.IPIP, Custom: nil, @@ -480,10 +465,10 @@ var _ = Describe("Node-related functionality of CNI", func() { })) By("checking IPIP tunnel interface attributes (Unnumbered)") - assertUnnumberedInterface(ipipSwIfIndex, "IPIP tunnel interface", vpp) + test.AssertUnnumberedInterface(ipipSwIfIndex, "IPIP tunnel interface", vpp) By("checking IPIP tunnel interface attributes (GSO+CNAT)") - assertInterfaceGSOCNat(ipipSwIfIndex, "IPIP tunnel interface", vpp) + test.AssertInterfaceGSOCNat(ipipSwIfIndex, "IPIP tunnel interface", vpp) By("checking IPIP tunnel interface attributes (Up state)") interfaceDetails, err := vpp.GetInterfaceDetails(ipipSwIfIndex) @@ -496,7 +481,7 @@ var _ = Describe("Node-related functionality of CNI", func() { Expect(routes).To(ContainElements( // when IPIP is created it makes steering route with for NextHop/ from Pod VRF gs.MatchFields(gs.IgnoreExtras, gs.Fields{ - "Dst": gs.PointTo(Equal(*ipNet(GatewayIP + "/32"))), + "Dst": gs.PointTo(Equal(*test.IpNet(GatewayIP + "/32"))), "Paths": ContainElements(gs.MatchFields(gs.IgnoreExtras, gs.Fields{ "SwIfIndex": Equal(ipipSwIfIndex), })), @@ -506,7 +491,7 @@ var _ = Describe("Node-related functionality of CNI", func() { Expect(routes).To(ContainElements( // steering route for NodeConnectivity.Dst using ipip that is leading to the added node gs.MatchFields(gs.IgnoreExtras, gs.Fields{ - "Dst": gs.PointTo(Equal(*ipNet(AddedNodeIP + "/24"))), // NodeConnectivity.Dst + "Dst": gs.PointTo(Equal(*test.IpNet(AddedNodeIP + "/24"))), // NodeConnectivity.Dst "Paths": ContainElements(gs.MatchFields(gs.IgnoreExtras, gs.Fields{ "SwIfIndex": Equal(ipipSwIfIndex), })), @@ -560,7 +545,7 @@ var _ = Describe("Node-related functionality of CNI", func() { It("must configure wireguard tunnel with one peer and routes to it", func() { By("Adding node") - configureBGPNodeIPAddresses(connectivityServer) + test.ConfigureBGPNodeIPAddresses(connectivityServer) err := connectivityServer.ForceProviderEnableDisable(connectivity.WIREGUARD, true) // creates the tunnel (this is normally called by Felix config change event handler) Expect(err).ToNot(HaveOccurred(), "could not call ForceProviderEnableDisable") @@ -570,7 +555,7 @@ var _ = Describe("Node-related functionality of CNI", func() { }, net.ParseIP(AddedNodeIP)) connectivityServer.ForceWGPublicKeyAddition(AddedNodeName, base64.StdEncoding.EncodeToString([]byte(addedNodePublicKey))) err = connectivityServer.UpdateIPConnectivity(&common.NodeConnectivity{ - Dst: *ipNet(AddedNodeIP + "/24"), + Dst: *test.IpNet(AddedNodeIP + "/24"), NextHop: net.ParseIP(AddedNodeIP), // wireguard impl uses nexthop as node IP ResolvedProvider: connectivity.WIREGUARD, Custom: nil, @@ -588,10 +573,10 @@ var _ = Describe("Node-related functionality of CNI", func() { Expect(wgTunnel.Addr).To(Equal(net.ParseIP(ThisNodeIP).To4()), "incorrectly set IP address of this node's wireguard tunnel interface") By("checking wireguard tunnel interface attributes (Unnumbered)") - assertUnnumberedInterface(wireguardSwIfIndex, "wireguard tunnel interface", vpp) + test.AssertUnnumberedInterface(wireguardSwIfIndex, "wireguard tunnel interface", vpp) By("checking wireguard tunnel interface attributes (GSO+CNAT)") - assertInterfaceGSOCNat(wireguardSwIfIndex, "wireguard tunnel interface", vpp) + test.AssertInterfaceGSOCNat(wireguardSwIfIndex, "wireguard tunnel interface", vpp) By("checking wireguard tunnel interface attributes (Up state)") interfaceDetails, err := vpp.GetInterfaceDetails(wireguardSwIfIndex) @@ -624,12 +609,12 @@ var _ = Describe("Node-related functionality of CNI", func() { "can't get wireguard peers from VPP") Expect(peers).To(ContainElement(gs.PointTo( gs.MatchFields(gs.IgnoreExtras, gs.Fields{ - "PublicKey": Equal(addPaddingTo32Bytes([]byte(addedNodePublicKey))), + "PublicKey": Equal(test.AddPaddingTo32Bytes([]byte(addedNodePublicKey))), "Port": Equal(uint16(felixConfig.WireguardListeningPort)), "TableID": Equal(uint32(0)), // default table "Addr": Equal(net.ParseIP(AddedNodeIP).To4()), "SwIfIndex": Equal(wireguardSwIfIndex), - "AllowedIps": ContainElements(*ipNet(AddedNodeIP + "/32")), + "AllowedIps": ContainElements(*test.IpNet(AddedNodeIP + "/32")), }), ))) @@ -639,7 +624,7 @@ var _ = Describe("Node-related functionality of CNI", func() { Expect(routes).To(ContainElements( // when wireguard is created it makes steering route with for NextHop/ from Pod VRF gs.MatchFields(gs.IgnoreExtras, gs.Fields{ - "Dst": gs.PointTo(Equal(*ipNet(AddedNodeIP + "/32"))), + "Dst": gs.PointTo(Equal(*test.IpNet(AddedNodeIP + "/32"))), "Paths": ContainElements(gs.MatchFields(gs.IgnoreExtras, gs.Fields{ "SwIfIndex": Equal(wireguardSwIfIndex), })), @@ -649,7 +634,7 @@ var _ = Describe("Node-related functionality of CNI", func() { Expect(routes).To(ContainElements( // steering route for NodeConnectivity.Dst using wireguard tunnel that is leading to the added node gs.MatchFields(gs.IgnoreExtras, gs.Fields{ - "Dst": gs.PointTo(Equal(*ipNet(AddedNodeIP + "/24"))), // NodeConnectivity.Dst + "Dst": gs.PointTo(Equal(*test.IpNet(AddedNodeIP + "/24"))), // NodeConnectivity.Dst "Paths": ContainElements(gs.MatchFields(gs.IgnoreExtras, gs.Fields{ "SwIfIndex": Equal(wireguardSwIfIndex), })), @@ -687,8 +672,8 @@ var _ = Describe("Node-related functionality of CNI", func() { agentConf.GetCalicoVppSrv6().PolicyPool = "C::/16" // also C::/112 subnet for BindingSIDs(=BSID=PolicyIP) for given node *agentConf.NodeName = ThisNodeName - // add node pool for SRv6 (subnet of agentConf.CalicoVppSrv6.LocalsidPool) - _, err := addIPPoolForCalicoClient(client, fmt.Sprintf("sr-localsids-pool-%s", *agentConf.NodeName), "B::1:0/112") + // add node pool for SRv6 (subnet of agentConf.SRv6localSidIPPool) + _, err := test.AddIPPoolForCalicoClient(client, fmt.Sprintf("sr-localsids-pool-%s", *agentConf.NodeName), "B::1:0/112") Expect(err).ToNot(HaveOccurred(), "could not call addIPPoolForCalicoClient") // SID/BSID format for testing: : @@ -711,7 +696,7 @@ var _ = Describe("Node-related functionality of CNI", func() { Expect(localsids).To(ContainElement(gs.PointTo( gs.MatchFields(gs.IgnoreExtras, gs.Fields{ // first IPAM-assigned IP for SRv6 node pool - "Localsid": Equal(iptypesIP6Address("B::1:1")), + "Localsid": Equal(test.IptypesIP6Address("B::1:1")), // exit tunnel by decapsulation + further routing using IPv4 routing table // (had to be Ipv4 traffic before encapsulation) "Behavior": Equal(types.SrBehaviorDT4), @@ -722,7 +707,7 @@ var _ = Describe("Node-related functionality of CNI", func() { Expect(localsids).To(ContainElement(gs.PointTo( gs.MatchFields(gs.IgnoreExtras, gs.Fields{ // second IPAM-assigned IP for SRv6 node pool - "Localsid": Equal(iptypesIP6Address("B::1:2")), + "Localsid": Equal(test.IptypesIP6Address("B::1:2")), // exit tunnel by decapsulation + further routing using IPv6 routing table // (had to be Ipv6 traffic before encapsulation) "Behavior": Equal(types.SrBehaviorDT6), @@ -737,12 +722,12 @@ var _ = Describe("Node-related functionality of CNI", func() { "encapsulated traffic forwarding", func() { // variables related to tunnel-end node (node id=2) _, tunnelEndNodeIPNet, _ := net.ParseCIDR(AddedNodeIP + "/24") - policyBsid := net.ParseIP("C::2:1") // normally generated by IPAM on tunnel end node - tunnelEndLocalSid := ipNet("B::2:1/128") // normally generated by IPAM on tunnel end node + policyBsid := net.ParseIP("C::2:1") // normally generated by IPAM on tunnel end node + tunnelEndLocalSid := test.IpNet("B::2:1/128") // normally generated by IPAM on tunnel end node By("Setting and checking encapsulation source for SRv6") // Note: encapsulation source sets source IP for traffic when exiting tunnel(=decapsulating) - configureBGPNodeIPAddresses(connectivityServer) + test.ConfigureBGPNodeIPAddresses(connectivityServer) err := connectivityServer.ForceRescanState(connectivity.SRv6) Expect(err).ToNot(HaveOccurred(), "can't rescan state of VPP and therefore "+ "can't properly set encapsulation source IP for this node") @@ -785,7 +770,7 @@ var _ = Describe("Node-related functionality of CNI", func() { SidLists: []types.Srv6SidList{{ NumSids: uint8(1), Weight: 1, - Sids: sidArray(types.ToVppIP6Address(tunnelEndLocalSid.IP)), + Sids: test.SidArray(types.ToVppIP6Address(tunnelEndLocalSid.IP)), }}, }, }, @@ -825,7 +810,7 @@ var _ = Describe("Node-related functionality of CNI", func() { SidLists: []types.Srv6SidList{{ NumSids: uint8(1), Weight: 1, - Sids: sidArray(types.ToVppIP6Address(tunnelEndLocalSid.IP)), + Sids: test.SidArray(types.ToVppIP6Address(tunnelEndLocalSid.IP)), }}, }), "Can't find SRv6 policy") @@ -848,229 +833,6 @@ var _ = Describe("Node-related functionality of CNI", func() { }) AfterEach(func() { - teardownVPP() + test.TeardownVPP() }) }) - -// startVPP creates docker container and runs inside the VPP -func startVPP() { - // prepare VPP configuration - vppBinaryConfigArg := `unix { - nodaemon - full-coredump - cli-listen /var/run/vpp/cli2.sock - pidfile /run/vpp/vpp2.pid - } - api-trace { on } - cpu { - workers 0 - } - socksvr { - socket-name /var/run/vpp/vpp-api-test.sock - } - plugins { - plugin default { enable } - plugin dpdk_plugin.so { disable } - plugin calico_plugin.so { enable } - plugin ping_plugin.so { disable } - } - buffers { - buffers-per-numa 131072 - }` - - // docker container cleanup (failed test that didn't properly clean up docker containers?) - err := exec.Command("docker", "rm", "-f", VPPContainerName).Run() - Expect(err).Should(BeNil(), "Failed to clean up old VPP docker container") - - // start VPP inside docker container - cmd := []string{"run", "-d", "--privileged", "--name", VPPContainerName, - "-v", "/tmp/" + VPPContainerName + ":/var/run/vpp/", - "-v", "/proc:/proc", // needed for manipulation of another docker container's network namespace - "--sysctl", "net.ipv6.conf.all.disable_ipv6=0"} // enable IPv6 in container (to set IPv6 on host's end of uplink) - cmd = append(cmd, vppContainerExtraArgs...) - cmd = append(cmd, "--entrypoint", vppBinary, vppImage, vppBinaryConfigArg) - err = exec.Command("docker", cmd...).Run() - Expect(err).Should(BeNil(), "Failed to start VPP inside docker container") -} - -// configureVPP connects to VPP and configures it with common configuration needed for tests -func configureVPP(log *logrus.Logger) (vpp *vpplink.VppLink, uplinkSwIfIndex uint32) { - // connect to VPP - vpp, err := common.CreateVppLinkInRetryLoop("/tmp/"+VPPContainerName+"/vpp-api-test.sock", - log.WithFields(logrus.Fields{"component": "vpp-api"}), 20*time.Second, 100*time.Millisecond) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Cannot create VPP client: %v", err)) - Expect(vpp).NotTo(BeNil()) - - // setup common VRF setup - for _, ipFamily := range vpplink.IpFamilies { //needed config for pod creation tests - err := vpp.AddVRF(common.PuntTableId, ipFamily.IsIp6, fmt.Sprintf("punt-table-%s", ipFamily.Str)) - if err != nil { - log.Fatal(errors.Wrapf(err, "Error creating punt vrf %s", ipFamily.Str)) - } - err = vpp.AddVRF(common.PodVRFIndex, ipFamily.IsIp6, fmt.Sprintf("calico-pods-%s", ipFamily.Str)) - if err != nil { - log.Fatal(err) - } - err = vpp.AddDefaultRouteViaTable(common.PodVRFIndex, common.DefaultVRFIndex, ipFamily.IsIp6) - if err != nil { - log.Fatal(err) - } - } - - // setup simplified mock version of uplink interface - // Note: for the real configuration of the uplink interface and other related things see - // UplinkDriver.CreateMainVppInterface(...) and VppRunner.configureVpp(...)) - uplinkSwIfIndex, err = vpp.CreateTapV2(&types.TapV2{ - GenericVppInterface: types.GenericVppInterface{ - HostInterfaceName: UplinkIfName, - HardwareAddr: mac("aa:bb:cc:dd:ee:01"), - }, - Tag: fmt.Sprintf("host-%s", UplinkIfName), - Flags: types.TapFlagNone, - // Host end of tap (it is located inside docker container) - HostMtu: 1500, - HostMacAddress: *mac("aa:bb:cc:dd:ee:02"), - }) - Expect(err).ToNot(HaveOccurred(), "Error creating mocked Uplink interface") - err = vpp.InterfaceAdminUp(uplinkSwIfIndex) - Expect(err).ToNot(HaveOccurred(), "Error setting state to UP for mocked Uplink interface") - err = vpp.AddInterfaceAddress(uplinkSwIfIndex, ipNet(UplinkIP+"/24")) - Expect(err).ToNot(HaveOccurred(), "Error adding IPv4 address to data interface") - err = vpp.AddInterfaceAddress(uplinkSwIfIndex, ipNet(UplinkIPv6+"/16")) - Expect(err).ToNot(HaveOccurred(), "Error adding IPv6 address to data interface") - err = exec.Command("docker", "exec", VPPContainerName, "ip", "address", "add", - GatewayIP+"/24", "dev", UplinkIfName).Run() - Expect(err).ToNot(HaveOccurred(), "Failed to set IPv4 address for host end of tap") - err = exec.Command("docker", "exec", VPPContainerName, "ip", "address", "add", - GatewayIPv6+"/16", "dev", UplinkIfName).Run() - Expect(err).ToNot(HaveOccurred(), "Failed to set IPv6 address for host end of tap") - err = exec.Command("docker", "exec", VPPContainerName, "ip", "link", "set", - UplinkIfName, "up").Run() - Expect(err).ToNot(HaveOccurred(), "Failed to set state to UP for host end of tap") - - return -} - -// teardownVPP removes container with running VPP to stop VPP and clean after it -func teardownVPP() { - err := exec.Command("docker", "rm", "-f", VPPContainerName).Run() - Expect(err).Should(BeNil(), "Failed to stop and remove VPP docker container") -} - -// assertUnnumberedInterface checks whether the provided interface is unnumbered and properly takes IP address -// from the correct interface (common.VppManagerInfo.GetMainSwIfIndex()). -func assertUnnumberedInterface(swIfIndex uint32, interfaceDescriptiveName string, vpp *vpplink.VppLink) { - unnumberedDetails, err := vpp.InterfaceGetUnnumbered(swIfIndex) - Expect(err).ToNot(HaveOccurred(), - fmt.Sprintf("can't get unnumbered details of %s", interfaceDescriptiveName)) - Expect(unnumberedDetails).ToNot(BeEmpty(), "can't find unnumbered interface") - Expect(unnumberedDetails[0].IPSwIfIndex).To(Equal( - interface_types.InterfaceIndex(common.VppManagerInfo.GetMainSwIfIndex())), - fmt.Sprintf("Unnumberred %s doesn't get IP address from expected interface", interfaceDescriptiveName)) -} - -// assertInterfaceGSOCNat check whether the given interface has properly set the GSO and CNAT attributes -func assertInterfaceGSOCNat(swIfIndex uint32, interfaceDescriptiveName string, vpp *vpplink.VppLink) { - // Note: no specialized binary api or VPP CLI for getting GSO on interface -> using - // Feature Arcs (https://wiki.fd.io/view/VPP/Feature_Arcs) to detect it (GSO is using them to steer - // traffic), Feature Arcs have no binary API -> using VPP's VPE binary API - featuresStr, err := vpp.RunCli(fmt.Sprintf("sh interface %d features", swIfIndex)) - Expect(err).ToNot(HaveOccurred(), - fmt.Sprintf("failed to get %s's configured features", interfaceDescriptiveName)) - featuresStr = strings.ToLower(featuresStr) - var GSOFeatureArcs = []string{"gso-ip4", "gso-ip6", "gso-l2-ip4", "gso-l2-ip6"} - for _, gsoStr := range GSOFeatureArcs { - // Note: not checking full Feature Arc (i.e. ipv4-unicast: gso-ipv4), just the destination - // of traffic steering. This is enough because without GSO enabled, the destination would not exist. - Expect(featuresStr).To(ContainSubstring(gsoStr), fmt.Sprintf("GSO not fully enabled "+ - "due to missing %s in configured features arcs %s", gsoStr, featuresStr)) - } - var CNATFeatureArcs = []string{"cnat-input-ip4", "cnat-input-ip6", "cnat-output-ip4", "cnat-output-ip6"} - for _, cnatStr := range CNATFeatureArcs { - // Note: could be enhanced by checking the full Feature Arc (from where we steer traffic to cnat) - Expect(featuresStr).To(ContainSubstring(cnatStr), fmt.Sprintf("CNAT not fully enabled "+ - "due to missing %s in configured features arcs %s", cnatStr, featuresStr)) - } -} - -// assertNextNodeLink asserts that in VPP graph the given node has linked the linkedNextNode as one of its -// "next" nodes for processing. It returns to node specific index of the checked next node. -func assertNextNodeLink(node, linkedNextNode string, vpp *vpplink.VppLink) int { - // get the node information from VPP (No VPP binary API for that -> using VPE) - nodeInfoStr, err := vpp.RunCli(fmt.Sprintf("show node %s", node)) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("failed to VPP graph info for node %s", node)) - - // asserting next node - Expect(strings.ToLower(nodeInfoStr)).To(ContainSubstring(strings.ToLower(linkedNextNode)), - fmt.Sprintf("can't find added next node %s in node %s information", linkedNextNode, node)) - - // getting next node's index that is relative to the given node (this is kind of brittle as we parse VPP CLI output) - linesToNextNode := strings.Split(strings.Split(nodeInfoStr, linkedNextNode)[0], "\n") - indexStrOfLinkedNextNode := strings.TrimSpace(linesToNextNode[len(linesToNextNode)-1][:10]) - nextNodeIndex, err := strconv.Atoi(indexStrOfLinkedNextNode) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("can't parse next node index "+ - "in given node from VPP CLI output %s", nodeInfoStr)) - - return nextNodeIndex -} - -func configureBGPNodeIPAddresses(connectivityServer *connectivity.ConnectivityServer) { - ip4, ip4net, _ := net.ParseCIDR(ThisNodeIP + "/24") - ip4net.IP = ip4 - ip6, ip6net, _ := net.ParseCIDR(ThisNodeIPv6 + "/128") - ip6net.IP = ip6 - connectivityServer.SetOurBGPSpec(&common.LocalNodeSpec{ - IPv4Address: ip4net, - IPv6Address: ip6net, - }) -} - -// addIPPoolForCalicoClient is convenience function for adding IPPool to mocked Calico IPAM Stub used -// in Calico client stub. This function doesn't set anything for the watchers.IpamCache implementation. -func addIPPoolForCalicoClient(client *calico.CalicoClientStub, poolName string, poolCIRD string) ( - *apiv3.IPPool, error) { - return client.IPPoolsStub.Create(context.Background(), &apiv3.IPPool{ - ObjectMeta: metav1.ObjectMeta{ - Name: poolName, - }, - Spec: apiv3.IPPoolSpec{ - CIDR: poolCIRD, - }, - }, options.SetOptions{}) -} - -func ipNet(ipNetCIDRStr string) *net.IPNet { - _, ipNet, err := net.ParseCIDR(ipNetCIDRStr) - Expect(err).To(BeNil()) - return ipNet -} - -func ipNetWithIPInIPv6Format(ipNetCIDRStr string) *net.IPNet { - _, ipNet, err := net.ParseCIDR(ipNetCIDRStr) - ipNet.IP = ipNet.IP.To16() - Expect(err).To(BeNil()) - return ipNet -} - -func mac(macStr string) *net.HardwareAddr { - mac, err := net.ParseMAC(macStr) - Expect(err).To(BeNil()) - return &mac -} - -func iptypesIP6Address(address string) ip_types.IP6Address { - addr, err := ip_types.ParseAddress(address) - Expect(err).ToNot(HaveOccurred(), "failed to parse ip_types.IP6Addess from string %s", addr) - return addr.Un.GetIP6() -} - -func sidArray(addresses ...ip_types.IP6Address) (sids [16]ip_types.IP6Address) { - copy(sids[:], addresses) - return sids -} - -func addPaddingTo32Bytes(value []byte) []byte { - result := [32]byte{} - copy(result[:], value) - return result[:] -} diff --git a/calico-vpp-agent/cni/cni_pod_test.go b/calico-vpp-agent/cni/cni_pod_test.go index 71f3561e..482aeaa3 100644 --- a/calico-vpp-agent/cni/cni_pod_test.go +++ b/calico-vpp-agent/cni/cni_pod_test.go @@ -15,19 +15,13 @@ package cni_test import ( "context" - "crypto/sha512" - "encoding/base64" "fmt" - "math" "net" "os" "os/exec" - "path/filepath" - "reflect" "strings" "syscall" - "github.com/containernetworking/plugins/pkg/ns" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" gs "github.com/onsi/gomega/gstruct" @@ -37,13 +31,11 @@ import ( "github.com/vishvananda/netlink" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/cni" - "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/cni/pod_interface" - "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/cni/storage" + test "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/common_tests" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/tests/mocks" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/watchers" "github.com/projectcalico/vpp-dataplane/config" - "github.com/projectcalico/vpp-dataplane/multinet-monitor/networkAttachmentDefinition" "github.com/projectcalico/vpp-dataplane/vpplink" "github.com/projectcalico/vpp-dataplane/vpplink/types" ) @@ -65,8 +57,8 @@ var _ = Describe("Pod-related functionality of CNI", func() { BeforeEach(func() { log = logrus.New() - startVPP() - vpp, _ = configureVPP(log) + test.StartVPP() + vpp, _ = test.ConfigureVPP(log) // setup connectivity server (functionality target of tests) if ipamStub == nil { ipamStub = mocks.NewIpamCacheStub() @@ -80,7 +72,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { Describe("Addition of the pod", func() { BeforeEach(func() { - createPod() + test.CreatePod() }) Context("When new pod is added", func() { @@ -132,25 +124,25 @@ var _ = Describe("Pod-related functionality of CNI", func() { "for IP address or doesn't exist at all") By("Checking existence of interface tunnel at VPP's end") - ifSwIfIndex := assertTunInterfaceExistence(vpp, newPod) + ifSwIfIndex := test.AssertTunInterfaceExistence(vpp, newPod) By("Checking correct IP address of interface tunnel at VPP's end") - assertTunnelInterfaceIPAddress(vpp, ifSwIfIndex, ipAddress) + test.AssertTunnelInterfaceIPAddress(vpp, ifSwIfIndex, ipAddress) By("Checking correct MTU for tunnel interface at VPP's end") - assertTunnelInterfaceMTU(vpp, ifSwIfIndex) + test.AssertTunnelInterfaceMTU(vpp, ifSwIfIndex) - runInPod(newPod.Netns, func() { + test.RunInPod(newPod.Netns, func() { By("Checking tun interface on pod side") _, err := netlink.LinkByName(interfaceName) Expect(err).ToNot(HaveOccurred(), "can't find tun interface in pod") }) By("Checking created pod RPF VRF") - RPFVRF := assertRPFVRFExistence(vpp, interfaceName, newPod.Netns) + RPFVRF := test.AssertRPFVRFExistence(vpp, interfaceName, newPod.Netns) By("Checking RPF routes are added") - assertRPFRoutes(vpp, RPFVRF, ifSwIfIndex, ipAddress) + test.AssertRPFRoutes(vpp, RPFVRF, ifSwIfIndex, ipAddress) }) }) @@ -199,13 +191,13 @@ var _ = Describe("Pod-related functionality of CNI", func() { fmt.Sprintf("Pod addition failed due to: %s", reply.ErrorMessage)) By("Checking existence of main interface tunnel to pod (at VPP's end)") - ifSwIfIndex := assertTunInterfaceExistence(vpp, newPod) + ifSwIfIndex := test.AssertTunInterfaceExistence(vpp, newPod) By("Checking main tunnel's tun interface for common interface attributes") - assertTunnelInterfaceIPAddress(vpp, ifSwIfIndex, ipAddress) - assertTunnelInterfaceMTU(vpp, ifSwIfIndex) + test.AssertTunnelInterfaceIPAddress(vpp, ifSwIfIndex, ipAddress) + test.AssertTunnelInterfaceMTU(vpp, ifSwIfIndex) - runInPod(newPod.Netns, func() { + test.RunInPod(newPod.Netns, func() { By("Checking main tunnel's tun interface on pod side") _, err := netlink.LinkByName(interfaceName) Expect(err).ToNot(HaveOccurred(), "can't find main interface in pod") @@ -213,13 +205,13 @@ var _ = Describe("Pod-related functionality of CNI", func() { By("Checking secondary tunnel's memif interface for existence") memifSwIfIndex, err := vpp.SearchInterfaceWithTag( - interfaceTagForLocalMemifTunnel(newPod.InterfaceName, newPod.Netns)) + test.InterfaceTagForLocalMemifTunnel(newPod.InterfaceName, newPod.Netns)) Expect(err).ShouldNot(HaveOccurred(), "Failed to get memif interface at VPP's end") By("Checking secondary tunnel's memif interface for common interface attributes") - assertTunnelInterfaceIPAddress(vpp, memifSwIfIndex, ipAddress) - assertTunnelInterfaceMTU(vpp, memifSwIfIndex) - assertInterfaceGSO(memifSwIfIndex, "secondary tunnel's memif interface", vpp) + test.AssertTunnelInterfaceIPAddress(vpp, memifSwIfIndex, ipAddress) + test.AssertTunnelInterfaceMTU(vpp, memifSwIfIndex) + test.AssertInterfaceGSO(memifSwIfIndex, "secondary tunnel's memif interface", vpp) By("Checking secondary tunnel's memif interface for memif attributes") memifs, err := vpp.ListMemifInterfaces() @@ -325,7 +317,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { fmt.Sprintf("Pod addition to primary network failed due to: %s", reply.ErrorMessage)) By("Adding Pod to secondary(multinet) network using CNI server") - secondaryIPAddress := firstIPinIPRange(networkDefinition.Range).String() + secondaryIPAddress := test.FirstIPinIPRange(networkDefinition.Range).String() newPodForSecondaryNetwork := &cniproto.AddRequest{ InterfaceName: secondaryInterfaceName, Netns: fmt.Sprintf("/proc/%s/ns/net", containerPidStr), // expecting mount of "/proc" from host @@ -334,7 +326,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { }}, Workload: &cniproto.WorkloadIDs{}, DataplaneOptions: map[string]string{ - dpoNetworkNameFieldName(): networkDefinition.Name, + test.DpoNetworkNameFieldName(): networkDefinition.Name, }, } reply, err = cniServer.Add(context.Background(), newPodForSecondaryNetwork) @@ -343,20 +335,20 @@ var _ = Describe("Pod-related functionality of CNI", func() { fmt.Sprintf("Pod addition to secondary network failed due to: %s", reply.ErrorMessage)) By("Checking existence of main tun interface tunnel to pod (at VPP's end)") - mainSwIfIndex := assertTunInterfaceExistence(vpp, newPodForPrimaryNetwork) + mainSwIfIndex := test.AssertTunInterfaceExistence(vpp, newPodForPrimaryNetwork) By("Checking main tunnel's tun interface for common interface attributes") - assertTunnelInterfaceIPAddress(vpp, mainSwIfIndex, ipAddress) - assertTunnelInterfaceMTU(vpp, mainSwIfIndex) + test.AssertTunnelInterfaceIPAddress(vpp, mainSwIfIndex, ipAddress) + test.AssertTunnelInterfaceMTU(vpp, mainSwIfIndex) By("Checking secondary tunnel's tun interface for existence") - secondarySwIfIndex := assertTunInterfaceExistence(vpp, newPodForSecondaryNetwork) + secondarySwIfIndex := test.AssertTunInterfaceExistence(vpp, newPodForSecondaryNetwork) By("Checking secondary tunnel's tun interface for common interface attributes") - assertTunnelInterfaceIPAddress(vpp, secondarySwIfIndex, secondaryIPAddress) - assertTunnelInterfaceMTU(vpp, secondarySwIfIndex) + test.AssertTunnelInterfaceIPAddress(vpp, secondarySwIfIndex, secondaryIPAddress) + test.AssertTunnelInterfaceMTU(vpp, secondarySwIfIndex) - runInPod(newPodForSecondaryNetwork.Netns, func() { + test.RunInPod(newPodForSecondaryNetwork.Netns, func() { By("Checking main tunnel's tun interface on pod side") _, err := netlink.LinkByName(mainInterfaceName) Expect(err).ToNot(HaveOccurred(), "can't find main interface in pod") @@ -370,7 +362,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { Expect(err).ToNot(HaveOccurred(), "can't get routes from pod") Expect(secTunLinkRoutes).To(ContainElements( gs.MatchFields(gs.IgnoreExtras, gs.Fields{ - "Dst": gs.PointTo(Equal(*ipNet(networkDefinition.Range))), + "Dst": gs.PointTo(Equal(*test.IpNet(networkDefinition.Range))), }), ), "can't find route in pod that steers all multinet network "+ "traffic into multinet tunnel interface in pod") @@ -382,19 +374,19 @@ var _ = Describe("Pod-related functionality of CNI", func() { gs.MatchFields(gs.IgnoreExtras, gs.Fields{ "Type": Equal(common.LocalPodAddressAdded), "New": gs.MatchFields(gs.IgnoreExtras, gs.Fields{ - "ContainerIP": gs.PointTo(Equal(*ipNetWithIPInIPv6Format(ipAddress + "/32"))), + "ContainerIP": gs.PointTo(Equal(*test.IpNetWithIPInIPv6Format(ipAddress + "/32"))), }), }), gs.MatchFields(gs.IgnoreExtras, gs.Fields{ "Type": Equal(common.LocalPodAddressAdded), "New": gs.MatchFields(gs.IgnoreExtras, gs.Fields{ - "ContainerIP": gs.PointTo(Equal(*ipNetWithIPInIPv6Format(secondaryIPAddress + "/32"))), + "ContainerIP": gs.PointTo(Equal(*test.IpNetWithIPInIPv6Format(secondaryIPAddress + "/32"))), }), }), )) By("Checking default route from pod-specific VRF to multinet network-specific vrf") - podVrf4ID, podVrf6ID, err := podVRFs(secondaryInterfaceName, newPodForSecondaryNetwork.Netns, vpp) + podVrf4ID, podVrf6ID, err := test.PodVRFs(secondaryInterfaceName, newPodForSecondaryNetwork.Netns, vpp) Expect(err).ToNot(HaveOccurred(), "can't find pod-specific VRFs") for idx, ipFamily := range vpplink.IpFamilies { podVrfID := podVrf4ID @@ -423,16 +415,16 @@ var _ = Describe("Pod-related functionality of CNI", func() { By("Checking steering route in multinet network VRF leading to pod " + "using multinet tunnel interface") // Note: should be checked for all container IPs of multinet, but we have only one - multinetVRFID := networkDefinition.VRF.Tables[ipFamilyIndex(vpplink.IpFamilyV4)] // secondaryIPAddress is from IpFamilyV4 + multinetVRFID := networkDefinition.VRF.Tables[test.IpFamilyIndex(vpplink.IpFamilyV4)] // secondaryIPAddress is from IpFamilyV4 routes, err := vpp.GetRoutes(multinetVRFID, false) Expect(err).ToNot(HaveOccurred(), "can't get ipv4 routes in multinet network-specific VRF") Expect(routes).To(ContainElements( types.Route{ - Dst: ipNet(secondaryIPAddress + "/32"), + Dst: test.IpNet(secondaryIPAddress + "/32"), Paths: []types.RoutePath{{ SwIfIndex: secondarySwIfIndex, - Gw: ipNet(secondaryIPAddress + "/32").IP, + Gw: test.IpNet(secondaryIPAddress + "/32").IP, }}, Table: multinetVRFID, }, @@ -444,231 +436,11 @@ var _ = Describe("Pod-related functionality of CNI", func() { }) AfterEach(func() { - teardownPod() + test.TeardownPod() }) }) AfterEach(func() { - teardownVPP() + test.TeardownVPP() }) }) - -func assertTunInterfaceExistence(vpp *vpplink.VppLink, newPod *cniproto.AddRequest) uint32 { - ifSwIfIndex, err := vpp.SearchInterfaceWithTag( - interfaceTagForLocalTunTunnel(newPod.InterfaceName, newPod.Netns)) - Expect(err).ShouldNot(HaveOccurred(), "Failed to get interface at VPP's end") - Expect(ifSwIfIndex).ToNot(Equal(vpplink.INVALID_SW_IF_INDEX), - "No interface at VPP's end is found") - Expect(ifSwIfIndex).NotTo(BeZero(), "No interface at VPP's end is found") - return ifSwIfIndex -} - -func assertTunnelInterfaceIPAddress(vpp *vpplink.VppLink, ifSwIfIndex uint32, expectedIPAddress string) { - couples, err := vpp.InterfaceGetUnnumbered(ifSwIfIndex) - Expect(err).ShouldNot(HaveOccurred(), "Failed to retrieve unnumbered interface "+ - "info dump for VPP's end of interface tunnel") - Expect(couples).ToNot(BeEmpty(), "can't find unnumbered interface") - addrList, err := vpp.AddrList(uint32(couples[0].IPSwIfIndex), false) - Expect(err).ShouldNot(HaveOccurred(), - "Failed to get addresses for unnumbered interfaces") - var correctAdress bool - for _, addr := range addrList { - if addr.IPNet.IP.Equal(net.ParseIP(expectedIPAddress)) { - correctAdress = true - } - } - Expect(correctAdress).To(BeTrue(), - "VPP's end of interface tunnel is not correctly configured for IP address") -} - -func assertTunnelInterfaceMTU(vpp *vpplink.VppLink, ifSwIfIndex uint32) { - details, err := vpp.GetInterfaceDetails(ifSwIfIndex) - Expect(err).ShouldNot(HaveOccurred(), - "Failed to retrieve interface details of VPP's end of interface tunnel") - Expect(int(details.Mtu[0])).To(Equal(vpplink.MAX_MTU), - "VPP's end of interface tunnel has not correctly configured MTU") -} - -// assertInterfaceGSOCNat check whether the given interface has properly set the GSO and CNAT attributes -func assertInterfaceGSO(swIfIndex uint32, interfaceDescriptiveName string, vpp *vpplink.VppLink) { - // Note: no specialized binary api or VPP CLI for getting GSO on interface -> using - // Feature Arcs (https://wiki.fd.io/view/VPP/Feature_Arcs) to detect it (GSO is using them to steer - // traffic), Feature Arcs have no binary API -> using VPP's VPE binary API - featuresStr, err := vpp.RunCli(fmt.Sprintf("sh interface %d features", swIfIndex)) - Expect(err).ToNot(HaveOccurred(), - fmt.Sprintf("failed to get %s's configured features", interfaceDescriptiveName)) - featuresStr = strings.ToLower(featuresStr) - var GSOFeatureArcs = []string{"gso-ip4", "gso-ip6", "gso-l2-ip4", "gso-l2-ip6"} - for _, gsoStr := range GSOFeatureArcs { - // Note: not checking full Feature Arc (i.e. ipv4-unicast: gso-ipv4), just the destination - // of traffic steering. This is enough because without GSO enabled, the destination would not exist. - Expect(featuresStr).To(ContainSubstring(gsoStr), fmt.Sprintf("GSO not fully enabled "+ - "due to missing %s in configured features arcs %s", gsoStr, featuresStr)) - } -} - -// assertRPFVRFExistence checks that dedicated VRF for RPF is created for interface in VPP -func assertRPFVRFExistence(vpp *vpplink.VppLink, interfaceName string, netnsName string) uint32 { - VRFs, err := vpp.ListVRFs() - Expect(err).ShouldNot(HaveOccurred(), - "Failed to retrieve list of VRFs in VPP") - hbytes := sha512.Sum512([]byte(fmt.Sprintf("%s%s%s%s", "4", netnsName, interfaceName, "RPF"))) - h := base64.StdEncoding.EncodeToString(hbytes[:])[:storage.VrfTagHashLen] - s := fmt.Sprintf("%s-%s-%s-%s", h, "4", interfaceName, filepath.Base(netnsName)) - vrfTag := storage.TruncateStr(s, storage.MaxApiTagLen) - foundRPFVRF := false - var vrfID uint32 - for _, VRF := range VRFs { - if VRF.Name == vrfTag { - foundRPFVRF = true - vrfID = VRF.VrfID - break - } - } - Expect(foundRPFVRF).Should(BeTrue(), - "Failed to find RPF VRF for interface") - return vrfID -} - -// assertRPFRoutes checks that a route to the pod is added in the RPFVRF and to addresses allowed -// to be spoofed -func assertRPFRoutes(vpp *vpplink.VppLink, vrfID uint32, swifindex uint32, ipAddress string) { - routes, err := vpp.GetRoutes(vrfID, false) - Expect(err).ShouldNot(HaveOccurred(), - "Failed to get routes from RPF VRF") - Expect(routes).To(ContainElements( - types.Route{ - Dst: ipNet(ipAddress + "/32"), - Paths: []types.RoutePath{{ - SwIfIndex: swifindex, - Gw: ipNet(ipAddress + "/32").IP, - }}, - Table: vrfID, - }, - types.Route{ - Dst: ipNet("172.16.104.7" + "/32"), - Paths: []types.RoutePath{{ - SwIfIndex: swifindex, - Gw: ipNet(ipAddress + "/32").IP, - }}, - Table: vrfID, - }, - types.Route{ - Dst: ipNet("3.4.5.6" + "/32"), - Paths: []types.RoutePath{{ - SwIfIndex: swifindex, - Gw: ipNet(ipAddress + "/32").IP, - }}, - Table: vrfID, - }, - ), "Cannot find route to pod in RPF VRF %s", ipAddress) - -} - -// createPod creates docker container that will be used as pod for CNI testing -func createPod() { - // docker container cleanup (failed test that didn't properly clean up docker containers?) - err := exec.Command("docker", "rm", "-f", PodMockContainerName).Run() - Expect(err).Should(BeNil(), "Failed to clean up old pod mock docker container") - - // start new pod mock (docker container) - err = exec.Command("docker", "run", "-d", "--network", "none", "--name", PodMockContainerName, - PodMockImage, "sleep", "10d").Run() - Expect(err).Should(BeNil(), "Failed to start new pod mock (docker container)") -} - -// teardownPod removes container that is used as Pod mock -func teardownPod() { - err := exec.Command("docker", "rm", "-f", PodMockContainerName).Run() - Expect(err).Should(BeNil(), "Failed to stop and remove pod mock (docker container)") -} - -// runInPod runs runner function in provided pod network namespace. This is the same as running -// networking commands inside pod. -func runInPod(podNetNS string, runner func()) { - err := ns.WithNetNSPath(podNetNS, func(hostNS ns.NetNS) error { - defer GinkgoRecover() // running in different goroutine -> needed for failed assertion retrieval - runner() - return nil - }) - Expect(err).Should(BeNil(), "Failed to runInPod") -} - -// dpoNetworkNameFieldName extracts JSON field name for NetworkName used in cniproto.AddRequest.DataplaneOptions -func dpoNetworkNameFieldName() string { - netNameField, found := reflect.TypeOf(networkAttachmentDefinition.NetConf{}.DpOptions).FieldByName("NetName") - Expect(found).To(BeTrue(), - "can't find network name field in NetworkAttachmentDefinition. Did that structure changed?") - jsonStr, isSet := netNameField.Tag.Lookup("json") - Expect(isSet).To(BeTrue(), "can't find json name for network name field in NetworkAttachmentDefinition") - return strings.Split(jsonStr, ",")[0] -} - -// interfaceTagForLocalTunTunnel constructs the tag for the VPP side of the tap tunnel the same way as cni server -func interfaceTagForLocalTunTunnel(interfaceName, netns string) string { - return interfaceTagForLocalTunnel(pod_interface.NewTunTapPodInterfaceDriver(nil, nil).Name, - interfaceName, netns) -} - -// interfaceTagForLocalMemifTunnel constructs the tag for the VPP side of the memif tunnel the same way as cni server -func interfaceTagForLocalMemifTunnel(interfaceName, netns string) string { - return interfaceTagForLocalTunnel(pod_interface.NewMemifPodInterfaceDriver(nil, nil).Name, - interfaceName, netns) -} - -// interfaceTagForLocalTunnel constructs the tag for the VPP side of the local tunnel the same way as cni server -func interfaceTagForLocalTunnel(prefix, interfaceName, netns string) string { - return (&storage.LocalPodSpec{ - NetnsName: netns, - InterfaceName: interfaceName, - }).GetInterfaceTag(prefix) -} - -// firstIPinIPRange computes first usable IPv4 address from the given subnet. The subnet definition IP address -// (ending with zero bits) is not considered as usable IPv4 address as it can have special meaning in certain situations. -func firstIPinIPRange(ipRangeCIDR string) net.IP { - ip, _, err := net.ParseCIDR(ipRangeCIDR) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("can't parse range subnet string %s as CIDR", ipRangeCIDR)) - ip = ip.To4() // expecting IPv4 address - ip[3]++ // incrementing last IP address byte to get the first usable IP address in subnet range (subnet x.y.z.0 -> first ip address x.y.z.1) - return ip -} - -// podVRFs gets ids of IPv4 and IPv6 pod-specific VRFs from VPP -func podVRFs(podInterface, podNetNSName string, vpp *vpplink.VppLink) (vrf4ID, vrf6ID uint32, err error) { - vrfs, err := vpp.ListVRFs() - Expect(err).ToNot(HaveOccurred(), "error listing VRFs to find all pod VRFs") - - podSpec := storage.LocalPodSpec{ - InterfaceName: podInterface, - NetnsName: podNetNSName, - V4VrfId: types.InvalidID, - V6VrfId: types.InvalidID, - } - for _, vrf := range vrfs { - for _, ipFamily := range vpplink.IpFamilies { - if vrf.Name == podSpec.GetVrfTag(ipFamily, "") { - podSpec.SetVrfId(vrf.VrfID, ipFamily) - } - } - if podSpec.V4VrfId != types.InvalidID && podSpec.V6VrfId != types.InvalidID { - return podSpec.V4VrfId, podSpec.V6VrfId, nil - } - } - - if (podSpec.V4VrfId != types.InvalidID) != (podSpec.V6VrfId != types.InvalidID) { - return podSpec.V4VrfId, podSpec.V6VrfId, - fmt.Errorf("partial VRF state v4=%d v6=%d key=%s", podSpec.V4VrfId, podSpec.V6VrfId, podSpec.Key()) - } - - return podSpec.V4VrfId, podSpec.V6VrfId, fmt.Errorf("not VRFs state (key=%s)", podSpec.Key()) -} - -func ipFamilyIndex(ipFamily vpplink.IpFamily) int { - for idx, family := range vpplink.IpFamilies { - if family == ipFamily { - return idx - } - } - return math.MaxInt -} diff --git a/calico-vpp-agent/common_tests/common_tests.go b/calico-vpp-agent/common_tests/common_tests.go new file mode 100644 index 00000000..be8c6985 --- /dev/null +++ b/calico-vpp-agent/common_tests/common_tests.go @@ -0,0 +1,507 @@ +package common_tests + +import ( + "context" + "crypto/sha512" + "encoding/base64" + "fmt" + "math" + "net" + "os/exec" + "path/filepath" + "reflect" + "strconv" + "strings" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + + "github.com/containernetworking/plugins/pkg/ns" + "github.com/projectcalico/calico/libcalico-go/lib/options" + "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/cni/pod_interface" + "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/cni/storage" + "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/connectivity" + cniproto "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" + "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/tests/mocks/calico" + "github.com/projectcalico/vpp-dataplane/multinet-monitor/networkAttachmentDefinition" + "github.com/projectcalico/vpp-dataplane/vpplink" + "github.com/projectcalico/vpp-dataplane/vpplink/binapi/vppapi/interface_types" + "github.com/projectcalico/vpp-dataplane/vpplink/binapi/vppapi/ip_types" + "github.com/projectcalico/vpp-dataplane/vpplink/types" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) +const ( + // PodMockContainerName is used container name for pod container mock + PodMockContainerName = "cni-tests-pod-mock" + // PodMockImage is docker image used for pod mocking + PodMockImage = "calicovpp/vpp-test-pod-mock:latest" + VPPContainerName = "cni-tests-vpp" + VppContainerExtraArgsName = "VPP_CONTAINER_EXTRA_ARGS" + ThisNodeName = "node1" + UplinkIfName = "uplink" + UplinkIP = "10.0.100.1" + UplinkIPv6 = "A::1:1" + GatewayIP = "10.0.100.254" + GatewayIPv6 = "A::1:254" + ThisNodeIP = UplinkIP + ThisNodeIPv6 = UplinkIPv6 + + AddedNodeName = "node2" + AddedNodeIP = "10.0.200.1" + AddedNodeIPv6 = "A::2:1" +) +var ( + // VppImage is the name of docker image containing VPP binary + VppImage string + // VppBinary is the full path to VPP binary inside docker image + VppBinary string + // vppContainerExtraArgs is a list of additionnal cli parameters for the VPP `docker run ...` + VppContainerExtraArgs []string = []string{} +) +func AssertTunInterfaceExistence(vpp *vpplink.VppLink, newPod *cniproto.AddRequest) uint32 { + ifSwIfIndex, err := vpp.SearchInterfaceWithTag( + InterfaceTagForLocalTunTunnel(newPod.InterfaceName, newPod.Netns)) + Expect(err).ShouldNot(HaveOccurred(), "Failed to get interface at VPP's end") + Expect(ifSwIfIndex).ToNot(Equal(vpplink.INVALID_SW_IF_INDEX), + "No interface at VPP's end is found") + Expect(ifSwIfIndex).NotTo(BeZero(), "No interface at VPP's end is found") + return ifSwIfIndex +} + +func AssertTunnelInterfaceIPAddress(vpp *vpplink.VppLink, ifSwIfIndex uint32, expectedIPAddress string) { + couples, err := vpp.InterfaceGetUnnumbered(ifSwIfIndex) + Expect(err).ShouldNot(HaveOccurred(), "Failed to retrieve unnumbered interface "+ + "info dump for VPP's end of interface tunnel") + Expect(couples).ToNot(BeEmpty(), "can't find unnumbered interface") + addrList, err := vpp.AddrList(uint32(couples[0].IPSwIfIndex), false) + Expect(err).ShouldNot(HaveOccurred(), + "Failed to get addresses for unnumbered interfaces") + var correctAdress bool + for _, addr := range addrList { + if addr.IPNet.IP.Equal(net.ParseIP(expectedIPAddress)) { + correctAdress = true + } + } + Expect(correctAdress).To(BeTrue(), + "VPP's end of interface tunnel is not correctly configured for IP address") +} + +func AssertTunnelInterfaceMTU(vpp *vpplink.VppLink, ifSwIfIndex uint32) { + details, err := vpp.GetInterfaceDetails(ifSwIfIndex) + Expect(err).ShouldNot(HaveOccurred(), + "Failed to retrieve interface details of VPP's end of interface tunnel") + Expect(int(details.Mtu[0])).To(Equal(vpplink.MAX_MTU), + "VPP's end of interface tunnel has not correctly configured MTU") +} + +// assertInterfaceGSOCNat check whether the given interface has properly set the GSO and CNAT attributes +func AssertInterfaceGSO(swIfIndex uint32, interfaceDescriptiveName string, vpp *vpplink.VppLink) { + // Note: no specialized binary api or VPP CLI for getting GSO on interface -> using + // Feature Arcs (https://wiki.fd.io/view/VPP/Feature_Arcs) to detect it (GSO is using them to steer + // traffic), Feature Arcs have no binary API -> using VPP's VPE binary API + featuresStr, err := vpp.RunCli(fmt.Sprintf("sh interface %d features", swIfIndex)) + Expect(err).ToNot(HaveOccurred(), + fmt.Sprintf("failed to get %s's configured features", interfaceDescriptiveName)) + featuresStr = strings.ToLower(featuresStr) + var GSOFeatureArcs = []string{"gso-ip4", "gso-ip6", "gso-l2-ip4", "gso-l2-ip6"} + for _, gsoStr := range GSOFeatureArcs { + // Note: not checking full Feature Arc (i.e. ipv4-unicast: gso-ipv4), just the destination + // of traffic steering. This is enough because without GSO enabled, the destination would not exist. + Expect(featuresStr).To(ContainSubstring(gsoStr), fmt.Sprintf("GSO not fully enabled "+ + "due to missing %s in configured features arcs %s", gsoStr, featuresStr)) + } +} + +// AssertRPFVRFExistence checks that dedicated VRF for RPF is created for interface in VPP +func AssertRPFVRFExistence(vpp *vpplink.VppLink, interfaceName string, netnsName string) uint32 { + VRFs, err := vpp.ListVRFs() + Expect(err).ShouldNot(HaveOccurred(), + "Failed to retrieve list of VRFs in VPP") + hbytes := sha512.Sum512([]byte(fmt.Sprintf("%s%s%s%s", "4", netnsName, interfaceName, "RPF"))) + h := base64.StdEncoding.EncodeToString(hbytes[:])[:storage.VrfTagHashLen] + s := fmt.Sprintf("%s-%s-%s-%s", h, "4", interfaceName, filepath.Base(netnsName)) + vrfTag := storage.TruncateStr(s, storage.MaxApiTagLen) + foundRPFVRF := false + var vrfID uint32 + for _, VRF := range VRFs { + if VRF.Name == vrfTag { + foundRPFVRF = true + vrfID = VRF.VrfID + break + } + } + Expect(foundRPFVRF).Should(BeTrue(), + "Failed to find RPF VRF for interface") + return vrfID +} + +// AssertRPFRoutes checks that a route to the pod is added in the RPFVRF and to addresses allowed +// to be spoofed +func AssertRPFRoutes(vpp *vpplink.VppLink, vrfID uint32, swifindex uint32, ipAddress string) { + routes, err := vpp.GetRoutes(vrfID, false) + Expect(err).ShouldNot(HaveOccurred(), + "Failed to get routes from RPF VRF") + Expect(routes).To(ContainElements( + types.Route{ + Dst: IpNet(ipAddress + "/32"), + Paths: []types.RoutePath{{ + SwIfIndex: swifindex, + Gw: IpNet(ipAddress + "/32").IP, + }}, + Table: vrfID, + }, + types.Route{ + Dst: IpNet("172.16.104.7" + "/32"), + Paths: []types.RoutePath{{ + SwIfIndex: swifindex, + Gw: IpNet(ipAddress + "/32").IP, + }}, + Table: vrfID, + }, + types.Route{ + Dst: IpNet("3.4.5.6" + "/32"), + Paths: []types.RoutePath{{ + SwIfIndex: swifindex, + Gw: IpNet(ipAddress + "/32").IP, + }}, + Table: vrfID, + }, + ), "Cannot find route to pod in RPF VRF %s", ipAddress) + +} + +// CreatePod creates docker container that will be used as pod for CNI testing +func CreatePod() { + // docker container cleanup (failed test that didn't properly clean up docker containers?) + err := exec.Command("docker", "rm", "-f", PodMockContainerName).Run() + Expect(err).Should(BeNil(), "Failed to clean up old pod mock docker container") + + // start new pod mock (docker container) + err = exec.Command("docker", "run", "-d", "--network", "none", "--name", PodMockContainerName, + PodMockImage, "sleep", "10d").Run() + Expect(err).Should(BeNil(), "Failed to start new pod mock (docker container)") +} + +// TeardownPod removes container that is used as Pod mock +func TeardownPod() { + err := exec.Command("docker", "rm", "-f", PodMockContainerName).Run() + Expect(err).Should(BeNil(), "Failed to stop and remove pod mock (docker container)") +} + +// RunInPod runs runner function in provided pod network namespace. This is the same as running +// networking commands inside pod. +func RunInPod(podNetNS string, runner func()) { + err := ns.WithNetNSPath(podNetNS, func(hostNS ns.NetNS) error { + defer GinkgoRecover() // running in different goroutine -> needed for failed assertion retrieval + runner() + return nil + }) + Expect(err).Should(BeNil(), "Failed to runInPod") +} + +// DpoNetworkNameFieldName extracts JSON field name for NetworkName used in proto.AddRequest.DataplaneOptions +func DpoNetworkNameFieldName() string { + netNameField, found := reflect.TypeOf(networkAttachmentDefinition.NetConf{}.DpOptions).FieldByName("NetName") + Expect(found).To(BeTrue(), + "can't find network name field in NetworkAttachmentDefinition. Did that structure changed?") + jsonStr, isSet := netNameField.Tag.Lookup("json") + Expect(isSet).To(BeTrue(), "can't find json name for network name field in NetworkAttachmentDefinition") + return strings.Split(jsonStr, ",")[0] +} + +// InterfaceTagForLocalTunTunnel constructs the tag for the VPP side of the tap tunnel the same way as cni server +func InterfaceTagForLocalTunTunnel(interfaceName, netns string) string { + return InterfaceTagForLocalTunnel(pod_interface.NewTunTapPodInterfaceDriver(nil, nil).Name, + interfaceName, netns) +} + +// InterfaceTagForLocalMemifTunnel constructs the tag for the VPP side of the memif tunnel the same way as cni server +func InterfaceTagForLocalMemifTunnel(interfaceName, netns string) string { + return InterfaceTagForLocalTunnel(pod_interface.NewMemifPodInterfaceDriver(nil, nil).Name, + interfaceName, netns) +} + +// InterfaceTagForLocalTunnel constructs the tag for the VPP side of the local tunnel the same way as cni server +func InterfaceTagForLocalTunnel(prefix, interfaceName, netns string) string { + return (&storage.LocalPodSpec{ + NetnsName: netns, + InterfaceName: interfaceName, + }).GetInterfaceTag(prefix) +} + +// FirstIPinIPRange computes first usable IPv4 address from the given subnet. The subnet definition IP address +// (ending with zero bits) is not considered as usable IPv4 address as it can have special meaning in certain situations. +func FirstIPinIPRange(ipRangeCIDR string) net.IP { + ip, _, err := net.ParseCIDR(ipRangeCIDR) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("can't parse range subnet string %s as CIDR", ipRangeCIDR)) + ip = ip.To4() // expecting IPv4 address + ip[3]++ // incrementing last IP address byte to get the first usable IP address in subnet range (subnet x.y.z.0 -> first ip address x.y.z.1) + return ip +} + +// PodVRFs gets ids of IPv4 and IPv6 pod-specific VRFs from VPP +func PodVRFs(podInterface, podNetNSName string, vpp *vpplink.VppLink) (vrf4ID, vrf6ID uint32, err error) { + vrfs, err := vpp.ListVRFs() + Expect(err).ToNot(HaveOccurred(), "error listing VRFs to find all pod VRFs") + + podSpec := storage.LocalPodSpec{ + InterfaceName: podInterface, + NetnsName: podNetNSName, + V4VrfId: types.InvalidID, + V6VrfId: types.InvalidID, + } + for _, vrf := range vrfs { + for _, ipFamily := range vpplink.IpFamilies { + if vrf.Name == podSpec.GetVrfTag(ipFamily, "") { + podSpec.SetVrfId(vrf.VrfID, ipFamily) + } + } + if podSpec.V4VrfId != types.InvalidID && podSpec.V6VrfId != types.InvalidID { + return podSpec.V4VrfId, podSpec.V6VrfId, nil + } + } + + if (podSpec.V4VrfId != types.InvalidID) != (podSpec.V6VrfId != types.InvalidID) { + return podSpec.V4VrfId, podSpec.V6VrfId, + fmt.Errorf("partial VRF state v4=%d v6=%d key=%s", podSpec.V4VrfId, podSpec.V6VrfId, podSpec.Key()) + } + + return podSpec.V4VrfId, podSpec.V6VrfId, fmt.Errorf("not VRFs state (key=%s)", podSpec.Key()) +} + +func IpFamilyIndex(ipFamily vpplink.IpFamily) int { + for idx, family := range vpplink.IpFamilies { + if family == ipFamily { + return idx + } + } + return math.MaxInt +} + +// StartVPP creates docker container and runs inside the VPP +func StartVPP() { + // prepare VPP configuration + vppBinaryConfigArg := `unix { + nodaemon + full-coredump + cli-listen /var/run/vpp/cli2.sock + pidfile /run/vpp/vpp2.pid + } + api-trace { on } + cpu { + workers 0 + } + socksvr { + socket-name /var/run/vpp/vpp-api-test.sock + } + plugins { + plugin default { enable } + plugin dpdk_plugin.so { disable } + plugin calico_plugin.so { enable } + plugin ping_plugin.so { disable } + } + buffers { + buffers-per-numa 131072 + }` + + // docker container cleanup (failed test that didn't properly clean up docker containers?) + err := exec.Command("docker", "rm", "-f", VPPContainerName).Run() + Expect(err).Should(BeNil(), "Failed to clean up old VPP docker container") + + // start VPP inside docker container + cmd := []string{"run", "-d", "--privileged", "--name", VPPContainerName, + "-v", "/tmp/" + VPPContainerName + ":/var/run/vpp/", + "-v", "/proc:/proc", // needed for manipulation of another docker container's network namespace + "--sysctl", "net.ipv6.conf.all.disable_ipv6=0"} // enable IPv6 in container (to set IPv6 on host's end of uplink) + cmd = append(cmd, VppContainerExtraArgs...) + cmd = append(cmd, "--entrypoint", VppBinary, VppImage, vppBinaryConfigArg) + err = exec.Command("docker", cmd...).Run() + Expect(err).Should(BeNil(), "Failed to start VPP inside docker container") +} + +// ConfigureVPP connects to VPP and configures it with common configuration needed for tests +func ConfigureVPP(log *logrus.Logger) (vpp *vpplink.VppLink, uplinkSwIfIndex uint32) { + // connect to VPP + vpp, err := common.CreateVppLinkInRetryLoop("/tmp/"+VPPContainerName+"/vpp-api-test.sock", + log.WithFields(logrus.Fields{"component": "vpp-api"}), 20*time.Second, 100*time.Millisecond) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Cannot create VPP client: %v", err)) + Expect(vpp).NotTo(BeNil()) + + // setup common VRF setup + for _, ipFamily := range vpplink.IpFamilies { //needed config for pod creation tests + err := vpp.AddVRF(common.PuntTableId, ipFamily.IsIp6, fmt.Sprintf("punt-table-%s", ipFamily.Str)) + if err != nil { + log.Fatal(errors.Wrapf(err, "Error creating punt vrf %s", ipFamily.Str)) + } + err = vpp.AddVRF(common.PodVRFIndex, ipFamily.IsIp6, fmt.Sprintf("calico-pods-%s", ipFamily.Str)) + if err != nil { + log.Fatal(err) + } + err = vpp.AddDefaultRouteViaTable(common.PodVRFIndex, common.DefaultVRFIndex, ipFamily.IsIp6) + if err != nil { + log.Fatal(err) + } + } + + // setup simplified mock version of uplink interface + // Note: for the real configuration of the uplink interface and other related things see + // UplinkDriver.CreateMainVppInterface(...) and VppRunner.configureVpp(...)) + uplinkSwIfIndex, err = vpp.CreateTapV2(&types.TapV2{ + GenericVppInterface: types.GenericVppInterface{ + HostInterfaceName: UplinkIfName, + HardwareAddr: Mac("aa:bb:cc:dd:ee:01"), + }, + Tag: fmt.Sprintf("main-%s", UplinkIfName), + Flags: types.TapFlagNone, + // Host end of tap (it is located inside docker container) + HostMtu: 1500, + HostMacAddress: *Mac("aa:bb:cc:dd:ee:02"), + }) + Expect(err).ToNot(HaveOccurred(), "Error creating mocked Uplink interface") + err = vpp.InterfaceAdminUp(uplinkSwIfIndex) + Expect(err).ToNot(HaveOccurred(), "Error setting state to UP for mocked Uplink interface") + err = vpp.AddInterfaceAddress(uplinkSwIfIndex, IpNet(UplinkIP+"/24")) + Expect(err).ToNot(HaveOccurred(), "Error adding IPv4 address to data interface") + err = vpp.AddInterfaceAddress(uplinkSwIfIndex, IpNet(UplinkIPv6+"/16")) + Expect(err).ToNot(HaveOccurred(), "Error adding IPv6 address to data interface") + err = exec.Command("docker", "exec", VPPContainerName, "ip", "address", "add", + GatewayIP+"/24", "dev", UplinkIfName).Run() + Expect(err).ToNot(HaveOccurred(), "Failed to set IPv4 address for host end of tap") + err = exec.Command("docker", "exec", VPPContainerName, "ip", "address", "add", + GatewayIPv6+"/16", "dev", UplinkIfName).Run() + Expect(err).ToNot(HaveOccurred(), "Failed to set IPv6 address for host end of tap") + err = exec.Command("docker", "exec", VPPContainerName, "ip", "link", "set", + UplinkIfName, "up").Run() + Expect(err).ToNot(HaveOccurred(), "Failed to set state to UP for host end of tap") + + return +} + +// TeardownVPP removes container with running VPP to stop VPP and clean after it +func TeardownVPP() { + err := exec.Command("docker", "rm", "-f", VPPContainerName).Run() + Expect(err).Should(BeNil(), "Failed to stop and remove VPP docker container") +} + +// AssertUnnumberedInterface checks whether the provided interface is unnumbered and properly takes IP address +// from the correct interface (common.VppManagerInfo.GetMainSwIfIndex()). +func AssertUnnumberedInterface(swIfIndex uint32, interfaceDescriptiveName string, vpp *vpplink.VppLink) { + unnumberedDetails, err := vpp.InterfaceGetUnnumbered(swIfIndex) + Expect(err).ToNot(HaveOccurred(), + fmt.Sprintf("can't get unnumbered details of %s", interfaceDescriptiveName)) + Expect(unnumberedDetails).ToNot(BeEmpty(), "can't find unnumbered interface") + Expect(unnumberedDetails[0].IPSwIfIndex).To(Equal( + interface_types.InterfaceIndex(common.VppManagerInfo.GetMainSwIfIndex())), + fmt.Sprintf("Unnumberred %s doesn't get IP address from expected interface", interfaceDescriptiveName)) +} + +// AssertInterfaceGSOCNat check whether the given interface has properly set the GSO and CNAT attributes +func AssertInterfaceGSOCNat(swIfIndex uint32, interfaceDescriptiveName string, vpp *vpplink.VppLink) { + // Note: no specialized binary api or VPP CLI for getting GSO on interface -> using + // Feature Arcs (https://wiki.fd.io/view/VPP/Feature_Arcs) to detect it (GSO is using them to steer + // traffic), Feature Arcs have no binary API -> using VPP's VPE binary API + featuresStr, err := vpp.RunCli(fmt.Sprintf("sh interface %d features", swIfIndex)) + Expect(err).ToNot(HaveOccurred(), + fmt.Sprintf("failed to get %s's configured features", interfaceDescriptiveName)) + featuresStr = strings.ToLower(featuresStr) + var GSOFeatureArcs = []string{"gso-ip4", "gso-ip6", "gso-l2-ip4", "gso-l2-ip6"} + for _, gsoStr := range GSOFeatureArcs { + // Note: not checking full Feature Arc (i.e. ipv4-unicast: gso-ipv4), just the destination + // of traffic steering. This is enough because without GSO enabled, the destination would not exist. + Expect(featuresStr).To(ContainSubstring(gsoStr), fmt.Sprintf("GSO not fully enabled "+ + "due to missing %s in configured features arcs %s", gsoStr, featuresStr)) + } + var CNATFeatureArcs = []string{"cnat-input-ip4", "cnat-input-ip6", "cnat-output-ip4", "cnat-output-ip6"} + for _, cnatStr := range CNATFeatureArcs { + // Note: could be enhanced by checking the full Feature Arc (from where we steer traffic to cnat) + Expect(featuresStr).To(ContainSubstring(cnatStr), fmt.Sprintf("CNAT not fully enabled "+ + "due to missing %s in configured features arcs %s", cnatStr, featuresStr)) + } +} + +// AssertNextNodeLink asserts that in VPP graph the given node has linked the linkedNextNode as one of its +// "next" nodes for processing. It returns to node specific index of the checked next node. +func AssertNextNodeLink(node, linkedNextNode string, vpp *vpplink.VppLink) int { + // get the node information from VPP (No VPP binary API for that -> using VPE) + nodeInfoStr, err := vpp.RunCli(fmt.Sprintf("show node %s", node)) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("failed to VPP graph info for node %s", node)) + + // asserting next node + Expect(strings.ToLower(nodeInfoStr)).To(ContainSubstring(strings.ToLower(linkedNextNode)), + fmt.Sprintf("can't find added next node %s in node %s information", linkedNextNode, node)) + + // getting next node's index that is relative to the given node (this is kind of brittle as we parse VPP CLI output) + linesToNextNode := strings.Split(strings.Split(nodeInfoStr, linkedNextNode)[0], "\n") + indexStrOfLinkedNextNode := strings.TrimSpace(linesToNextNode[len(linesToNextNode)-1][:10]) + nextNodeIndex, err := strconv.Atoi(indexStrOfLinkedNextNode) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("can't parse next node index "+ + "in given node from VPP CLI output %s", nodeInfoStr)) + + return nextNodeIndex +} + +func ConfigureBGPNodeIPAddresses(connectivityServer *connectivity.ConnectivityServer) { + ip4, ip4net, _ := net.ParseCIDR(ThisNodeIP + "/24") + ip4net.IP = ip4 + ip6, ip6net, _ := net.ParseCIDR(ThisNodeIPv6 + "/128") + ip6net.IP = ip6 + connectivityServer.SetOurBGPSpec(&common.LocalNodeSpec{ + IPv4Address: ip4net, + IPv6Address: ip6net, + }) +} + +// AddIPPoolForCalicoClient is convenience function for adding IPPool to mocked Calico IPAM Stub used +// in Calico client stub. This function doesn't set anything for the watchers.IpamCache implementation. +func AddIPPoolForCalicoClient(client *calico.CalicoClientStub, poolName string, poolCIRD string) ( + *apiv3.IPPool, error) { + return client.IPPoolsStub.Create(context.Background(), &apiv3.IPPool{ + ObjectMeta: metav1.ObjectMeta{ + Name: poolName, + }, + Spec: apiv3.IPPoolSpec{ + CIDR: poolCIRD, + }, + }, options.SetOptions{}) +} + +func IpNet(ipNetCIDRStr string) *net.IPNet { + _, ipNet, err := net.ParseCIDR(ipNetCIDRStr) + Expect(err).To(BeNil()) + return ipNet +} + +func IpNetWithIPInIPv6Format(ipNetCIDRStr string) *net.IPNet { + _, ipNet, err := net.ParseCIDR(ipNetCIDRStr) + ipNet.IP = ipNet.IP.To16() + Expect(err).To(BeNil()) + return ipNet +} + +func Mac(macStr string) *net.HardwareAddr { + mac, err := net.ParseMAC(macStr) + Expect(err).To(BeNil()) + return &mac +} + +func IptypesIP6Address(address string) ip_types.IP6Address { + addr, err := ip_types.ParseAddress(address) + Expect(err).ToNot(HaveOccurred(), "failed to parse ip_types.IP6Addess from string %s", addr) + return addr.Un.GetIP6() +} + +func SidArray(addresses ...ip_types.IP6Address) (sids [16]ip_types.IP6Address) { + copy(sids[:], addresses) + return sids +} + +func AddPaddingTo32Bytes(value []byte) []byte { + result := [32]byte{} + copy(result[:], value) + return result[:] +} diff --git a/calico-vpp-agent/policy/host_endpoint.go b/calico-vpp-agent/policy/host_endpoint.go index 52028d32..3b57e100 100644 --- a/calico-vpp-agent/policy/host_endpoint.go +++ b/calico-vpp-agent/policy/host_endpoint.go @@ -51,7 +51,6 @@ func (he *HostEndpoint) String() string { s += types.StrListToString(" expectedIPs=", he.expectedIPs) s += types.IntListToString(" uplink=", he.UplinkSwIfIndexes) s += types.IntListToString(" tap=", he.TapSwIfIndexes) - s += types.IntListToString(" uplink=", he.UplinkSwIfIndexes) s += types.IntListToString(" tunnel=", he.TunnelSwIfIndexes) s += types.StrListToString(" profiles=", he.Profiles) s += types.StrableListToString(" tiers=", he.Tiers) diff --git a/calico-vpp-agent/policy/policy_server.go b/calico-vpp-agent/policy/policy_server.go index 8645d054..e36bb3f7 100644 --- a/calico-vpp-agent/policy/policy_server.go +++ b/calico-vpp-agent/policy/policy_server.go @@ -114,7 +114,7 @@ type Server struct { } // NewServer creates a policy server -func NewPolicyServer(vpp *vpplink.VppLink, log *logrus.Entry) (*Server, error) { +func NewPolicyServer(vpp *vpplink.VppLink, log *logrus.Entry, withInstallPlugin bool) (*Server, error) { var err error server := &Server{ @@ -165,9 +165,11 @@ func NewPolicyServer(vpp *vpplink.VppLink, log *logrus.Entry) (*Server, error) { return nil, errors.Wrapf(err, "Could not delete socket %s", config.FelixDataplaneSocket) } - err = InstallFelixPlugin() - if err != nil { - return nil, errors.Wrap(err, "could not install felix plugin") + if withInstallPlugin { + err = InstallFelixPlugin() + if err != nil { + return nil, errors.Wrap(err, "could not install felix plugin") + } } return server, nil diff --git a/calico-vpp-agent/cmd/watch_dog.go b/calico-vpp-agent/watch_dog/watch_dog.go similarity index 98% rename from calico-vpp-agent/cmd/watch_dog.go rename to calico-vpp-agent/watch_dog/watch_dog.go index 7ae75388..0eb09d7a 100644 --- a/calico-vpp-agent/cmd/watch_dog.go +++ b/calico-vpp-agent/watch_dog/watch_dog.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package watchdog import ( "time" diff --git a/go.mod b/go.mod index 7cd0c5fb..1d2e5e2f 100644 --- a/go.mod +++ b/go.mod @@ -33,6 +33,8 @@ require ( sigs.k8s.io/controller-runtime v0.12.0 ) +require github.com/osrg/gobgp v2.0.0+incompatible + require ( cloud.google.com/go/compute v1.12.1 // indirect cloud.google.com/go/compute/metadata v0.2.1 // indirect diff --git a/go.sum b/go.sum index 5bc6c88d..c4e2c390 100644 --- a/go.sum +++ b/go.sum @@ -740,6 +740,8 @@ github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuh github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/orijtech/prometheus-go-metrics-exporter v0.0.6 h1:ExkpQsyDDcyp0U3zhoNUQaCQ/o0Ovq7e1jRCL9lQ/4o= github.com/orijtech/prometheus-go-metrics-exporter v0.0.6/go.mod h1:BiTx/ugZex8LheBk3j53tktWaRdFjV5FCfT2o0P7msE= +github.com/osrg/gobgp v2.0.0+incompatible h1:91ARQbE1AtO0U4TIxHPJ7wYVZIqduyBwS1+FjlHlmrY= +github.com/osrg/gobgp v2.0.0+incompatible/go.mod h1:vGVJPLW6JFDD7WA1vJsjB8OKmbbC2TKwHtr90CZS/u4= github.com/osrg/gobgp/v3 v3.10.0 h1:gBFOKEFX4ih8IubY7SFuXPwE33HWX+VqSuXyQb6vqyU= github.com/osrg/gobgp/v3 v3.10.0/go.mod h1:/q/mr+dOuPf5hDlLiatVll1P7BX4pmiXQxd4ZZpUbkg= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= diff --git a/test/integration-tests/run-tests.sh b/test/integration-tests/run-tests.sh index 60fdef22..ac94d289 100755 --- a/test/integration-tests/run-tests.sh +++ b/test/integration-tests/run-tests.sh @@ -42,9 +42,13 @@ echo "Running Calico VPP Agent - CNI tests..." prepageImageWithVPPBinary preparePodMockImage # Note: some pod tests expect elevated user privileges -> using sudo +if [ $1 = cni ]; then INTEGRATION_TEST="." VPP_IMAGE=$VPP_IMAGE VPP_BINARY="/usr/bin/vpp" \ sudo -E env "PATH=$PATH" go test -v -run Integration ../../calico-vpp-agent/cni -ginkgo.v || result=$? - +else +INTEGRATION_TEST="." VPP_IMAGE=$VPP_IMAGE VPP_BINARY="/usr/bin/vpp" \ +sudo -E env "PATH=$PATH" go test -v -run Integration ../../calico-vpp-agent/policy -ginkgo.v || result=$? +fi if [ $result -ne 0 ]; then From 3b1ff014a49bf383a34b973388443430d51c5bbd Mon Sep 17 00:00:00 2001 From: hedi bouattour Date: Fri, 10 Feb 2023 16:54:13 +0000 Subject: [PATCH 2/2] agent: add first fv tests version --- Makefile | 1 + calico-vpp-agent/cmd/calico_vpp_dataplane.go | 2 +- calico-vpp-agent/cni/cni_node_test.go | 2 +- calico-vpp-agent/cni/cni_pod_test.go | 4 +- calico-vpp-agent/common_tests/common_tests.go | 71 ++++- calico-vpp-agent/policy/policy_test.go | 301 ++++++++++++++++++ felix/Makefile | 11 + felix/README.md | 4 + 8 files changed, 376 insertions(+), 20 deletions(-) create mode 100644 calico-vpp-agent/policy/policy_test.go create mode 100644 felix/Makefile create mode 100644 felix/README.md diff --git a/Makefile b/Makefile index ec0bc575..e4d5651c 100644 --- a/Makefile +++ b/Makefile @@ -197,6 +197,7 @@ run-integration-tests-cni: .PHONY: run-integration-tests-policy run-integration-tests-policy: + $(MAKE) -C felix fv-prereqs cd test/integration-tests;./run-tests.sh policy .PHONY: test diff --git a/calico-vpp-agent/cmd/calico_vpp_dataplane.go b/calico-vpp-agent/cmd/calico_vpp_dataplane.go index 3093aa24..a2b89569 100644 --- a/calico-vpp-agent/cmd/calico_vpp_dataplane.go +++ b/calico-vpp-agent/cmd/calico_vpp_dataplane.go @@ -40,8 +40,8 @@ import ( "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/services" "github.com/projectcalico/vpp-dataplane/config" + watchdog "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/watch_dog" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/watchers" - "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/watch_dog" ) /* diff --git a/calico-vpp-agent/cni/cni_node_test.go b/calico-vpp-agent/cni/cni_node_test.go index 48a8dd41..705497c3 100644 --- a/calico-vpp-agent/cni/cni_node_test.go +++ b/calico-vpp-agent/cni/cni_node_test.go @@ -124,7 +124,7 @@ var _ = Describe("Node-related functionality of CNI", func() { JustBeforeEach(func() { test.StartVPP() - vpp, uplinkSwIfIndex = test.ConfigureVPP(log) + vpp, uplinkSwIfIndex = test.ConfigureVPP(log, false) // setup connectivity server (functionality target of tests) if ipamStub == nil { diff --git a/calico-vpp-agent/cni/cni_pod_test.go b/calico-vpp-agent/cni/cni_pod_test.go index 482aeaa3..7294081e 100644 --- a/calico-vpp-agent/cni/cni_pod_test.go +++ b/calico-vpp-agent/cni/cni_pod_test.go @@ -31,8 +31,8 @@ import ( "github.com/vishvananda/netlink" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/cni" - test "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/common_tests" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/common" + test "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/common_tests" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/tests/mocks" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/watchers" "github.com/projectcalico/vpp-dataplane/config" @@ -58,7 +58,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { BeforeEach(func() { log = logrus.New() test.StartVPP() - vpp, _ = test.ConfigureVPP(log) + vpp, _ = test.ConfigureVPP(log, false) // setup connectivity server (functionality target of tests) if ipamStub == nil { ipamStub = mocks.NewIpamCacheStub() diff --git a/calico-vpp-agent/common_tests/common_tests.go b/calico-vpp-agent/common_tests/common_tests.go index be8c6985..126e63c0 100644 --- a/calico-vpp-agent/common_tests/common_tests.go +++ b/calico-vpp-agent/common_tests/common_tests.go @@ -20,41 +20,46 @@ import ( "github.com/sirupsen/logrus" "github.com/containernetworking/plugins/pkg/ns" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + cniproto "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/cni/pod_interface" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/cni/storage" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/connectivity" - cniproto "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/tests/mocks/calico" "github.com/projectcalico/vpp-dataplane/multinet-monitor/networkAttachmentDefinition" "github.com/projectcalico/vpp-dataplane/vpplink" "github.com/projectcalico/vpp-dataplane/vpplink/binapi/vppapi/interface_types" "github.com/projectcalico/vpp-dataplane/vpplink/binapi/vppapi/ip_types" "github.com/projectcalico/vpp-dataplane/vpplink/types" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) + const ( // PodMockContainerName is used container name for pod container mock PodMockContainerName = "cni-tests-pod-mock" // PodMockImage is docker image used for pod mocking - PodMockImage = "calicovpp/vpp-test-pod-mock:latest" - VPPContainerName = "cni-tests-vpp" - VppContainerExtraArgsName = "VPP_CONTAINER_EXTRA_ARGS" - ThisNodeName = "node1" - UplinkIfName = "uplink" - UplinkIP = "10.0.100.1" - UplinkIPv6 = "A::1:1" - GatewayIP = "10.0.100.254" - GatewayIPv6 = "A::1:254" - ThisNodeIP = UplinkIP - ThisNodeIPv6 = UplinkIPv6 + PodMockImage = "calicovpp/vpp-test-pod-mock:latest" + VPPContainerName = "cni-tests-vpp" + VppContainerExtraArgsName = "VPP_CONTAINER_EXTRA_ARGS" + ThisNodeName = "node1" + UplinkIfName = "uplink" + Uplink2IfName = "uplink2" + UplinkIP = "10.0.100.1" + UplinkIPv6 = "A::1:1" + Uplink2IP = "11.0.100.1" + Uplink2IPv6 = "B::1:1" + GatewayIP = "10.0.100.254" + GatewayIPv6 = "A::1:254" + ThisNodeIP = UplinkIP + ThisNodeIPv6 = UplinkIPv6 AddedNodeName = "node2" AddedNodeIP = "10.0.200.1" AddedNodeIPv6 = "A::2:1" ) + var ( // VppImage is the name of docker image containing VPP binary VppImage string @@ -63,6 +68,7 @@ var ( // vppContainerExtraArgs is a list of additionnal cli parameters for the VPP `docker run ...` VppContainerExtraArgs []string = []string{} ) + func AssertTunInterfaceExistence(vpp *vpplink.VppLink, newPod *cniproto.AddRequest) uint32 { ifSwIfIndex, err := vpp.SearchInterfaceWithTag( InterfaceTagForLocalTunTunnel(newPod.InterfaceName, newPod.Netns)) @@ -289,8 +295,8 @@ func StartVPP() { vppBinaryConfigArg := `unix { nodaemon full-coredump - cli-listen /var/run/vpp/cli2.sock - pidfile /run/vpp/vpp2.pid + cli-listen /var/run/vpp/cli.sock + pidfile /run/vpp/vpp.pid } api-trace { on } cpu { @@ -325,7 +331,7 @@ func StartVPP() { } // ConfigureVPP connects to VPP and configures it with common configuration needed for tests -func ConfigureVPP(log *logrus.Logger) (vpp *vpplink.VppLink, uplinkSwIfIndex uint32) { +func ConfigureVPP(log *logrus.Logger, additionaluplink bool) (vpp *vpplink.VppLink, uplinkSwIfIndex uint32) { // connect to VPP vpp, err := common.CreateVppLinkInRetryLoop("/tmp/"+VPPContainerName+"/vpp-api-test.sock", log.WithFields(logrus.Fields{"component": "vpp-api"}), 20*time.Second, 100*time.Millisecond) @@ -379,6 +385,39 @@ func ConfigureVPP(log *logrus.Logger) (vpp *vpplink.VppLink, uplinkSwIfIndex uin UplinkIfName, "up").Run() Expect(err).ToNot(HaveOccurred(), "Failed to set state to UP for host end of tap") + if additionaluplink { + // setup simplified mock version of uplink interface + // Note: for the real configuration of the uplink interface and other related things see + // UplinkDriver.CreateMainVppInterface(...) and VppRunner.configureVpp(...)) + uplinkSwIfIndex, err = vpp.CreateTapV2(&types.TapV2{ + GenericVppInterface: types.GenericVppInterface{ + HostInterfaceName: Uplink2IfName, + HardwareAddr: Mac("aa:bb:cc:dd:ee:03"), + }, + Tag: fmt.Sprintf("main-%s", Uplink2IfName), + Flags: types.TapFlagNone, + // Host end of tap (it is located inside docker container) + HostMtu: 1500, + HostMacAddress: *Mac("aa:bb:cc:dd:ee:04"), + }) + Expect(err).ToNot(HaveOccurred(), "Error creating mocked Uplink interface") + err = vpp.InterfaceAdminUp(uplinkSwIfIndex) + Expect(err).ToNot(HaveOccurred(), "Error setting state to UP for mocked Uplink interface") + err = vpp.AddInterfaceAddress(uplinkSwIfIndex, IpNet(Uplink2IP+"/24")) + Expect(err).ToNot(HaveOccurred(), "Error adding IPv4 address to data interface") + err = vpp.AddInterfaceAddress(uplinkSwIfIndex, IpNet(Uplink2IPv6+"/16")) + Expect(err).ToNot(HaveOccurred(), "Error adding IPv6 address to data interface") + err = exec.Command("docker", "exec", VPPContainerName, "ip", "address", "add", + GatewayIP+"/24", "dev", Uplink2IfName).Run() + Expect(err).ToNot(HaveOccurred(), "Failed to set IPv4 address for host end of tap") + err = exec.Command("docker", "exec", VPPContainerName, "ip", "address", "add", + GatewayIPv6+"/16", "dev", Uplink2IfName).Run() + Expect(err).ToNot(HaveOccurred(), "Failed to set IPv6 address for host end of tap") + err = exec.Command("docker", "exec", VPPContainerName, "ip", "link", "set", + Uplink2IfName, "up").Run() + Expect(err).ToNot(HaveOccurred(), "Failed to set state to UP for host end of tap") + + } return } diff --git a/calico-vpp-agent/policy/policy_test.go b/calico-vpp-agent/policy/policy_test.go new file mode 100644 index 00000000..73a972e3 --- /dev/null +++ b/calico-vpp-agent/policy/policy_test.go @@ -0,0 +1,301 @@ +// Copyright (c) 2022 Cisco and/or its affiliates. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policy_test + +import ( + "context" + "fmt" + "os" + "os/exec" + "strings" + "syscall" + "testing" + + felixconfig "github.com/projectcalico/calico/felix/config" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + //felixconfig "github.com/projectcalico/calico/felix/config" + "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" + "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/cni" + "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/common" + test "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/common_tests" + "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/policy" + watchdog "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/watch_dog" + "github.com/projectcalico/vpp-dataplane/config" + agentConf "github.com/projectcalico/vpp-dataplane/config" + "github.com/projectcalico/vpp-dataplane/vpplink" + "github.com/projectcalico/vpp-dataplane/vpplink/types" + "github.com/sirupsen/logrus" + tomb "gopkg.in/tomb.v2" +) + +// Names of integration tests arguments +const ( + IntegrationTestEnableArgName = "INTEGRATION_TEST" + VppImageArgName = "VPP_IMAGE" + VppBinaryArgName = "VPP_BINARY" + VppContainerExtraArgsName = "VPP_CONTAINER_EXTRA_ARGS" + testTimeout = "40s" +) + +var t tomb.Tomb + +// TestCniIntegration runs all the ginkgo integration test inside CNI package +func TestCniIntegration(t *testing.T) { + // skip test if test run is not integration test run (prevent accidental run of integration tests using go test ./...) + _, isIntegrationTestRun := os.LookupEnv(IntegrationTestEnableArgName) + if !isIntegrationTestRun { + t.Skip("skipping CNI integration tests (set INTEGRATION_TEST env variable to run these tests)") + } + + // integrate gomega and ginkgo -> register all CNI integration tests + RegisterFailHandler(Fail) + RunSpecs(t, "CNI Integration Suite") +} + +var _ = BeforeSuite(func() { + // extract common input for CNI integration tests + var found bool + test.VppImage, found = os.LookupEnv(VppImageArgName) + if !found { + Expect(test.VppImage).ToNot(BeEmpty(), fmt.Sprintf("Please specify docker image containing "+ + "VPP binary using %s environment variable.", VppImageArgName)) + } + test.VppBinary, found = os.LookupEnv(VppBinaryArgName) + if !found { + Expect(test.VppBinary).ToNot(BeEmpty(), fmt.Sprintf("Please specify VPP binary (full path) "+ + "inside docker image %s using %s environment variable.", test.VppImage, VppBinaryArgName)) + } + + vppContainerExtraArgsList, found := os.LookupEnv(VppContainerExtraArgsName) + if found { + test.VppContainerExtraArgs = append(test.VppContainerExtraArgs, strings.Split(vppContainerExtraArgsList, ",")...) + } + +}) + +var _ = Describe("Functionality of policy server using felix", func() { + var ( + log *logrus.Logger + vpp *vpplink.VppLink + policyServer *policy.Server + err error + cniServer *cni.Server + cmd *exec.Cmd + watchDog *watchdog.WatchDog + ) + BeforeEach(func() { + log = logrus.New() + common.ThePubSub = common.NewPubSub(log.WithFields(logrus.Fields{"component": "pubsub"})) + }) + + JustBeforeEach(func() { + test.StartVPP() + vpp, _ = test.ConfigureVPP(log, true) + // Additional configuration specific to policies test: add tap for host endpoints + _, err = vpp.CreateTapV2(&types.TapV2{ + GenericVppInterface: types.GenericVppInterface{ + HostInterfaceName: test.UplinkIfName, + HardwareAddr: test.Mac("aa:bb:cc:dd:ee:01"), + }, + Tag: fmt.Sprintf("host-%s", test.UplinkIfName), + Flags: types.TapFlagNone, + // Host end of tap (it is located inside docker container) + HostMtu: 1500, + HostMacAddress: *test.Mac("aa:bb:cc:dd:ee:02"), + }) + Expect(err).ToNot(HaveOccurred(), "Error creating mocked tap interface") + + _, err = vpp.CreateTapV2(&types.TapV2{ + GenericVppInterface: types.GenericVppInterface{ + HostInterfaceName: test.Uplink2IfName, + HardwareAddr: test.Mac("aa:bb:cc:dd:ee:03"), + }, + Tag: fmt.Sprintf("host-%s", test.Uplink2IfName), + Flags: types.TapFlagNone, + // Host end of tap (it is located inside docker container) + HostMtu: 1500, + HostMacAddress: *test.Mac("aa:bb:cc:dd:ee:04"), + }) + Expect(err).ToNot(HaveOccurred(), "Error creating mocked tap interface") + + config.GetCalicoVppDebug().PoliciesEnabled = &config.True + common.VppManagerInfo = &agentConf.VppManagerInfo{UplinkStatuses: []agentConf.UplinkStatus{{IsMain: true, SwIfIndex: 1}}} + policyServer, err = policy.NewPolicyServer(vpp, log.WithFields(logrus.Fields{"component": "policy"}), false) + Expect(err).ToNot(HaveOccurred(), "Failed to create policy server") + cniServer = cni.NewCNIServer(vpp, policyServer, log.WithFields(logrus.Fields{"component": "cni"})) + Go(policyServer.ServePolicy) + log.Info("WAITING FOR FELIX CONFIG... please run felix") + cmd = exec.Command("make", "fv") + cmd.Env = os.Environ() + cmd.Dir = "../../felix" + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + watchDog = watchdog.NewWatchDog(log.WithFields(logrus.Fields{"component": "watchDog"}), &t) + }) + + Describe("Creation of a workload endpoint", func() { + Context("With creation of the pod in cni server", func() { + const ( + ipAddress = "1.2.3.44" + interfaceName = "newInterface" + ) + JustBeforeEach(func() { + test.CreatePod() + By("Getting Pod mock container's PID") + containerPidOutput, err := exec.Command("docker", "inspect", "-f", "{{.State.Pid}}", + test.PodMockContainerName).Output() + Expect(err).Should(BeNil(), "Failed to get pod mock container's PID string") + containerPidStr := strings.ReplaceAll(string(containerPidOutput), "\n", "") + By("Adding pod using CNI server") + newPod := &proto.AddRequest{ + InterfaceName: interfaceName, + Netns: fmt.Sprintf("/proc/%s/ns/net", containerPidStr), // expecting mount of "/proc" from host + ContainerIps: []*proto.IPConfig{{Address: ipAddress + "/24"}}, + Workload: &proto.WorkloadIDs{ + // these values come from fv test + Orchestrator: "k8s", + Endpoint: "eth0", + Namespace: "default", + Pod: "test-pod-44445555-idx1", + }, + } + cniServer.SetFelixConfig((&felixconfig.Config{})) + cniServer.FetchBufferConfig() + config.GetCalicoVppInterfaces().DefaultPodIfSpec = &config.InterfaceSpec{} + config.GetCalicoVppFeatureGates().IPSecEnabled = &config.False + config.GetCalicoVppDebug().GSOEnabled = &config.True + reply, err := cniServer.Add(context.Background(), newPod) + Expect(err).ToNot(HaveOccurred(), "Pod addition failed") + Expect(reply.Successful).To(BeTrue(), + fmt.Sprintf("Pod addition failed due to: %s", reply.ErrorMessage)) + By("Checking existence (and IP address) of interface tunnel at added pod's end") + interfaceDetails, err := exec.Command("docker", "exec", test.PodMockContainerName, + "ip", "address", "show", "dev", interfaceName).Output() + log.Infof("%s", interfaceDetails) + Expect(err).Should(BeNil(), "Failed to get added interface details from pod container") + Expect(string(interfaceDetails)).Should(ContainSubstring(ipAddress), + "Interface tunnel on new pod's end is either wrong configured "+ + "for IP address or doesn't exist at all") + }) + Context("With creation of the workload in fv tests", func() { + It("should configure a pod with policies", func() { + cmd.Env = append(cmd.Env, "GINKGO_FOCUS=should create a pod with a policy") + err = cmd.Start() + Expect(err).Should(BeNil(), "Failed to start felix %+v", err) + _ = watchDog.Wait(policyServer.FelixConfigChan, "Waiting for FelixConfig to be provided by the calico pod") + Eventually(vpp.RunCli, testTimeout).WithArguments("show capo int").Should(ContainSubstring("addr=" + ipAddress)) + Eventually(vpp.RunCli, testTimeout).WithArguments("show capo int").Should(ContainSubstring("1.9.9.1")) + st, _ := vpp.RunCli("show capo int") + fmt.Printf(st) + }) + It("should configure a pod with default profiles", func() { + cmd.Env = append(cmd.Env, "GINKGO_FOCUS=should create a pod without") + err = cmd.Start() + Expect(err).Should(BeNil(), "Failed to start felix %+v", err) + _ = watchDog.Wait(policyServer.FelixConfigChan, "Waiting for FelixConfig to be provided by the calico pod") + Eventually(vpp.RunCli, testTimeout).WithArguments("show capo int").Should(ContainSubstring("addr=" + ipAddress + "]\n profiles")) + st, _ := vpp.RunCli("show capo int") + fmt.Printf(st) + }) + }) + }) + }) + Describe("Creation of a host endpoint", func() { + It("should have default policies on host interfaces", func() { + Eventually(vpp.RunCli, testTimeout).WithArguments("show capo int").Should(ContainSubstring("tx:[rule#0;deny][src==[ipset#0;ip;],]")) + st, _ := vpp.RunCli("show capo int") + fmt.Printf(st) + }) + It("should configure empty host endpoint", func() { + // this should change as we fix the behaviour of an empty host endpoint (deny) + cmd.Env = append(cmd.Env, "GINKGO_FOCUS=should create an empty host endpoint") + err := cmd.Start() + Expect(err).Should(BeNil(), "Failed to start felix %+v", err) + _ = watchDog.Wait(policyServer.FelixConfigChan, "Waiting for FelixConfig to be provided by the calico pod") + Eventually(vpp.RunCli, testTimeout).WithArguments("show capo int").Should(ContainSubstring("invertedaddr=10.0.100.0]\n[")) + st, _ := vpp.RunCli("show capo int") + fmt.Printf(st) + }) + Context("with a policy not applied on forward", func() { + JustBeforeEach(func() { + cmd.Env = append(cmd.Env, "GINKGO_FOCUS=should create a host endpoint with a policy not on forward") + err := cmd.Start() + Expect(err).Should(BeNil(), "Failed to start felix %+v", err) + _ = watchDog.Wait(policyServer.FelixConfigChan, "Waiting for FelixConfig to be provided by the calico pod") + }) + It("should configure host endpoint with policy not applied on forward", func() { + // uplink should be empty + Eventually(vpp.RunCli, testTimeout).WithArguments("show capo int").Should(ContainSubstring("invertedaddr=10.0.100.0]\n[")) + // vpptap should have policy applied + Eventually(vpp.RunCli, testTimeout).WithArguments("show capo int").Should(ContainSubstring("1.9.9.1")) + st, _ := vpp.RunCli("show capo int") + fmt.Printf(st) + }) + It("should configure failsafe policies", func() { + Eventually(vpp.RunCli, testTimeout).WithArguments("show capo int").Should(ContainSubstring("rx:[rule#8;allow][proto==TCP,dst==179,dst==2379,dst==2380,dst==5473,dst==6443,dst==6666,dst==6667,]")) + st, _ := vpp.RunCli("show capo int") + fmt.Printf(st) + }) + }) + It("should configure host endpoint with policy and apply on forward", func() { + cmd.Env = append(cmd.Env, "GINKGO_FOCUS=should create a host endpoint with a policy on forward") + err := cmd.Start() + Expect(err).Should(BeNil(), "Failed to start felix %+v", err) + _ = watchDog.Wait(policyServer.FelixConfigChan, "Waiting for FelixConfig to be provided by the calico pod") + Eventually(vpp.RunCli, testTimeout).WithArguments("show capo int").Should(ContainSubstring("1.9.9.1")) + st, _ := vpp.RunCli("show capo int") + fmt.Printf(st) + }) + It("should configure wildcard host endpoint", func() { + cmd.Env = append(cmd.Env, "GINKGO_FOCUS=should create a wildcard host endpoint") + err := cmd.Start() + Expect(err).Should(BeNil(), "Failed to start felix %+v", err) + _ = watchDog.Wait(policyServer.FelixConfigChan, "Waiting for FelixConfig to be provided by the calico pod") + Eventually(vpp.RunCli, testTimeout).WithArguments("show capo int").Should(ContainSubstring("[tap0 sw_if_index=1 invertedaddr=10.0.100.0]")) + st, _ := vpp.RunCli("show capo int") + fmt.Printf(st) + }) + }) + /*Describe("Felix Configuration functionalities", func() { + It("should change default endpoint to host action", func() { + cmd.Env = append(cmd.Env, "GINKGO_FOCUS=should change default endpoint to host action to ACCEPT") + err := cmd.Start() + Expect(err).Should(BeNil(), "Failed to start felix %+v", err) + _ = watchDog.Wait(policyServer.FelixConfigChan, "Waiting for FelixConfig to be provided by the calico pod") + Eventually(vpp.RunCli, testTimeout).WithArguments("show capo int").Should(ContainSubstring("tx:[rule#0;allow][src==[ipset#0;ip;],]")) + st, _ := vpp.RunCli("show capo int") + fmt.Printf(st) + }) + })*/ + AfterEach(func() { + //cmd.Process.Kill() + //syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + //log.Info(cmd.Process.Pid) + test.TeardownVPP() + test.TeardownPod() + }) +}) + +func Go(f func(t *tomb.Tomb) error) { + t.Go(func() error { + defer GinkgoRecover() + err := f(&t) + if err != nil { + Expect(err).Should(BeNil(), "Tomb function errored with %s", err) + } + return err + }) +} diff --git a/felix/Makefile b/felix/Makefile new file mode 100644 index 00000000..be4baa33 --- /dev/null +++ b/felix/Makefile @@ -0,0 +1,11 @@ + +.PHONY: fv +fv: + cd calico/felix && make fv-no-prereqs + +.PHONY: fv-prereqs +fv-prereqs: + sudo mkdir -p /var/run/calico + git clone https://github.com/calico-vpp/calico 2> /dev/null || echo + (cd calico/ ; git checkout origin/feature/wip-fv-tests-vpp) + cd calico/felix && make fv-prereqs diff --git a/felix/README.md b/felix/README.md new file mode 100644 index 00000000..f144c0b9 --- /dev/null +++ b/felix/README.md @@ -0,0 +1,4 @@ + +### Prerequisits for running integration tests for policy server using felix fv tests: +* docker buildx plugin installed +*