From f463346d1c216807432fcbcb8841e15c1b7d4e4d Mon Sep 17 00:00:00 2001 From: hedi bouattour Date: Fri, 10 Feb 2023 16:54:13 +0000 Subject: [PATCH] agent: add first fv tests version --- Makefile | 1 + calico-vpp-agent/cmd/calico_vpp_dataplane.go | 2 +- calico-vpp-agent/cni/cni_pod_test.go | 2 +- calico-vpp-agent/common_tests/common_tests.go | 29 ++- calico-vpp-agent/policy/policy_test.go | 233 ++++++++++++++++++ felix/Makefile | 11 + felix/README.md | 4 + 7 files changed, 267 insertions(+), 15 deletions(-) create mode 100644 calico-vpp-agent/policy/policy_test.go create mode 100644 felix/Makefile create mode 100644 felix/README.md diff --git a/Makefile b/Makefile index ec0bc575..e4d5651c 100644 --- a/Makefile +++ b/Makefile @@ -197,6 +197,7 @@ run-integration-tests-cni: .PHONY: run-integration-tests-policy run-integration-tests-policy: + $(MAKE) -C felix fv-prereqs cd test/integration-tests;./run-tests.sh policy .PHONY: test diff --git a/calico-vpp-agent/cmd/calico_vpp_dataplane.go b/calico-vpp-agent/cmd/calico_vpp_dataplane.go index 3093aa24..a2b89569 100644 --- a/calico-vpp-agent/cmd/calico_vpp_dataplane.go +++ b/calico-vpp-agent/cmd/calico_vpp_dataplane.go @@ -40,8 +40,8 @@ import ( "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/services" "github.com/projectcalico/vpp-dataplane/config" + watchdog "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/watch_dog" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/watchers" - "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/watch_dog" ) /* diff --git a/calico-vpp-agent/cni/cni_pod_test.go b/calico-vpp-agent/cni/cni_pod_test.go index 482aeaa3..2f5499ca 100644 --- a/calico-vpp-agent/cni/cni_pod_test.go +++ b/calico-vpp-agent/cni/cni_pod_test.go @@ -31,8 +31,8 @@ import ( "github.com/vishvananda/netlink" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/cni" - test "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/common_tests" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/common" + test "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/common_tests" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/tests/mocks" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/watchers" "github.com/projectcalico/vpp-dataplane/config" diff --git a/calico-vpp-agent/common_tests/common_tests.go b/calico-vpp-agent/common_tests/common_tests.go index be8c6985..b1cae16d 100644 --- a/calico-vpp-agent/common_tests/common_tests.go +++ b/calico-vpp-agent/common_tests/common_tests.go @@ -20,41 +20,43 @@ import ( "github.com/sirupsen/logrus" "github.com/containernetworking/plugins/pkg/ns" + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + cniproto "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/cni/pod_interface" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/cni/storage" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/connectivity" - cniproto "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/tests/mocks/calico" "github.com/projectcalico/vpp-dataplane/multinet-monitor/networkAttachmentDefinition" "github.com/projectcalico/vpp-dataplane/vpplink" "github.com/projectcalico/vpp-dataplane/vpplink/binapi/vppapi/interface_types" "github.com/projectcalico/vpp-dataplane/vpplink/binapi/vppapi/ip_types" "github.com/projectcalico/vpp-dataplane/vpplink/types" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) + const ( // PodMockContainerName is used container name for pod container mock PodMockContainerName = "cni-tests-pod-mock" // PodMockImage is docker image used for pod mocking - PodMockImage = "calicovpp/vpp-test-pod-mock:latest" - VPPContainerName = "cni-tests-vpp" - VppContainerExtraArgsName = "VPP_CONTAINER_EXTRA_ARGS" - ThisNodeName = "node1" - UplinkIfName = "uplink" - UplinkIP = "10.0.100.1" - UplinkIPv6 = "A::1:1" - GatewayIP = "10.0.100.254" - GatewayIPv6 = "A::1:254" - ThisNodeIP = UplinkIP - ThisNodeIPv6 = UplinkIPv6 + PodMockImage = "calicovpp/vpp-test-pod-mock:latest" + VPPContainerName = "cni-tests-vpp" + VppContainerExtraArgsName = "VPP_CONTAINER_EXTRA_ARGS" + ThisNodeName = "node1" + UplinkIfName = "uplink" + UplinkIP = "10.0.100.1" + UplinkIPv6 = "A::1:1" + GatewayIP = "10.0.100.254" + GatewayIPv6 = "A::1:254" + ThisNodeIP = UplinkIP + ThisNodeIPv6 = UplinkIPv6 AddedNodeName = "node2" AddedNodeIP = "10.0.200.1" AddedNodeIPv6 = "A::2:1" ) + var ( // VppImage is the name of docker image containing VPP binary VppImage string @@ -63,6 +65,7 @@ var ( // vppContainerExtraArgs is a list of additionnal cli parameters for the VPP `docker run ...` VppContainerExtraArgs []string = []string{} ) + func AssertTunInterfaceExistence(vpp *vpplink.VppLink, newPod *cniproto.AddRequest) uint32 { ifSwIfIndex, err := vpp.SearchInterfaceWithTag( InterfaceTagForLocalTunTunnel(newPod.InterfaceName, newPod.Netns)) diff --git a/calico-vpp-agent/policy/policy_test.go b/calico-vpp-agent/policy/policy_test.go new file mode 100644 index 00000000..5df25ccd --- /dev/null +++ b/calico-vpp-agent/policy/policy_test.go @@ -0,0 +1,233 @@ +// Copyright (c) 2022 Cisco and/or its affiliates. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policy_test + +import ( + "fmt" + "os" + "os/exec" + "strings" + "testing" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + felixconfig "github.com/projectcalico/calico/felix/config" + "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/cni" + "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/common" + test "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/common_tests" + "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/policy" + watchdog "github.com/projectcalico/vpp-dataplane/calico-vpp-agent/watch_dog" + "github.com/projectcalico/vpp-dataplane/config" + agentConf "github.com/projectcalico/vpp-dataplane/config" + "github.com/projectcalico/vpp-dataplane/vpplink" + "github.com/projectcalico/vpp-dataplane/vpplink/types" + "github.com/sirupsen/logrus" + tomb "gopkg.in/tomb.v2" +) + +// Names of integration tests arguments +const ( + IntegrationTestEnableArgName = "INTEGRATION_TEST" + VppImageArgName = "VPP_IMAGE" + VppBinaryArgName = "VPP_BINARY" + VppContainerExtraArgsName = "VPP_CONTAINER_EXTRA_ARGS" +) + +var t tomb.Tomb + +// TestCniIntegration runs all the ginkgo integration test inside CNI package +func TestCniIntegration(t *testing.T) { + // skip test if test run is not integration test run (prevent accidental run of integration tests using go test ./...) + _, isIntegrationTestRun := os.LookupEnv(IntegrationTestEnableArgName) + if !isIntegrationTestRun { + t.Skip("skipping CNI integration tests (set INTEGRATION_TEST env variable to run these tests)") + } + + // integrate gomega and ginkgo -> register all CNI integration tests + RegisterFailHandler(Fail) + RunSpecs(t, "CNI Integration Suite") +} + +var _ = BeforeSuite(func() { + // extract common input for CNI integration tests + var found bool + test.VppImage, found = os.LookupEnv(VppImageArgName) + if !found { + Expect(test.VppImage).ToNot(BeEmpty(), fmt.Sprintf("Please specify docker image containing "+ + "VPP binary using %s environment variable.", VppImageArgName)) + } + test.VppBinary, found = os.LookupEnv(VppBinaryArgName) + if !found { + Expect(test.VppBinary).ToNot(BeEmpty(), fmt.Sprintf("Please specify VPP binary (full path) "+ + "inside docker image %s using %s environment variable.", test.VppImage, VppBinaryArgName)) + } + + vppContainerExtraArgsList, found := os.LookupEnv(VppContainerExtraArgsName) + if found { + test.VppContainerExtraArgs = append(test.VppContainerExtraArgs, strings.Split(vppContainerExtraArgsList, ",")...) + } + +}) + +var _ = Describe("Functionality of policy server using felix", func() { + var ( + log *logrus.Logger + vpp *vpplink.VppLink + policyServer *policy.Server + err error + cniServer *cni.Server + ) + BeforeEach(func() { + log = logrus.New() + common.ThePubSub = common.NewPubSub(log.WithFields(logrus.Fields{"component": "pubsub"})) + }) + + JustBeforeEach(func() { + test.StartVPP() + var uplinkSwIfIndex uint32 + vpp, _ = test.ConfigureVPP(log) + // Additional configuration specific to policies test: add tap for host endpoints + uplinkSwIfIndex, err = vpp.CreateTapV2(&types.TapV2{ + GenericVppInterface: types.GenericVppInterface{ + HostInterfaceName: test.UplinkIfName, + HardwareAddr: test.Mac("aa:bb:cc:dd:ee:01"), + }, + Tag: fmt.Sprintf("host-%s", test.UplinkIfName), + Flags: types.TapFlagNone, + // Host end of tap (it is located inside docker container) + HostMtu: 1500, + HostMacAddress: *test.Mac("aa:bb:cc:dd:ee:02"), + }) + Expect(err).ToNot(HaveOccurred(), "Error creating mocked tap interface") + log.Infof("tap swifindex: %+v", uplinkSwIfIndex) + + config.GetCalicoVppDebug().PoliciesEnabled = &config.True + common.VppManagerInfo = &agentConf.VppManagerInfo{UplinkStatuses: []agentConf.UplinkStatus{{IsMain: true, SwIfIndex: 1}}} + policyServer, err = policy.NewPolicyServer(vpp, log.WithFields(logrus.Fields{"component": "policy"}), false) + Expect(err).ToNot(HaveOccurred(), "Failed to create policy server") + cniServer = cni.NewCNIServer(vpp, policyServer, log.WithFields(logrus.Fields{"component": "cni"})) + Go(policyServer.ServePolicy) + log.Info("WAITING FOR FELIX CONFIG... please run felix") + + /*cmd := exec.Command("./run-batches") + cmd.Env = os.Environ() + cmd.Dir = "/home/ubuntu/vpp-dataplane/felix/fv" + cmd.Env = append(cmd.Env, + "FV_FELIXIMAGE=felix:latest-amd64", + "FV_ETCDIMAGE=quay.io/coreos/etcd:v3.5.1", + "FV_TYPHAIMAGE=felix-test/typha:latest-amd64", + "FV_K8SIMAGE=calico/go-build:v0.78", + "FV_NUM_BATCHES=1", + "FV_BATCHES_TO_RUN=\"1\"", + "FV_FELIX_LOG_LEVEL=\"\"", + "CERTS_PATH=/home/ubuntu/calico/hack/test/certs", + "PRIVATE_KEY=/home/ubuntu/vpp-dataplane/felix/fv/private.key", + "GINKGO_ARGS=''", + "GINKGO_FOCUS=", + "FELIX_FV_ENABLE_BPF=\"\"", + "FV_RACE_DETECTOR_ENABLED=true", + "FV_BINARY=/home/ubuntu/vpp-dataplane/felix/fv/bin/calico-felix-race-amd64", + "FELIX_FV_WIREGUARD_AVAILABLE=`./wireguard-available >/dev/null && echo true || echo false`", + ) + + out, err := cmd.Output() + log.Infof("OOOOO %s -- %+v", out, err) + Expect(err).Should(BeNil(), "Failed to start felix %+v", err)*/ + + // now run felix leveraging fv tests from calico + cmd := exec.Command("make", "fv") + cmd.Env = os.Environ() + cmd.Dir = "../../felix" + cmd.Env = append(cmd.Env, "GINKGO_FOCUS=should create a host endpoint with a policy" ) + err := cmd.Run() + + Expect(err).Should(BeNil(), "Failed to start felix %+v", err) + + watchDog := watchdog.NewWatchDog(log.WithFields(logrus.Fields{"component": "watchDog"}), &t) + felixConfig := watchDog.Wait(policyServer.FelixConfigChan, "Waiting for FelixConfig to be provided by the calico pod") + + //stopped here until the felix config is sent + log.Info("WAITING FOR FELIX CONFIG... please run felix") + + cniServer.SetFelixConfig(felixConfig.(*felixconfig.Config)) + cniServer.FetchBufferConfig() + + }) + + Describe("Addition of a policy on a pod", func() { + Context("With creation of the pod in cni server", func() { + /*It("should create pod", func() { + test.CreatePod() + const ( + ipAddress = "1.2.3.44" + interfaceName = "newInterface" + ) + By("Getting Pod mock container's PID") + containerPidOutput, err := exec.Command("docker", "inspect", "-f", "{{.State.Pid}}", + test.PodMockContainerName).Output() + Expect(err).Should(BeNil(), "Failed to get pod mock container's PID string") + containerPidStr := strings.ReplaceAll(string(containerPidOutput), "\n", "") + + By("Adding pod using CNI server") + newPod := &proto.AddRequest{ + InterfaceName: interfaceName, + Netns: fmt.Sprintf("/proc/%s/ns/net", containerPidStr), // expecting mount of "/proc" from host + ContainerIps: []*proto.IPConfig{{Address: ipAddress + "/24"}}, + Workload: &proto.WorkloadIDs{ + Annotations: map[string]string{ + "cni.projectcalico.org/AllowedSourcePrefixes": "[\"172.16.104.7\", \"3.4.5.6\"]", + }, + }, + } + reply, err := cniServer.Add(context.Background(), newPod) + Expect(err).ToNot(HaveOccurred(), "Pod addition failed") + Expect(reply.Successful).To(BeTrue(), + fmt.Sprintf("Pod addition failed due to: %s", reply.ErrorMessage)) + + By("Checking existence (and IP address) of interface tunnel at added pod's end") + interfaceDetails, err := exec.Command("docker", "exec", test.PodMockContainerName, + "ip", "address", "show", "dev", interfaceName).Output() + time.Sleep(time.Second * 300) + log.Infof("%+v", interfaceDetails) + })*/ + + }) + }) + Describe("Addition of a policy on a host", func() { + It("should configure host endpoint", func() { + time.Sleep(time.Second * 10) + policiesStr, err := vpp.RunCli("show capo int") + Expect(err).ToNot(HaveOccurred(), "failed to get policies on interfaces") + Expect(policiesStr).To(ContainSubstring("1.9.9.1")) + time.Sleep(time.Second * 5) + + }) + }) + + AfterEach(func() { + test.TeardownVPP() + }) +}) + +func Go(f func(t *tomb.Tomb) error) { + t.Go(func() error { + defer GinkgoRecover() + err := f(&t) + if err != nil { + Expect(err).Should(BeNil(), "Tomb function errored with %s", err) + } + return err + }) +} diff --git a/felix/Makefile b/felix/Makefile new file mode 100644 index 00000000..be4baa33 --- /dev/null +++ b/felix/Makefile @@ -0,0 +1,11 @@ + +.PHONY: fv +fv: + cd calico/felix && make fv-no-prereqs + +.PHONY: fv-prereqs +fv-prereqs: + sudo mkdir -p /var/run/calico + git clone https://github.com/calico-vpp/calico 2> /dev/null || echo + (cd calico/ ; git checkout origin/feature/wip-fv-tests-vpp) + cd calico/felix && make fv-prereqs diff --git a/felix/README.md b/felix/README.md new file mode 100644 index 00000000..f144c0b9 --- /dev/null +++ b/felix/README.md @@ -0,0 +1,4 @@ + +### Prerequisits for running integration tests for policy server using felix fv tests: +* docker buildx plugin installed +*