diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml index 8277f966cf3..0e827b0ef40 100644 --- a/.github/workflows/build-x86-image.yaml +++ b/.github/workflows/build-x86-image.yaml @@ -1083,6 +1083,124 @@ jobs: name: kube-ovn-ic-conformance-e2e-${{ matrix.ip-family }}-ko-log path: kube-ovn-ic-conformance-e2e-${{ matrix.ip-family }}-ko-log.tar.gz + multus-conformance-e2e: + name: Multus Conformance E2E + needs: + - build-kube-ovn + - build-e2e-binaries + runs-on: ubuntu-22.04 + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + ip-family: + - ipv4 + - ipv6 + - dual + steps: + - uses: jlumbroso/free-disk-space@v1.3.1 + with: + android: true + dotnet: true + haskell: true + docker-images: false + large-packages: false + tool-cache: false + swap-storage: false + + - uses: actions/checkout@v4 + + - name: Create the default branch directory + if: (github.base_ref || github.ref_name) != github.event.repository.default_branch + run: mkdir -p test/e2e/source + + - name: Check out the default branch + if: (github.base_ref || github.ref_name) != github.event.repository.default_branch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.repository.default_branch }} + fetch-depth: 1 + path: test/e2e/source + + - name: Export E2E directory + run: | + if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then + echo "E2E_DIR=." >> "$GITHUB_ENV" + else + echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV" + fi + + - uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION || '' }} + go-version-file: ${{ env.E2E_DIR }}/go.mod + check-latest: true + cache: false + + - name: Export Go full version + run: echo "GO_FULL_VER=$(go version | awk '{print $3}')" >> "$GITHUB_ENV" + + - name: Go cache + uses: actions/cache/restore@v4 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }} + restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86- + + - name: Install kind + uses: helm/kind-action@v1.9.0 + with: + version: ${{ env.KIND_VERSION }} + install_only: true + + - name: Install ginkgo + working-directory: ${{ env.E2E_DIR }} + run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo + + - name: Download kube-ovn image + uses: actions/download-artifact@v4 + with: + name: kube-ovn + + - name: Load images + run: docker load -i kube-ovn.tar + + - name: Create kind cluster + run: | + sudo pip3 install j2cli + sudo pip3 install "j2cli[yaml]" + sudo PATH=~/.local/bin:$PATH make kind-init-${{ matrix.ip-family }} + sudo cp -r /root/.kube/ ~/.kube/ + sudo chown -R $(id -un). ~/.kube/ + + - name: Install Kube-OVN + run: make kind-install-${{ matrix.ip-family }} + + - name: Install Multus + run: make kind-install-multus + + - name: Run E2E + working-directory: ${{ env.E2E_DIR }} + env: + E2E_BRANCH: ${{ github.base_ref || github.ref_name }} + E2E_IP_FAMILY: ${{ matrix.ip-family }} + run: make kube-ovn-multus-conformance-e2e + + - name: kubectl ko log + if: failure() + run: | + make kubectl-ko-log + mv kubectl-ko-log.tar.gz multus-conformance-e2e-${{ matrix.ip-family }}-ko-log.tar.gz + + - name: upload kubectl ko log + uses: actions/upload-artifact@v4 + if: failure() + with: + name: multus-conformance-e2e-${{ matrix.ip-family }}-ko-log + path: multus-conformance-e2e-${{ matrix.ip-family }}-ko-log.tar.gz + chart-test: name: Chart Installation/Uninstallation Test needs: build-kube-ovn @@ -2217,6 +2335,7 @@ jobs: - cyclonus-netpol-e2e - kube-ovn-conformance-e2e - kube-ovn-ic-conformance-e2e + - multus-conformance-e2e - ovn-vpc-nat-gw-conformance-e2e - iptables-vpc-nat-gw-conformance-e2e - webhook-e2e diff --git a/Makefile.e2e b/Makefile.e2e index 2560237ba6d..61768883c73 100644 --- a/Makefile.e2e +++ b/Makefile.e2e @@ -71,6 +71,7 @@ e2e-build: ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/k8s-network ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/kube-ovn ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/ovn-ic + ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/multus ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/lb-svc ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/vip ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/iptables-vpc-nat-gw @@ -134,6 +135,15 @@ kube-ovn-submariner-conformance-e2e: --context kind-kube-ovn --tocontext kind-kube-ovn1 \ --verbose --disruptive-tests +.PHONY: kube-ovn-multus-conformance-e2e +kube-ovn-multus-conformance-e2e: + ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/multus + E2E_BRANCH=$(E2E_BRANCH) \ + E2E_IP_FAMILY=$(E2E_IP_FAMILY) \ + E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \ + ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v \ + --focus=CNI:Kube-OVN ./test/e2e/multus/multus.test -- $(TEST_BIN_ARGS) + .PHONY: kube-ovn-lb-svc-conformance-e2e kube-ovn-lb-svc-conformance-e2e: ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/lb-svc diff --git a/cmd/cni/cni.go b/cmd/cni/cni.go index 4e1ac4da79a..ff437d49e77 100644 --- a/cmd/cni/cni.go +++ b/cmd/cni/cni.go @@ -13,6 +13,7 @@ import ( "github.com/containernetworking/cni/pkg/version" kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/netconf" "github.com/kubeovn/kube-ovn/pkg/request" "github.com/kubeovn/kube-ovn/pkg/util" "github.com/kubeovn/kube-ovn/versions" @@ -74,6 +75,7 @@ func generateCNIResult(cniResponse *request.CniResponse, netns string) current.R result := current.Result{ CNIVersion: current.ImplementedSpecVersion, DNS: cniResponse.DNS, + Routes: parseRoutes(cniResponse.Routes), } _, mask, _ := net.ParseCIDR(cniResponse.CIDR) podIface := current.Interface{ @@ -85,20 +87,21 @@ func generateCNIResult(cniResponse *request.CniResponse, netns string) current.R case kubeovnv1.ProtocolIPv4: ip, route := assignV4Address(cniResponse.IPAddress, cniResponse.Gateway, mask) result.IPs = []*current.IPConfig{ip} - if route != nil { + if len(result.Routes) == 0 && route != nil { result.Routes = []*types.Route{route} } result.Interfaces = []*current.Interface{&podIface} case kubeovnv1.ProtocolIPv6: ip, route := assignV6Address(cniResponse.IPAddress, cniResponse.Gateway, mask) result.IPs = []*current.IPConfig{ip} - if route != nil { + if len(result.Routes) == 0 && route != nil { result.Routes = []*types.Route{route} } result.Interfaces = []*current.Interface{&podIface} case kubeovnv1.ProtocolDual: var netMask *net.IPNet var gwStr string + addRoutes := len(result.Routes) == 0 for _, cidrBlock := range strings.Split(cniResponse.CIDR, ",") { _, netMask, _ = net.ParseCIDR(cidrBlock) gwStr = "" @@ -110,7 +113,7 @@ func generateCNIResult(cniResponse *request.CniResponse, netns string) current.R ip, route := assignV4Address(ipStr, gwStr, netMask) result.IPs = append(result.IPs, ip) - if route != nil { + if addRoutes && route != nil { result.Routes = append(result.Routes, route) } } else if util.CheckProtocol(cidrBlock) == kubeovnv1.ProtocolIPv6 { @@ -121,7 +124,7 @@ func generateCNIResult(cniResponse *request.CniResponse, netns string) current.R ip, route := assignV6Address(ipStr, gwStr, netMask) result.IPs = append(result.IPs, ip) - if route != nil { + if addRoutes && route != nil { result.Routes = append(result.Routes, route) } } @@ -132,6 +135,24 @@ func generateCNIResult(cniResponse *request.CniResponse, netns string) current.R return result } +func parseRoutes(routes []request.Route) []*types.Route { + parsedRoutes := make([]*types.Route, len(routes)) + for i, r := range routes { + if r.Destination == "" { + if util.CheckProtocol(r.Gateway) == kubeovnv1.ProtocolIPv4 { + r.Destination = "0.0.0.0/0" + } else { + r.Destination = "::/0" + } + } + parsedRoutes[i] = &types.Route{GW: net.ParseIP(r.Gateway)} + if _, cidr, err := net.ParseCIDR(r.Destination); err == nil { + parsedRoutes[i].Dst = *cidr + } + } + return parsedRoutes +} + func cmdDel(args *skel.CmdArgs) error { netConf, _, err := loadNetConf(args.StdinData) if err != nil { @@ -169,13 +190,8 @@ func cmdDel(args *skel.CmdArgs) error { return nil } -type ipamConf struct { - ServerSocket string `json:"server_socket"` - Provider string `json:"provider"` -} - -func loadNetConf(bytes []byte) (*netConf, string, error) { - n := &netConf{} +func loadNetConf(bytes []byte) (*netconf.NetConf, string, error) { + n := &netconf.NetConf{} if err := json.Unmarshal(bytes, n); err != nil { return nil, "", types.NewError(types.ErrDecodingFailure, "failed to load netconf", err.Error()) } @@ -183,6 +199,7 @@ func loadNetConf(bytes []byte) (*netConf, string, error) { if n.Type != util.CniTypeName && n.IPAM != nil { n.Provider = n.IPAM.Provider n.ServerSocket = n.IPAM.ServerSocket + n.Routes = n.IPAM.Routes } if n.ServerSocket == "" { @@ -193,7 +210,7 @@ func loadNetConf(bytes []byte) (*netConf, string, error) { n.Provider = util.OvnProvider } - n.postLoad() + n.PostLoad() return n, n.CNIVersion, nil } diff --git a/pkg/daemon/handler.go b/pkg/daemon/handler.go index dc5be010d64..af042a11743 100644 --- a/pkg/daemon/handler.go +++ b/pkg/daemon/handler.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "net" "net/http" "strconv" "strings" @@ -225,6 +226,7 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon return } + routes = append(podRequest.Routes, routes...) if strings.HasSuffix(podRequest.Provider, util.OvnProvider) && subnet != "" { podSubnet, err := csh.Controller.subnetsLister.Get(subnet) if err != nil { @@ -298,17 +300,17 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon } } - routes = append(podRequest.Routes, routes...) macAddr = pod.Annotations[fmt.Sprintf(util.MacAddressAnnotationTemplate, podRequest.Provider)] klog.Infof("create container interface %s mac %s, ip %s, cidr %s, gw %s, custom routes %v", ifName, macAddr, ipAddr, cidr, gw, routes) podNicName = ifName switch nicType { case util.InternalType: - podNicName, err = csh.configureNicWithInternalPort(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, ifName, macAddr, mtu, ipAddr, gw, isDefaultRoute, detectIPConflict, routes, podRequest.DNS.Nameservers, podRequest.DNS.Search, ingress, egress, podRequest.DeviceID, nicType, latency, limit, loss, jitter, gatewayCheckMode, u2oInterconnectionIP) + podNicName, routes, err = csh.configureNicWithInternalPort(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, ifName, macAddr, mtu, ipAddr, gw, isDefaultRoute, detectIPConflict, routes, podRequest.DNS.Nameservers, podRequest.DNS.Search, ingress, egress, podRequest.DeviceID, nicType, latency, limit, loss, jitter, gatewayCheckMode, u2oInterconnectionIP) case util.DpdkType: err = csh.configureDpdkNic(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, ifName, macAddr, mtu, ipAddr, gw, ingress, egress, getShortSharedDir(pod.UID, podRequest.VhostUserSocketVolumeName), podRequest.VhostUserSocketName, podRequest.VhostUserSocketConsumption) + routes = nil default: - err = csh.configureNic(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, podRequest.VfDriver, ifName, macAddr, mtu, ipAddr, gw, isDefaultRoute, detectIPConflict, routes, podRequest.DNS.Nameservers, podRequest.DNS.Search, ingress, egress, podRequest.DeviceID, nicType, latency, limit, loss, jitter, gatewayCheckMode, u2oInterconnectionIP, oldPodName) + routes, err = csh.configureNic(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, podRequest.VfDriver, ifName, macAddr, mtu, ipAddr, gw, isDefaultRoute, detectIPConflict, routes, podRequest.DNS.Nameservers, podRequest.DNS.Search, ingress, egress, podRequest.DeviceID, nicType, latency, limit, loss, jitter, gatewayCheckMode, u2oInterconnectionIP, oldPodName) } if err != nil { errMsg := fmt.Errorf("configure nic failed %v", err) @@ -333,6 +335,30 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon } return } + } else if len(routes) != 0 { + hasDefaultRoute := make(map[string]bool, 2) + for _, r := range routes { + if r.Destination == "" { + hasDefaultRoute[util.CheckProtocol(r.Gateway)] = true + continue + } + if _, cidr, err := net.ParseCIDR(r.Destination); err == nil { + if ones, _ := cidr.Mask.Size(); ones == 0 { + hasDefaultRoute[util.CheckProtocol(r.Gateway)] = true + } + } + } + if len(hasDefaultRoute) != 0 { + // remove existing default route so other CNI plugins, such as macvlan, can add the new default route correctly + if err = csh.removeDefaultRoute(podRequest.NetNs, hasDefaultRoute[kubeovnv1.ProtocolIPv4], hasDefaultRoute[kubeovnv1.ProtocolIPv6]); err != nil { + errMsg := fmt.Errorf("failed to remove existing default route for interface %s of pod %s/%s: %v", podRequest.IfName, podRequest.PodNamespace, podRequest.PodName, err) + klog.Error(errMsg) + if err = resp.WriteHeaderAndEntity(http.StatusInternalServerError, request.CniResponse{Err: errMsg.Error()}); err != nil { + klog.Errorf("failed to write response: %v", err) + } + return + } + } } response := &request.CniResponse{ @@ -341,6 +367,7 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon MacAddress: macAddr, CIDR: cidr, PodNicName: podNicName, + Routes: routes, } if isDefaultRoute { response.Gateway = gw diff --git a/pkg/daemon/ovs_linux.go b/pkg/daemon/ovs_linux.go index 3ee71866f5c..2b941f4d111 100644 --- a/pkg/daemon/ovs_linux.go +++ b/pkg/daemon/ovs_linux.go @@ -63,20 +63,20 @@ func (csh cniServerHandler) configureDpdkNic(podName, podNamespace, provider, ne return ovs.SetInterfaceBandwidth(podName, podNamespace, ifaceID, egress, ingress) } -func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, containerID, vfDriver, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute, detectIPConflict bool, routes []request.Route, _, _ []string, ingress, egress, deviceID, nicType, latency, limit, loss, jitter string, gwCheckMode int, u2oInterconnectionIP, oldPodName string) error { +func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, containerID, vfDriver, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute, detectIPConflict bool, routes []request.Route, _, _ []string, ingress, egress, deviceID, nicType, latency, limit, loss, jitter string, gwCheckMode int, u2oInterconnectionIP, oldPodName string) ([]request.Route, error) { var err error var hostNicName, containerNicName string if deviceID == "" { hostNicName, containerNicName, err = setupVethPair(containerID, ifName, mtu) if err != nil { klog.Errorf("failed to create veth pair %v", err) - return err + return nil, err } } else { hostNicName, containerNicName, err = setupSriovInterface(containerID, deviceID, vfDriver, ifName, mtu, mac) if err != nil { klog.Errorf("failed to create sriov interfaces %v", err) - return err + return nil, err } } @@ -92,7 +92,7 @@ func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, fmt.Sprintf("external_ids:ip=%s", ipStr), fmt.Sprintf("external_ids:pod_netns=%s", netns)) if err != nil { - return fmt.Errorf("add nic to ovs failed %v: %q", err, output) + return nil, fmt.Errorf("add nic to ovs failed %v: %q", err, output) } // add hostNicName and containerNicName into pod annotations @@ -106,7 +106,7 @@ func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, pod, err := csh.Controller.podsLister.Pods(podNamespace).Get(podNameNew) if err != nil { klog.Errorf("failed to generate patch for pod %s/%s: %v", podNameNew, podNamespace, err) - return err + return nil, err } oriPod := pod.DeepCopy() pod.Annotations[fmt.Sprintf(util.VfRepresentorNameTemplate, provider)] = hostNicName @@ -114,53 +114,53 @@ func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, patch, err := util.GenerateMergePatchPayload(oriPod, pod) if err != nil { klog.Errorf("failed to generate patch for pod %s/%s: %v", podNameNew, podNamespace, err) - return err + return nil, err } if _, err := csh.Config.KubeClient.CoreV1().Pods(podNamespace).Patch(context.Background(), podNameNew, types.MergePatchType, patch, metav1.PatchOptions{}, ""); err != nil { klog.Errorf("patch pod %s/%s failed: %v", podNameNew, podNamespace, err) - return err + return nil, err } } // lsp and container nic must use same mac address, otherwise ovn will reject these packets by default macAddr, err := net.ParseMAC(mac) if err != nil { - return fmt.Errorf("failed to parse mac %s %v", macAddr, err) + return nil, fmt.Errorf("failed to parse mac %s %v", macAddr, err) } if err = configureHostNic(hostNicName); err != nil { klog.Error(err) - return err + return nil, err } if err = ovs.SetInterfaceBandwidth(podName, podNamespace, ifaceID, egress, ingress); err != nil { klog.Error(err) - return err + return nil, err } if err = ovs.SetNetemQos(podName, podNamespace, ifaceID, latency, limit, loss, jitter); err != nil { klog.Error(err) - return err + return nil, err } if containerNicName == "" { - return nil + return nil, nil } isUserspaceDP, err := ovs.IsUserspaceDataPath() if err != nil { klog.Error(err) - return err + return nil, err } if isUserspaceDP { // turn off tx checksum if err = turnOffNicTxChecksum(containerNicName); err != nil { klog.Error(err) - return err + return nil, err } } podNS, err := ns.GetNS(netns) if err != nil { - return fmt.Errorf("failed to open netns %q: %v", netns, err) + return nil, fmt.Errorf("failed to open netns %q: %v", netns, err) } return configureContainerNic(containerNicName, ifName, ip, gateway, isDefaultRoute, detectIPConflict, routes, macAddr, podNS, mtu, nicType, gwCheckMode, u2oInterconnectionIP) } @@ -311,23 +311,24 @@ func configureHostNic(nicName string) error { return nil } -func configureContainerNic(nicName, ifName, ipAddr, gateway string, isDefaultRoute, detectIPConflict bool, routes []request.Route, macAddr net.HardwareAddr, netns ns.NetNS, mtu int, nicType string, gwCheckMode int, u2oInterconnectionIP string) error { +func configureContainerNic(nicName, ifName, ipAddr, gateway string, isDefaultRoute, detectIPConflict bool, routes []request.Route, macAddr net.HardwareAddr, netns ns.NetNS, mtu int, nicType string, gwCheckMode int, u2oInterconnectionIP string) ([]request.Route, error) { containerLink, err := netlink.LinkByName(nicName) if err != nil { - return fmt.Errorf("can not find container nic %s: %v", nicName, err) + return nil, fmt.Errorf("can not find container nic %s: %v", nicName, err) } // Set link alias to its origin link name for fastpath to recognize and bypass netfilter if err := netlink.LinkSetAlias(containerLink, nicName); err != nil { klog.Errorf("failed to set link alias for container nic %s: %v", nicName, err) - return err + return nil, err } if err = netlink.LinkSetNsFd(containerLink, int(netns.Fd())); err != nil { - return fmt.Errorf("failed to move link to netns: %v", err) + return nil, fmt.Errorf("failed to move link to netns: %v", err) } - return ns.WithNetNSPath(netns.Path(), func(_ ns.NetNS) error { + var finalRoutes []request.Route + err = ns.WithNetNSPath(netns.Path(), func(_ ns.NetNS) error { if nicType != util.InternalType { if err = netlink.LinkSetName(containerLink, ifName); err != nil { klog.Error(err) @@ -415,6 +416,35 @@ func configureContainerNic(nicName, ifName, ipAddr, gateway string, isDefaultRou } } + linkRoutes, err := netlink.RouteList(containerLink, netlink.FAMILY_ALL) + if err != nil { + return fmt.Errorf("failed to get routes on interface %s: %v", ifName, err) + } + + for _, r := range linkRoutes { + if r.Family != netlink.FAMILY_V4 && r.Family != netlink.FAMILY_V6 { + continue + } + if r.Dst == nil && r.Gw == nil { + continue + } + if r.Dst != nil && r.Dst.IP.IsLinkLocalUnicast() { + if _, bits := r.Dst.Mask.Size(); bits == net.IPv6len*8 { + // skip fe80::/10 + continue + } + } + + var route request.Route + if r.Dst != nil { + route.Destination = r.Dst.String() + } + if r.Gw != nil { + route.Gateway = r.Gw.String() + } + finalRoutes = append(finalRoutes, route) + } + if gwCheckMode != gatewayModeDisabled { var ( underlayGateway = gwCheckMode == gatewayCheckModeArping || gwCheckMode == gatewayCheckModeArpingNotConcerned @@ -436,6 +466,8 @@ func configureContainerNic(nicName, ifName, ipAddr, gateway string, isDefaultRou return nil }) + + return finalRoutes, err } func checkGatewayReady(gwCheckMode int, intr, ipAddr, gateway string, underlayGateway, verbose bool) error { @@ -1438,7 +1470,7 @@ func renameLink(curName, newName string) error { return netlink.LinkSetUp(link) } -func (csh cniServerHandler) configureNicWithInternalPort(podName, podNamespace, provider, netns, containerID, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute, detectIPConflict bool, routes []request.Route, _, _ []string, ingress, egress, _, nicType, latency, limit, loss, jitter string, gwCheckMode int, u2oInterconnectionIP string) (string, error) { +func (csh cniServerHandler) configureNicWithInternalPort(podName, podNamespace, provider, netns, containerID, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute, detectIPConflict bool, routes []request.Route, _, _ []string, ingress, egress, _, nicType, latency, limit, loss, jitter string, gwCheckMode int, u2oInterconnectionIP string) (string, []request.Route, error) { _, containerNicName := generateNicName(containerID, ifName) ipStr := util.GetIPWithoutMask(ip) ifaceID := ovs.PodNameToPortName(podName, podNamespace, provider) @@ -1454,31 +1486,65 @@ func (csh cniServerHandler) configureNicWithInternalPort(podName, podNamespace, fmt.Sprintf("external_ids:ip=%s", ipStr), fmt.Sprintf("external_ids:pod_netns=%s", netns)) if err != nil { - return containerNicName, fmt.Errorf("add nic to ovs failed %v: %q", err, output) + return containerNicName, nil, fmt.Errorf("add nic to ovs failed %v: %q", err, output) } // container nic must use same mac address from pod annotation, otherwise ovn will reject these packets by default macAddr, err := net.ParseMAC(mac) if err != nil { - return containerNicName, fmt.Errorf("failed to parse mac %s %v", macAddr, err) + return containerNicName, nil, fmt.Errorf("failed to parse mac %s %v", macAddr, err) } if err = ovs.SetInterfaceBandwidth(podName, podNamespace, ifaceID, egress, ingress); err != nil { - return containerNicName, err + return containerNicName, nil, err } if err = ovs.SetNetemQos(podName, podNamespace, ifaceID, latency, limit, loss, jitter); err != nil { - return containerNicName, err + return containerNicName, nil, err } podNS, err := ns.GetNS(netns) if err != nil { - return containerNicName, fmt.Errorf("failed to open netns %q: %v", netns, err) + return containerNicName, nil, fmt.Errorf("failed to open netns %q: %v", netns, err) } - if err = configureContainerNic(containerNicName, ifName, ip, gateway, isDefaultRoute, detectIPConflict, routes, macAddr, podNS, mtu, nicType, gwCheckMode, u2oInterconnectionIP); err != nil { - return containerNicName, err + routes, err = configureContainerNic(containerNicName, ifName, ip, gateway, isDefaultRoute, detectIPConflict, routes, macAddr, podNS, mtu, nicType, gwCheckMode, u2oInterconnectionIP) + return containerNicName, routes, err +} + +func (csh cniServerHandler) removeDefaultRoute(netns string, ipv4, ipv6 bool) error { + podNS, err := ns.GetNS(netns) + if err != nil { + return fmt.Errorf("failed to open netns %q: %v", netns, err) } - return containerNicName, nil + + return ns.WithNetNSPath(podNS.Path(), func(_ ns.NetNS) error { + routes, err := netlink.RouteList(nil, netlink.FAMILY_ALL) + if err != nil { + return fmt.Errorf("failed to get all routes: %v", err) + } + + for _, r := range routes { + if r.Dst != nil { + if ones, _ := r.Dst.Mask.Size(); ones != 0 { + continue + } + } + if ipv4 && r.Family == netlink.FAMILY_V4 { + klog.Infof("deleting default ipv4 route %+v", r) + if err = netlink.RouteDel(&r); err != nil { + return fmt.Errorf("failed to delete route %+v: %v", r, err) + } + continue + } + if ipv6 && r.Family == netlink.FAMILY_V6 { + klog.Infof("deleting default ipv6 route %+v", r) + if err = netlink.RouteDel(&r); err != nil { + return fmt.Errorf("failed to delete route %+v: %v", r, err) + } + } + } + return nil + }) } // https://github.com/antrea-io/antrea/issues/1691 diff --git a/pkg/daemon/ovs_windows.go b/pkg/daemon/ovs_windows.go index d49441c61ae..0fd513f1a6f 100644 --- a/pkg/daemon/ovs_windows.go +++ b/pkg/daemon/ovs_windows.go @@ -22,29 +22,30 @@ func (csh cniServerHandler) configureDpdkNic(podName, podNamespace, provider, ne return errors.New("DPDK is not supported on Windows") } -func (csh cniServerHandler) configureNicWithInternalPort(podName, podNamespace, provider, netns, containerID, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute, detectIPConflict bool, routes []request.Route, dnsServer, dnsSuffix []string, ingress, egress, DeviceID, nicType, latency, limit, loss, jitter string, gwCheckMode int, u2oInterconnectionIP string) (string, error) { - return ifName, csh.configureNic(podName, podNamespace, provider, netns, containerID, "", ifName, mac, mtu, ip, gateway, isDefaultRoute, detectIPConflict, routes, dnsServer, dnsSuffix, ingress, egress, DeviceID, nicType, latency, limit, loss, jitter, gwCheckMode, u2oInterconnectionIP, "") +func (csh cniServerHandler) configureNicWithInternalPort(podName, podNamespace, provider, netns, containerID, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute, detectIPConflict bool, routes []request.Route, dnsServer, dnsSuffix []string, ingress, egress, DeviceID, nicType, latency, limit, loss, jitter string, gwCheckMode int, u2oInterconnectionIP string) (string, []request.Route, error) { + routes, err := csh.configureNic(podName, podNamespace, provider, netns, containerID, "", ifName, mac, mtu, ip, gateway, isDefaultRoute, detectIPConflict, routes, dnsServer, dnsSuffix, ingress, egress, DeviceID, nicType, latency, limit, loss, jitter, gwCheckMode, u2oInterconnectionIP, "") + return ifName, routes, err } -func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, containerID, vfDriver, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute, detectIPConflict bool, routes []request.Route, dnsServer, dnsSuffix []string, ingress, egress, DeviceID, nicType, latency, limit, loss, jitter string, gwCheckMode int, u2oInterconnectionIP, _ string) error { +func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, containerID, vfDriver, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute, detectIPConflict bool, routes []request.Route, dnsServer, dnsSuffix []string, ingress, egress, DeviceID, nicType, latency, limit, loss, jitter string, gwCheckMode int, u2oInterconnectionIP, _ string) ([]request.Route, error) { if DeviceID != "" { - return errors.New("SR-IOV is not supported on Windows") + return nil, errors.New("SR-IOV is not supported on Windows") } hnsNetwork, err := hcsshim.GetHNSNetworkByName(util.HnsNetwork) if err != nil { klog.Errorf("failed to get HNS network %s: %v", util.HnsNetwork) - return err + return nil, err } if hnsNetwork == nil { err = fmt.Errorf("HNS network %s does not exist", util.HnsNetwork) klog.Error(err) - return err + return nil, err } if !strings.EqualFold(hnsNetwork.Type, "Transparent") { err = fmt.Errorf(`type of HNS network %s is "%s", while "Transparent" is required`, util.HnsNetwork, hnsNetwork.Type) klog.Error(err) - return err + return nil, err } ipAddr := util.GetIPWithoutMask(ip) @@ -79,22 +80,22 @@ func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, }) if err != nil { klog.Errorf("failed to add HNS endpoint: %v", err) - return err + return nil, err } if containerID != sandbox { // pause container, return here - return nil + return nil, nil } // add OVS port exists, err := ovs.PortExists(epName) if err != nil { klog.Error(err) - return err + return nil, err } if exists { - return nil + return nil, nil } timeout := 5 @@ -109,7 +110,7 @@ func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, _mtu := uint32(mtu) if err = util.SetNetIPInterface(adapter.InterfaceIndex, nil, &_mtu, nil, nil); err != nil { klog.Errorf("failed to set MTU of %s to %d: %v", adapterName, mtu, err) - return err + return nil, err } ifaceID := ovs.PodNameToPortName(podName, podNamespace, provider) @@ -121,18 +122,18 @@ func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, fmt.Sprintf("external_ids:pod_namespace=%s", podNamespace), fmt.Sprintf("external_ids:ip=%s", ipAddr)) if err != nil { - return fmt.Errorf("failed to add OVS port %s, %v: %q", epName, err, output) + return nil, fmt.Errorf("failed to add OVS port %s, %v: %q", epName, err, output) } if err = ovs.SetInterfaceBandwidth(podName, podNamespace, ifaceID, egress, ingress); err != nil { klog.Error(err) - return err + return nil, err } - return nil + return nil, nil } - return fmt.Errorf(`failed to get network adapter "%s" after %d seconds`, adapterName, timeout) + return nil, fmt.Errorf(`failed to get network adapter "%s" after %d seconds`, adapterName, timeout) } func configureNic(name, ip string, mac net.HardwareAddr, mtu int) error { @@ -222,6 +223,10 @@ func configureNic(name, ip string, mac net.HardwareAddr, mtu int) error { return nil } +func (csh cniServerHandler) removeDefaultRoute(netns string, ipv4, ipv6 bool) error { + return nil +} + func (csh cniServerHandler) deleteNic(podName, podNamespace, containerID, netns, deviceID, ifName, nicType, _ string) error { epName := hns.ConstructEndpointName(containerID, netns, util.HnsNetwork)[:12] // remove ovs port diff --git a/pkg/netconf/ipam_conf.go b/pkg/netconf/ipam_conf.go new file mode 100644 index 00000000000..f6557b920f2 --- /dev/null +++ b/pkg/netconf/ipam_conf.go @@ -0,0 +1,10 @@ +package netconf + +import "github.com/kubeovn/kube-ovn/pkg/request" + +type IPAMConf struct { + Type string `json:"type"` + ServerSocket string `json:"server_socket"` + Provider string `json:"provider"` + Routes []request.Route `json:"routes"` +} diff --git a/cmd/cni/netconf.go b/pkg/netconf/netconf.go similarity index 85% rename from cmd/cni/netconf.go rename to pkg/netconf/netconf.go index dd0f6c0d46a..52c44aab78f 100644 --- a/cmd/cni/netconf.go +++ b/pkg/netconf/netconf.go @@ -1,7 +1,7 @@ //go:build !windows // +build !windows -package main +package netconf import ( "github.com/containernetworking/cni/pkg/types" @@ -9,12 +9,12 @@ import ( "github.com/kubeovn/kube-ovn/pkg/request" ) -type netConf struct { +type NetConf struct { types.NetConf ServerSocket string `json:"server_socket"` Provider string `json:"provider"` Routes []request.Route `json:"routes"` - IPAM *ipamConf `json:"ipam"` + IPAM *IPAMConf `json:"ipam"` // PciAddrs in case of using sriov DeviceID string `json:"deviceID"` VfDriver string `json:"vf_driver"` @@ -24,6 +24,6 @@ type netConf struct { VhostUserSocketConsumption string `json:"vhost_user_socket_consumption"` } -func (n *netConf) postLoad() { +func (n *NetConf) PostLoad() { // nothing to do on linux } diff --git a/cmd/cni/netconf_windows.go b/pkg/netconf/netconf_windows.go similarity index 87% rename from cmd/cni/netconf_windows.go rename to pkg/netconf/netconf_windows.go index 6dadc8ed85d..4c6fd7a39f8 100644 --- a/cmd/cni/netconf_windows.go +++ b/pkg/netconf/netconf_windows.go @@ -1,4 +1,4 @@ -package main +package netconf import ( "github.com/containernetworking/plugins/pkg/hns" @@ -6,12 +6,12 @@ import ( "github.com/kubeovn/kube-ovn/pkg/request" ) -type netConf struct { +type NetConf struct { hns.NetConf ServerSocket string `json:"server_socket"` Provider string `json:"provider"` Routes []request.Route `json:"routes"` - IPAM *ipamConf `json:"ipam"` + IPAM *IPAMConf `json:"ipam"` // PciAddrs in case of using sriov DeviceID string `json:"deviceID"` VfDriver string `json:"vf_driver"` @@ -21,7 +21,7 @@ type netConf struct { VhostUserSocketConsumption string `json:"vhost_user_socket_consumption"` } -func (n *netConf) postLoad() { +func (n *NetConf) PostLoad() { if len(n.DNS.Nameservers) == 0 { n.DNS.Nameservers = n.RuntimeConfig.DNS.Nameservers } diff --git a/pkg/request/cniserver.go b/pkg/request/cniserver.go index 2d3912ab32e..585b24ea63b 100644 --- a/pkg/request/cniserver.go +++ b/pkg/request/cniserver.go @@ -46,6 +46,7 @@ type CniResponse struct { MacAddress string `json:"mac_address"` CIDR string `json:"cidr"` Gateway string `json:"gateway"` + Routes []Route `json:"routes"` Mtu int `json:"mtu"` PodNicName string `json:"nicname"` DNS types.DNS `json:"dns"` diff --git a/test/e2e/framework/network-attachment-definition.go b/test/e2e/framework/network-attachment-definition.go index 28b54aa1338..9ca4c3f8280 100644 --- a/test/e2e/framework/network-attachment-definition.go +++ b/test/e2e/framework/network-attachment-definition.go @@ -32,7 +32,7 @@ func (c *NetworkAttachmentDefinitionClient) Get(name string) *apiv1.NetworkAttac func (c *NetworkAttachmentDefinitionClient) Create(nad *apiv1.NetworkAttachmentDefinition) *apiv1.NetworkAttachmentDefinition { nad, err := c.NetworkAttachmentDefinitionInterface.Create(context.TODO(), nad, metav1.CreateOptions{}) ExpectNoError(err, "Error creating nad") - return nad.DeepCopy() + return c.Get(nad.Name) } // Delete deletes a nad if the nad exists diff --git a/test/e2e/framework/pod.go b/test/e2e/framework/pod.go index 5457c096d38..4afd7d40cb0 100644 --- a/test/e2e/framework/pod.go +++ b/test/e2e/framework/pod.go @@ -47,7 +47,8 @@ func (c *PodClient) Delete(name string) error { } func (c *PodClient) DeleteSync(name string) { - c.PodClient.DeleteSync(context.Background(), name, metav1.DeleteOptions{}, timeout) + var gps int64 = 1 + c.PodClient.DeleteSync(context.Background(), name, metav1.DeleteOptions{GracePeriodSeconds: &gps}, timeout) } func (c *PodClient) Patch(original, modified *corev1.Pod) *corev1.Pod { diff --git a/test/e2e/multus/e2e_test.go b/test/e2e/multus/e2e_test.go new file mode 100644 index 00000000000..795312d4cd5 --- /dev/null +++ b/test/e2e/multus/e2e_test.go @@ -0,0 +1,400 @@ +package multus + +import ( + "encoding/json" + "flag" + "fmt" + "testing" + + "k8s.io/klog/v2" + "k8s.io/kubernetes/test/e2e" + k8sframework "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/config" + + "github.com/containernetworking/cni/pkg/types" + "github.com/onsi/ginkgo/v2" + + apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/netconf" + "github.com/kubeovn/kube-ovn/pkg/request" + "github.com/kubeovn/kube-ovn/pkg/util" + "github.com/kubeovn/kube-ovn/test/e2e/framework" + "github.com/kubeovn/kube-ovn/test/e2e/framework/iproute" +) + +const CNIVersion = "0.3.1" + +// https://github.com/containernetworking/plugins/blob/main/plugins/main/macvlan/macvlan.go#L37 +type MacvlanNetConf struct { + netconf.NetConf + Master string `json:"master"` + Mode string `json:"mode"` + MTU int `json:"mtu"` + Mac string `json:"mac,omitempty"` + LinkContNs bool `json:"linkInContainer,omitempty"` + + RuntimeConfig struct { + Mac string `json:"mac,omitempty"` + } `json:"runtimeConfig,omitempty"` +} + +func init() { + klog.SetOutput(ginkgo.GinkgoWriter) + + // Register flags. + config.CopyFlags(config.Flags, flag.CommandLine) + k8sframework.RegisterCommonFlags(flag.CommandLine) + k8sframework.RegisterClusterFlags(flag.CommandLine) +} + +func TestE2E(t *testing.T) { + k8sframework.AfterReadingAllFlags(&k8sframework.TestContext) + e2e.RunE2ETests(t) +} + +var _ = framework.SerialDescribe("[group:multus]", func() { + f := framework.NewDefaultFramework("multus") + + var podClient *framework.PodClient + var subnetClient *framework.SubnetClient + var nadClient *framework.NetworkAttachmentDefinitionClient + var nadName, podName, subnetName, namespaceName, image string + var cidr string + var subnet *apiv1.Subnet + ginkgo.BeforeEach(func() { + namespaceName = f.Namespace.Name + nadName = "nad-" + framework.RandomSuffix() + podName = "pod-" + framework.RandomSuffix() + subnetName = "subnet-" + framework.RandomSuffix() + cidr = framework.RandomCIDR(f.ClusterIPFamily) + podClient = f.PodClient() + subnetClient = f.SubnetClient() + nadClient = f.NetworkAttachmentDefinitionClient(namespaceName) + + if image == "" { + image = framework.GetKubeOvnImage(f.ClientSet) + } + }) + ginkgo.AfterEach(func() { + ginkgo.By("Deleting pod " + podName) + podClient.DeleteSync(podName) + + ginkgo.By("Deleting subnet " + subnetName) + subnetClient.DeleteSync(subnetName) + + ginkgo.By("Deleting network attachment definition " + nadName) + nadClient.Delete(nadName) + }) + + framework.ConformanceIt("should be able to create attachment interface", func() { + provider := fmt.Sprintf("%s.%s.%s", nadName, namespaceName, util.OvnProvider) + + ginkgo.By("Constructing network attachment definition config") + config := &netconf.NetConf{ + NetConf: types.NetConf{ + CNIVersion: CNIVersion, + Type: util.CniTypeName, + }, + ServerSocket: "/run/openvswitch/kube-ovn-daemon.sock", + Provider: provider, + } + buf, err := json.MarshalIndent(config, "", " ") + framework.ExpectNoError(err) + + ginkgo.By("Creating network attachment definition " + nadName) + nad := framework.MakeNetworkAttachmentDefinition(nadName, namespaceName, string(buf)) + nad = nadClient.Create(nad) + framework.Logf("created network attachment definition config:\n%s", nad.Spec.Config) + + ginkgo.By("Creating subnet " + subnetName) + subnet = framework.MakeSubnet(subnetName, "", cidr, "", "", "", nil, nil, nil) + subnet.Spec.Provider = provider + subnet = subnetClient.CreateSync(subnet) + + ginkgo.By("Creating pod " + podName) + annotations := map[string]string{"k8s.v1.cni.cncf.io/networks": fmt.Sprintf("%s/%s", nad.Namespace, nad.Name)} + cmd := []string{"sh", "-c", "sleep infinity"} + pod := framework.MakePod(namespaceName, podName, nil, annotations, image, cmd, nil) + pod = podClient.CreateSync(pod) + + ginkgo.By("Retrieving pod routes") + podRoutes, err := iproute.RouteShow("", "", func(cmd ...string) ([]byte, []byte, error) { + return framework.KubectlExec(namespaceName, podName, cmd...) + }) + framework.ExpectNoError(err) + + ginkgo.By("Validating pod routes") + actualRoutes := make([]request.Route, 0, len(podRoutes)) + for _, r := range podRoutes { + if r.Gateway != "" || r.Dst != "" { + actualRoutes = append(actualRoutes, request.Route{Destination: r.Dst, Gateway: r.Gateway}) + } + } + ipv4CIDR, ipv6CIDR := util.SplitStringIP(pod.Annotations[util.CidrAnnotation]) + ipv4Gateway, ipv6Gateway := util.SplitStringIP(pod.Annotations[util.GatewayAnnotation]) + nadIPv4CIDR, nadIPv6CIDR := util.SplitStringIP(subnet.Spec.CIDRBlock) + nadIPv4Gateway, nadIPv6Gateway := util.SplitStringIP(subnet.Spec.Gateway) + if f.HasIPv4() { + framework.ExpectContainElement(actualRoutes, request.Route{Destination: ipv4CIDR}) + framework.ExpectContainElement(actualRoutes, request.Route{Destination: "default", Gateway: ipv4Gateway}) + framework.ExpectContainElement(actualRoutes, request.Route{Destination: nadIPv4CIDR}) + framework.ExpectNotContainElement(actualRoutes, request.Route{Destination: "default", Gateway: nadIPv4Gateway}) + } + if f.HasIPv6() { + framework.ExpectContainElement(actualRoutes, request.Route{Destination: ipv6CIDR}) + framework.ExpectContainElement(actualRoutes, request.Route{Destination: "default", Gateway: ipv6Gateway}) + framework.ExpectContainElement(actualRoutes, request.Route{Destination: nadIPv6CIDR}) + framework.ExpectNotContainElement(actualRoutes, request.Route{Destination: "default", Gateway: nadIPv6Gateway}) + } + }) + + framework.ConformanceIt("should be able to create attachment interface with custom routes", func() { + provider := fmt.Sprintf("%s.%s.%s", nadName, namespaceName, util.OvnProvider) + + ginkgo.By("Creating subnet " + subnetName) + subnet = framework.MakeSubnet(subnetName, "", cidr, "", "", "", nil, nil, nil) + subnet.Spec.Provider = provider + subnet = subnetClient.CreateSync(subnet) + + ginkgo.By("Constructing network attachment definition config") + var routeDst string + for i := 0; i < 3; i++ { + routeDst = framework.RandomCIDR(f.ClusterIPFamily) + if routeDst != subnet.Spec.CIDRBlock { + break + } + } + framework.ExpectNotEqual(routeDst, subnet.Spec.CIDRBlock) + routeGw := framework.RandomIPs(subnet.Spec.CIDRBlock, "", 1) + nadIPv4Gateway, nadIPv6Gateway := util.SplitStringIP(subnet.Spec.Gateway) + ipv4RouteDst, ipv6RouteDst := util.SplitStringIP(routeDst) + ipv4RouteGw, ipv6RouteGw := util.SplitStringIP(routeGw) + routes := make([]request.Route, 0, 4) + if f.HasIPv4() { + routes = append(routes, request.Route{Gateway: nadIPv4Gateway}) + routes = append(routes, request.Route{Destination: ipv4RouteDst, Gateway: ipv4RouteGw}) + } + if f.HasIPv6() { + routes = append(routes, request.Route{Gateway: nadIPv6Gateway}) + routes = append(routes, request.Route{Destination: ipv6RouteDst, Gateway: ipv6RouteGw}) + } + + config := &netconf.NetConf{ + NetConf: types.NetConf{ + CNIVersion: CNIVersion, + Type: util.CniTypeName, + }, + ServerSocket: "/run/openvswitch/kube-ovn-daemon.sock", + Provider: provider, + Routes: routes, + } + buf, err := json.MarshalIndent(config, "", " ") + framework.ExpectNoError(err) + + ginkgo.By("Creating network attachment definition " + nadName) + nad := framework.MakeNetworkAttachmentDefinition(nadName, namespaceName, string(buf)) + nad = nadClient.Create(nad) + framework.Logf("created network attachment definition config:\n%s", nad.Spec.Config) + + ginkgo.By("Creating pod " + podName) + annotations := map[string]string{"k8s.v1.cni.cncf.io/networks": fmt.Sprintf("%s/%s", nad.Namespace, nad.Name)} + cmd := []string{"sh", "-c", "sleep infinity"} + pod := framework.MakePod(namespaceName, podName, nil, annotations, image, cmd, nil) + pod = podClient.CreateSync(pod) + + ginkgo.By("Retrieving pod routes") + podRoutes, err := iproute.RouteShow("", "", func(cmd ...string) ([]byte, []byte, error) { + return framework.KubectlExec(namespaceName, podName, cmd...) + }) + framework.ExpectNoError(err) + + ginkgo.By("Validating pod routes") + actualRoutes := make([]request.Route, 0, len(podRoutes)) + for _, r := range podRoutes { + if r.Gateway != "" || r.Dst != "" { + actualRoutes = append(actualRoutes, request.Route{Destination: r.Dst, Gateway: r.Gateway}) + } + } + ipv4CIDR, ipv6CIDR := util.SplitStringIP(pod.Annotations[util.CidrAnnotation]) + ipv4Gateway, ipv6Gateway := util.SplitStringIP(pod.Annotations[util.GatewayAnnotation]) + nadIPv4CIDR, nadIPv6CIDR := util.SplitStringIP(subnet.Spec.CIDRBlock) + if f.HasIPv4() { + framework.ExpectContainElement(actualRoutes, request.Route{Destination: ipv4CIDR}) + framework.ExpectNotContainElement(actualRoutes, request.Route{Destination: "default", Gateway: ipv4Gateway}) + framework.ExpectContainElement(actualRoutes, request.Route{Destination: nadIPv4CIDR}) + framework.ExpectContainElement(actualRoutes, request.Route{Destination: "default", Gateway: nadIPv4Gateway}) + framework.ExpectContainElement(actualRoutes, request.Route{Destination: ipv4RouteDst, Gateway: ipv4RouteGw}) + } + if f.HasIPv6() { + framework.ExpectContainElement(actualRoutes, request.Route{Destination: ipv6CIDR}) + framework.ExpectNotContainElement(actualRoutes, request.Route{Destination: "default", Gateway: ipv6Gateway}) + framework.ExpectContainElement(actualRoutes, request.Route{Destination: nadIPv6CIDR}) + framework.ExpectContainElement(actualRoutes, request.Route{Destination: "default", Gateway: nadIPv6Gateway}) + framework.ExpectContainElement(actualRoutes, request.Route{Destination: ipv6RouteDst, Gateway: ipv6RouteGw}) + } + }) + + framework.ConformanceIt("should be able to provide IPAM for macvlan", func() { + provider := fmt.Sprintf("%s.%s", nadName, namespaceName) + + ginkgo.By("Constructing network attachment definition config") + config := &MacvlanNetConf{ + NetConf: netconf.NetConf{ + NetConf: types.NetConf{ + CNIVersion: CNIVersion, + Type: "macvlan", + }, + IPAM: &netconf.IPAMConf{ + Type: util.CniTypeName, + ServerSocket: "/run/openvswitch/kube-ovn-daemon.sock", + Provider: provider, + }, + }, + Master: "eth0", + Mode: "bridge", + LinkContNs: true, + } + buf, err := json.MarshalIndent(config, "", " ") + framework.ExpectNoError(err) + + ginkgo.By("Creating network attachment definition " + nadName) + nad := framework.MakeNetworkAttachmentDefinition(nadName, namespaceName, string(buf)) + nad = nadClient.Create(nad) + framework.Logf("created network attachment definition config:\n%s", nad.Spec.Config) + + ginkgo.By("Creating subnet " + subnetName) + subnet = framework.MakeSubnet(subnetName, "", cidr, "", "", "", nil, nil, nil) + subnet.Spec.Provider = provider + subnet = subnetClient.CreateSync(subnet) + + ginkgo.By("Creating pod " + podName) + annotations := map[string]string{"k8s.v1.cni.cncf.io/networks": fmt.Sprintf("%s/%s", nad.Namespace, nad.Name)} + cmd := []string{"sh", "-c", "sleep infinity"} + pod := framework.MakePod(namespaceName, podName, nil, annotations, image, cmd, nil) + pod = podClient.CreateSync(pod) + + ginkgo.By("Retrieving pod routes") + podRoutes, err := iproute.RouteShow("", "", func(cmd ...string) ([]byte, []byte, error) { + return framework.KubectlExec(namespaceName, podName, cmd...) + }) + framework.ExpectNoError(err) + + ginkgo.By("Validating pod routes") + actualRoutes := make([]request.Route, 0, len(podRoutes)) + for _, r := range podRoutes { + if r.Gateway != "" || r.Dst != "" { + actualRoutes = append(actualRoutes, request.Route{Destination: r.Dst, Gateway: r.Gateway}) + } + } + ipv4CIDR, ipv6CIDR := util.SplitStringIP(pod.Annotations[util.CidrAnnotation]) + ipv4Gateway, ipv6Gateway := util.SplitStringIP(pod.Annotations[util.GatewayAnnotation]) + nadIPv4CIDR, nadIPv6CIDR := util.SplitStringIP(subnet.Spec.CIDRBlock) + nadIPv4Gateway, nadIPv6Gateway := util.SplitStringIP(subnet.Spec.Gateway) + if f.HasIPv4() { + framework.ExpectContainElement(actualRoutes, request.Route{Destination: ipv4CIDR}) + framework.ExpectContainElement(actualRoutes, request.Route{Destination: "default", Gateway: ipv4Gateway}) + framework.ExpectContainElement(actualRoutes, request.Route{Destination: nadIPv4CIDR}) + framework.ExpectNotContainElement(actualRoutes, request.Route{Destination: "default", Gateway: nadIPv4Gateway}) + } + if f.HasIPv6() { + framework.ExpectContainElement(actualRoutes, request.Route{Destination: ipv6CIDR}) + framework.ExpectContainElement(actualRoutes, request.Route{Destination: "default", Gateway: ipv6Gateway}) + framework.ExpectContainElement(actualRoutes, request.Route{Destination: nadIPv6CIDR}) + framework.ExpectNotContainElement(actualRoutes, request.Route{Destination: "default", Gateway: nadIPv6Gateway}) + } + }) + + framework.ConformanceIt("should be able to provide IPAM with custom routes for macvlan", func() { + provider := fmt.Sprintf("%s.%s", nadName, namespaceName) + + ginkgo.By("Creating subnet " + subnetName) + subnet = framework.MakeSubnet(subnetName, "", cidr, "", "", "", nil, nil, nil) + subnet.Spec.Provider = provider + subnet = subnetClient.CreateSync(subnet) + + ginkgo.By("Constructing network attachment definition config") + var routeDst string + for i := 0; i < 3; i++ { + routeDst = framework.RandomCIDR(f.ClusterIPFamily) + if routeDst != subnet.Spec.CIDRBlock { + break + } + } + framework.ExpectNotEqual(routeDst, subnet.Spec.CIDRBlock) + routeGw := framework.RandomIPs(subnet.Spec.CIDRBlock, "", 1) + nadIPv4Gateway, nadIPv6Gateway := util.SplitStringIP(subnet.Spec.Gateway) + ipv4RouteDst, ipv6RouteDst := util.SplitStringIP(routeDst) + ipv4RouteGw, ipv6RouteGw := util.SplitStringIP(routeGw) + routes := make([]request.Route, 0, 4) + if f.HasIPv4() { + routes = append(routes, request.Route{Gateway: nadIPv4Gateway}) + routes = append(routes, request.Route{Destination: ipv4RouteDst, Gateway: ipv4RouteGw}) + } + if f.HasIPv6() { + routes = append(routes, request.Route{Gateway: nadIPv6Gateway}) + routes = append(routes, request.Route{Destination: ipv6RouteDst, Gateway: ipv6RouteGw}) + } + + config := &MacvlanNetConf{ + NetConf: netconf.NetConf{ + NetConf: types.NetConf{ + CNIVersion: CNIVersion, + Type: "macvlan", + }, + IPAM: &netconf.IPAMConf{ + Type: util.CniTypeName, + ServerSocket: "/run/openvswitch/kube-ovn-daemon.sock", + Provider: provider, + Routes: routes, + }, + }, + Master: "eth0", + Mode: "bridge", + LinkContNs: true, + } + buf, err := json.MarshalIndent(config, "", " ") + framework.ExpectNoError(err) + + ginkgo.By("Creating network attachment definition " + nadName) + nad := framework.MakeNetworkAttachmentDefinition(nadName, namespaceName, string(buf)) + nad = nadClient.Create(nad) + framework.Logf("created network attachment definition config:\n%s", nad.Spec.Config) + + ginkgo.By("Creating pod " + podName) + annotations := map[string]string{"k8s.v1.cni.cncf.io/networks": fmt.Sprintf("%s/%s", nad.Namespace, nad.Name)} + cmd := []string{"sh", "-c", "sleep infinity"} + pod := framework.MakePod(namespaceName, podName, nil, annotations, image, cmd, nil) + pod = podClient.CreateSync(pod) + + ginkgo.By("Retrieving pod routes") + podRoutes, err := iproute.RouteShow("", "", func(cmd ...string) ([]byte, []byte, error) { + return framework.KubectlExec(namespaceName, podName, cmd...) + }) + framework.ExpectNoError(err) + + ginkgo.By("Validating pod routes") + actualRoutes := make([]request.Route, 0, len(podRoutes)) + for _, r := range podRoutes { + if r.Gateway != "" || r.Dst != "" { + actualRoutes = append(actualRoutes, request.Route{Destination: r.Dst, Gateway: r.Gateway}) + } + } + ipv4CIDR, ipv6CIDR := util.SplitStringIP(pod.Annotations[util.CidrAnnotation]) + ipv4Gateway, ipv6Gateway := util.SplitStringIP(pod.Annotations[util.GatewayAnnotation]) + nadIPv4CIDR, nadIPv6CIDR := util.SplitStringIP(subnet.Spec.CIDRBlock) + if f.HasIPv4() { + framework.ExpectContainElement(actualRoutes, request.Route{Destination: ipv4CIDR}) + framework.ExpectNotContainElement(actualRoutes, request.Route{Destination: "default", Gateway: ipv4Gateway}) + framework.ExpectContainElement(actualRoutes, request.Route{Destination: nadIPv4CIDR}) + framework.ExpectContainElement(actualRoutes, request.Route{Destination: "default", Gateway: nadIPv4Gateway}) + framework.ExpectContainElement(actualRoutes, request.Route{Destination: ipv4RouteDst, Gateway: ipv4RouteGw}) + } + if f.HasIPv6() { + framework.ExpectContainElement(actualRoutes, request.Route{Destination: ipv6CIDR}) + framework.ExpectNotContainElement(actualRoutes, request.Route{Destination: "default", Gateway: ipv6Gateway}) + framework.ExpectContainElement(actualRoutes, request.Route{Destination: nadIPv6CIDR}) + framework.ExpectContainElement(actualRoutes, request.Route{Destination: "default", Gateway: nadIPv6Gateway}) + framework.ExpectContainElement(actualRoutes, request.Route{Destination: ipv6RouteDst, Gateway: ipv6RouteGw}) + } + }) +})