From 6db8c840b159e0d710487ff1c0b80ed7118eba8e Mon Sep 17 00:00:00 2001 From: bobz965 Date: Wed, 31 Jan 2024 14:13:58 +0800 Subject: [PATCH] Support ip creation (#3667) * support ip creation * merge node gateway static routes handling * fix webhook --------- Signed-off-by: bobz965 --- pkg/controller/controller.go | 15 +- pkg/controller/external_gw.go | 2 +- pkg/controller/gc.go | 35 ++-- pkg/controller/init.go | 25 ++- pkg/controller/ip.go | 289 ++++++++++++++++++++++++++++- pkg/controller/node.go | 103 +--------- pkg/controller/ovn_eip.go | 4 +- pkg/controller/pod.go | 84 +++++---- pkg/controller/pod_iptables_eip.go | 4 +- pkg/controller/pod_iptables_fip.go | 6 +- pkg/controller/subnet.go | 11 +- pkg/controller/vip.go | 73 +------- pkg/controller/vpc.go | 22 +-- pkg/controller/vpc_nat_gw_eip.go | 4 +- pkg/controller/vpc_nat_gw_nat.go | 2 +- pkg/daemon/handler.go | 26 +-- pkg/ipam/ipam.go | 6 + pkg/util/const.go | 2 + pkg/util/net.go | 18 +- pkg/webhook/ip.go | 116 ++++++++++++ pkg/webhook/static_ip.go | 10 +- pkg/webhook/webhook.go | 3 + test/e2e/webhook/pod/pod.go | 6 +- 23 files changed, 553 insertions(+), 313 deletions(-) create mode 100644 pkg/webhook/ip.go diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 149ff346c20..46cb71c5c0b 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -798,7 +798,7 @@ func (c *Controller) Run(ctx context.Context) { } // sync ip crd before initIPAM since ip crd will be used to restore vm and statefulset pod in initIPAM - if err := c.initSyncCrdIPs(); err != nil { + if err := c.syncIPCR(); err != nil { util.LogFatalAndExit(err, "failed to sync crd ips") } @@ -806,19 +806,16 @@ func (c *Controller) Run(ctx context.Context) { util.LogFatalAndExit(err, "failed to initialize ipam") } - if err := c.initNodeRoutes(); err != nil { + if err := c.syncNodeRoutes(); err != nil { util.LogFatalAndExit(err, "failed to initialize node routes") } - if err := c.initSyncCrdSubnets(); err != nil { + if err := c.syncSubnetCR(); err != nil { util.LogFatalAndExit(err, "failed to sync crd subnets") } - if err := c.initSyncCrdVlans(); err != nil { - util.LogFatalAndExit(err, "failed to sync crd vlans") - } - if err := c.addNodeGwStaticRoute(); err != nil { - util.LogFatalAndExit(err, "failed to add static route for node gateway") + if err := c.syncVlanCR(); err != nil { + util.LogFatalAndExit(err, "failed to sync crd vlans") } // start workers to do all the network operations @@ -1167,7 +1164,7 @@ func (c *Controller) initResourceOnce() { } if c.config.PodDefaultFipType == util.IptablesFip { - if err := c.initSyncCrdVpcNatGw(); err != nil { + if err := c.syncVpcNatGatewayCR(); err != nil { util.LogFatalAndExit(err, "failed to sync crd vpc nat gateways") } } diff --git a/pkg/controller/external_gw.go b/pkg/controller/external_gw.go index 79296d707c2..8659eaf5b7e 100644 --- a/pkg/controller/external_gw.go +++ b/pkg/controller/external_gw.go @@ -203,7 +203,7 @@ func (c *Controller) createDefaultVpcLrpEip() (string, string, error) { klog.Errorf("failed to acquire ip address for default vpc lrp %s, %v", lrpEipName, err) return "", "", err } - if err := c.createOrUpdateCrdOvnEip(lrpEipName, c.config.ExternalGatewaySwitch, v4ip, v6ip, mac, util.OvnEipTypeLRP); err != nil { + if err := c.createOrUpdateOvnEipCR(lrpEipName, c.config.ExternalGatewaySwitch, v4ip, v6ip, mac, util.OvnEipTypeLRP); err != nil { klog.Errorf("failed to create ovn eip cr for lrp %s, %v", lrpEipName, err) return "", "", err } diff --git a/pkg/controller/gc.go b/pkg/controller/gc.go index eb0f6ebffdc..beee37fc092 100644 --- a/pkg/controller/gc.go +++ b/pkg/controller/gc.go @@ -358,8 +358,7 @@ func (c *Controller) markAndCleanLSP() error { klog.Errorf("failed to delete lsp %s: %v", lsp.Name, err) return err } - klog.Infof("gc ip %s", lsp.Name) - ipCr, err := c.config.KubeOvnClient.KubeovnV1().IPs().Get(context.Background(), lsp.Name, metav1.GetOptions{}) + ipCR, err := c.config.KubeOvnClient.KubeovnV1().IPs().Get(context.Background(), lsp.Name, metav1.GetOptions{}) if err != nil { if k8serrors.IsNotFound(err) { // ip cr not found, skip lsp gc @@ -368,22 +367,26 @@ func (c *Controller) markAndCleanLSP() error { klog.Errorf("failed to get ip %s, %v", lsp.Name, err) return err } - klog.Infof("gc ip %s", ipCr.Name) - if err := c.config.KubeOvnClient.KubeovnV1().IPs().Delete(context.Background(), ipCr.Name, metav1.DeleteOptions{}); err != nil { - if k8serrors.IsNotFound(err) { - // ip cr not found, skip lsp gc + if ipCR.Labels[util.IPReservedLabel] != "true" { + klog.Infof("gc ip %s", ipCR.Name) + if err := c.config.KubeOvnClient.KubeovnV1().IPs().Delete(context.Background(), ipCR.Name, metav1.DeleteOptions{}); err != nil { + if k8serrors.IsNotFound(err) { + // ip cr not found, skip lsp gc + continue + } + klog.Errorf("failed to delete ip %s, %v", ipCR.Name, err) + return err + } + if ipCR.Spec.Subnet == "" { + klog.Errorf("ip %s has no subnet", ipCR.Name) + // ip cr no subnet, skip lsp gc continue } - klog.Errorf("failed to delete ip %s, %v", ipCr.Name, err) - return err - } - if ipCr.Spec.Subnet == "" { - klog.Errorf("ip %s has no subnet", ipCr.Name) - // ip cr no subnet, skip lsp gc - continue - } - if key := lsp.ExternalIDs["pod"]; key != "" { - c.ipam.ReleaseAddressByPod(key, ipCr.Spec.Subnet) + if key := lsp.ExternalIDs["pod"]; key != "" { + c.ipam.ReleaseAddressByPod(key, ipCR.Spec.Subnet) + } + } else { + klog.Infof("gc skip reserved ip %s", ipCR.Name) } } lastNoPodLSP = noPodLSP diff --git a/pkg/controller/init.go b/pkg/controller/init.go index d2c3de85226..91f5406b377 100644 --- a/pkg/controller/init.go +++ b/pkg/controller/init.go @@ -321,7 +321,7 @@ func (c *Controller) InitIPAM() error { for _, ip := range ips { // recover sts and kubevirt vm ip, other ip recover in later pod loop - if ip.Spec.PodType != "StatefulSet" && ip.Spec.PodType != util.VM { + if ip.Spec.PodType != util.StatefulSet && ip.Spec.PodType != util.VM { continue } @@ -365,7 +365,7 @@ func (c *Controller) InitIPAM() error { if err != nil { klog.Errorf("failed to init pod %s.%s address %s: %v", podName, pod.Namespace, pod.Annotations[fmt.Sprintf(util.IPAddressAnnotationTemplate, podNet.ProviderName)], err) } else { - err = c.createOrUpdateCrdIPs(podName, ip, mac, podNet.Subnet.Name, pod.Namespace, pod.Spec.NodeName, podNet.ProviderName, podType) + err = c.createOrUpdateIPCR(portName, podName, ip, mac, podNet.Subnet.Name, pod.Namespace, pod.Spec.NodeName, podType) if err != nil { klog.Errorf("failed to create/update ips CR %s.%s with ip address %s: %v", podName, pod.Namespace, ip, err) } @@ -563,7 +563,7 @@ func (c *Controller) initDefaultVlan() error { return nil } -func (c *Controller) initSyncCrdIPs() error { +func (c *Controller) syncIPCR() error { klog.Info("start to sync ips") ips, err := c.ipsLister.List(labels.Everything()) if err != nil { @@ -575,8 +575,8 @@ func (c *Controller) initSyncCrdIPs() error { ipMap := strset.New(c.getVMLsps()...) - for _, ipCr := range ips { - ip := ipCr.DeepCopy() + for _, ipCR := range ips { + ip := ipCR.DeepCopy() changed := false if ipMap.Has(ip.Name) && ip.Spec.PodType == "" { ip.Spec.PodType = util.VM @@ -587,9 +587,9 @@ func (c *Controller) initSyncCrdIPs() error { if ip.Spec.V4IPAddress == v4IP && ip.Spec.V6IPAddress == v6IP && !changed { continue } + ip.Spec.V4IPAddress = v4IP ip.Spec.V6IPAddress = v6IP - _, err := c.config.KubeOvnClient.KubeovnV1().IPs().Update(context.Background(), ip, metav1.UpdateOptions{}) if err != nil { klog.Errorf("failed to sync crd ip %s: %v", ip.Spec.IPAddress, err) @@ -599,7 +599,7 @@ func (c *Controller) initSyncCrdIPs() error { return nil } -func (c *Controller) initSyncCrdSubnets() error { +func (c *Controller) syncSubnetCR() error { klog.Info("start to sync subnets") subnets, err := c.subnetsLister.List(labels.Everything()) if err != nil { @@ -639,7 +639,7 @@ func (c *Controller) initSyncCrdSubnets() error { return nil } -func (c *Controller) initSyncCrdVpcNatGw() error { +func (c *Controller) syncVpcNatGatewayCR() error { klog.Info("start to sync crd vpc nat gw") // get vpc nat gateway enable state cm, err := c.configMapsLister.ConfigMaps(c.config.PodNamespace).Get(util.VpcNatGatewayConfig) @@ -681,7 +681,7 @@ func (c *Controller) initSyncCrdVpcNatGw() error { return nil } -func (c *Controller) initSyncCrdVlans() error { +func (c *Controller) syncVlanCR() error { klog.Info("start to sync vlans") vlans, err := c.vlansLister.List(labels.Everything()) if err != nil { @@ -717,6 +717,7 @@ func (c *Controller) initSyncCrdVlans() error { } func (c *Controller) migrateNodeRoute(af int, node, ip, nexthop string) error { + // default vpc use static route in old version, migrate to policy route var ( match = fmt.Sprintf("ip%d.dst == %s", af, ip) action = kubeovnv1.PolicyRouteActionReroute @@ -768,7 +769,7 @@ func (c *Controller) migrateNodeRoute(af int, node, ip, nexthop string) error { return nil } -func (c *Controller) initNodeRoutes() error { +func (c *Controller) syncNodeRoutes() error { nodes, err := c.nodesLister.List(labels.Everything()) if err != nil { klog.Errorf("failed to list nodes: %v", err) @@ -792,6 +793,10 @@ func (c *Controller) initNodeRoutes() error { } } + if err := c.addNodeGatewayStaticRoute(); err != nil { + klog.Errorf("failed to add static route for node gateway") + return err + } return nil } diff --git a/pkg/controller/ip.go b/pkg/controller/ip.go index b3fc8ebd4ae..f5ad6351372 100644 --- a/pkg/controller/ip.go +++ b/pkg/controller/ip.go @@ -2,7 +2,9 @@ package controller import ( "context" + "encoding/json" "fmt" + "net" "reflect" "strings" @@ -15,6 +17,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/ovs" "github.com/kubeovn/kube-ovn/pkg/util" ) @@ -107,7 +110,7 @@ func (c *Controller) processNextAddIPWorkItem() bool { utilruntime.HandleError(fmt.Errorf("expected string in workqueue but got %#v", obj)) return nil } - if err := c.handleAddIP(key); err != nil { + if err := c.handleAddReservedIP(key); err != nil { c.addIPQueue.AddRateLimited(key) return fmt.Errorf("error syncing '%s': %s, requeuing", key, err.Error()) } @@ -179,19 +182,88 @@ func (c *Controller) processNextDeleteIPWorkItem() bool { return true } -func (c *Controller) handleAddIP(key string) error { - cachedIP, err := c.ipsLister.Get(key) +func (c *Controller) handleAddReservedIP(key string) error { + ip, err := c.ipsLister.Get(key) if err != nil { if k8serrors.IsNotFound(err) { return nil } return err } - klog.V(3).Infof("handle add ip %s", cachedIP.Name) - if err := c.handleAddIPFinalizer(cachedIP, util.ControllerName); err != nil { - klog.Errorf("failed to handle add ip finalizer %v", err) + klog.V(3).Infof("handle add reserved ip %s", ip.Name) + if ip.Spec.Subnet == "" { + err := fmt.Errorf("subnet parameter cannot be empty") + klog.Error(err) + return err + } + if ip.Spec.PodType != "" && ip.Spec.PodType != util.VM && ip.Spec.PodType != util.StatefulSet { + err := fmt.Errorf("podType %s is not supported", ip.Spec.PodType) + klog.Error(err) + return err + } + + subnet, err := c.subnetsLister.Get(ip.Spec.Subnet) + if err != nil { + err = fmt.Errorf("failed to get subnet %s: %v", ip.Spec.Subnet, err) + klog.Error(err) + return err + } + + portName := ovs.PodNameToPortName(ip.Spec.PodName, ip.Spec.Namespace, subnet.Spec.Provider) + if portName != ip.Name { + // invalid ip or node ip, no need to handle it here + klog.V(3).Infof("port name %s is not equal to ip name %s", portName, ip.Name) + return nil + } + + // not handle add the ip, which created in pod process, lsp created before ip + lsp, err := c.OVNNbClient.GetLogicalSwitchPort(portName, true) + if err != nil { + klog.Errorf("failed to list logical switch ports %s, %v", portName, err) + return err + } + if lsp != nil { + // port already exists means the ip already created + klog.V(3).Infof("ip %s is ready", portName) + return nil + } + + v4IP, v6IP, mac, err := c.ipAcquireAddress(ip, subnet) + if err != nil { + err = fmt.Errorf("failed to acquire ip address %v", err) + klog.Error(err) + return err + } + ipStr := util.GetStringIP(v4IP, v6IP) + if err := c.createOrUpdateIPCR(ip.Name, ip.Spec.PodName, ipStr, mac, subnet.Name, ip.Spec.Namespace, ip.Spec.NodeName, ip.Spec.PodType); err != nil { + err = fmt.Errorf("failed to create ips CR %s.%s: %v", ip.Spec.PodName, ip.Spec.Namespace, err) + klog.Error(err) return err } + if ip.Labels[util.IPReservedLabel] != "false" { + cachedIP, err := c.ipsLister.Get(key) + if err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + return err + } + ip = cachedIP.DeepCopy() + ip.Labels[util.IPReservedLabel] = "true" + patchPayloadTemplate := `[{ "op": "%s", "path": "/metadata/labels", "value": %s }]` + raw, err := json.Marshal(ip.Labels) + if err != nil { + klog.Error(err) + return err + } + op := "replace" + patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw) + if _, err := c.config.KubeOvnClient.KubeovnV1().IPs().Patch(context.Background(), ip.Name, + types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{}); err != nil { + klog.Errorf("failed to patch label for ip %s, %v", ip.Name, err) + return err + } + } return nil } @@ -245,7 +317,7 @@ func (c *Controller) handleUpdateIP(key string) error { } } if cleanIPAM { - klog.V(3).Infof("release ipam for deleted ip %s from subnet %s", cachedIP.Name, cachedIP.Spec.Subnet) + klog.V(3).Infof("release ipam address %s for deleted ip %s from subnet %s", cachedIP.Spec.IPAddress, cachedIP.Name, cachedIP.Spec.Subnet) c.ipam.ReleaseAddressByPod(cachedIP.Name, cachedIP.Spec.Subnet) } if err = c.handleDelIPFinalizer(cachedIP, util.ControllerName); err != nil { @@ -257,8 +329,7 @@ func (c *Controller) handleUpdateIP(key string) error { } func (c *Controller) handleDelIP(ip *kubeovnv1.IP) error { - klog.V(3).Infof("handle delete ip %s", ip.Name) - klog.V(3).Infof("enqueue update status subnet %s", ip.Spec.Subnet) + klog.V(3).Infof("handle delete ip %s from subnet %s", ip.Name, ip.Spec.Subnet) c.updateSubnetStatusQueue.Add(ip.Spec.Subnet) for _, as := range ip.Spec.AttachSubnets { klog.V(3).Infof("enqueue update attach status for subnet %s", as) @@ -312,3 +383,203 @@ func (c *Controller) handleDelIPFinalizer(cachedIP *kubeovnv1.IP, finalizer stri } return nil } + +func (c *Controller) acquireIPAddress(subnetName, name, nicName string) (string, string, string, error) { + var skippedAddrs []string + var v4ip, v6ip, mac string + checkConflict := true + var err error + for { + v4ip, v6ip, mac, err = c.ipam.GetRandomAddress(name, nicName, nil, subnetName, "", skippedAddrs, checkConflict) + if err != nil { + klog.Error(err) + return "", "", "", err + } + + ipv4OK, ipv6OK, err := c.validatePodIP(name, subnetName, v4ip, v6ip) + if err != nil { + klog.Error(err) + return "", "", "", err + } + + if ipv4OK && ipv6OK { + return v4ip, v6ip, mac, nil + } + + if !ipv4OK { + skippedAddrs = append(skippedAddrs, v4ip) + } + if !ipv6OK { + skippedAddrs = append(skippedAddrs, v6ip) + } + } +} + +func (c *Controller) acquireStaticIPAddress(subnetName, name, nicName, ip string) (string, string, string, error) { + checkConflict := true + var v4ip, v6ip, mac string + var err error + for _, ipStr := range strings.Split(ip, ",") { + if net.ParseIP(ipStr) == nil { + return "", "", "", fmt.Errorf("failed to parse vip ip %s", ipStr) + } + } + + if v4ip, v6ip, mac, err = c.ipam.GetStaticAddress(name, nicName, ip, nil, subnetName, checkConflict); err != nil { + klog.Errorf("failed to get static virtual ip '%s', mac '%s', subnet '%s', %v", ip, mac, subnetName, err) + return "", "", "", err + } + return v4ip, v6ip, mac, nil +} + +func (c *Controller) createOrUpdateIPCR(ipCRName, podName, ip, mac, subnetName, ns, nodeName, podType string) error { + // `ipCRName`: pod or vm IP name must set ip CR name when creating ip CR + var key, ipName string + if ipCRName != "" { + // pod IP + key = podName + ipName = ipCRName + } else { + // node IP or interconn IP + switch { + case subnetName == c.config.NodeSwitch: + key = nodeName + ipName = fmt.Sprintf("node-%s", nodeName) + case strings.HasPrefix(podName, util.U2OInterconnName[0:19]): + key = podName // interconn IP name + ipName = podName + } + } + + var err error + var ipCR *kubeovnv1.IP + ipCR, err = c.ipsLister.Get(ipName) + if err != nil { + if !k8serrors.IsNotFound(err) { + err := fmt.Errorf("failed to get ip CR %s: %v", ipName, err) + klog.Error(err) + return err + } + // the returned pointer is not nil if the CR does not exist + ipCR = nil + } + + v4IP, v6IP := util.SplitStringIP(ip) + if ipCR == nil { + ipCR, err = c.config.KubeOvnClient.KubeovnV1().IPs().Create(context.Background(), &kubeovnv1.IP{ + ObjectMeta: metav1.ObjectMeta{ + Name: ipName, + Labels: map[string]string{ + util.SubnetNameLabel: subnetName, + util.NodeNameLabel: nodeName, + subnetName: "", + util.IPReservedLabel: "false", // ip create with pod or node, ip not reserved + }, + }, + Spec: kubeovnv1.IPSpec{ + PodName: key, + Subnet: subnetName, + NodeName: nodeName, + Namespace: ns, + IPAddress: ip, + V4IPAddress: v4IP, + V6IPAddress: v6IP, + MacAddress: mac, + AttachIPs: []string{}, + AttachMacs: []string{}, + AttachSubnets: []string{}, + PodType: podType, + }, + }, metav1.CreateOptions{}) + if err != nil { + errMsg := fmt.Errorf("failed to create ip CR %s: %v", ipName, err) + klog.Error(errMsg) + return errMsg + } + } else { + newIPCR := ipCR.DeepCopy() + if newIPCR.Labels != nil { + newIPCR.Labels[util.SubnetNameLabel] = subnetName + newIPCR.Labels[util.NodeNameLabel] = nodeName + } else { + newIPCR.Labels = map[string]string{ + util.SubnetNameLabel: subnetName, + util.NodeNameLabel: nodeName, + } + // update not touch IP Reserved Label + } + newIPCR.Spec.PodName = key + newIPCR.Spec.Namespace = ns + newIPCR.Spec.Subnet = subnetName + newIPCR.Spec.NodeName = nodeName + newIPCR.Spec.IPAddress = ip + newIPCR.Spec.V4IPAddress = v4IP + newIPCR.Spec.V6IPAddress = v6IP + newIPCR.Spec.MacAddress = mac + newIPCR.Spec.AttachIPs = []string{} + newIPCR.Spec.AttachMacs = []string{} + newIPCR.Spec.AttachSubnets = []string{} + newIPCR.Spec.PodType = podType + if reflect.DeepEqual(newIPCR.Labels, ipCR.Labels) && reflect.DeepEqual(newIPCR.Spec, ipCR.Spec) { + return nil + } + + ipCR, err = c.config.KubeOvnClient.KubeovnV1().IPs().Update(context.Background(), newIPCR, metav1.UpdateOptions{}) + if err != nil { + err := fmt.Errorf("failed to update ip CR %s: %v", newIPCR.Name, err) + klog.Error(err) + return err + } + } + + if err := c.handleAddIPFinalizer(ipCR, util.ControllerName); err != nil { + klog.Errorf("failed to handle add ip finalizer %v", err) + return err + } + + return nil +} + +func (c *Controller) subnetCountIP(subnet *kubeovnv1.Subnet) error { + var err error + if util.CheckProtocol(subnet.Spec.CIDRBlock) == kubeovnv1.ProtocolDual { + _, err = c.calcDualSubnetStatusIP(subnet) + } else { + _, err = c.calcSubnetStatusIP(subnet) + } + if err != nil { + klog.Error(err) + return err + } + return nil +} + +func (c *Controller) ipAcquireAddress(ip *kubeovnv1.IP, subnet *kubeovnv1.Subnet) (string, string, string, error) { + key := fmt.Sprintf("%s/%s", ip.Spec.Namespace, ip.Spec.PodName) + portName := ovs.PodNameToPortName(ip.Spec.PodName, ip.Spec.Namespace, subnet.Spec.Provider) + ipStr := util.GetStringIP(ip.Spec.V4IPAddress, ip.Spec.V6IPAddress) + + var v4IP, v6IP, mac string + var err error + if ipStr == "" { + // allocate address + v4IP, v6IP, mac, err = c.acquireIPAddress(subnet.Name, ip.Name, portName) + if err == nil { + return v4IP, v6IP, mac, err + } + err = fmt.Errorf("failed to get random address for ip %s, %v", ip.Name, err) + } else { + // static address + if ip.Spec.MacAddress == "" { + v4IP, v6IP, mac, err = c.acquireStaticAddress(key, portName, ipStr, nil, subnet.Name, true) + } else { + v4IP, v6IP, mac, err = c.acquireStaticAddress(key, portName, ipStr, &ip.Spec.MacAddress, subnet.Name, true) + } + if err == nil { + return v4IP, v6IP, mac, nil + } + err = fmt.Errorf("failed to get static address for ip %s, %v", ip.Name, err) + } + klog.Error(err) + return "", "", "", err +} diff --git a/pkg/controller/node.go b/pkg/controller/node.go index cb4de2997c0..4ace3c76a12 100644 --- a/pkg/controller/node.go +++ b/pkg/controller/node.go @@ -305,7 +305,7 @@ func (c *Controller) handleAddNode(key string) error { } } - if err := c.addNodeGwStaticRoute(); err != nil { + if err := c.addNodeGatewayStaticRoute(); err != nil { klog.Errorf("failed to add static route for node gw: %v", err) return err } @@ -336,7 +336,7 @@ func (c *Controller) handleAddNode(key string) error { return err } - if err := c.createOrUpdateCrdIPs("", ipStr, mac, c.config.NodeSwitch, "", node.Name, "", ""); err != nil { + if err := c.createOrUpdateIPCR("", "", ipStr, mac, c.config.NodeSwitch, "", node.Name, ""); err != nil { klog.Errorf("failed to create or update IPs node-%s: %v", key, err) return err } @@ -643,103 +643,6 @@ func (c *Controller) handleUpdateNode(key string) error { return nil } -func (c *Controller) createOrUpdateCrdIPs(podName, ip, mac, subnetName, ns, nodeName, providerName, podType string) error { - var key, ipName string - - switch { - case subnetName == c.config.NodeSwitch: - key = nodeName - ipName = fmt.Sprintf("node-%s", nodeName) - case strings.HasPrefix(podName, util.U2OInterconnName[0:19]): - key = podName - ipName = podName - default: - key = podName - ipName = ovs.PodNameToPortName(podName, ns, providerName) - } - - var err error - var ipCr *kubeovnv1.IP - ipCr, err = c.ipsLister.Get(ipName) - if err != nil { - if !k8serrors.IsNotFound(err) { - errMsg := fmt.Errorf("failed to get ip CR %s: %v", ipName, err) - klog.Error(errMsg) - return errMsg - } - // the returned pointer is not nil if the CR does not exist - ipCr = nil - } - - v4IP, v6IP := util.SplitStringIP(ip) - if ipCr == nil { - _, err = c.config.KubeOvnClient.KubeovnV1().IPs().Create(context.Background(), &kubeovnv1.IP{ - ObjectMeta: metav1.ObjectMeta{ - Name: ipName, - Labels: map[string]string{ - util.SubnetNameLabel: subnetName, - util.NodeNameLabel: nodeName, - subnetName: "", - }, - }, - Spec: kubeovnv1.IPSpec{ - PodName: key, - Subnet: subnetName, - NodeName: nodeName, - Namespace: ns, - IPAddress: ip, - V4IPAddress: v4IP, - V6IPAddress: v6IP, - MacAddress: mac, - AttachIPs: []string{}, - AttachMacs: []string{}, - AttachSubnets: []string{}, - PodType: podType, - }, - }, metav1.CreateOptions{}) - if err != nil { - errMsg := fmt.Errorf("failed to create ip CR %s: %v", ipName, err) - klog.Error(errMsg) - return errMsg - } - } else { - newIPCr := ipCr.DeepCopy() - if newIPCr.Labels != nil { - newIPCr.Labels[util.SubnetNameLabel] = subnetName - newIPCr.Labels[util.NodeNameLabel] = nodeName - } else { - newIPCr.Labels = map[string]string{ - util.SubnetNameLabel: subnetName, - util.NodeNameLabel: nodeName, - } - } - newIPCr.Spec.PodName = key - newIPCr.Spec.Namespace = ns - newIPCr.Spec.Subnet = subnetName - newIPCr.Spec.NodeName = nodeName - newIPCr.Spec.IPAddress = ip - newIPCr.Spec.V4IPAddress = v4IP - newIPCr.Spec.V6IPAddress = v6IP - newIPCr.Spec.MacAddress = mac - newIPCr.Spec.AttachIPs = []string{} - newIPCr.Spec.AttachMacs = []string{} - newIPCr.Spec.AttachSubnets = []string{} - newIPCr.Spec.PodType = podType - if reflect.DeepEqual(newIPCr.Labels, ipCr.Labels) && reflect.DeepEqual(newIPCr.Spec, ipCr.Spec) { - return nil - } - - _, err := c.config.KubeOvnClient.KubeovnV1().IPs().Update(context.Background(), newIPCr, metav1.UpdateOptions{}) - if err != nil { - errMsg := fmt.Errorf("failed to update ip CR %s: %v", newIPCr.Name, err) - klog.Error(errMsg) - return errMsg - } - } - - return nil -} - func (c *Controller) CheckGatewayReady() { if err := c.checkGatewayReady(); err != nil { klog.Errorf("failed to check gateway ready %v", err) @@ -1017,7 +920,7 @@ func (c *Controller) UpdateChassisTag(node *v1.Node) error { return nil } -func (c *Controller) addNodeGwStaticRoute() error { +func (c *Controller) addNodeGatewayStaticRoute() error { // If user not manage static route for default vpc, just add route about ovn-default to join if vpc, err := c.vpcsLister.Get(c.config.ClusterRouter); err != nil || vpc.Spec.StaticRoutes != nil { existRoute, err := c.OVNNbClient.ListLogicalRouterStaticRoutes(c.config.ClusterRouter, nil, nil, "", nil) diff --git a/pkg/controller/ovn_eip.go b/pkg/controller/ovn_eip.go index 8e602564b8e..c17fa3f38c3 100644 --- a/pkg/controller/ovn_eip.go +++ b/pkg/controller/ovn_eip.go @@ -256,7 +256,7 @@ func (c *Controller) handleAddOvnEip(key string) error { cachedEip.Spec.Type = util.OvnEipTypeNAT } - if err = c.createOrUpdateCrdOvnEip(key, subnet.Name, v4ip, v6ip, mac, cachedEip.Spec.Type); err != nil { + if err = c.createOrUpdateOvnEipCR(key, subnet.Name, v4ip, v6ip, mac, cachedEip.Spec.Type); err != nil { klog.Errorf("failed to create or update ovn eip '%s', %v", cachedEip.Name, err) return err } @@ -351,7 +351,7 @@ func (c *Controller) handleDelOvnEip(key string) error { return nil } -func (c *Controller) createOrUpdateCrdOvnEip(key, subnet, v4ip, v6ip, mac, usage string) error { +func (c *Controller) createOrUpdateOvnEipCR(key, subnet, v4ip, v6ip, mac, usage string) error { cachedEip, err := c.ovnEipsLister.Get(key) if err != nil { if k8serrors.IsNotFound(err) { diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index 7bc79d5a2e0..2f072b14ecc 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -536,7 +536,7 @@ func (c *Controller) getPodKubeovnNets(pod *v1.Pod) ([]*kubeovnNet, error) { func (c *Controller) changeVMSubnet(vmName, namespace, providerName, subnetName string) error { ipName := ovs.PodNameToPortName(vmName, namespace, providerName) - ipCr, err := c.ipsLister.Get(ipName) + ipCR, err := c.ipsLister.Get(ipName) if err != nil { if k8serrors.IsNotFound(err) { return nil @@ -545,10 +545,10 @@ func (c *Controller) changeVMSubnet(vmName, namespace, providerName, subnetName klog.Error(err) return err } - if ipCr.Spec.Subnet != subnetName { + if ipCR.Spec.Subnet != subnetName { key := fmt.Sprintf("%s/%s", namespace, vmName) - klog.Infof("release ipam for vm %s from old subnet %s", key, ipCr.Spec.Subnet) - c.ipam.ReleaseAddressByPod(key, ipCr.Spec.Subnet) + klog.Infof("release ipam for vm %s from old subnet %s", key, ipCR.Spec.Subnet) + c.ipam.ReleaseAddressByPod(key, ipCR.Spec.Subnet) klog.Infof("gc logical switch port %s", key) if err := c.OVNNbClient.DeleteLogicalSwitchPort(key); err != nil { klog.Errorf("failed to delete lsp %s, %v", key, err) @@ -747,8 +747,9 @@ func (c *Controller) reconcileAllocateSubnets(cachedPod, pod *v1.Pod, needAlloca c.syncVirtualPortsQueue.Add(podNet.Subnet.Name) } } - // CreatePort may fail, so put ip cr creation after CreatePort - if err := c.createOrUpdateCrdIPs(podName, ipStr, mac, subnet.Name, pod.Namespace, pod.Spec.NodeName, podNet.ProviderName, podType); err != nil { + // CreatePort may fail, so put ip CR creation after CreatePort + ipCRName := ovs.PodNameToPortName(podName, pod.Namespace, podNet.ProviderName) + if err := c.createOrUpdateIPCR(ipCRName, podName, ipStr, mac, subnet.Name, pod.Namespace, pod.Spec.NodeName, podType); err != nil { err = fmt.Errorf("failed to create ips CR %s.%s: %v", podName, pod.Namespace, err) klog.Error(err) return nil, err @@ -993,7 +994,7 @@ func (c *Controller) handleDeletePod(key string) error { var keepIPCR bool if ok, sts := isStatefulSetPod(pod); ok { toDel := isStatefulSetPodToDel(c.config.KubeClient, pod, sts) - isDelete, err := appendCheckPodToDel(c, pod, sts, "StatefulSet") + isDelete, err := appendCheckPodToDel(c, pod, sts, util.StatefulSet) if pod.DeletionTimestamp != nil { // triggered by delete event if !(toDel || (isDelete && err == nil)) { @@ -1081,13 +1082,29 @@ func (c *Controller) handleDeletePod(key string) error { return err } } - klog.Infof("release all ip address for deleting pod %s", podKey) + klog.Infof("try release all ip address for deleting pod %s", podKey) for _, podNet := range podNets { - if err = c.deleteCrdIPs(podName, pod.Namespace, podNet.ProviderName); err != nil { - klog.Errorf("failed to delete ip for pod %s, %v, please delete manually", pod.Name, err) + portName := ovs.PodNameToPortName(podName, pod.Namespace, podNet.ProviderName) + ipCR, err := c.ipsLister.Get(portName) + if err != nil { + if k8serrors.IsNotFound(err) { + continue + } + klog.Errorf("failed to get ip %s, %v", portName, err) + return err + } + if ipCR.Labels[util.IPReservedLabel] != "true" { + klog.Infof("delete ip CR %s", ipCR.Name) + if err := c.config.KubeOvnClient.KubeovnV1().IPs().Delete(context.Background(), ipCR.Name, metav1.DeleteOptions{}); err != nil { + if !k8serrors.IsNotFound(err) { + klog.Errorf("failed to delete ip %s, %v", ipCR.Name, err) + return err + } + } + // release ipam address after delete ip CR + c.ipam.ReleaseAddressByPod(podKey, podNet.Subnet.Name) } } - c.ipam.ReleaseAddressByPod(podKey, "") if pod.Annotations[util.VipAnnotation] != "" { if err = c.releaseVip(pod.Annotations[util.VipAnnotation]); err != nil { klog.Errorf("failed to clean label from vip %s, %v", pod.Annotations[util.VipAnnotation], err) @@ -1112,18 +1129,6 @@ func (c *Controller) handleDeletePod(key string) error { return nil } -func (c *Controller) deleteCrdIPs(podName, ns, providerName string) error { - portName := ovs.PodNameToPortName(podName, ns, providerName) - klog.Infof("delete cr ip '%s' for pod %s/%s", portName, ns, podName) - if err := c.config.KubeOvnClient.KubeovnV1().IPs().Delete(context.Background(), portName, metav1.DeleteOptions{}); err != nil { - if !k8serrors.IsNotFound(err) { - klog.Errorf("failed to delete ip %s, %v", portName, err) - return err - } - } - return nil -} - func (c *Controller) handleUpdatePodSecurity(key string) error { namespace, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { @@ -1272,7 +1277,7 @@ func (c *Controller) syncKubeOvnNet(cachedPod, pod *v1.Pod, podNets []*kubeovnNe func isStatefulSetPod(pod *v1.Pod) (bool, string) { for _, owner := range pod.OwnerReferences { - if owner.Kind == "StatefulSet" && strings.HasPrefix(owner.APIVersion, "apps/") { + if owner.Kind == util.StatefulSet && strings.HasPrefix(owner.APIVersion, "apps/") { if strings.HasPrefix(pod.Name, owner.Name) { return true, owner.Name } @@ -1642,18 +1647,17 @@ func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, st return vip.Status.V4ip, vip.Status.V6ip, vip.Status.Mac, podNet.Subnet, nil } - var macStr *string + var macPointer *string if isOvnSubnet(podNet.Subnet) { - mac := pod.Annotations[fmt.Sprintf(util.MacAddressAnnotationTemplate, podNet.ProviderName)] - if mac != "" { - if _, err := net.ParseMAC(mac); err != nil { + annoMAC := pod.Annotations[fmt.Sprintf(util.MacAddressAnnotationTemplate, podNet.ProviderName)] + if annoMAC != "" { + if _, err := net.ParseMAC(annoMAC); err != nil { return "", "", "", podNet.Subnet, err } - macStr = &mac + macPointer = &annoMAC } } else { - macStr = new(string) - *macStr = "" + macPointer = nil } ippoolStr := pod.Annotations[fmt.Sprintf(util.IPPoolAnnotationTemplate, podNet.ProviderName)] @@ -1675,7 +1679,7 @@ func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, st for { portName := ovs.PodNameToPortName(podName, pod.Namespace, podNet.ProviderName) - ipv4, ipv6, mac, err := c.ipam.GetRandomAddress(key, portName, macStr, podNet.Subnet.Name, "", skippedAddrs, !podNet.AllowLiveMigration) + ipv4, ipv6, mac, err := c.ipam.GetRandomAddress(key, portName, macPointer, podNet.Subnet.Name, "", skippedAddrs, !podNet.AllowLiveMigration) if err != nil { klog.Error(err) return "", "", "", podNet.Subnet, err @@ -1710,7 +1714,7 @@ func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, st ipStr := pod.Annotations[fmt.Sprintf(util.IPAddressAnnotationTemplate, podNet.ProviderName)] for _, net := range nsNets { - v4IP, v6IP, mac, err = c.acquireStaticAddress(key, portName, ipStr, macStr, net.Subnet.Name, net.AllowLiveMigration) + v4IP, v6IP, mac, err = c.acquireStaticAddress(key, portName, ipStr, macPointer, net.Subnet.Name, net.AllowLiveMigration) if err == nil { return v4IP, v6IP, mac, net.Subnet, nil } @@ -1737,12 +1741,14 @@ func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, st var skippedAddrs []string for { portName := ovs.PodNameToPortName(podName, pod.Namespace, podNet.ProviderName) - ipv4, ipv6, mac, err := c.ipam.GetRandomAddress(key, portName, macStr, podNet.Subnet.Name, ipPool[0], skippedAddrs, !podNet.AllowLiveMigration) + ipv4, ipv6, mac, err := c.ipam.GetRandomAddress(key, portName, macPointer, podNet.Subnet.Name, ipPool[0], skippedAddrs, !podNet.AllowLiveMigration) if err != nil { + klog.Error(err) return "", "", "", podNet.Subnet, err } ipv4OK, ipv6OK, err := c.validatePodIP(pod.Name, podNet.Subnet.Name, ipv4, ipv6) if err != nil { + klog.Error(err) return "", "", "", podNet.Subnet, err } if ipv4OK && ipv6OK { @@ -1774,7 +1780,7 @@ func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, st continue } - v4IP, v6IP, mac, err = c.acquireStaticAddress(key, portName, staticIP, macStr, net.Subnet.Name, net.AllowLiveMigration) + v4IP, v6IP, mac, err = c.acquireStaticAddress(key, portName, staticIP, macPointer, net.Subnet.Name, net.AllowLiveMigration) if err == nil { return v4IP, v6IP, mac, net.Subnet, nil } @@ -1788,7 +1794,7 @@ func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, st if index < len(ipPool) { for _, net := range nsNets { - v4IP, v6IP, mac, err = c.acquireStaticAddress(key, portName, ipPool[index], macStr, net.Subnet.Name, net.AllowLiveMigration) + v4IP, v6IP, mac, err = c.acquireStaticAddress(key, portName, ipPool[index], macPointer, net.Subnet.Name, net.AllowLiveMigration) if err == nil { return v4IP, v6IP, mac, net.Subnet, nil } @@ -1797,7 +1803,7 @@ func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, st } } } - klog.Errorf("alloc address for %s failed, return NoAvailableAddress", key) + klog.Errorf("allocate address for %s failed, return NoAvailableAddress", key) return "", "", "", podNet.Subnet, ipam.ErrNoAvailable } @@ -1829,7 +1835,7 @@ func appendCheckPodToDel(c *Controller, pod *v1.Pod, ownerRefName, ownerRefKind var ownerRefSubnetExist bool var ownerRefSubnet string switch ownerRefKind { - case "StatefulSet": + case util.StatefulSet: ss, err := c.config.KubeClient.AppsV1().StatefulSets(pod.Namespace).Get(context.Background(), ownerRefName, metav1.GetOptions{}) if err != nil { if k8serrors.IsNotFound(err) { @@ -2003,7 +2009,7 @@ func (c *Controller) getNsAvailableSubnets(pod *v1.Pod, podNet *kubeovnNet) ([]* func getPodType(pod *v1.Pod) string { if ok, _ := isStatefulSetPod(pod); ok { - return "StatefulSet" + return util.StatefulSet } if isVMPod, _ := isVMPod(pod); isVMPod { diff --git a/pkg/controller/pod_iptables_eip.go b/pkg/controller/pod_iptables_eip.go index 8f9a35de347..e1a4e936722 100644 --- a/pkg/controller/pod_iptables_eip.go +++ b/pkg/controller/pod_iptables_eip.go @@ -133,7 +133,7 @@ func (c *Controller) enqueueDeletePodAnnotatedIptablesEip(obj interface{}) { c.delPodAnnotatedIptablesEipQueue.Add(obj) return } - if isDelete, err := appendCheckPodToDel(c, p, statefulSetName, "StatefulSet"); isDelete && err == nil { + if isDelete, err := appendCheckPodToDel(c, p, statefulSetName, util.StatefulSet); isDelete && err == nil { c.delPodAnnotatedIptablesEipQueue.Add(obj) return } @@ -280,7 +280,7 @@ func (c *Controller) handleAddPodAnnotatedIptablesEip(key string) error { return err } externalNetwork := util.GetExternalNetwork(eip.Spec.ExternalSubnet) - if err := c.createOrUpdateCrdEip(eipName, "", "", "", "", natGw, externalNetwork); err != nil { + if err := c.createOrUpdateEipCR(eipName, "", "", "", "", natGw, externalNetwork); err != nil { klog.Errorf("failed to create eip %s: %v", eipName, err) return err } diff --git a/pkg/controller/pod_iptables_fip.go b/pkg/controller/pod_iptables_fip.go index b9de9b64645..e87216f3042 100644 --- a/pkg/controller/pod_iptables_fip.go +++ b/pkg/controller/pod_iptables_fip.go @@ -121,7 +121,7 @@ func (c *Controller) enqueueDeletePodAnnotatedIptablesFip(obj interface{}) { c.delPodAnnotatedIptablesFipQueue.Add(obj) return } - if isDelete, err := appendCheckPodToDel(c, p, statefulSetName, "StatefulSet"); isDelete && err == nil { + if isDelete, err := appendCheckPodToDel(c, p, statefulSetName, util.StatefulSet); isDelete && err == nil { c.delPodAnnotatedIptablesFipQueue.Add(obj) return } @@ -261,7 +261,7 @@ func (c *Controller) handleAddPodAnnotatedIptablesFip(key string) error { return err } klog.V(3).Infof("handle add pod annotated iptables fip %s", fipName) - if err := c.createOrUpdateCrdFip(fipName, fipName, cachedPod.Annotations[util.IPAddressAnnotation]); err != nil { + if err := c.createOrUpdateFipCR(fipName, fipName, cachedPod.Annotations[util.IPAddressAnnotation]); err != nil { klog.Errorf("failed to create fip %s: %v", fipName, err) return err } @@ -289,7 +289,7 @@ func (c *Controller) handleDeletePodAnnotatedIptablesFip(pod *v1.Pod) error { var keepFipCR bool klog.V(3).Infof("handle delete annotated iptables fip for pod %s/%s", pod.Namespace, pod.Name) if ok, sts := isStatefulSetPod(pod); ok { - isDelete, err := appendCheckPodToDel(c, pod, sts, "StatefulSet") + isDelete, err := appendCheckPodToDel(c, pod, sts, util.StatefulSet) keepFipCR = !isStatefulSetPodToDel(c.config.KubeClient, pod, sts) && !isDelete && err == nil } if !keepFipCR { diff --git a/pkg/controller/subnet.go b/pkg/controller/subnet.go index fd86c06e813..041d80810b9 100644 --- a/pkg/controller/subnet.go +++ b/pkg/controller/subnet.go @@ -1893,7 +1893,7 @@ func (c *Controller) reconcileU2OInterconnectionIP(subnet *kubeovnv1.Subnet) err case kubeovnv1.ProtocolDual: subnet.Status.U2OInterconnectionIP = fmt.Sprintf("%s,%s", v4ip, v6ip) } - if err := c.createOrUpdateCrdIPs(u2oInterconnName, subnet.Status.U2OInterconnectionIP, "", subnet.Name, "default", "", "", ""); err != nil { + if err := c.createOrUpdateIPCR("", u2oInterconnName, subnet.Status.U2OInterconnectionIP, "", subnet.Name, "default", "", ""); err != nil { klog.Errorf("failed to create or update IPs of %s : %v", u2oInterconnLrpName, err) return err } @@ -2758,6 +2758,10 @@ func (c *Controller) deletePolicyRouteForU2OInterconn(subnet *kubeovnv1.Subnet) } func (c *Controller) addStaticRouteForU2OInterconn(subnet *kubeovnv1.Subnet) error { + if subnet.Spec.Vpc == "" { + return nil + } + var v4Gw, v6Gw, v4Cidr, v6Cidr string for _, gw := range strings.Split(subnet.Spec.Gateway, ",") { switch util.CheckProtocol(gw) { @@ -2807,6 +2811,10 @@ func (c *Controller) addStaticRouteForU2OInterconn(subnet *kubeovnv1.Subnet) err } func (c *Controller) deleteStaticRouteForU2OInterconn(subnet *kubeovnv1.Subnet) error { + if subnet.Spec.Vpc == "" { + return nil + } + var v4Gw, v6Gw, v4Cidr, v6Cidr string for _, gw := range strings.Split(subnet.Spec.Gateway, ",") { switch util.CheckProtocol(gw) { @@ -2846,6 +2854,7 @@ func (c *Controller) deleteStaticRouteForU2OInterconn(subnet *kubeovnv1.Subnet) v6Gw, kubeovnv1.PolicySrc, ); err != nil { + klog.Errorf("failed to delete static route, %v", err) return err } } diff --git a/pkg/controller/vip.go b/pkg/controller/vip.go index f754fd3d211..644797df00f 100644 --- a/pkg/controller/vip.go +++ b/pkg/controller/vip.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "net" "reflect" "slices" "strings" @@ -280,7 +279,7 @@ func (c *Controller) handleAddVirtualIP(key string) error { parentV6ip = vip.Spec.ParentV6ip parentMac = vip.Spec.ParentMac } - if err = c.createOrUpdateCrdVip(key, vip.Spec.Namespace, subnet.Name, v4ip, v6ip, mac, parentV4ip, parentV6ip, parentMac); err != nil { + if err = c.createOrUpdateVipCR(key, vip.Spec.Namespace, subnet.Name, v4ip, v6ip, mac, parentV4ip, parentV6ip, parentMac); err != nil { klog.Errorf("failed to create or update vip '%s', %v", vip.Name, err) return err } @@ -323,7 +322,7 @@ func (c *Controller) handleUpdateVirtualIP(key string) error { // should update if vip.Status.Mac == "" { // TODO:// add vip in its parent port aap list - if err = c.createOrUpdateCrdVip(key, vip.Spec.Namespace, vip.Spec.Subnet, + if err = c.createOrUpdateVipCR(key, vip.Spec.Namespace, vip.Spec.Subnet, vip.Spec.V4ip, vip.Spec.V6ip, vip.Spec.MacAddress, vip.Spec.ParentV4ip, vip.Spec.ParentV6ip, vip.Spec.MacAddress); err != nil { return err @@ -367,54 +366,6 @@ func (c *Controller) handleDelVirtualIP(vip *kubeovnv1.Vip) error { return nil } -func (c *Controller) acquireStaticIPAddress(subnetName, name, nicName, ip string) (string, string, string, error) { - checkConflict := true - var v4ip, v6ip, mac string - var err error - for _, ipStr := range strings.Split(ip, ",") { - if net.ParseIP(ipStr) == nil { - return "", "", "", fmt.Errorf("failed to parse vip ip %s", ipStr) - } - } - - if v4ip, v6ip, mac, err = c.ipam.GetStaticAddress(name, nicName, ip, nil, subnetName, checkConflict); err != nil { - klog.Errorf("failed to get static virtual ip '%s', mac '%s', subnet '%s', %v", ip, mac, subnetName, err) - return "", "", "", err - } - return v4ip, v6ip, mac, nil -} - -func (c *Controller) acquireIPAddress(subnetName, name, nicName string) (string, string, string, error) { - var skippedAddrs []string - var v4ip, v6ip, mac string - checkConflict := true - var err error - for { - v4ip, v6ip, mac, err = c.ipam.GetRandomAddress(name, nicName, nil, subnetName, "", skippedAddrs, checkConflict) - if err != nil { - klog.Error(err) - return "", "", "", err - } - - ipv4OK, ipv6OK, err := c.validatePodIP(name, subnetName, v4ip, v6ip) - if err != nil { - klog.Error(err) - return "", "", "", err - } - - if ipv4OK && ipv6OK { - return v4ip, v6ip, mac, nil - } - - if !ipv4OK { - skippedAddrs = append(skippedAddrs, v4ip) - } - if !ipv6OK { - skippedAddrs = append(skippedAddrs, v6ip) - } - } -} - func (c *Controller) handleUpdateVirtualParents(key string) error { cachedVip, err := c.virtualIpsLister.Get(key) if err != nil { @@ -484,22 +435,8 @@ func (c *Controller) handleUpdateVirtualParents(key string) error { return nil } -func (c *Controller) subnetCountIP(subnet *kubeovnv1.Subnet) error { - var err error - if util.CheckProtocol(subnet.Spec.CIDRBlock) == kubeovnv1.ProtocolDual { - _, err = c.calcDualSubnetStatusIP(subnet) - } else { - _, err = c.calcSubnetStatusIP(subnet) - } - if err != nil { - klog.Error(err) - return err - } - return nil -} - -func (c *Controller) createOrUpdateCrdVip(key, ns, subnet, v4ip, v6ip, mac, pV4ip, pV6ip, pmac string) error { - vipCr, err := c.virtualIpsLister.Get(key) +func (c *Controller) createOrUpdateVipCR(key, ns, subnet, v4ip, v6ip, mac, pV4ip, pV6ip, pmac string) error { + vipCR, err := c.virtualIpsLister.Get(key) if err != nil { if k8serrors.IsNotFound(err) { if _, err := c.config.KubeOvnClient.KubeovnV1().Vips().Create(context.Background(), &kubeovnv1.Vip{ @@ -532,7 +469,7 @@ func (c *Controller) createOrUpdateCrdVip(key, ns, subnet, v4ip, v6ip, mac, pV4i return errMsg } } else { - vip := vipCr.DeepCopy() + vip := vipCR.DeepCopy() if vip.Status.Mac == "" && mac != "" { // vip not support to update, just delete and create vip.Spec.Namespace = ns diff --git a/pkg/controller/vpc.go b/pkg/controller/vpc.go index 2ec8402ba28..5c3be03017d 100644 --- a/pkg/controller/vpc.go +++ b/pkg/controller/vpc.go @@ -727,9 +727,8 @@ func (c *Controller) addStaticRouteToVpc(name string, route *kubeovnv1.StaticRou func (c *Controller) deleteStaticRouteFromVpc(name, table, cidr, nextHop string, policy kubeovnv1.RoutePolicy) error { var ( - vpc, cachedVpc *kubeovnv1.Vpc - policyStr string - err error + policyStr string + err error ) policyStr = convertPolicy(policy) @@ -738,21 +737,6 @@ func (c *Controller) deleteStaticRouteFromVpc(name, table, cidr, nextHop string, return err } - cachedVpc, err = c.vpcsLister.Get(name) - if err != nil { - if k8serrors.IsNotFound(err) { - return nil - } - klog.Error(err) - return err - } - vpc = cachedVpc.DeepCopy() - // make sure custom policies not be deleted - _, err = c.config.KubeOvnClient.KubeovnV1().Vpcs().Update(context.Background(), vpc, metav1.UpdateOptions{}) - if err != nil { - klog.Error(err) - return err - } return nil } @@ -1108,7 +1092,7 @@ func (c *Controller) handleAddVpcExternalSubnet(key, subnet string) error { klog.Errorf("failed to acquire ip address for lrp eip %s, %v", lrpEipName, err) return err } - if err := c.createOrUpdateCrdOvnEip(lrpEipName, subnet, v4ip, v6ip, mac, util.OvnEipTypeLRP); err != nil { + if err := c.createOrUpdateOvnEipCR(lrpEipName, subnet, v4ip, v6ip, mac, util.OvnEipTypeLRP); err != nil { klog.Errorf("failed to create ovn eip for lrp %s: %v", lrpEipName, err) return err } diff --git a/pkg/controller/vpc_nat_gw_eip.go b/pkg/controller/vpc_nat_gw_eip.go index 2f74581f8b3..562df443a54 100644 --- a/pkg/controller/vpc_nat_gw_eip.go +++ b/pkg/controller/vpc_nat_gw_eip.go @@ -255,7 +255,7 @@ func (c *Controller) handleAddIptablesEip(key string) error { return err } } - if err = c.createOrUpdateCrdEip(key, v4ip, v6ip, mac, cachedEip.Spec.NatGwDp, cachedEip.Spec.QoSPolicy, externalNetwork); err != nil { + if err = c.createOrUpdateEipCR(key, v4ip, v6ip, mac, cachedEip.Spec.NatGwDp, cachedEip.Spec.QoSPolicy, externalNetwork); err != nil { klog.Errorf("failed to update eip %s, %v", key, err) return err } @@ -645,7 +645,7 @@ func (c *Controller) GetGwBySubnet(name string) (string, string, error) { return "", "", fmt.Errorf("failed to get subnet %s", name) } -func (c *Controller) createOrUpdateCrdEip(key, v4ip, v6ip, mac, natGwDp, qos, externalNet string) error { +func (c *Controller) createOrUpdateEipCR(key, v4ip, v6ip, mac, natGwDp, qos, externalNet string) error { needCreate := false cachedEip, err := c.iptablesEipsLister.Get(key) if err != nil { diff --git a/pkg/controller/vpc_nat_gw_nat.go b/pkg/controller/vpc_nat_gw_nat.go index 5c6b5de3852..8e45fab00ef 100644 --- a/pkg/controller/vpc_nat_gw_nat.go +++ b/pkg/controller/vpc_nat_gw_nat.go @@ -1806,7 +1806,7 @@ func (c *Controller) isDnatDuplicated(gwName, eipName, dnatName, externalPort st return false, nil } -func (c *Controller) createOrUpdateCrdFip(key, eipName, internalIP string) error { +func (c *Controller) createOrUpdateFipCR(key, eipName, internalIP string) error { cachedFip, err := c.iptablesFipsLister.Get(key) if err != nil { klog.V(3).Infof("create fip cr %s", key) diff --git a/pkg/daemon/handler.go b/pkg/daemon/handler.go index ee47582b62a..a7fbaf8c242 100644 --- a/pkg/daemon/handler.go +++ b/pkg/daemon/handler.go @@ -209,7 +209,7 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon if subnet == "" && podSubnet != nil { subnet = podSubnet.Name } - if err := csh.UpdateIPCr(podRequest, subnet, ip); err != nil { + if err := csh.UpdateIPCR(podRequest, subnet, ip); err != nil { if err := resp.WriteHeaderAndEntity(http.StatusInternalServerError, request.CniResponse{Err: err.Error()}); err != nil { klog.Errorf("failed to write response, %v", err) } @@ -350,22 +350,22 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon } } -func (csh cniServerHandler) UpdateIPCr(podRequest request.CniRequest, subnet, ip string) error { - ipCrName := ovs.PodNameToPortName(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider) +func (csh cniServerHandler) UpdateIPCR(podRequest request.CniRequest, subnet, ip string) error { + ipCRName := ovs.PodNameToPortName(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider) for i := 0; i < 20; i++ { - oriIPCr, err := csh.KubeOvnClient.KubeovnV1().IPs().Get(context.Background(), ipCrName, metav1.GetOptions{}) + ipCR, err := csh.KubeOvnClient.KubeovnV1().IPs().Get(context.Background(), ipCRName, metav1.GetOptions{}) if err != nil { err = fmt.Errorf("failed to get ip crd for %s, %v", ip, err) // maybe create a backup pod with previous annotations klog.Error(err) - } else if oriIPCr.Spec.NodeName != csh.Config.NodeName { - ipCr := oriIPCr.DeepCopy() - ipCr.Spec.NodeName = csh.Config.NodeName - ipCr.Spec.AttachIPs = []string{} - ipCr.Labels[subnet] = "" - ipCr.Spec.AttachSubnets = []string{} - ipCr.Spec.AttachMacs = []string{} - if _, err := csh.KubeOvnClient.KubeovnV1().IPs().Update(context.Background(), ipCr, metav1.UpdateOptions{}); err != nil { + } else if ipCR.Spec.NodeName != csh.Config.NodeName { + ipCR := ipCR.DeepCopy() + ipCR.Spec.NodeName = csh.Config.NodeName + ipCR.Spec.AttachIPs = []string{} + ipCR.Labels[subnet] = "" + ipCR.Spec.AttachSubnets = []string{} + ipCR.Spec.AttachMacs = []string{} + if _, err := csh.KubeOvnClient.KubeovnV1().IPs().Update(context.Background(), ipCR, metav1.UpdateOptions{}); err != nil { err = fmt.Errorf("failed to update ip crd for %s, %v", ip, err) klog.Error(err) } else { @@ -373,7 +373,7 @@ func (csh cniServerHandler) UpdateIPCr(podRequest request.CniRequest, subnet, ip } } if err != nil { - klog.Warningf("wait pod ip %s to be ready", ipCrName) + klog.Warningf("wait pod ip %s to be ready", ipCRName) time.Sleep(1 * time.Second) } else { return nil diff --git a/pkg/ipam/ipam.go b/pkg/ipam/ipam.go index e2c548cd111..2b6097205d5 100644 --- a/pkg/ipam/ipam.go +++ b/pkg/ipam/ipam.go @@ -97,6 +97,12 @@ func (ipam *IPAM) GetStaticAddress(podName, nicName, ip string, mac *string, sub return "", "", "", err } + if macStr == "" { + err := fmt.Errorf("failed to allocate static mac for %s", podName) + klog.Error(err) + return "", "", "", ErrNoAvailable + } + switch subnet.Protocol { case kubeovnv1.ProtocolIPv4: klog.Infof("allocate v4 %s, mac %s for %s from subnet %s", ip, macStr, podName, subnetName) diff --git a/pkg/util/const.go b/pkg/util/const.go index 9d92ea2d9b8..3ff8f1413fb 100644 --- a/pkg/util/const.go +++ b/pkg/util/const.go @@ -219,6 +219,8 @@ const ( VM = "VirtualMachine" VMInstance = "VirtualMachineInstance" + StatefulSet = "StatefulSet" + MirrorControlAnnotation = "ovn.kubernetes.io/mirror" MirrorDefaultName = "m0" diff --git a/pkg/util/net.go b/pkg/util/net.go index 03834b48707..2fd237326e3 100644 --- a/pkg/util/net.go +++ b/pkg/util/net.go @@ -286,16 +286,14 @@ func SplitIpsByProtocol(excludeIps []string) ([]string, []string) { } func GetStringIP(v4IP, v6IP string) string { - var ipStr string - switch { - case IsValidIP(v4IP) && IsValidIP(v6IP): - ipStr = v4IP + "," + v6IP - case IsValidIP(v4IP): - ipStr = v4IP - case IsValidIP(v6IP): - ipStr = v6IP - } - return ipStr + var ipList []string + if IsValidIP(v4IP) { + ipList = append(ipList, v4IP) + } + if IsValidIP(v6IP) { + ipList = append(ipList, v6IP) + } + return strings.Join(ipList, ",") } func GetIPAddrWithMask(ip, cidr string) (string, error) { diff --git a/pkg/webhook/ip.go b/pkg/webhook/ip.go new file mode 100644 index 00000000000..c08d6a25656 --- /dev/null +++ b/pkg/webhook/ip.go @@ -0,0 +1,116 @@ +package webhook + +import ( + "context" + "fmt" + "net" + "net/http" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrlwebhook "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + ovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +var ipGVK = metav1.GroupVersionKind{Group: ovnv1.SchemeGroupVersion.Group, Version: ovnv1.SchemeGroupVersion.Version, Kind: "IP"} + +func (v *ValidatingHook) IPCreateHook(ctx context.Context, req admission.Request) admission.Response { + ip := ovnv1.IP{} + if err := v.decoder.DecodeRaw(req.Object, &ip); err != nil { + return ctrlwebhook.Errored(http.StatusBadRequest, err) + } + + if err := v.ValidateIP(ctx, &ip); err != nil { + return ctrlwebhook.Errored(http.StatusBadRequest, err) + } + return ctrlwebhook.Allowed("by pass") +} + +func (v *ValidatingHook) IPUpdateHook(ctx context.Context, req admission.Request) admission.Response { + ipOld := ovnv1.IP{} + if err := v.decoder.DecodeRaw(req.OldObject, &ipOld); err != nil { + return ctrlwebhook.Errored(http.StatusBadRequest, err) + } + + ipNew := ovnv1.IP{} + if err := v.decoder.DecodeRaw(req.Object, &ipNew); err != nil { + return ctrlwebhook.Errored(http.StatusBadRequest, err) + } + + if err := v.ValidateIP(ctx, &ipNew); err != nil { + return ctrlwebhook.Errored(http.StatusBadRequest, err) + } + + // ip can not change these specs below + if ipOld.Spec.Subnet != "" && ipNew.Spec.Subnet != ipOld.Spec.Subnet { + err := fmt.Errorf("ip %s subnet can not change", ipNew.Name) + return ctrlwebhook.Errored(http.StatusBadRequest, err) + } + if ipOld.Spec.Namespace != "" && ipNew.Spec.Namespace != ipOld.Spec.Namespace { + err := fmt.Errorf("ip %s namespace can not change", ipNew.Name) + return ctrlwebhook.Errored(http.StatusBadRequest, err) + } + if ipOld.Spec.PodName != "" && ipNew.Spec.PodName != ipOld.Spec.PodName { + err := fmt.Errorf("ip %s podName can not change", ipNew.Name) + return ctrlwebhook.Errored(http.StatusBadRequest, err) + } + if ipOld.Spec.PodType != "" && ipNew.Spec.PodType != ipOld.Spec.PodType { + err := fmt.Errorf("ip %s podType can not change", ipNew.Name) + return ctrlwebhook.Errored(http.StatusBadRequest, err) + } + return ctrlwebhook.Allowed("by pass") +} + +func (v *ValidatingHook) ValidateIP(ctx context.Context, ip *ovnv1.IP) error { + if ip.Spec.Subnet == "" { + err := fmt.Errorf("subnet parameter cannot be empty") + return err + } + + subnet := &ovnv1.Subnet{} + key := types.NamespacedName{Name: ip.Spec.Subnet} + if err := v.cache.Get(ctx, key, subnet); err != nil { + return err + } + + if ip.Spec.V4IPAddress != "" { + if net.ParseIP(ip.Spec.V4IPAddress) == nil { + err := fmt.Errorf("%s is not a valid ip", ip.Spec.V4IPAddress) + return err + } + + if !util.CIDRContainIP(subnet.Spec.CIDRBlock, ip.Spec.V4IPAddress) { + err := fmt.Errorf("the V4ip %s is not in the range of subnet %s, cidr %v", + ip.Spec.V4IPAddress, subnet.Name, subnet.Spec.CIDRBlock) + return err + } + } + + if ip.Spec.V6IPAddress != "" { + if net.ParseIP(ip.Spec.V6IPAddress) == nil { + err := fmt.Errorf("%s is not a valid ip", ip.Spec.V6IPAddress) + return err + } + + if !util.CIDRContainIP(subnet.Spec.CIDRBlock, ip.Spec.V6IPAddress) { + err := fmt.Errorf("the ip %s is not in the range of subnet %s, cidr %v", + ip.Spec.V6IPAddress, subnet.Name, subnet.Spec.CIDRBlock) + return err + } + } + + if ip.Spec.Subnet == "" { + err := fmt.Errorf("subnet parameter cannot be empty") + return err + } + + if ip.Spec.PodType != "" && ip.Spec.PodType != util.VM && ip.Spec.PodType != util.StatefulSet { + err := fmt.Errorf("podType %s is not supported", ip.Spec.PodType) + return err + } + + return nil +} diff --git a/pkg/webhook/static_ip.go b/pkg/webhook/static_ip.go index cadc4c21448..21c286813af 100644 --- a/pkg/webhook/static_ip.go +++ b/pkg/webhook/static_ip.go @@ -208,17 +208,17 @@ func (v *ValidatingHook) checkIPConflict(ipAddress, annoSubnet, name string, ipL return fmt.Errorf("invalid static ip/ippool annotation value: %s", ipAddress) } - for _, ipCr := range ipList { - if annoSubnet != "" && ipCr.Spec.Subnet != annoSubnet { + for _, ipCR := range ipList { + if annoSubnet != "" && ipCR.Spec.Subnet != annoSubnet { continue } - v4IP, v6IP := util.SplitStringIP(ipCr.Spec.IPAddress) + v4IP, v6IP := util.SplitStringIP(ipCR.Spec.IPAddress) if ipAddr.String() == v4IP || ipAddr.String() == v6IP { - if name == ipCr.Spec.PodName { + if name == ipCR.Spec.PodName { klog.Infof("get same ip crd for %s", name) } else { - err := fmt.Errorf("annotation static-ip %s is conflict with ip crd %s, ip %s", ipAddr.String(), ipCr.Name, ipCr.Spec.IPAddress) + err := fmt.Errorf("annotation static-ip %s is conflict with ip crd %s, ip %s", ipAddr.String(), ipCR.Name, ipCR.Spec.IPAddress) return err } } diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go index 65fd8bf4eb1..c9bc7f0e0fb 100644 --- a/pkg/webhook/webhook.go +++ b/pkg/webhook/webhook.go @@ -48,6 +48,9 @@ func NewValidatingHook(client client.Client, scheme *runtime.Scheme, cache cache updateHooks[vpcGVK] = v.VpcUpdateHook deleteHooks[vpcGVK] = v.VpcDeleteHook + createHooks[ipGVK] = v.IPCreateHook + updateHooks[ipGVK] = v.IPUpdateHook + createHooks[vipGVK] = v.VipCreateHook updateHooks[vipGVK] = v.VipUpdateHook diff --git a/test/e2e/webhook/pod/pod.go b/test/e2e/webhook/pod/pod.go index 051c41e7285..ba44f7baaa6 100644 --- a/test/e2e/webhook/pod/pod.go +++ b/test/e2e/webhook/pod/pod.go @@ -106,7 +106,7 @@ var _ = framework.Describe("[group:webhook-pod]", func() { } pod.Annotations = annotations _ = podClient.CreateSync(pod) - ipCr := podName + "." + namespaceName + ipCR := podName + "." + namespaceName framework.WaitUntil(2*time.Second, time.Minute, func(ctx context.Context) (bool, error) { checkPod, err := podClient.PodInterface.Get(ctx, podName, metav1.GetOptions{}) @@ -115,9 +115,9 @@ var _ = framework.Describe("[group:webhook-pod]", func() { }, fmt.Sprintf("pod's annotation %s is true", util.RoutedAnnotation)) ginkgo.By("validate pod ip conflict") - framework.Logf("validate ip conflict, pod %s, ip cr %s, conflict pod %s", podName, ipCr, conflictName) + framework.Logf("validate ip conflict, pod %s, ip cr %s, conflict pod %s", podName, ipCR, conflictName) conflictPod := framework.MakePod(namespaceName, conflictName, nil, annotations, image, cmd, nil) _, err = podClient.PodInterface.Create(context.TODO(), conflictPod, metav1.CreateOptions{}) - framework.ExpectError(err, "annotation static-ip %s is conflict with ip crd %s, ip %s", staticIP, ipCr, staticIP) + framework.ExpectError(err, "annotation static-ip %s is conflict with ip crd %s, ip %s", staticIP, ipCR, staticIP) }) })