From add55c5752802182365b0b872e66e325cd8d5a03 Mon Sep 17 00:00:00 2001 From: changluyi Date: Thu, 18 Jan 2024 18:04:08 +0800 Subject: [PATCH] ecmp refactor Signed-off-by: changluyi --- Makefile | 66 ++-- charts/templates/ic-controller-deploy.yaml | 99 +++++ cmd/cmdmain.go | 4 + cmd/ovn_ic_controller/ovn_ic_controller.go | 24 ++ dist/images/Dockerfile | 3 +- dist/images/cleanup.sh | 2 + dist/images/install-ic-server.sh | 148 +++++++ dist/images/install.sh | 106 +++++ dist/images/kubectl-ko | 28 ++ dist/images/ovn-ic-healthcheck.sh | 28 ++ dist/images/start-ic-controller.sh | 60 +++ dist/images/start-ic-db.sh | 194 ++++++++- pkg/controller/controller.go | 8 - pkg/controller/ovn_ic_test.go | 40 -- pkg/daemon/gateway_linux.go | 2 - pkg/ovn_ic_controller/config.go | 142 +++++++ pkg/ovn_ic_controller/controller.go | 115 ++++++ .../ovn_ic_controller.go} | 369 +++++++++++------- pkg/ovn_leader_checker/ovn.go | 158 +++++++- pkg/ovs/ovn-ic-nbctl.go | 18 + pkg/ovs/ovn-ic-sbctl.go | 17 + test/e2e/framework/framework.go | 8 + test/e2e/ovn-ic/e2e_test.go | 143 ++++++- yamls/kind.yaml.j2 | 2 + 24 files changed, 1529 insertions(+), 255 deletions(-) create mode 100644 charts/templates/ic-controller-deploy.yaml create mode 100644 cmd/ovn_ic_controller/ovn_ic_controller.go create mode 100755 dist/images/install-ic-server.sh create mode 100755 dist/images/ovn-ic-healthcheck.sh create mode 100755 dist/images/start-ic-controller.sh delete mode 100644 pkg/controller/ovn_ic_test.go create mode 100644 pkg/ovn_ic_controller/config.go create mode 100644 pkg/ovn_ic_controller/controller.go rename pkg/{controller/ovn_ic.go => ovn_ic_controller/ovn_ic_controller.go} (70%) diff --git a/Makefile b/Makefile index 1eb7f9c790a..051286342b9 100644 --- a/Makefile +++ b/Makefile @@ -357,19 +357,19 @@ kind-init-ovn-ic: kind-init-ovn-ic-ipv4 .PHONY: kind-init-ovn-ic-ipv4 kind-init-ovn-ic-ipv4: kind-clean-ovn-ic - @ovn_ic=true $(MAKE) kind-init + @ha=true $(MAKE) kind-init @ovn_ic=true $(MAKE) kind-generate-config $(call kind_create_cluster,yamls/kind.yaml,kube-ovn1,1) .PHONY: kind-init-ovn-ic-ipv6 kind-init-ovn-ic-ipv6: kind-clean-ovn-ic - @ovn_ic=true $(MAKE) kind-init-ipv6 + @ha=true $(MAKE) kind-init-ipv6 @ovn_ic=true ip_family=ipv6 $(MAKE) kind-generate-config $(call kind_create_cluster,yamls/kind.yaml,kube-ovn1,1) .PHONY: kind-init-ovn-ic-dual kind-init-ovn-ic-dual: kind-clean-ovn-ic - @ovn_ic=true $(MAKE) kind-init-dual + @ha=true $(MAKE) kind-init-dual @ovn_ic=true ip_family=dual $(MAKE) kind-generate-config $(call kind_create_cluster,yamls/kind.yaml,kube-ovn1,1) @@ -439,6 +439,7 @@ kind-install-chart: kind-load-image kind-untaint-control-plane --set global.images.kubeovn.tag=$(VERSION) \ --set replicaCount=$$(echo $$ips | awk -F ',' '{print NF}') \ --set MASTER_NODES="$$(echo $$ips | sed 's/,/\\,/g')" + --set func.ENABLE_IC=$$(kubectl get node --show-labels | grep -q "ovn.kubernetes.io/ic-gw" && echo true || echo false) sleep 60 kubectl -n kube-system rollout status --timeout=1s deployment/ovn-central kubectl -n kube-system rollout status --timeout=1s daemonset/ovs-ovn @@ -453,6 +454,7 @@ kind-upgrade-chart: kind-load-image --set global.images.kubeovn.tag=$(VERSION) \ --set replicaCount=$$(echo $(OVN_DB_IPS) | awk -F ',' '{print NF}') \ --set MASTER_NODES='$(OVN_DB_IPS)' + --set func.ENABLE_IC=$$(kubectl get node --show-labels | grep -q "ovn.kubernetes.io/ic-gw" && echo true || echo false) sleep 90 kubectl -n kube-system rollout status --timeout=1s deployment/ovn-central kubectl -n kube-system wait pod --for=condition=ready -l app=ovs @@ -493,7 +495,8 @@ kind-install-overlay-ipv4: kind-install kind-install-ovn-ic: kind-install-ovn-ic-ipv4 .PHONY: kind-install-ovn-ic-ipv4 -kind-install-ovn-ic-ipv4: kind-install +kind-install-ovn-ic-ipv4: + @ENABLE_IC=true $(MAKE) kind-install $(call kind_load_image,kube-ovn1,$(REGISTRY)/kube-ovn:$(VERSION)) kubectl config use-context kind-kube-ovn1 $(MAKE) kind-untaint-control-plane @@ -501,23 +504,24 @@ kind-install-ovn-ic-ipv4: kind-install -e 's/10.96.0/10.98.0/g' \ -e 's/100.64.0/100.68.0/g' \ -e 's/VERSION=.*/VERSION=$(VERSION)/' \ - dist/images/install.sh | bash + dist/images/install.sh | ENABLE_IC=true bash kubectl describe no - docker run -d --name ovn-ic-db --network kind $(REGISTRY)/kube-ovn:$(VERSION) bash start-ic-db.sh - @set -e; \ - ic_db_host=$$(docker inspect ovn-ic-db -f "{{.NetworkSettings.Networks.kind.IPAddress}}"); \ - zone=az0 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn-worker,kube-ovn-worker2;kube-ovn-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \ - zone=az1 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn1-worker,kube-ovn1-worker2;kube-ovn1-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml kubectl config use-context kind-kube-ovn + sed 's/VERSION=.*/VERSION=$(VERSION)/' dist/images/install-ic-server.sh | bash + + @set -e; \ + ic_db_host=$$(kubectl get deployment ovn-ic-server -n kube-system -o jsonpath='{range .spec.template.spec.containers[0].env[?(@.name=="NODE_IPS")]}{.value}{end}'); \ + ic_db_host=$${ic_db_host%?}; \ + zone=az0 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn-worker,kube-ovn-worker2,kube-ovn-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \ + zone=az1 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn1-worker,kube-ovn1-worker2,kube-ovn1-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml kubectl apply -f ovn-ic-0.yaml kubectl config use-context kind-kube-ovn1 kubectl apply -f ovn-ic-1.yaml - sleep 6 - docker exec ovn-ic-db ovn-ic-sbctl show .PHONY: kind-install-ovn-ic-ipv6 -kind-install-ovn-ic-ipv6: kind-install-ipv6 +kind-install-ovn-ic-ipv6: + @ENABLE_IC=true $(MAKE) kind-install-ipv6 $(call kind_load_image,kube-ovn1,$(REGISTRY)/kube-ovn:$(VERSION)) kubectl config use-context kind-kube-ovn1 @$(MAKE) kind-untaint-control-plane @@ -526,23 +530,24 @@ kind-install-ovn-ic-ipv6: kind-install-ipv6 -e 's/fd00:100:64:/fd00:100:68:/g' \ -e 's/VERSION=.*/VERSION=$(VERSION)/' \ dist/images/install.sh | \ - IPV6=true bash + IPV6=true ENABLE_IC=true bash kubectl describe no - docker run -d --name ovn-ic-db --network kind -e PROTOCOL="ipv6" $(REGISTRY)/kube-ovn:$(VERSION) bash start-ic-db.sh - @set -e; \ - ic_db_host=$$(docker inspect ovn-ic-db -f "{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}"); \ - zone=az0 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn-worker,kube-ovn-worker2;kube-ovn-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \ - zone=az1 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn1-worker,kube-ovn1-worker2;kube-ovn1-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml kubectl config use-context kind-kube-ovn + sed 's/VERSION=.*/VERSION=$(VERSION)/' dist/images/install-ic-server.sh | bash + + @set -e; \ + ic_db_host=$$(kubectl get deployment ovn-ic-server -n kube-system -o jsonpath='{range .spec.template.spec.containers[0].env[?(@.name=="NODE_IPS")]}{.value}{end}'); \ + ic_db_host=$${ic_db_host%?}; \ + zone=az0 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn-worker,kube-ovn-worker2,kube-ovn-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \ + zone=az1 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn1-worker,kube-ovn1-worker2,kube-ovn1-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml kubectl apply -f ovn-ic-0.yaml kubectl config use-context kind-kube-ovn1 kubectl apply -f ovn-ic-1.yaml - sleep 6 - docker exec ovn-ic-db ovn-ic-sbctl show .PHONY: kind-install-ovn-ic-dual -kind-install-ovn-ic-dual: kind-install-dual +kind-install-ovn-ic-dual: + @ENABLE_IC=true $(MAKE) kind-install-dual $(call kind_load_image,kube-ovn1,$(REGISTRY)/kube-ovn:$(VERSION)) kubectl config use-context kind-kube-ovn1 @$(MAKE) kind-untaint-control-plane @@ -554,21 +559,20 @@ kind-install-ovn-ic-dual: kind-install-dual -e 's/fd00:100:64:/fd00:100:68:/g' \ -e 's/VERSION=.*/VERSION=$(VERSION)/' \ dist/images/install.sh | \ - DUAL_STACK=true bash + DUAL_STACK=true ENABLE_IC=true bash kubectl describe no - docker run -d --name ovn-ic-db --network kind -e PROTOCOL="dual" $(REGISTRY)/kube-ovn:$(VERSION) bash start-ic-db.sh - @set -e; \ - - ic_db_host=$$(docker inspect ovn-ic-db -f "{{.NetworkSettings.Networks.kind.IPAddress}}"); \ - zone=az0 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn-worker,kube-ovn-worker2;kube-ovn-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \ - zone=az1 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn1-worker,kube-ovn1-worker2;kube-ovn1-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml kubectl config use-context kind-kube-ovn + sed 's/VERSION=.*/VERSION=$(VERSION)/' dist/images/install-ic-server.sh | bash + + @set -e; \ + ic_db_host=$$(kubectl get deployment ovn-ic-server -n kube-system -o jsonpath='{range .spec.template.spec.containers[0].env[?(@.name=="NODE_IPS")]}{.value}{end}'); \ + ic_db_host=$${ic_db_host%?}; \ + zone=az0 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn-worker,kube-ovn-worker2,kube-ovn-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \ + zone=az1 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn1-worker,kube-ovn1-worker2,kube-ovn1-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml kubectl apply -f ovn-ic-0.yaml kubectl config use-context kind-kube-ovn1 kubectl apply -f ovn-ic-1.yaml - sleep 6 - docker exec ovn-ic-db ovn-ic-sbctl show .PHONY: kind-install-ovn-submariner kind-install-ovn-submariner: kind-install diff --git a/charts/templates/ic-controller-deploy.yaml b/charts/templates/ic-controller-deploy.yaml new file mode 100644 index 00000000000..b224a0f0855 --- /dev/null +++ b/charts/templates/ic-controller-deploy.yaml @@ -0,0 +1,99 @@ +{{- if eq .Values.func.ENABLE_IC true }} +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ovn-ic-controller + namespace: kube-system + annotations: + kubernetes.io/description: | + OVN IC Client +spec: + replicas: 1 + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app: ovn-ic-controller + template: + metadata: + labels: + app: ovn-ic-controller + component: network + type: infra + spec: + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: ovn-ic-controller + topologyKey: kubernetes.io/hostname + priorityClassName: system-cluster-critical + serviceAccountName: ovn + hostNetwork: true + containers: + - name: ovn-ic-controller + image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["/kube-ovn/start-ic-controller.sh"] + securityContext: + capabilities: + add: ["SYS_NICE"] + env: + - name: ENABLE_SSL + value: "{{ .Values.networking.ENABLE_SSL }}" + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OVN_DB_IPS + value: "{{ .Values.MASTER_NODES }}" + resources: + requests: + cpu: 300m + memory: 200Mi + limits: + cpu: 3 + memory: 1Gi + volumeMounts: + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /etc/ovn + name: host-config-ovn + - mountPath: /var/log/ovn + name: host-log-ovn + - mountPath: /etc/localtime + name: localtime + - mountPath: /var/run/tls + name: kube-ovn-tls + nodeSelector: + kubernetes.io/os: "linux" + kube-ovn/role: "master" + volumes: + - name: host-run-ovn + hostPath: + path: /run/ovn + - name: host-config-ovn + hostPath: + path: /etc/origin/ovn + - name: host-log-ovn + hostPath: + path: /var/log/ovn + - name: localtime + hostPath: + path: /etc/localtime + - name: kube-ovn-tls + secret: + optional: true + secretName: kube-ovn-tls +{{- end }} diff --git a/cmd/cmdmain.go b/cmd/cmdmain.go index 3887715cab1..f9462c28065 100644 --- a/cmd/cmdmain.go +++ b/cmd/cmdmain.go @@ -14,6 +14,7 @@ import ( "github.com/kubeovn/kube-ovn/cmd/controller" "github.com/kubeovn/kube-ovn/cmd/controller_health_check" "github.com/kubeovn/kube-ovn/cmd/daemon" + "github.com/kubeovn/kube-ovn/cmd/ovn_ic_controller" "github.com/kubeovn/kube-ovn/cmd/ovn_leader_checker" "github.com/kubeovn/kube-ovn/cmd/ovn_monitor" "github.com/kubeovn/kube-ovn/cmd/pinger" @@ -29,6 +30,7 @@ const ( CmdSpeaker = "kube-ovn-speaker" CmdControllerHealthCheck = "kube-ovn-controller-healthcheck" CmdOvnLeaderChecker = "kube-ovn-leader-checker" + CmdOvnICController = "kube-ovn-ic-controller" ) const timeFormat = "2006-01-02_15:04:05" @@ -95,6 +97,8 @@ func main() { controller_health_check.CmdMain() case CmdOvnLeaderChecker: ovn_leader_checker.CmdMain() + case CmdOvnICController: + ovn_ic_controller.CmdMain() default: util.LogFatalAndExit(nil, "%s is an unknown command", cmd) } diff --git a/cmd/ovn_ic_controller/ovn_ic_controller.go b/cmd/ovn_ic_controller/ovn_ic_controller.go new file mode 100644 index 00000000000..570637898bd --- /dev/null +++ b/cmd/ovn_ic_controller/ovn_ic_controller.go @@ -0,0 +1,24 @@ +package ovn_ic_controller + +import ( + "k8s.io/klog/v2" + "k8s.io/sample-controller/pkg/signals" + + "github.com/kubeovn/kube-ovn/pkg/ovn_ic_controller" + "github.com/kubeovn/kube-ovn/pkg/util" + "github.com/kubeovn/kube-ovn/versions" +) + +func CmdMain() { + defer klog.Flush() + + klog.Infof(versions.String()) + config, err := ovn_ic_controller.ParseFlags() + if err != nil { + util.LogFatalAndExit(err, "failed to parse config") + } + + stopCh := signals.SetupSignalHandler().Done() + ctl := ovn_ic_controller.NewController(config) + ctl.Run(stopCh) +} diff --git a/dist/images/Dockerfile b/dist/images/Dockerfile index e91fe57ca8d..c22d33a42a1 100644 --- a/dist/images/Dockerfile +++ b/dist/images/Dockerfile @@ -26,4 +26,5 @@ RUN ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-controller && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-pinger && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-speaker && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-controller-healthcheck && \ - ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-leader-checker + ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-leader-checker && \ + ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-ic-controller diff --git a/dist/images/cleanup.sh b/dist/images/cleanup.sh index c7a2a149900..7fcd8dea0ba 100644 --- a/dist/images/cleanup.sh +++ b/dist/images/cleanup.sh @@ -92,6 +92,8 @@ kubectl delete --ignore-not-found deploy kube-ovn-monitor -n kube-system kubectl delete --ignore-not-found cm ovn-config ovn-ic-config ovn-external-gw-config -n kube-system kubectl delete --ignore-not-found svc kube-ovn-pinger kube-ovn-controller kube-ovn-cni kube-ovn-monitor -n kube-system kubectl delete --ignore-not-found deploy kube-ovn-controller -n kube-system +kubectl delete --ignore-not-found deploy ovn-ic-controller -n kube-system +kubectl delete --ignore-not-found deploy ovn-ic-server -n kube-system # wait for provier-networks to be deleted before deleting kube-ovn-cni sleep 5 diff --git a/dist/images/install-ic-server.sh b/dist/images/install-ic-server.sh new file mode 100755 index 00000000000..7358efe59bd --- /dev/null +++ b/dist/images/install-ic-server.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +set -euo pipefail + +REGISTRY="kubeovn" +VERSION="v1.13.0" +TS_NUM=${TS_NUM:-3} +IMAGE_PULL_POLICY="IfNotPresent" +addresses=$(kubectl get no -lkube-ovn/role=master --no-headers -o wide | awk '{print $6}' | tr \\n ',') +count=$(kubectl get no -lkube-ovn/role=master --no-headers | wc -l) +OVN_LEADER_PROBE_INTERVAL=${OVN_LEADER_PROBE_INTERVAL:-5} + +cat < ovn-ic-server.yaml +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ovn-ic-server + namespace: kube-system + annotations: + kubernetes.io/description: | + OVN IC Server +spec: + replicas: $count + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app: ovn-ic-server + template: + metadata: + labels: + app: ovn-ic-server + component: network + type: infra + spec: + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: ovn-ic-server + topologyKey: kubernetes.io/hostname + priorityClassName: system-cluster-critical + serviceAccountName: ovn + hostNetwork: true + containers: + - name: ovn-ic-server + image: "$REGISTRY/kube-ovn:$VERSION" + imagePullPolicy: $IMAGE_PULL_POLICY + command: ["/kube-ovn/start-ic-db.sh"] + securityContext: + capabilities: + add: ["SYS_NICE"] + env: + - name: ENABLE_SSL + value: "false" + - name: TS_NUM + value: "$TS_NUM" + - name: NODE_IPS + value: $addresses + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: OVN_LEADER_PROBE_INTERVAL + value: "$OVN_LEADER_PROBE_INTERVAL" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IPS + valueFrom: + fieldRef: + fieldPath: status.podIPs + resources: + requests: + cpu: 300m + memory: 200Mi + limits: + cpu: 3 + memory: 1Gi + volumeMounts: + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /etc/ovn + name: host-config-ovn + - mountPath: /var/log/ovn + name: host-log-ovn + - mountPath: /etc/localtime + name: localtime + - mountPath: /var/run/tls + name: kube-ovn-tls + readinessProbe: + exec: + command: + - bash + - /kube-ovn/ovn-ic-healthcheck.sh + periodSeconds: 15 + timeoutSeconds: 45 + livenessProbe: + exec: + command: + - bash + - /kube-ovn/ovn-ic-healthcheck.sh + initialDelaySeconds: 30 + periodSeconds: 15 + failureThreshold: 5 + timeoutSeconds: 4 + nodeSelector: + kubernetes.io/os: "linux" + kube-ovn/role: "master" + volumes: + - name: host-run-ovn + hostPath: + path: /run/ovn + - name: host-config-ovn + hostPath: + path: /etc/origin/ovn + - name: host-log-ovn + hostPath: + path: /var/log/ovn + - name: localtime + hostPath: + path: /etc/localtime + - name: kube-ovn-tls + secret: + optional: true + secretName: kube-ovn-tls +EOF + +kubectl apply -f ovn-ic-server.yaml +kubectl rollout status deployment/ovn-ic-server -n kube-system --timeout 600s + +echo "OVN IC Server installed Successfully" diff --git a/dist/images/install.sh b/dist/images/install.sh index 82d5435d17d..d834f9ad412 100755 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -22,6 +22,7 @@ ENABLE_NAT_GW=${ENABLE_NAT_GW:-false} ENABLE_KEEP_VM_IP=${ENABLE_KEEP_VM_IP:-true} ENABLE_ARP_DETECT_IP_CONFLICT=${ENABLE_ARP_DETECT_IP_CONFLICT:-true} NODE_LOCAL_DNS_IP=${NODE_LOCAL_DNS_IP:-} +ENABLE_IC=${ENABLE_IC:-$(kubectl get node --show-labels | grep -q "ovn.kubernetes.io/ic-gw" && echo true || echo false)} # exchange link names of OVS bridge and the provider nic # in the default provider-network EXCHANGE_LINK_NAME=${EXCHANGE_LINK_NAME:-false} @@ -4564,6 +4565,111 @@ EOF kubectl apply -f kube-ovn.yaml kubectl rollout status deployment/kube-ovn-controller -n kube-system --timeout 300s kubectl rollout status daemonset/kube-ovn-cni -n kube-system --timeout 300s + +if $ENABLE_IC; then + +cat < ovn-ic-controller.yaml +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ovn-ic-controller + namespace: kube-system + annotations: + kubernetes.io/description: | + OVN IC Controller +spec: + replicas: 1 + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app: ovn-ic-controller + template: + metadata: + labels: + app: ovn-ic-controller + component: network + type: infra + spec: + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: ovn-ic-controller + topologyKey: kubernetes.io/hostname + priorityClassName: system-cluster-critical + serviceAccountName: ovn + hostNetwork: true + containers: + - name: ovn-ic-controller + image: "$REGISTRY/kube-ovn:$VERSION" + imagePullPolicy: $IMAGE_PULL_POLICY + command: ["/kube-ovn/start-ic-controller.sh"] + securityContext: + capabilities: + add: ["SYS_NICE"] + env: + - name: ENABLE_SSL + value: "$ENABLE_SSL" + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OVN_DB_IPS + value: $addresses + resources: + requests: + cpu: 300m + memory: 200Mi + limits: + cpu: 3 + memory: 1Gi + volumeMounts: + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /etc/ovn + name: host-config-ovn + - mountPath: /var/log/ovn + name: host-log-ovn + - mountPath: /etc/localtime + name: localtime + - mountPath: /var/run/tls + name: kube-ovn-tls + nodeSelector: + kubernetes.io/os: "linux" + kube-ovn/role: "master" + volumes: + - name: host-run-ovn + hostPath: + path: /run/ovn + - name: host-config-ovn + hostPath: + path: /etc/origin/ovn + - name: host-log-ovn + hostPath: + path: /var/log/ovn + - name: localtime + hostPath: + path: /etc/localtime + - name: kube-ovn-tls + secret: + optional: true + secretName: kube-ovn-tls +EOF +kubectl apply -f ovn-ic-controller.yaml +fi + echo "-------------------------------" echo "" diff --git a/dist/images/kubectl-ko b/dist/images/kubectl-ko index 608c83e8efe..96f85aebc16 100755 --- a/dist/images/kubectl-ko +++ b/dist/images/kubectl-ko @@ -5,6 +5,8 @@ KUBE_OVN_NS=kube-system WITHOUT_KUBE_PROXY=${WITHOUT_KUBE_PROXY:-false} OVN_NB_POD= OVN_SB_POD= +OVN_IC_NB_POD= +OVN_IC_SB_POD= OVN_NORTHD_POD= KUBE_OVN_VERSION= REGISTRY="docker.io/kubeovn" @@ -38,6 +40,8 @@ showHelp(){ echo " reload restart all kube-ovn components" echo " log {kube-ovn|ovn|ovs|linux|all} save log to ./kubectl-ko-log/" echo " perf [image] performance test default image is docker.io/kubeovn/test:v1.13.0" + echo " icnbctl [ovn-nbctl options ...] invoke ovn-ic-nbctl" + echo " icsbctl [ovn-sbctl options ...] invoke ovn-ic-sbctl" } # usage: ipv4_to_hex 192.168.0.1 @@ -770,6 +774,22 @@ getOvnCentralPod(){ KUBE_OVN_VERSION=$(basename $image | awk -F ':' '{print $2}') } +getOVNICNBPod(){ + OVN_IC_NB_POD=$(kubectl get pod -n $KUBE_OVN_NS -l ovn-ic-nb-leader=true | grep ovn-ic-server | head -n 1 | awk '{print $1}') + if [ -z "$OVN_IC_NB_POD" ]; then + echo "ic nb leader not exists" + exit 1 + fi +} + +getOVNICSBPod(){ + OVN_IC_SB_POD=$(kubectl get pod -n $KUBE_OVN_NS -l ovn-ic-sb-leader=true | grep ovn-ic-server | head -n 1 | awk '{print $1}') + if [ -z "$OVN_IC_SB_POD" ]; then + echo "ic sb leader not exists" + exit 1 + fi +} + getOvnCentralDbStatus(){ NB_PODS=$(kubectl get pod -n $KUBE_OVN_NS | grep ovn-central | awk '{print $1}') for pod in $NB_PODS @@ -1643,6 +1663,14 @@ case $subcommand in getOvnCentralPod kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-sbctl "$@" ;; + icnbctl) + getOVNICNBPod + kubectl exec "$OVN_IC_NB_POD" -n $KUBE_OVN_NS -- ovn-ic-nbctl "$@" + ;; + icsbctl) + getOVNICSBPod + kubectl exec "$OVN_IC_SB_POD" -n $KUBE_OVN_NS -- ovn-ic-sbctl "$@" + ;; vsctl|ofctl|dpctl|appctl) xxctl "$subcommand" "$@" ;; diff --git a/dist/images/ovn-ic-healthcheck.sh b/dist/images/ovn-ic-healthcheck.sh new file mode 100755 index 00000000000..ef0b95f22b6 --- /dev/null +++ b/dist/images/ovn-ic-healthcheck.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -euo pipefail +shopt -s expand_aliases + +alias ovn-ctl='/usr/share/ovn/scripts/ovn-ctl' + +ic_nb_status=$(ovn-appctl -t /var/run/ovn/ovn_ic_nb_db.ctl cluster/status OVN_IC_Northbound | grep Status) +ic_sb_status=$(ovn-appctl -t /var/run/ovn/ovn_ic_sb_db.ctl cluster/status OVN_IC_Southbound | grep Status) +ic_nb_role=$(ovn-appctl -t /var/run/ovn/ovn_ic_nb_db.ctl cluster/status OVN_IC_Northbound | grep Role) +ic_sb_role=$(ovn-appctl -t /var/run/ovn/ovn_ic_sb_db.ctl cluster/status OVN_IC_Southbound | grep Role) + +if ! echo ${ic_nb_status} | grep -v "failed"; then + echo "nb health check failed" + exit 1 +fi +if ! echo ${ic_sb_status} | grep -v "failed"; then + echo "sb health check failed" + exit 1 +fi + +if echo ${ic_nb_status} | grep "disconnected" && echo ${ic_nb_role} | grep "candidate"; then + echo "nb health check failed" + exit 1 +fi +if echo ${ic_sb_status} | grep "disconnected" && echo ${ic_sb_role} | grep "candidate"; then + echo "sb health check failed" + exit 1 +fi diff --git a/dist/images/start-ic-controller.sh b/dist/images/start-ic-controller.sh new file mode 100755 index 00000000000..764032656a7 --- /dev/null +++ b/dist/images/start-ic-controller.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +set -euo pipefail +ENABLE_SSL=${ENABLE_SSL:-false} +OVN_DB_IPS=${OVN_DB_IPS:-} + +function gen_conn_str { + if [[ -z "${OVN_DB_IPS}" ]]; then + if [[ "$1" == "6641" ]]; then + if [[ "$ENABLE_SSL" == "false" ]]; then + x="tcp:[${OVN_NB_SERVICE_HOST}]:${OVN_NB_SERVICE_PORT}" + else + x="ssl:[${OVN_NB_SERVICE_HOST}]:${OVN_NB_SERVICE_PORT}" + fi + else + if [[ "$ENABLE_SSL" == "false" ]]; then + x="tcp:[${OVN_SB_SERVICE_HOST}]:${OVN_SB_SERVICE_PORT}" + else + x="ssl:[${OVN_SB_SERVICE_HOST}]:${OVN_SB_SERVICE_PORT}" + fi + fi + else + t=$(echo -n "${OVN_DB_IPS}" | sed 's/[[:space:]]//g' | sed 's/,/ /g') + if [[ "$ENABLE_SSL" == "false" ]]; then + x=$(for i in ${t}; do echo -n "tcp:[$i]:$1",; done| sed 's/,$//') + else + x=$(for i in ${t}; do echo -n "ssl:[$i]:$1",; done| sed 's/,$//') + fi + fi + echo "$x" +} + +nb_addr="$(gen_conn_str 6641)" +sb_addr="$(gen_conn_str 6642)" + +for ((i=0; i<3; i++)); do + if [[ "$ENABLE_SSL" == "false" ]]; then + OVN_NB_DAEMON=$(ovn-nbctl --db="$nb_addr" --pidfile --detach --overwrite-pidfile) + else + OVN_NB_DAEMON=$(ovn-nbctl -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert --db="$nb_addr" --pidfile --detach --overwrite-pidfile) + fi + if echo -n "${OVN_NB_DAEMON}" | grep -qE '^/var/run/ovn/ovn-nbctl\.[0-9]+\.ctl$'; then + export OVN_NB_DAEMON + break + fi + if [ $(echo ${OVN_NB_DAEMON} | wc -c) -gt 64 ]; then + OVN_NB_DAEMON="$(echo ${OVN_NB_DAEMON} | cut -c1-64)..." + fi + echo "invalid ovn-nbctl daemon socket: \"${OVN_NB_DAEMON}\"" + unset OVN_NB_DAEMON + pkill -f ovn-nbctl +done + +if [ -z "${OVN_NB_DAEMON}" ]; then + echo "failed to start ovn-nbctl daemon" + exit 1 +fi + +exec ./kube-ovn-ic-controller --ovn-nb-addr="$nb_addr" \ + --ovn-sb-addr="$sb_addr" \ + $@ diff --git a/dist/images/start-ic-db.sh b/dist/images/start-ic-db.sh index 3927b3b171d..9d86c086a8e 100644 --- a/dist/images/start-ic-db.sh +++ b/dist/images/start-ic-db.sh @@ -1,45 +1,215 @@ #!/bin/bash set -eo pipefail +TS_NAME=${TS_NAME:-ts} +LOCAL_IP=${LOCAL_IP:-$POD_IP} +TS_NUM=${TS_NUM:-ts} +ENABLE_BIND_LOCAL_IP=${ENABLE_BIND_LOCAL_IP:-true} + +DB_ADDR=:: +if [[ $ENABLE_BIND_LOCAL_IP == "true" ]]; then + DB_ADDR="$POD_IP" +fi + +function get_leader_ip { + t=$(echo -n "${NODE_IPS}" | sed 's/[[:space:]]//g' | sed 's/,/ /g') + echo -n "${t}" | cut -f 1 -d " " +} + function quit { /usr/share/ovn/scripts/ovn-ctl stop_ic_ovsdb exit 0 } +function ovndb_query_leader { + local db="" + local db_eval="" + case $1 in + nb) + db=OVN_Northbound + db_eval="NB" + ;; + sb) + db=OVN_Southbound + db_eval="SB" + ;; + *) + echo "invalid database: $1" + exit 1 + ;; + esac + + eval port="\$${db_eval}_PORT" + query='["_Server",{"table":"Database","where":[["name","==","'$db'"]],"columns":["leader"],"op":"select"}]' + if [[ "$ENABLE_SSL" == "false" ]]; then + timeout 10 ovsdb-client query $(gen_conn_addr $i $port) "$query" + else + timeout 10 ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query $(gen_conn_addr $i $port) "$query" + fi +} + function gen_conn_str { t=$(echo -n "${NODE_IPS}" | sed 's/[[:space:]]//g' | sed 's/,/ /g') - x=$(for i in ${t}; do echo -n "tcp:[$i]:$1",; done| sed 's/,$//') + if [[ "$ENABLE_SSL" == "false" ]]; then + x=$(for i in ${t}; do echo -n "tcp:[$i]:$1",; done| sed 's/,$//') + else + x=$(for i in ${t}; do echo -n "ssl:[$i]:$1",; done| sed 's/,$//') + fi echo "$x" } +function ovn_db_pre_start() { + local db="" + local port="" + case $1 in + ic_nb) + db=OVN_IC_Northbound + port=6645 + ;; + ic_sb) + db=OVN_IC_Southbound + port=6646 + ;; + *) + echo "invalid database: $1" + exit 1 + ;; + esac + + local db_file="/etc/ovn/ovn${1}_db.db" + [ ! -e "$db_file" ] && return + ! ovsdb-tool db-is-clustered "$db_file" && return + ovsdb-tool check-cluster "$db_file" && return + + local db_bak="$db_file.backup-$(date +%s)-$(random_str)" + echo "backup $db_file to $db_bak" + cp "$db_file" "$db_bak" || return 1 + + echo "detected database corruption for file $db_file, try to fix it." + ovsdb-tool fix-cluster "$db_file" && return + + echo "failed to fix database file $db_file, rebuild it." + local sid=$(ovsdb-tool db-sid "$db_file") + if ! echo -n "$sid" | grep -qE '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'; then + echo "failed to get sid from $1 db file $db_file" + return 1 + fi + echo "get local server id $sid" + + local local_addr="$(gen_conn_addr $POD_IP $port)" + echo "local address: $local_addr" + + local remote_addr=() + local ips=$(echo -n "${NODE_IPS}" | sed 's/,/ /g') + for ip in ${ips[*]}; do + if [ ! "$POD_IP" = "$ip" ]; then + remote_addr=(${remote_addr[*]} "$(gen_conn_addr $ip $port)") + fi + done + echo "remote addresses: ${remote_addr[*]}" + + local db_new="$db_file.init-$(random_str)" + echo "generating new database file $db_new" + ovsdb-tool --sid $sid join-cluster "$db_new" $db $local_addr ${remote_addr[*]} || return 1 + + echo "use new database file $db_new" + mv "$db_new" "$db_file" +} + + +function is_clustered { + t=$(echo -n "${NODE_IPS}" | sed 's/,/ /g') + if [[ "$ENABLE_SSL" == "false" ]]; then + x=$(for i in ${t}; do echo -n "tcp:[${i}]:6645,"; done | sed 's/,/ /g') + for i in ${x}; + do + nb_leader=$(timeout 10 ovsdb-client query ${i} "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_IC_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]") + if [[ $nb_leader =~ "true" ]] + then + return 0 + fi + done + else + x=$(for i in ${t}; do echo -n "ssl:[${i}]:6645,"; done| sed 's/,/ /g') + for i in ${x}; + do + nb_leader=$(timeout 10 ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query ${i} "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_IC_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]") + if [[ $nb_leader =~ "true" ]] + then + return 0 + fi + done + fi + return 1 +} + trap quit EXIT +/usr/share/ovn/scripts/ovn-ctl stop_ic_ovsdb +ovn_db_pre_start ic_nb +ovn_db_pre_start ic_sb + if [[ -z "$NODE_IPS" && -z "$LOCAL_IP" ]]; then /usr/share/ovn/scripts/ovn-ctl --db-ic-nb-create-insecure-remote=yes --db-ic-sb-create-insecure-remote=yes --db-ic-nb-addr="[::]" --db-ic-sb-addr="[::]" start_ic_ovsdb /usr/share/ovn/scripts/ovn-ctl status_ic_ovsdb - tail --follow=name --retry /var/log/ovn/ovsdb-server-ic-nb.log else - if [[ -z "$LEADER_IP" ]]; then + ic_nb_leader_ip=$(get_leader_ip nb) + ic_sb_leader_ip=$(get_leader_ip sb) + set +eo pipefail + is_clustered + result=$? + set -eo pipefail + # leader up only when no cluster and on first node + if [[ ${result} -eq 1 && "$ic_nb_leader_ip" == "${POD_IP}" ]]; then echo "leader start with local ${LOCAL_IP} and cluster $(gen_conn_str 6647)" /usr/share/ovn/scripts/ovn-ctl --db-ic-nb-create-insecure-remote=yes \ --db-ic-sb-create-insecure-remote=yes \ - --db-ic-sb-cluster-local-addr="${LOCAL_IP}" \ - --db-ic-nb-cluster-local-addr="${LOCAL_IP}" \ + --db-ic-sb-cluster-local-addr="[${LOCAL_IP}]" \ + --db-ic-nb-cluster-local-addr="[${LOCAL_IP}]" \ --ovn-ic-nb-db="$(gen_conn_str 6647)" \ --ovn-ic-sb-db="$(gen_conn_str 6648)" \ + --db-ic-nb-addr=[$DB_ADDR] \ + --db-ic-sb-addr=[$DB_ADDR] \ start_ic_ovsdb /usr/share/ovn/scripts/ovn-ctl status_ic_ovsdb - tail --follow=name --retry /var/log/ovn/ovsdb-server-ic-nb.log else - echo "follower start with local ${LOCAL_IP}, leader ${LEADER_IP} and cluster $(gen_conn_str 6647)" + # known leader always first + set +eo pipefail + if [ ${result} -eq 0 ]; then + t=$(echo -n "${NODE_IPS}" | sed 's/,/ /g') + for i in ${t}; + do + nb_leader=$(ovndb_query_leader nb $i) + if [[ $nb_leader =~ "true" ]] + then + nb_leader_ip=${i} + break + fi + done + for i in ${t}; + do + sb_leader=$(ovndb_query_leader sb $i) + if [[ $sb_leader =~ "true" ]] + then + sb_leader_ip=${i} + break + fi + done + fi + set -eo pipefail + echo "follower start with local ${LOCAL_IP}, ovn-ic-nb leader ${ic_nb_leader_ip} ovn-ic-sb leader ${ic_sb_leader_ip}" /usr/share/ovn/scripts/ovn-ctl --db-ic-nb-create-insecure-remote=yes \ --db-ic-sb-create-insecure-remote=yes \ - --db-ic-sb-cluster-local-addr="${LOCAL_IP}" \ - --db-ic-nb-cluster-local-addr="${LOCAL_IP}" \ - --db-ic-nb-cluster-remote-addr="${LEADER_IP}" \ - --db-ic-sb-cluster-remote-addr="${LEADER_IP}" \ + --db-ic-sb-cluster-local-addr="[${LOCAL_IP}]" \ + --db-ic-nb-cluster-local-addr="[${LOCAL_IP}]" \ + --db-ic-nb-cluster-remote-addr="[${ic_nb_leader_ip}]" \ + --db-ic-sb-cluster-remote-addr="[${ic_sb_leader_ip}]" \ --ovn-ic-nb-db="$(gen_conn_str 6647)" \ --ovn-ic-sb-db="$(gen_conn_str 6648)" \ + --db-ic-nb-addr=[$DB_ADDR] \ + --db-ic-sb-addr=[$DB_ADDR] \ start_ic_ovsdb - tail --follow=name --retry /var/log/ovn/ovsdb-server-ic-nb.log fi fi + +chmod 600 /etc/ovn/* +/kube-ovn/kube-ovn-leader-checker --probeInterval=${OVN_LEADER_PROBE_INTERVAL} --isICDBServer=true diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index c8e8bff05cf..78f32b8d253 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -1030,14 +1030,6 @@ func (c *Controller) startWorkers(ctx context.Context) { go wait.Until(c.runUpdateVlanWorker, time.Second, ctx.Done()) } - go wait.Until(func() { - c.resyncInterConnection() - }, time.Second, ctx.Done()) - - go wait.Until(func() { - c.SynRouteToPolicy() - }, 5*time.Second, ctx.Done()) - go wait.Until(func() { c.resyncExternalGateway() }, time.Second, ctx.Done()) diff --git a/pkg/controller/ovn_ic_test.go b/pkg/controller/ovn_ic_test.go deleted file mode 100644 index 5d9db014c94..00000000000 --- a/pkg/controller/ovn_ic_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package controller - -import ( - "testing" - - "github.com/golang/mock/gomock" - "github.com/scylladb/go-set/strset" - "github.com/stretchr/testify/require" - - "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" -) - -func Test_listRemoteLogicalSwitchPortAddress(t *testing.T) { - t.Parallel() - - fakeController := newFakeController(t) - ctrl := fakeController.fakeController - mockOvnClient := fakeController.mockOvnClient - - mockLsp := func(addresses []string) ovnnb.LogicalSwitchPort { - return ovnnb.LogicalSwitchPort{ - Addresses: addresses, - } - } - - lsps := []ovnnb.LogicalSwitchPort{ - mockLsp([]string{"00:00:00:53:21:6F 10.244.0.17 fc00::af4:11"}), - mockLsp([]string{"00:00:00:53:21:6F 10.244.0.18"}), - mockLsp([]string{"00:00:00:53:21:6E 10.244.0.10"}), - mockLsp([]string{"00:00:00:53:21:6F"}), - mockLsp([]string{""}), - mockLsp([]string{}), - } - - mockOvnClient.EXPECT().ListLogicalSwitchPorts(gomock.Any(), gomock.Any(), gomock.Any()).Return(lsps, nil) - - addresses, err := ctrl.listRemoteLogicalSwitchPortAddress() - require.NoError(t, err) - require.True(t, addresses.IsEqual(strset.New("10.244.0.10", "10.244.0.18"))) -} diff --git a/pkg/daemon/gateway_linux.go b/pkg/daemon/gateway_linux.go index a3878b86625..bc7f047bea3 100644 --- a/pkg/daemon/gateway_linux.go +++ b/pkg/daemon/gateway_linux.go @@ -672,7 +672,6 @@ func (c *Controller) setIptables() error { ) } } - _, subnetCidrs, err := c.getDefaultVpcSubnetsCIDR(protocol) if err != nil { klog.Errorf("get subnets failed, %+v", err) @@ -713,7 +712,6 @@ func (c *Controller) setIptables() error { } } } - var natPreroutingRules, natPostroutingRules, ovnMasqueradeRules, manglePostroutingRules []util.IPTableRule for _, rule := range iptablesRules { if rule.Table == NAT { diff --git a/pkg/ovn_ic_controller/config.go b/pkg/ovn_ic_controller/config.go new file mode 100644 index 00000000000..71ce4aa29b0 --- /dev/null +++ b/pkg/ovn_ic_controller/config.go @@ -0,0 +1,142 @@ +package ovn_ic_controller + +import ( + "flag" + "fmt" + "os" + + "github.com/spf13/pflag" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" + + clientset "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +// Configuration is the controller conf +type Configuration struct { + KubeConfigFile string + KubeClient kubernetes.Interface + KubeOvnClient clientset.Interface + + PodNamespace string + OvnNbAddr string + OvnSbAddr string + OvnTimeout int + + NodeSwitch string + ClusterRouter string + NodeSwitchCIDR string + + ClusterTCPLoadBalancer string + ClusterUDPLoadBalancer string + ClusterTCPSessionLoadBalancer string + ClusterUDPSessionLoadBalancer string +} + +func ParseFlags() (*Configuration, error) { + var ( + argKubeConfigFile = pflag.String("kubeconfig", "", "Path to kubeconfig file with authorization and master location information. If not set use the inCluster token.") + + argOvnNbAddr = pflag.String("ovn-nb-addr", "", "ovn-nb address") + argOvnSbAddr = pflag.String("ovn-sb-addr", "", "ovn-sb address") + argOvnTimeout = pflag.Int("ovn-timeout", 60, "") + + argClusterRouter = pflag.String("cluster-router", util.DefaultVpc, "The router name for cluster router") + argNodeSwitch = pflag.String("node-switch", "join", "The name of node gateway switch which help node to access pod network") + argNodeSwitchCIDR = pflag.String("node-switch-cidr", "100.64.0.0/16", "The cidr for node switch") + + argClusterTCPLoadBalancer = pflag.String("cluster-tcp-loadbalancer", "cluster-tcp-loadbalancer", "The name for cluster tcp loadbalancer") + argClusterUDPLoadBalancer = pflag.String("cluster-udp-loadbalancer", "cluster-udp-loadbalancer", "The name for cluster udp loadbalancer") + argClusterTCPSessionLoadBalancer = pflag.String("cluster-tcp-session-loadbalancer", "cluster-tcp-session-loadbalancer", "The name for cluster tcp session loadbalancer") + argClusterUDPSessionLoadBalancer = pflag.String("cluster-udp-session-loadbalancer", "cluster-udp-session-loadbalancer", "The name for cluster udp session loadbalancer") + ) + + klogFlags := flag.NewFlagSet("klog", flag.ContinueOnError) + klog.InitFlags(klogFlags) + + // Sync the glog and klog flags. + pflag.CommandLine.VisitAll(func(f1 *pflag.Flag) { + f2 := klogFlags.Lookup(f1.Name) + if f2 != nil { + value := f1.Value.String() + if err := f2.Value.Set(value); err != nil { + klog.Errorf("failed to set flag %v", err) + } + } + }) + + // change the behavior of cmdline + // not exit. not good + pflag.CommandLine.Init(os.Args[0], pflag.ContinueOnError) + pflag.CommandLine.AddGoFlagSet(klogFlags) + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + + if err := pflag.CommandLine.Parse(os.Args[1:]); err != nil { + return nil, err + } + + config := &Configuration{ + KubeConfigFile: *argKubeConfigFile, + + PodNamespace: os.Getenv("POD_NAMESPACE"), + OvnNbAddr: *argOvnNbAddr, + OvnSbAddr: *argOvnSbAddr, + OvnTimeout: *argOvnTimeout, + + ClusterRouter: *argClusterRouter, + NodeSwitch: *argNodeSwitch, + NodeSwitchCIDR: *argNodeSwitchCIDR, + + ClusterTCPLoadBalancer: *argClusterTCPLoadBalancer, + ClusterUDPLoadBalancer: *argClusterUDPLoadBalancer, + ClusterTCPSessionLoadBalancer: *argClusterTCPSessionLoadBalancer, + ClusterUDPSessionLoadBalancer: *argClusterUDPSessionLoadBalancer, + } + + if err := config.initKubeClient(); err != nil { + return nil, fmt.Errorf("failed to init kube client, %v", err) + } + + return config, nil +} + +func (config *Configuration) initKubeClient() error { + var cfg *rest.Config + var err error + if config.KubeConfigFile == "" { + klog.Infof("no --kubeconfig, use in-cluster kubernetes config") + cfg, err = rest.InClusterConfig() + if err != nil { + klog.Errorf("use in cluster config failed %v", err) + return err + } + } else { + cfg, err = clientcmd.BuildConfigFromFlags("", config.KubeConfigFile) + if err != nil { + klog.Errorf("use --kubeconfig %s failed %v", config.KubeConfigFile, err) + return err + } + } + cfg.QPS = 1000 + cfg.Burst = 2000 + + kubeOvnClient, err := clientset.NewForConfig(cfg) + if err != nil { + klog.Errorf("init kubeovn client failed %v", err) + return err + } + config.KubeOvnClient = kubeOvnClient + + cfg.ContentType = "application/vnd.kubernetes.protobuf" + cfg.AcceptContentTypes = "application/vnd.kubernetes.protobuf,application/json" + kubeClient, err := kubernetes.NewForConfig(cfg) + if err != nil { + klog.Errorf("init kubernetes client failed %v", err) + return err + } + config.KubeClient = kubeClient + return nil +} diff --git a/pkg/ovn_ic_controller/controller.go b/pkg/ovn_ic_controller/controller.go new file mode 100644 index 00000000000..95067a7e817 --- /dev/null +++ b/pkg/ovn_ic_controller/controller.go @@ -0,0 +1,115 @@ +package ovn_ic_controller + +import ( + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + kubeinformers "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/scheme" + typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + listerv1 "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + kubeovninformer "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions" + kubeovnlister "github.com/kubeovn/kube-ovn/pkg/client/listers/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/ovs" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +const controllerAgentName = "ovn-ic-controller" + +type Controller struct { + config *Configuration + + subnetsLister kubeovnlister.SubnetLister + subnetSynced cache.InformerSynced + nodesLister listerv1.NodeLister + nodesSynced cache.InformerSynced + configMapsLister listerv1.ConfigMapLister + configMapsSynced cache.InformerSynced + vpcsLister kubeovnlister.VpcLister + vpcSynced cache.InformerSynced + + informerFactory kubeinformers.SharedInformerFactory + kubeovnInformerFactory kubeovninformer.SharedInformerFactory + recorder record.EventRecorder + + ovnLegacyClient *ovs.LegacyClient + OVNNbClient ovs.NbClient + OVNSbClient ovs.SbClient +} + +func NewController(config *Configuration) *Controller { + utilruntime.Must(kubeovnv1.AddToScheme(scheme.Scheme)) + klog.V(4).Info("Creating event broadcaster") + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartLogging(klog.Infof) + eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: config.KubeClient.CoreV1().Events("")}) + recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: controllerAgentName}) + + informerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(config.KubeClient, 0, + kubeinformers.WithTweakListOptions(func(listOption *metav1.ListOptions) { + listOption.AllowWatchBookmarks = true + })) + kubeovnInformerFactory := kubeovninformer.NewSharedInformerFactoryWithOptions(config.KubeOvnClient, 0, + kubeovninformer.WithTweakListOptions(func(listOption *metav1.ListOptions) { + listOption.AllowWatchBookmarks = true + })) + + vpcInformer := kubeovnInformerFactory.Kubeovn().V1().Vpcs() + nodeInformer := informerFactory.Core().V1().Nodes() + subnetInformer := kubeovnInformerFactory.Kubeovn().V1().Subnets() + configMapInformer := informerFactory.Core().V1().ConfigMaps() + + controller := &Controller{ + config: config, + + vpcsLister: vpcInformer.Lister(), + vpcSynced: vpcInformer.Informer().HasSynced, + subnetsLister: subnetInformer.Lister(), + subnetSynced: subnetInformer.Informer().HasSynced, + nodesLister: nodeInformer.Lister(), + nodesSynced: nodeInformer.Informer().HasSynced, + configMapsLister: configMapInformer.Lister(), + configMapsSynced: configMapInformer.Informer().HasSynced, + + informerFactory: informerFactory, + kubeovnInformerFactory: kubeovnInformerFactory, + recorder: recorder, + + ovnLegacyClient: ovs.NewLegacyClient(config.OvnTimeout), + } + + var err error + if controller.OVNNbClient, err = ovs.NewOvnNbClient(config.OvnNbAddr, config.OvnTimeout); err != nil { + util.LogFatalAndExit(err, "failed to create ovn nb client") + } + if controller.OVNSbClient, err = ovs.NewOvnSbClient(config.OvnSbAddr, config.OvnTimeout); err != nil { + util.LogFatalAndExit(err, "failed to create ovn sb client") + } + + return controller +} + +func (c *Controller) Run(stopCh <-chan struct{}) { + defer utilruntime.HandleCrash() + c.informerFactory.Start(stopCh) + c.kubeovnInformerFactory.Start(stopCh) + + if !cache.WaitForCacheSync(stopCh, c.subnetSynced, c.nodesSynced) { + util.LogFatalAndExit(nil, "failed to wait for caches to sync") + return + } + + klog.Info("Started workers") + go wait.Until(c.resyncInterConnection, time.Second, stopCh) + go wait.Until(c.SynRouteToPolicy, 5*time.Second, stopCh) + <-stopCh + klog.Info("Shutting down workers") +} diff --git a/pkg/controller/ovn_ic.go b/pkg/ovn_ic_controller/ovn_ic_controller.go similarity index 70% rename from pkg/controller/ovn_ic.go rename to pkg/ovn_ic_controller/ovn_ic_controller.go index a381b04e77f..9f26b153e89 100644 --- a/pkg/controller/ovn_ic.go +++ b/pkg/ovn_ic_controller/ovn_ic_controller.go @@ -1,4 +1,4 @@ -package controller +package ovn_ic_controller import ( "context" @@ -7,6 +7,7 @@ import ( "os" "os/exec" "reflect" + "sort" "strings" "time" @@ -18,7 +19,6 @@ import ( "k8s.io/klog/v2" kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" - "github.com/kubeovn/kube-ovn/pkg/ovs" "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" "github.com/kubeovn/kube-ovn/pkg/util" ) @@ -26,45 +26,33 @@ import ( var ( icEnabled = "unknown" lastIcCm map[string]string + lastTSs []string + curTSs []string ) -func (c *Controller) resyncInterConnection() { - cm, err := c.configMapsLister.ConfigMaps(c.config.PodNamespace).Get(util.InterconnectionConfig) - if err != nil && !k8serrors.IsNotFound(err) { - klog.Errorf("failed to get ovn-ic-config, %v", err) +const ( + icNoAction = iota + icFirstEstablish + icConfigChange +) + +func (c *Controller) disableOVNIC(azName string) { + if err := c.removeInterConnection(azName); err != nil { + klog.Errorf("failed to remove ovn-ic, %v", err) return } - - if k8serrors.IsNotFound(err) || cm.Data["enable-ic"] == "false" { - if icEnabled == "false" { - return - } - klog.Info("start to remove ovn-ic") - azName := "" - if cm != nil { - azName = cm.Data["az-name"] - } else if lastIcCm != nil { - azName = lastIcCm["az-name"] - } - if err := c.removeInterConnection(azName); err != nil { - klog.Errorf("failed to remove ovn-ic, %v", err) - return - } - if err := c.delLearnedRoute(); err != nil { - klog.Errorf("failed to remove learned static routes, %v", err) - return - } - icEnabled = "false" - lastIcCm = nil - - klog.Info("finish removing ovn-ic") + if err := c.delLearnedRoute(); err != nil { + klog.Errorf("failed to remove learned static routes, %v", err) return } - blackList := []string{} - autoRoute := false - if cm.Data["auto-route"] == "true" { - autoRoute = true + + if err := c.RemoveOldChassisInSbDB(azName); err != nil { + klog.Errorf("failed to remove remote chassis: %v", err) } +} + +func (c *Controller) setAutoRoute(autoRoute bool) { + var blackList []string subnets, err := c.subnetsLister.List(labels.Everything()) if err != nil { klog.Errorf("failed to list subnets, %v", err) @@ -93,48 +81,146 @@ func (c *Controller) resyncInterConnection() { klog.Errorf("failed to config auto route, %v", err) return } +} + +func (c *Controller) DeleteICResources(azName string) error { + icTSs := make([]string, 0) + if err := c.OVNNbClient.DeleteLogicalSwitchPorts(nil, func(lsp *ovnnb.LogicalSwitchPort) bool { + // add the code below because azName may have multi "-" + firstIndex := strings.Index(lsp.Name, "-") + if firstIndex != -1 { + firstPart := lsp.Name[:firstIndex] + secondPart := lsp.Name[firstIndex+1:] + needDelete := secondPart == azName && strings.HasPrefix(firstPart, util.InterconnectionSwitch) + if needDelete { + icTSs = append(icTSs, firstPart) + } + return needDelete + } + return false + }); err != nil { + return err + } + + if err := c.OVNNbClient.DeleteLogicalRouterPorts(nil, func(lrp *ovnnb.LogicalRouterPort) bool { + lastIndex := strings.LastIndex(lrp.Name, "-") + if lastIndex != -1 { + firstPart := lrp.Name[:lastIndex] + secondPart := lrp.Name[lastIndex+1:] + return firstPart == azName && strings.HasPrefix(secondPart, util.InterconnectionSwitch) + } + return false + }); err != nil { + return err + } - isCMEqual := reflect.DeepEqual(cm.Data, lastIcCm) - if icEnabled == "true" && lastIcCm != nil && isCMEqual { + for _, icTS := range icTSs { + if err := c.OVNNbClient.DeleteLogicalSwitch(icTS); err != nil { + return err + } + } + return nil +} + +func (c *Controller) getICState(cmData, lastcmData map[string]string) int { + isCMEqual := reflect.DeepEqual(cmData, lastcmData) + if icEnabled != "true" && len(lastcmData) == 0 && cmData["enable-ic"] == "true" { + return icFirstEstablish + } + + if icEnabled == "true" && lastcmData != nil && isCMEqual { + var err error + c.ovnLegacyClient.OvnICNbAddress = genHostAddress(cmData["ic-db-host"], cmData["ic-nb-port"]) + curTSs, err = c.ovnLegacyClient.GetTs() + if err != nil { + klog.Errorf("failed to get Transit_Switch, %v", err) + return icNoAction + } + isTsEqual := reflect.DeepEqual(lastTSs, curTSs) + if isTsEqual { + return icNoAction + } + } + return icConfigChange +} + +func (c *Controller) resyncInterConnection() { + cm, err := c.configMapsLister.ConfigMaps(c.config.PodNamespace).Get(util.InterconnectionConfig) + if err != nil && !k8serrors.IsNotFound(err) { + klog.Errorf("failed to get ovn-ic-config, %v", err) return } - if icEnabled == "true" && lastIcCm != nil && !isCMEqual { - if err := c.removeInterConnection(lastIcCm["az-name"]); err != nil { - klog.Errorf("failed to remove ovn-ic, %v", err) + + if k8serrors.IsNotFound(err) || cm.Data["enable-ic"] == "false" { + if icEnabled == "false" { return } - if err := c.delLearnedRoute(); err != nil { - klog.Errorf("failed to remove learned static routes, %v", err) - return + klog.Info("start to remove ovn-ic") + azName := "" + icDBHost := "" + if cm != nil { + azName = cm.Data["az-name"] + icDBHost = cm.Data["ic-db-host"] + } else if lastIcCm != nil { + azName = lastIcCm["az-name"] + icDBHost = lastIcCm["ic-db-host"] + } + + if icDBHost != "" { + c.ovnLegacyClient.OvnICSbAddress = genHostAddress(icDBHost, cm.Data["ic-sb-port"]) + c.ovnLegacyClient.OvnICNbAddress = genHostAddress(icDBHost, cm.Data["ic-nb-port"]) } - c.ovnLegacyClient.OvnICSbAddress = genHostAddress(cm.Data["ic-db-host"], cm.Data["ic-sb-port"]) + c.disableOVNIC(azName) + icEnabled = "false" + lastIcCm = nil + + klog.Info("finish removing ovn-ic") + return + } + + autoRoute := false + if cm.Data["auto-route"] == "true" { + autoRoute = true + } + c.setAutoRoute(autoRoute) + + switch c.getICState(cm.Data, lastIcCm) { + case icNoAction: + return + case icFirstEstablish: c.ovnLegacyClient.OvnICNbAddress = genHostAddress(cm.Data["ic-db-host"], cm.Data["ic-nb-port"]) + klog.Info("start to establish ovn-ic") + if err := c.establishInterConnection(cm.Data); err != nil { + klog.Errorf("failed to establish ovn-ic, %v", err) + return + } + curTSs, err := c.ovnLegacyClient.GetTs() + if err != nil { + klog.Errorf("failed to get Transit_Switch, %v", err) + return + } + icEnabled = "true" + lastIcCm = cm.Data + lastTSs = curTSs + klog.Info("finish establishing ovn-ic") + return + case icConfigChange: + c.ovnLegacyClient.OvnICSbAddress = genHostAddress(lastIcCm["ic-db-host"], cm.Data["ic-sb-port"]) + c.ovnLegacyClient.OvnICNbAddress = genHostAddress(lastIcCm["ic-db-host"], cm.Data["ic-nb-port"]) + c.disableOVNIC(lastIcCm["az-name"]) klog.Info("start to reestablish ovn-ic") if err := c.establishInterConnection(cm.Data); err != nil { klog.Errorf("failed to reestablish ovn-ic, %v", err) return } - if err := c.RemoveOldChassisInSbDB(lastIcCm["az-name"]); err != nil { - klog.Errorf("failed to remove remote chassis: %v", err) - } - icEnabled = "true" lastIcCm = cm.Data + lastTSs = curTSs klog.Info("finish reestablishing ovn-ic") return } - - c.ovnLegacyClient.OvnICNbAddress = genHostAddress(cm.Data["ic-db-host"], cm.Data["ic-nb-port"]) - klog.Info("start to establish ovn-ic") - if err := c.establishInterConnection(cm.Data); err != nil { - klog.Errorf("failed to establish ovn-ic, %v", err) - return - } - icEnabled = "true" - lastIcCm = cm.Data - klog.Info("finish establishing ovn-ic") } func (c *Controller) removeInterConnection(azName string) error { @@ -166,38 +252,18 @@ func (c *Controller) removeInterConnection(azName string) error { } } - if azName != "" { - if err := c.OVNNbClient.DeleteLogicalSwitchPorts(nil, func(lsp *ovnnb.LogicalSwitchPort) bool { - // add the code below because azName may have multi "-" - firstIndex := strings.Index(lsp.Name, "-") - if firstIndex != -1 { - firstPart := lsp.Name[:firstIndex] - secondPart := lsp.Name[firstIndex+1:] - return secondPart == azName && strings.HasPrefix(firstPart, util.InterconnectionSwitch) - } - return false - }); err != nil { - return err - } - - if err := c.OVNNbClient.DeleteLogicalRouterPorts(nil, func(lrp *ovnnb.LogicalRouterPort) bool { - lastIndex := strings.LastIndex(lrp.Name, "-") - if lastIndex != -1 { - firstPart := lrp.Name[:lastIndex] - secondPart := lrp.Name[lastIndex+1:] - return firstPart == azName && strings.HasPrefix(secondPart, util.InterconnectionSwitch) - } - return false - }); err != nil { - return err - } - } - if err := c.stopOVNIC(); err != nil { klog.Errorf("failed to stop ovn-ic, %v", err) return err } + if azName != "" { + if err := c.DeleteICResources(azName); err != nil { + klog.Errorf("failed to delete ovn-ic resource on az %s , %v", azName, err) + return err + } + } + return nil } @@ -212,11 +278,20 @@ func (c *Controller) establishInterConnection(config map[string]string) error { return err } - groups := strings.Split(config["gw-nodes"], ";") - for i, group := range groups { - gwNodes := strings.Split(group, ",") - chassises := make([]string, len(gwNodes)) - for j, gw := range gwNodes { + tsNames, err := c.ovnLegacyClient.GetTs() + if err != nil { + klog.Errorf("failed to list ic logical switch. %v ", err) + return err + } + + sort.Strings(tsNames) + + gwNodes := strings.Split(config["gw-nodes"], ",") + chassises := make([]string, len(gwNodes)) + + for i, tsName := range tsNames { + gwNodesOrdered := moveElements(gwNodes, i) + for j, gw := range gwNodesOrdered { gw = strings.TrimSpace(gw) chassis, err := c.OVNSbClient.GetChassisByHost(gw) if err != nil { @@ -254,18 +329,6 @@ func (c *Controller) establishInterConnection(config map[string]string) error { } } - tsName := c.getTSName(i) - cidr, err := c.getTSCidr(i) - if err != nil { - klog.Errorf("failed to get ts cidr index %d err: %v", i, err) - return err - } - - if err := c.createTS(genHostAddress(config["ic-db-host"], config["ic-nb-port"]), tsName, cidr); err != nil { - klog.Errorf("failed to create ts %q: %v", tsName, err) - return err - } - tsPort := fmt.Sprintf("%s-%s", tsName, config["az-name"]) exist, err := c.OVNNbClient.LogicalSwitchPortExists(tsPort) if err != nil { @@ -321,7 +384,6 @@ func (c *Controller) acquireLrpAddress(ts string) (string, error) { if !existAddress.Has(random) { return random, nil } - klog.Infof("random ip %s already exists", random) time.Sleep(1 * time.Second) } @@ -361,27 +423,6 @@ func (c *Controller) stopOVNIC() error { return nil } -func (c *Controller) createTS(icNBAddr, tsName, subnet string) error { - cmd := exec.Command("ovn-ic-nbctl", - fmt.Sprintf("--db=%s", icNBAddr), - ovs.MayExist, "ts-add", tsName, - "--", "set", "Transit_Switch", tsName, fmt.Sprintf(`external_ids:subnet="%s"`, subnet)) - if os.Getenv("ENABLE_SSL") == "true" { - cmd = exec.Command("ovn-ic-nbctl", - fmt.Sprintf("--db=%s", icNBAddr), - "--private-key=/var/run/tls/key", - "--certificate=/var/run/tls/cert", - "--ca-cert=/var/run/tls/cacert", - ovs.MayExist, "ts-add", tsName, - "--", "set", "Transit_Switch", tsName, fmt.Sprintf(`external_ids:subnet="%s"`, subnet)) - } - output, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("output: %s, err: %v", output, err) - } - return nil -} - func (c *Controller) delLearnedRoute() error { lrList, err := c.OVNNbClient.ListLogicalRouter(false, nil) if err != nil { @@ -399,6 +440,7 @@ func (c *Controller) delLearnedRoute() error { if r.Policy != nil { policy = *r.Policy } + if err = c.deleteStaticRouteFromVpc( lr.Name, r.RouteTable, @@ -409,6 +451,7 @@ func (c *Controller) delLearnedRoute() error { klog.Errorf("failed to delete learned static route %#v on logical router %s: %v", r, lr.Name, err) return err } + } } @@ -416,6 +459,37 @@ func (c *Controller) delLearnedRoute() error { return nil } +func (c *Controller) deleteStaticRouteFromVpc(name, table, cidr, nextHop string, policy kubeovnv1.RoutePolicy) error { + var ( + vpc, cachedVpc *kubeovnv1.Vpc + policyStr string + err error + ) + + policyStr = convertPolicy(policy) + if err = c.OVNNbClient.DeleteLogicalRouterStaticRoute(name, &table, &policyStr, cidr, nextHop); err != nil { + klog.Errorf("del vpc %s static route failed, %v", name, err) + return err + } + + cachedVpc, err = c.vpcsLister.Get(name) + if err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + klog.Error(err) + return err + } + vpc = cachedVpc.DeepCopy() + // make sure custom policies not be deleted + _, err = c.config.KubeOvnClient.KubeovnV1().Vpcs().Update(context.Background(), vpc, metav1.UpdateOptions{}) + if err != nil { + klog.Error(err) + return err + } + return nil +} + func genHostAddress(host, port string) (hostAddress string) { hostList := strings.Split(host, ",") if len(hostList) == 1 { @@ -441,12 +515,21 @@ func (c *Controller) SynRouteToPolicy() { } func (c *Controller) RemoveOldChassisInSbDB(azName string) error { + if azName == "" { + return nil + } + azUUID, err := c.ovnLegacyClient.GetAzUUID(azName) if err != nil { klog.Errorf("failed to get UUID of AZ %s: %v", lastIcCm["az-name"], err) return err } + if azUUID == "" { + klog.Infof("%s have already been deleted", azName) + return nil + } + gateways, err := c.ovnLegacyClient.GetGatewayUUIDsInOneAZ(azUUID) if err != nil { klog.Errorf("failed to get gateway UUIDs in AZ %s: %v", azUUID, err) @@ -459,6 +542,13 @@ func (c *Controller) RemoveOldChassisInSbDB(azName string) error { return err } + portBindings, err := c.ovnLegacyClient.GetPortBindingUUIDsInOneAZ(azUUID) + if err != nil { + klog.Errorf("failed to get Port_Binding UUIDs in AZ %s: %v", azUUID, err) + return err + } + + c.ovnLegacyClient.DestroyPortBindings(portBindings) c.ovnLegacyClient.DestroyGateways(gateways) c.ovnLegacyClient.DestroyRoutes(routes) return c.ovnLegacyClient.DestroyChassis(azUUID) @@ -480,7 +570,7 @@ func stripPrefix(policyMatch string) (string, error) { func (c *Controller) syncOneRouteToPolicy(key, value string) { lr, err := c.OVNNbClient.GetLogicalRouter(c.config.ClusterRouter, false) if err != nil { - klog.Errorf("logical router does not exist %v at %v", err, time.Now()) + klog.Infof("logical router %s is not ready at %v", util.DefaultVpc, time.Now()) return } lrRouteList, err := c.OVNNbClient.ListLogicalRouterStaticRoutesByOption(lr.Name, util.MainRouteTable, key, value) @@ -571,27 +661,24 @@ func (c *Controller) listRemoteLogicalSwitchPortAddress() (*strset.Set, error) { return existAddress, nil } -func (c *Controller) getTSName(index int) string { - if index == 0 { - return util.InterconnectionSwitch +func moveElements(arr []string, order int) []string { + if order >= len(arr) { + order %= len(arr) } - return fmt.Sprintf("%s%d", util.InterconnectionSwitch, index) + + return append(arr[order:], arr[:order]...) } -func (c *Controller) getTSCidr(index int) (string, error) { - defaultSubnet, err := c.subnetsLister.Get(c.config.DefaultLogicalSwitch) - if err != nil { - return "", err +func convertPolicy(origin kubeovnv1.RoutePolicy) string { + if origin == kubeovnv1.PolicyDst { + return ovnnb.LogicalRouterStaticRoutePolicyDstIP } + return ovnnb.LogicalRouterStaticRoutePolicySrcIP +} - var cidr string - switch defaultSubnet.Spec.Protocol { - case kubeovnv1.ProtocolIPv4: - cidr = fmt.Sprintf("169.254.%d.0/24", 100+index) - case kubeovnv1.ProtocolIPv6: - cidr = fmt.Sprintf("fe80:a9fe:%02x::/112", 100+index) - case kubeovnv1.ProtocolDual: - cidr = fmt.Sprintf("169.254.%d.0/24,fe80:a9fe:%02x::/112", 100+index, 100+index) +func reversePolicy(origin ovnnb.LogicalRouterStaticRoutePolicy) kubeovnv1.RoutePolicy { + if origin == ovnnb.LogicalRouterStaticRoutePolicyDstIP { + return kubeovnv1.PolicyDst } - return cidr, nil + return kubeovnv1.PolicySrc } diff --git a/pkg/ovn_leader_checker/ovn.go b/pkg/ovn_leader_checker/ovn.go index fd4cab036ea..200cdfcfd4a 100755 --- a/pkg/ovn_leader_checker/ovn.go +++ b/pkg/ovn_leader_checker/ovn.go @@ -23,6 +23,8 @@ import ( "k8s.io/client-go/tools/clientcmd" "k8s.io/klog/v2" + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/ovs" "github.com/kubeovn/kube-ovn/pkg/util" ) @@ -44,6 +46,7 @@ type Configuration struct { KubeClient kubernetes.Interface ProbeInterval int EnableCompact bool + ISICDBServer bool } // ParseFlags parses cmd args then init kubeclient and conf @@ -53,6 +56,7 @@ func ParseFlags() (*Configuration, error) { argKubeConfigFile = pflag.String("kubeconfig", "", "Path to kubeconfig file with authorization and master location information. If not set use the inCluster token.") argProbeInterval = pflag.Int("probeInterval", DefaultProbeInterval, "interval of probing leader in seconds") argEnableCompact = pflag.Bool("enableCompact", true, "is enable compact") + argIsICDBServer = pflag.Bool("isICDBServer", false, "is ic db server ") ) klogFlags := flag.NewFlagSet("klog", flag.ContinueOnError) @@ -83,6 +87,7 @@ func ParseFlags() (*Configuration, error) { KubeConfigFile: *argKubeConfigFile, ProbeInterval: *argProbeInterval, EnableCompact: *argEnableCompact, + ISICDBServer: *argIsICDBServer, } return config, nil } @@ -352,7 +357,7 @@ func doOvnLeaderCheck(cfg *Configuration, podName, podNamespace string) { util.LogFatalAndExit(nil, "preValidChkCfg: invalid cfg") } - if !checkOvnIsAlive() { + if !cfg.ISICDBServer && !checkOvnIsAlive() { klog.Errorf("ovn is not alive") return } @@ -367,26 +372,45 @@ func doOvnLeaderCheck(cfg *Configuration, podName, podNamespace string) { for k, v := range cachedPod.Labels { labels[k] = v } - nbLeader := isDBLeader("OVN_Northbound", 6641) - sbLeader := isDBLeader("OVN_Southbound", 6642) - northdLeader := checkNorthdActive() - updatePodLabels(labels, "ovn-nb-leader", nbLeader) - updatePodLabels(labels, "ovn-sb-leader", sbLeader) - updatePodLabels(labels, "ovn-northd-leader", northdLeader) - if err = patchPodLabels(cfg, cachedPod, labels); err != nil { - klog.Errorf("patch label error %v", err) - return - } - if sbLeader && checkNorthdSvcExist(cfg, podNamespace, "ovn-northd") { - if !checkNorthdEpAlive(cfg, podNamespace, "ovn-northd") { - klog.Warning("no available northd leader, try to release the lock") - stealLock() + + if !cfg.ISICDBServer { + nbLeader := isDBLeader("OVN_Northbound", 6641) + sbLeader := isDBLeader("OVN_Southbound", 6642) + northdLeader := checkNorthdActive() + updatePodLabels(labels, "ovn-nb-leader", nbLeader) + updatePodLabels(labels, "ovn-sb-leader", sbLeader) + updatePodLabels(labels, "ovn-northd-leader", northdLeader) + if err = patchPodLabels(cfg, cachedPod, labels); err != nil { + klog.Errorf("patch label error %v", err) + return + } + if sbLeader && checkNorthdSvcExist(cfg, podNamespace, "ovn-northd") { + if !checkNorthdEpAlive(cfg, podNamespace, "ovn-northd") { + klog.Warning("no available northd leader, try to release the lock") + stealLock() + } } - } - if cfg.EnableCompact { - compactOvnDatabase("nb") - compactOvnDatabase("sb") + if cfg.EnableCompact { + compactOvnDatabase("nb") + compactOvnDatabase("sb") + } + } else { + icNbLeader := isDBLeader("OVN_IC_Northbound", 6645) + icSbLeader := isDBLeader("OVN_IC_Southbound", 6646) + updatePodLabels(labels, "ovn-ic-nb-leader", icNbLeader) + updatePodLabels(labels, "ovn-ic-sb-leader", icSbLeader) + if err = patchPodLabels(cfg, cachedPod, labels); err != nil { + klog.Errorf("patch label error %v", err) + return + } + + if icNbLeader { + if err := updateTS(); err != nil { + klog.Errorf("update ts num failed err: %v ", err) + return + } + } } } @@ -399,3 +423,99 @@ func StartOvnLeaderCheck(cfg *Configuration) { time.Sleep(interval) } } + +func getTSName(index int) string { + if index == 0 { + return util.InterconnectionSwitch + } + return fmt.Sprintf("%s%d", util.InterconnectionSwitch, index) +} + +func getTSCidr(index int) (string, error) { + var proto, cidr string + podIpsEnv := os.Getenv("POD_IPS") + podIps := strings.Split(podIpsEnv, ",") + if len(podIps) == 1 { + if util.CheckProtocol(podIps[0]) == kubeovnv1.ProtocolIPv6 { + proto = kubeovnv1.ProtocolIPv6 + } else { + proto = kubeovnv1.ProtocolIPv4 + } + } else if len(podIps) == 2 { + proto = kubeovnv1.ProtocolDual + } + + switch proto { + case kubeovnv1.ProtocolIPv4: + cidr = fmt.Sprintf("169.254.%d.0/24", 100+index) + case kubeovnv1.ProtocolIPv6: + cidr = fmt.Sprintf("fe80:a9fe:%02x::/112", 100+index) + case kubeovnv1.ProtocolDual: + cidr = fmt.Sprintf("169.254.%d.0/24,fe80:a9fe:%02x::/112", 100+index, 100+index) + } + return cidr, nil +} + +func updateTS() error { + cmd := exec.Command("ovn-ic-nbctl", "show") + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("ovn-ic-nbctl show output: %s, err: %v", output, err) + } + var existTSCount int + if lines := strings.TrimSpace(string(output)); lines != "" { + existTSCount = len(strings.Split(lines, "\n")) + } + expectTSCount, err := strconv.Atoi(os.Getenv("TS_NUM")) + if err != nil { + return fmt.Errorf("expectTSCount atoi failed output: %s, err: %v", output, err) + } + if expectTSCount == existTSCount { + klog.V(3).Infof("expectTSCount %d no changes required.", expectTSCount) + return nil + } + + if expectTSCount > existTSCount { + for i := expectTSCount - 1; i > existTSCount-1; i-- { + tsName := getTSName(i) + subnet, err := getTSCidr(i) + if err != nil { + return err + } + cmd := exec.Command("ovn-ic-nbctl", + ovs.MayExist, "ts-add", tsName, + "--", "set", "Transit_Switch", tsName, fmt.Sprintf(`external_ids:subnet="%s"`, subnet)) + if os.Getenv("ENABLE_SSL") == "true" { + cmd = exec.Command("ovn-ic-nbctl", + "--private-key=/var/run/tls/key", + "--certificate=/var/run/tls/cert", + "--ca-cert=/var/run/tls/cacert", + ovs.MayExist, "ts-add", tsName, + "--", "set", "Transit_Switch", tsName, fmt.Sprintf(`external_ids:subnet="%s"`, subnet)) + } + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("output: %s, err: %v", output, err) + } + } + } else { + for i := existTSCount - 1; i >= expectTSCount; i-- { + tsName := getTSName(i) + cmd := exec.Command("ovn-ic-nbctl", + "ts-del", tsName) + if os.Getenv("ENABLE_SSL") == "true" { + cmd = exec.Command("ovn-ic-nbctl", + "--private-key=/var/run/tls/key", + "--certificate=/var/run/tls/cert", + "--ca-cert=/var/run/tls/cacert", + "ts-del", tsName) + } + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("output: %s, err: %v", output, err) + } + } + } + + return nil +} diff --git a/pkg/ovs/ovn-ic-nbctl.go b/pkg/ovs/ovn-ic-nbctl.go index 75292a4baa8..b559a0f7fd4 100644 --- a/pkg/ovs/ovn-ic-nbctl.go +++ b/pkg/ovs/ovn-ic-nbctl.go @@ -45,3 +45,21 @@ func (c LegacyClient) GetTsSubnet(ts string) (string, error) { } return subnet, nil } + +func (c LegacyClient) GetTs() ([]string, error) { + cmd := []string{"--format=csv", "--data=bare", "--no-heading", "--columns=name", "find", "Transit_Switch"} + output, err := c.ovnIcNbCommand(cmd...) + if err != nil { + klog.Errorf("failed to list logical switch port, %v", err) + return nil, err + } + lines := strings.Split(output, "\n") + result := make([]string, 0, len(lines)) + for _, l := range lines { + if len(strings.TrimSpace(l)) == 0 { + continue + } + result = append(result, strings.TrimSpace(l)) + } + return result, nil +} diff --git a/pkg/ovs/ovn-ic-sbctl.go b/pkg/ovs/ovn-ic-sbctl.go index b36f6cbcbab..92e60b71161 100644 --- a/pkg/ovs/ovn-ic-sbctl.go +++ b/pkg/ovs/ovn-ic-sbctl.go @@ -97,6 +97,14 @@ func (c LegacyClient) GetRouteUUIDsInOneAZ(uuid string) ([]string, error) { return routes, nil } +func (c LegacyClient) GetPortBindingUUIDsInOneAZ(uuid string) ([]string, error) { + portBindings, err := c.FindUUIDWithAttrInTable("availability_zone", uuid, "Port_Binding") + if err != nil { + return nil, fmt.Errorf("failed to get ovn-ic-sb Port_Binding with uuid %v: %v", uuid, err) + } + return portBindings, nil +} + func (c LegacyClient) DestroyGateways(uuids []string) { for _, uuid := range uuids { if err := c.DestroyTableWithUUID(uuid, "gateway"); err != nil { @@ -115,6 +123,15 @@ func (c LegacyClient) DestroyRoutes(uuids []string) { } } +func (c LegacyClient) DestroyPortBindings(uuids []string) { + for _, uuid := range uuids { + if err := c.DestroyTableWithUUID(uuid, "Port_Binding"); err != nil { + klog.Errorf("failed to delete Port_Binding %v: %v", uuid, err) + } + continue + } +} + func (c LegacyClient) DestroyChassis(uuid string) error { if err := c.DestroyTableWithUUID(uuid, "availability_zone"); err != nil { klog.Error(err) diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index 97c9df16426..d57c6bb9beb 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -102,6 +102,14 @@ func NewFrameworkWithContext(baseName, kubeContext string) *Framework { f.ClusterVersion = os.Getenv("E2E_BRANCH") f.ClusterNetworkMode = os.Getenv("E2E_NETWORK_MODE") + if strings.HasPrefix(f.ClusterVersion, "release-") { + n, err := fmt.Sscanf(f.ClusterVersion, "release-%d.%d", &f.ClusterVersionMajor, &f.ClusterVersionMinor) + ExpectNoError(err) + ExpectEqual(n, 2) + } else { + f.ClusterVersionMajor, f.ClusterVersionMinor = 999, 999 + } + ginkgo.BeforeEach(func() { framework.TestContext.Host = "" }) diff --git a/test/e2e/ovn-ic/e2e_test.go b/test/e2e/ovn-ic/e2e_test.go index dd15ae9f02e..986c1c06086 100644 --- a/test/e2e/ovn-ic/e2e_test.go +++ b/test/e2e/ovn-ic/e2e_test.go @@ -77,7 +77,7 @@ func execPodOrDie(kubeContext, namespace, pod, cmd string) string { return e2epodoutput.RunHostCmdOrDie(namespace, pod, cmd) } -var _ = framework.OrderedDescribe("[group:ovn-ic]", func() { +var _ = framework.SerialDescribe("[group:ovn-ic]", func() { frameworks := make([]*framework.Framework, len(clusters)) for i := range clusters { frameworks[i] = framework.NewFrameworkWithContext("ovn-ic", "kind-"+clusters[i]) @@ -212,4 +212,145 @@ var _ = framework.OrderedDescribe("[group:ovn-ic]", func() { fnCheckPodHTTP() }) + + framework.ConformanceIt("Should Support ECMP OVN Interconnection", func() { + frameworks[0].SkipVersionPriorTo(1, 11, "This feature was introduced in v1.11") + ginkgo.By("case 1: ecmp gateway network test") + if frameworks[0].ClusterIPFamily == "dual" { + checkECMPCount(6) + } else { + checkECMPCount(3) + } + fnCheckPodHTTP() + + ginkgo.By("case 2: reduce two clusters from 3 gateway to 1 gateway") + oldGatewayStr := make([]string, len(clusters)) + gwNodes := make([]string, len(clusters)) + for i := range clusters { + ginkgo.By("fetching the ConfigMap in cluster " + clusters[i]) + cm, err := clientSets[i].CoreV1().ConfigMaps(framework.KubeOvnNamespace).Get(context.TODO(), util.InterconnectionConfig, metav1.GetOptions{}) + framework.ExpectNoError(err, "failed to get ConfigMap") + gwNodes[i] = cm.Data["gw-nodes"] + oldGatewayStr[i] = cm.Data["gw-nodes"] + gws := strings.Split(gwNodes[i], ",") + newGatewayStr := strings.Join(gws[0:len(gws)-2], ",") + + configMapPatchPayload, _ := json.Marshal(corev1.ConfigMap{ + Data: map[string]string{ + "gw-nodes": newGatewayStr, + }, + }) + _, err = clientSets[i].CoreV1().ConfigMaps(framework.KubeOvnNamespace).Patch(context.TODO(), util.InterconnectionConfig, k8stypes.StrategicMergePatchType, configMapPatchPayload, metav1.PatchOptions{}) + framework.ExpectNoError(err, "patch ovn-ic-config failed") + } + fnCheckPodHTTP() + + ginkgo.By("case 3: recover two clusters from 1 gateway to 3 gateway") + for i := range clusters { + ginkgo.By("fetching the ConfigMap in cluster " + clusters[i]) + + configMapPatchPayload, _ := json.Marshal(corev1.ConfigMap{ + Data: map[string]string{ + "gw-nodes": oldGatewayStr[i], + }, + }) + + _, err := clientSets[i].CoreV1().ConfigMaps(framework.KubeOvnNamespace).Patch(context.TODO(), util.InterconnectionConfig, k8stypes.StrategicMergePatchType, configMapPatchPayload, metav1.PatchOptions{}) + framework.ExpectNoError(err, "patch ovn-ic-config failed") + } + fnCheckPodHTTP() + + ginkgo.By("case 4: scale ecmp path from 3 to 5 ") + switchCmd := "kubectl config use-context kind-kube-ovn" + _, err := exec.Command("bash", "-c", switchCmd).CombinedOutput() + framework.ExpectNoError(err, "switch to kube-ovn cluster failed") + + patchCmd := "kubectl patch deployment ovn-ic-server -n kube-system --type='json' -p=\"[{'op': 'replace', 'path': '/spec/template/spec/containers/0/env/1/value', 'value': '5'}]\"" + _, _ = exec.Command("bash", "-c", patchCmd).CombinedOutput() + if frameworks[0].ClusterIPFamily == "dual" { + checkECMPCount(10) + } else { + checkECMPCount(5) + } + fnCheckPodHTTP() + + ginkgo.By("case 5: reduce ecmp path from 5 to 3 ") + patchCmd = "kubectl patch deployment ovn-ic-server -n kube-system --type='json' -p=\"[{'op': 'replace', 'path': '/spec/template/spec/containers/0/env/1/value', 'value': '3'}]\"" + _, _ = exec.Command("bash", "-c", patchCmd).CombinedOutput() + if frameworks[0].ClusterIPFamily == "dual" { + checkECMPCount(6) + } else { + checkECMPCount(3) + } + fnCheckPodHTTP() + + ginkgo.By("case 6: disable gateway kube-ovn1-worker gateway") + switchCmd = "kubectl config use-context kind-kube-ovn1" + _, err = exec.Command("bash", "-c", switchCmd).CombinedOutput() + framework.ExpectNoError(err, "switch to kube-ovn1 cluster failed") + + disableNetworkCmd := "docker exec kube-ovn1-worker iptables -I INPUT -p udp --dport 6081 -j DROP" + _, err = exec.Command("bash", "-c", disableNetworkCmd).CombinedOutput() + framework.ExpectNoError(err, "disable kube-ovn1-worker gateway failed") + + taintCmd := "kubectl taint nodes kube-ovn1-worker e2e=test:NoSchedule" + _, _ = exec.Command("bash", "-c", taintCmd).CombinedOutput() + fnCheckPodHTTP() + + ginkgo.By("case 7: disable gateway kube-ovn1-worker2 gateway") + switchCmd = "kubectl config use-context kind-kube-ovn1" + _, err = exec.Command("bash", "-c", switchCmd).CombinedOutput() + framework.ExpectNoError(err, "switch to kube-ovn1 cluster failed") + + disableNetworkCmd = "docker exec kube-ovn1-worker2 iptables -I INPUT -p udp --dport 6081 -j DROP" + _, err = exec.Command("bash", "-c", disableNetworkCmd).CombinedOutput() + framework.ExpectNoError(err, "disable kube-ovn1-worker2 gateway failed") + + taintCmd = "kubectl taint nodes kube-ovn1-worker2 e2e=test:NoSchedule" + _, _ = exec.Command("bash", "-c", taintCmd).CombinedOutput() + fnCheckPodHTTP() + + ginkgo.By("case 8: enable gateway kube-ovn1-worker gateway") + switchCmd = "kubectl config use-context kind-kube-ovn1" + _, err = exec.Command("bash", "-c", switchCmd).CombinedOutput() + framework.ExpectNoError(err, "switch to kube-ovn1 cluster failed") + + disableNetworkCmd = "docker exec kube-ovn1-worker iptables -D INPUT -p udp --dport 6081 -j DROP" + _, err = exec.Command("bash", "-c", disableNetworkCmd).CombinedOutput() + framework.ExpectNoError(err, "enable kube-ovn1-worker gateway failed") + + taintCmd = "kubectl taint nodes kube-ovn1-worker e2e=test:NoSchedule-" + _, _ = exec.Command("bash", "-c", taintCmd).CombinedOutput() + fnCheckPodHTTP() + + ginkgo.By("case 9: enable gateway kube-ovn1-worker2 gateway") + switchCmd = "kubectl config use-context kind-kube-ovn1" + _, err = exec.Command("bash", "-c", switchCmd).CombinedOutput() + framework.ExpectNoError(err, "switch to kube-ovn1 cluster failed") + + disableNetworkCmd = "docker exec kube-ovn1-worker2 iptables -D INPUT -p udp --dport 6081 -j DROP" + _, err = exec.Command("bash", "-c", disableNetworkCmd).CombinedOutput() + framework.ExpectNoError(err, "enable kube-ovn1-worker2 gateway failed") + + taintCmd = "kubectl taint nodes kube-ovn1-worker2 e2e=test:NoSchedule-" + _, _ = exec.Command("bash", "-c", taintCmd).CombinedOutput() + fnCheckPodHTTP() + }) }) + +func checkECMPCount(expectCount int) { + ecmpCount := 0 + maxRetryTimes := 10 + for i := 0; i < maxRetryTimes; i++ { + time.Sleep(3 * time.Second) + execCmd := "kubectl ko nbctl lr-route-list ovn-cluster " + output, err := exec.Command("bash", "-c", execCmd).CombinedOutput() + framework.ExpectNoError(err) + ecmpCount = strings.Count(string(output), "ecmp") + if ecmpCount == expectCount { + break + } + } + + framework.ExpectEqual(ecmpCount, expectCount) +} diff --git a/yamls/kind.yaml.j2 b/yamls/kind.yaml.j2 index d852397d593..21c35b5bb8f 100644 --- a/yamls/kind.yaml.j2 +++ b/yamls/kind.yaml.j2 @@ -82,6 +82,8 @@ nodes: labels: kube-ovn/role: master {%- elif ovn_ic is equalto "true" %} + - role: worker + image: kindest/node:{{ k8s_version }} - role: worker image: kindest/node:{{ k8s_version }} {%- endif %}