Skip to content

Commit

Permalink
ecmp refactor
Browse files Browse the repository at this point in the history
Signed-off-by: changluyi <[email protected]>
  • Loading branch information
changluyi committed Jan 18, 2024
1 parent 141ceaa commit add55c5
Show file tree
Hide file tree
Showing 24 changed files with 1,529 additions and 255 deletions.
66 changes: 35 additions & 31 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -357,19 +357,19 @@ kind-init-ovn-ic: kind-init-ovn-ic-ipv4

.PHONY: kind-init-ovn-ic-ipv4
kind-init-ovn-ic-ipv4: kind-clean-ovn-ic
@ovn_ic=true $(MAKE) kind-init
@ha=true $(MAKE) kind-init
@ovn_ic=true $(MAKE) kind-generate-config
$(call kind_create_cluster,yamls/kind.yaml,kube-ovn1,1)

.PHONY: kind-init-ovn-ic-ipv6
kind-init-ovn-ic-ipv6: kind-clean-ovn-ic
@ovn_ic=true $(MAKE) kind-init-ipv6
@ha=true $(MAKE) kind-init-ipv6
@ovn_ic=true ip_family=ipv6 $(MAKE) kind-generate-config
$(call kind_create_cluster,yamls/kind.yaml,kube-ovn1,1)

.PHONY: kind-init-ovn-ic-dual
kind-init-ovn-ic-dual: kind-clean-ovn-ic
@ovn_ic=true $(MAKE) kind-init-dual
@ha=true $(MAKE) kind-init-dual
@ovn_ic=true ip_family=dual $(MAKE) kind-generate-config
$(call kind_create_cluster,yamls/kind.yaml,kube-ovn1,1)

Expand Down Expand Up @@ -439,6 +439,7 @@ kind-install-chart: kind-load-image kind-untaint-control-plane
--set global.images.kubeovn.tag=$(VERSION) \
--set replicaCount=$$(echo $$ips | awk -F ',' '{print NF}') \
--set MASTER_NODES="$$(echo $$ips | sed 's/,/\\,/g')"
--set func.ENABLE_IC=$$(kubectl get node --show-labels | grep -q "ovn.kubernetes.io/ic-gw" && echo true || echo false)
sleep 60
kubectl -n kube-system rollout status --timeout=1s deployment/ovn-central
kubectl -n kube-system rollout status --timeout=1s daemonset/ovs-ovn
Expand All @@ -453,6 +454,7 @@ kind-upgrade-chart: kind-load-image
--set global.images.kubeovn.tag=$(VERSION) \
--set replicaCount=$$(echo $(OVN_DB_IPS) | awk -F ',' '{print NF}') \
--set MASTER_NODES='$(OVN_DB_IPS)'
--set func.ENABLE_IC=$$(kubectl get node --show-labels | grep -q "ovn.kubernetes.io/ic-gw" && echo true || echo false)
sleep 90
kubectl -n kube-system rollout status --timeout=1s deployment/ovn-central
kubectl -n kube-system wait pod --for=condition=ready -l app=ovs
Expand Down Expand Up @@ -493,31 +495,33 @@ kind-install-overlay-ipv4: kind-install
kind-install-ovn-ic: kind-install-ovn-ic-ipv4

.PHONY: kind-install-ovn-ic-ipv4
kind-install-ovn-ic-ipv4: kind-install
kind-install-ovn-ic-ipv4:
@ENABLE_IC=true $(MAKE) kind-install
$(call kind_load_image,kube-ovn1,$(REGISTRY)/kube-ovn:$(VERSION))
kubectl config use-context kind-kube-ovn1
$(MAKE) kind-untaint-control-plane
sed -e 's/10.16.0/10.18.0/g' \
-e 's/10.96.0/10.98.0/g' \
-e 's/100.64.0/100.68.0/g' \
-e 's/VERSION=.*/VERSION=$(VERSION)/' \
dist/images/install.sh | bash
dist/images/install.sh | ENABLE_IC=true bash
kubectl describe no

docker run -d --name ovn-ic-db --network kind $(REGISTRY)/kube-ovn:$(VERSION) bash start-ic-db.sh
@set -e; \
ic_db_host=$$(docker inspect ovn-ic-db -f "{{.NetworkSettings.Networks.kind.IPAddress}}"); \
zone=az0 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn-worker,kube-ovn-worker2;kube-ovn-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \
zone=az1 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn1-worker,kube-ovn1-worker2;kube-ovn1-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml
kubectl config use-context kind-kube-ovn
sed 's/VERSION=.*/VERSION=$(VERSION)/' dist/images/install-ic-server.sh | bash

@set -e; \
ic_db_host=$$(kubectl get deployment ovn-ic-server -n kube-system -o jsonpath='{range .spec.template.spec.containers[0].env[?(@.name=="NODE_IPS")]}{.value}{end}'); \
ic_db_host=$${ic_db_host%?}; \
zone=az0 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn-worker,kube-ovn-worker2,kube-ovn-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \
zone=az1 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn1-worker,kube-ovn1-worker2,kube-ovn1-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml
kubectl apply -f ovn-ic-0.yaml
kubectl config use-context kind-kube-ovn1
kubectl apply -f ovn-ic-1.yaml
sleep 6
docker exec ovn-ic-db ovn-ic-sbctl show

.PHONY: kind-install-ovn-ic-ipv6
kind-install-ovn-ic-ipv6: kind-install-ipv6
kind-install-ovn-ic-ipv6:
@ENABLE_IC=true $(MAKE) kind-install-ipv6
$(call kind_load_image,kube-ovn1,$(REGISTRY)/kube-ovn:$(VERSION))
kubectl config use-context kind-kube-ovn1
@$(MAKE) kind-untaint-control-plane
Expand All @@ -526,23 +530,24 @@ kind-install-ovn-ic-ipv6: kind-install-ipv6
-e 's/fd00:100:64:/fd00:100:68:/g' \
-e 's/VERSION=.*/VERSION=$(VERSION)/' \
dist/images/install.sh | \
IPV6=true bash
IPV6=true ENABLE_IC=true bash
kubectl describe no

docker run -d --name ovn-ic-db --network kind -e PROTOCOL="ipv6" $(REGISTRY)/kube-ovn:$(VERSION) bash start-ic-db.sh
@set -e; \
ic_db_host=$$(docker inspect ovn-ic-db -f "{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}"); \
zone=az0 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn-worker,kube-ovn-worker2;kube-ovn-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \
zone=az1 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn1-worker,kube-ovn1-worker2;kube-ovn1-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml
kubectl config use-context kind-kube-ovn
sed 's/VERSION=.*/VERSION=$(VERSION)/' dist/images/install-ic-server.sh | bash

@set -e; \
ic_db_host=$$(kubectl get deployment ovn-ic-server -n kube-system -o jsonpath='{range .spec.template.spec.containers[0].env[?(@.name=="NODE_IPS")]}{.value}{end}'); \
ic_db_host=$${ic_db_host%?}; \
zone=az0 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn-worker,kube-ovn-worker2,kube-ovn-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \
zone=az1 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn1-worker,kube-ovn1-worker2,kube-ovn1-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml
kubectl apply -f ovn-ic-0.yaml
kubectl config use-context kind-kube-ovn1
kubectl apply -f ovn-ic-1.yaml
sleep 6
docker exec ovn-ic-db ovn-ic-sbctl show

.PHONY: kind-install-ovn-ic-dual
kind-install-ovn-ic-dual: kind-install-dual
kind-install-ovn-ic-dual:
@ENABLE_IC=true $(MAKE) kind-install-dual
$(call kind_load_image,kube-ovn1,$(REGISTRY)/kube-ovn:$(VERSION))
kubectl config use-context kind-kube-ovn1
@$(MAKE) kind-untaint-control-plane
Expand All @@ -554,21 +559,20 @@ kind-install-ovn-ic-dual: kind-install-dual
-e 's/fd00:100:64:/fd00:100:68:/g' \
-e 's/VERSION=.*/VERSION=$(VERSION)/' \
dist/images/install.sh | \
DUAL_STACK=true bash
DUAL_STACK=true ENABLE_IC=true bash
kubectl describe no

docker run -d --name ovn-ic-db --network kind -e PROTOCOL="dual" $(REGISTRY)/kube-ovn:$(VERSION) bash start-ic-db.sh
@set -e; \

ic_db_host=$$(docker inspect ovn-ic-db -f "{{.NetworkSettings.Networks.kind.IPAddress}}"); \
zone=az0 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn-worker,kube-ovn-worker2;kube-ovn-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \
zone=az1 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn1-worker,kube-ovn1-worker2;kube-ovn1-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml
kubectl config use-context kind-kube-ovn
sed 's/VERSION=.*/VERSION=$(VERSION)/' dist/images/install-ic-server.sh | bash

@set -e; \
ic_db_host=$$(kubectl get deployment ovn-ic-server -n kube-system -o jsonpath='{range .spec.template.spec.containers[0].env[?(@.name=="NODE_IPS")]}{.value}{end}'); \
ic_db_host=$${ic_db_host%?}; \
zone=az0 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn-worker,kube-ovn-worker2,kube-ovn-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \
zone=az1 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn1-worker,kube-ovn1-worker2,kube-ovn1-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml
kubectl apply -f ovn-ic-0.yaml
kubectl config use-context kind-kube-ovn1
kubectl apply -f ovn-ic-1.yaml
sleep 6
docker exec ovn-ic-db ovn-ic-sbctl show

.PHONY: kind-install-ovn-submariner
kind-install-ovn-submariner: kind-install
Expand Down
99 changes: 99 additions & 0 deletions charts/templates/ic-controller-deploy.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
{{- if eq .Values.func.ENABLE_IC true }}
kind: Deployment
apiVersion: apps/v1
metadata:
name: ovn-ic-controller
namespace: kube-system
annotations:
kubernetes.io/description: |
OVN IC Client
spec:
replicas: 1
strategy:
rollingUpdate:
maxSurge: 0
maxUnavailable: 1
type: RollingUpdate
selector:
matchLabels:
app: ovn-ic-controller
template:
metadata:
labels:
app: ovn-ic-controller
component: network
type: infra
spec:
tolerations:
- effect: NoSchedule
operator: Exists
- effect: NoExecute
operator: Exists
- key: CriticalAddonsOnly
operator: Exists
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
app: ovn-ic-controller
topologyKey: kubernetes.io/hostname
priorityClassName: system-cluster-critical
serviceAccountName: ovn
hostNetwork: true
containers:
- name: ovn-ic-controller
image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
command: ["/kube-ovn/start-ic-controller.sh"]
securityContext:
capabilities:
add: ["SYS_NICE"]
env:
- name: ENABLE_SSL
value: "{{ .Values.networking.ENABLE_SSL }}"
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: OVN_DB_IPS
value: "{{ .Values.MASTER_NODES }}"
resources:
requests:
cpu: 300m
memory: 200Mi
limits:
cpu: 3
memory: 1Gi
volumeMounts:
- mountPath: /var/run/ovn
name: host-run-ovn
- mountPath: /etc/ovn
name: host-config-ovn
- mountPath: /var/log/ovn
name: host-log-ovn
- mountPath: /etc/localtime
name: localtime
- mountPath: /var/run/tls
name: kube-ovn-tls
nodeSelector:
kubernetes.io/os: "linux"
kube-ovn/role: "master"
volumes:
- name: host-run-ovn
hostPath:
path: /run/ovn
- name: host-config-ovn
hostPath:
path: /etc/origin/ovn
- name: host-log-ovn
hostPath:
path: /var/log/ovn
- name: localtime
hostPath:
path: /etc/localtime
- name: kube-ovn-tls
secret:
optional: true
secretName: kube-ovn-tls
{{- end }}
4 changes: 4 additions & 0 deletions cmd/cmdmain.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"github.com/kubeovn/kube-ovn/cmd/controller"
"github.com/kubeovn/kube-ovn/cmd/controller_health_check"
"github.com/kubeovn/kube-ovn/cmd/daemon"
"github.com/kubeovn/kube-ovn/cmd/ovn_ic_controller"
"github.com/kubeovn/kube-ovn/cmd/ovn_leader_checker"
"github.com/kubeovn/kube-ovn/cmd/ovn_monitor"
"github.com/kubeovn/kube-ovn/cmd/pinger"
Expand All @@ -29,6 +30,7 @@ const (
CmdSpeaker = "kube-ovn-speaker"
CmdControllerHealthCheck = "kube-ovn-controller-healthcheck"
CmdOvnLeaderChecker = "kube-ovn-leader-checker"
CmdOvnICController = "kube-ovn-ic-controller"
)

const timeFormat = "2006-01-02_15:04:05"
Expand Down Expand Up @@ -95,6 +97,8 @@ func main() {
controller_health_check.CmdMain()
case CmdOvnLeaderChecker:
ovn_leader_checker.CmdMain()
case CmdOvnICController:
ovn_ic_controller.CmdMain()
default:
util.LogFatalAndExit(nil, "%s is an unknown command", cmd)
}
Expand Down
24 changes: 24 additions & 0 deletions cmd/ovn_ic_controller/ovn_ic_controller.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package ovn_ic_controller

import (
"k8s.io/klog/v2"
"k8s.io/sample-controller/pkg/signals"

"github.com/kubeovn/kube-ovn/pkg/ovn_ic_controller"
"github.com/kubeovn/kube-ovn/pkg/util"
"github.com/kubeovn/kube-ovn/versions"
)

func CmdMain() {
defer klog.Flush()

klog.Infof(versions.String())
config, err := ovn_ic_controller.ParseFlags()
if err != nil {
util.LogFatalAndExit(err, "failed to parse config")
}

stopCh := signals.SetupSignalHandler().Done()
ctl := ovn_ic_controller.NewController(config)
ctl.Run(stopCh)
}
3 changes: 2 additions & 1 deletion dist/images/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,5 @@ RUN ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-controller && \
ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-pinger && \
ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-speaker && \
ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-controller-healthcheck && \
ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-leader-checker
ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-leader-checker && \
ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-ic-controller
2 changes: 2 additions & 0 deletions dist/images/cleanup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,8 @@ kubectl delete --ignore-not-found deploy kube-ovn-monitor -n kube-system
kubectl delete --ignore-not-found cm ovn-config ovn-ic-config ovn-external-gw-config -n kube-system
kubectl delete --ignore-not-found svc kube-ovn-pinger kube-ovn-controller kube-ovn-cni kube-ovn-monitor -n kube-system
kubectl delete --ignore-not-found deploy kube-ovn-controller -n kube-system
kubectl delete --ignore-not-found deploy ovn-ic-controller -n kube-system
kubectl delete --ignore-not-found deploy ovn-ic-server -n kube-system

# wait for provier-networks to be deleted before deleting kube-ovn-cni
sleep 5
Expand Down
Loading

0 comments on commit add55c5

Please sign in to comment.