diff --git a/.gitignore b/.gitignore index 99218d88a69d..b2ef3ccdf845 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,8 @@ kwok-node.yaml broker-info.subm broker-info.subm.* broker-info-internal.subm +yamls/clab-bgp.yaml +yamls/clab-bgp-ha.yaml kube-ovn.tar vpc-nat-gateway.tar image-amd64.tar diff --git a/Makefile b/Makefile index b2e2b1673819..c94f765363b0 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,8 @@ endif CONTROL_PLANE_TAINTS = node-role.kubernetes.io/master node-role.kubernetes.io/control-plane +CLAB_IMAGE = ghcr.io/srl-labs/clab:latest + MULTUS_VERSION = v4.0.2 MULTUS_IMAGE = ghcr.io/k8snetworkplumbingwg/multus-cni:$(MULTUS_VERSION)-thick MULTUS_YAML = https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/$(MULTUS_VERSION)/deployments/multus-daemonset-thick.yml @@ -401,6 +403,32 @@ kind-init-ipv6: kind-init-dual: @ip_family=dual $(MAKE) kind-init +.PHONY: kind-init-bgp +kind-init-bgp: kind-clean-bgp kind-init + kube_ovn_version=$(VERSION) j2 yamls/clab-bgp.yaml.j2 -o yamls/clab-bgp.yaml + docker run --rm --privileged \ + --name kube-ovn-bgp \ + --network host \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /var/run/netns:/var/run/netns \ + -v /var/lib/docker/containers:/var/lib/docker/containers \ + --pid=host \ + -v $(CURDIR)/yamls/clab-bgp.yaml:/clab.yaml \ + $(CLAB_IMAGE) clab deploy -t /clab.yaml + +.PHONY: kind-init-bgp-ha +kind-init-bgp-ha: kind-clean-bgp kind-init + kube_ovn_version=$(VERSION) j2 yamls/clab-bgp-ha.yaml.j2 -o yamls/clab-bgp-ha.yaml + docker run --rm --privileged \ + --name kube-ovn-bgp \ + --network host \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /var/run/netns:/var/run/netns \ + -v /var/lib/docker/containers:/var/lib/docker/containers \ + --pid=host \ + -v $(CURDIR)/yamls/clab-bgp-ha.yaml:/clab.yaml \ + $(CLAB_IMAGE) clab deploy -t /clab.yaml + .PHONY: kind-load-image kind-load-image: $(call kind_load_image,kube-ovn,$(REGISTRY)/kube-ovn:$(VERSION)) @@ -767,6 +795,31 @@ kind-install-cilium-chaining: kind-load-image kind-untaint-control-plane ENABLE_LB=false ENABLE_NP=false CNI_CONFIG_PRIORITY=10 bash kubectl describe no +.PHONY: kind-install-bgp +kind-install-bgp: kind-install + kubectl label node --all ovn.kubernetes.io/bgp=true + kubectl annotate subnet ovn-default ovn.kubernetes.io/bgp=local + sed -e 's#image: .*#image: $(REGISTRY)/kube-ovn:$(VERSION)#' \ + -e 's/--neighbor-address=.*/--neighbor-address=10.0.1.1/' \ + -e 's/--neighbor-as=.*/--neighbor-as=65001/' \ + -e 's/--cluster-as=.*/--cluster-as=65002/' yamls/speaker.yaml | \ + kubectl apply -f - + kubectl -n kube-system rollout status ds kube-ovn-speaker --timeout 60s + docker exec clab-bgp-router vtysh -c "show ip route bgp" + +.PHONY: kind-install-bgp-ha +kind-install-bgp-ha: kind-install + kubectl label node --all ovn.kubernetes.io/bgp=true + kubectl annotate subnet ovn-default ovn.kubernetes.io/bgp=local + sed -e 's#image: .*#image: $(REGISTRY)/kube-ovn:$(VERSION)#' \ + -e 's/--neighbor-address=.*/--neighbor-address=10.0.1.1,10.0.1.2/' \ + -e 's/--neighbor-as=.*/--neighbor-as=65001/' \ + -e 's/--cluster-as=.*/--cluster-as=65002/' yamls/speaker.yaml | \ + kubectl apply -f - + kubectl -n kube-system rollout status ds kube-ovn-speaker --timeout 60s + docker exec clab-bgp-router-1 vtysh -c "show ip route bgp" + docker exec clab-bgp-router-2 vtysh -c "show ip route bgp" + .PHONY: kind-install-deepflow kind-install-deepflow: kind-install helm repo add deepflow $(DEEPFLOW_CHART_REPO) @@ -818,6 +871,34 @@ kind-clean-ovn-ic: kind-clean kind-clean-ovn-submariner: kind-clean kind delete cluster --name=kube-ovn1 +.PHONY: kind-clean-bgp +kind-clean-bgp: kind-clean-bgp-ha + kube_ovn_version=$(VERSION) j2 yamls/clab-bgp.yaml.j2 -o yamls/clab-bgp.yaml + docker run --rm --privileged \ + --name kube-ovn-bgp \ + --network host \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /var/run/netns:/var/run/netns \ + -v /var/lib/docker/containers:/var/lib/docker/containers \ + --pid=host \ + -v $(CURDIR)/yamls/clab-bgp.yaml:/clab.yaml \ + $(CLAB_IMAGE) clab destroy -t /clab.yaml + @$(MAKE) kind-clean + +.PHONY: kind-clean-bgp-ha +kind-clean-bgp-ha: + kube_ovn_version=$(VERSION) j2 yamls/clab-bgp-ha.yaml.j2 -o yamls/clab-bgp-ha.yaml + docker run --rm --privileged \ + --name kube-ovn-bgp \ + --network host \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /var/run/netns:/var/run/netns \ + -v /var/lib/docker/containers:/var/lib/docker/containers \ + --pid=host \ + -v $(CURDIR)/yamls/clab-bgp-ha.yaml:/clab.yaml \ + $(CLAB_IMAGE) clab destroy -t /clab.yaml + @$(MAKE) kind-clean + .PHONY: uninstall uninstall: bash dist/images/cleanup.sh @@ -861,6 +942,7 @@ ipam-bench: clean: $(RM) dist/images/kube-ovn dist/images/kube-ovn-cmd $(RM) yamls/kind.yaml + $(RM) yamls/clab-bgp.yaml yamls/clab-bgp-ha.yaml $(RM) ovn.yaml kube-ovn.yaml kube-ovn-crd.yaml $(RM) ovn-ic-0.yaml ovn-ic-1.yaml $(RM) kustomization.yaml kwok.yaml kwok-node.yaml diff --git a/go.mod b/go.mod index a42fe2654aa1..25339fc2f193 100644 --- a/go.mod +++ b/go.mod @@ -230,13 +230,14 @@ require ( go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.25.0 // indirect - golang.org/x/crypto v0.13.0 // indirect - golang.org/x/net v0.15.0 // indirect - golang.org/x/oauth2 v0.12.0 // indirect - golang.org/x/sync v0.3.0 // indirect - golang.org/x/term v0.12.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect + golang.org/x/net v0.16.0 // indirect + golang.org/x/oauth2 v0.13.0 // indirect + golang.org/x/sync v0.4.0 // indirect + golang.org/x/term v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect - golang.org/x/tools v0.13.0 // indirect + golang.org/x/tools v0.14.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/api v0.126.0 // indirect google.golang.org/appengine v1.6.8 // indirect diff --git a/go.sum b/go.sum index efebb23bff3a..c22aa3c5a9f8 100644 --- a/go.sum +++ b/go.sum @@ -2146,8 +2146,8 @@ golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2164,8 +2164,8 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -2299,8 +2299,8 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos= +golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2330,8 +2330,8 @@ golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= -golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= -golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= +golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= +golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -2350,8 +2350,8 @@ golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -2503,8 +2503,8 @@ golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= -golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2632,8 +2632,8 @@ golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= -golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/speaker/config.go b/pkg/speaker/config.go index 48f90cedb520..9cd6db02a009 100644 --- a/pkg/speaker/config.go +++ b/pkg/speaker/config.go @@ -7,6 +7,7 @@ import ( "fmt" "net" "os" + "strings" "time" api "github.com/osrg/gobgp/v3/api" @@ -39,8 +40,8 @@ type Configuration struct { GrpcPort uint32 ClusterAs uint32 RouterID string - NeighborAddress string - NeighborIPv6Address string + NeighborAddresses []string + NeighborIPv6Addresses []string NeighborAs uint32 AuthPassword string HoldTime float64 @@ -52,6 +53,7 @@ type Configuration struct { PassiveMode bool EbgpMultihopTTL uint8 + NodeName string KubeConfigFile string KubeClient kubernetes.Interface KubeOvnClient clientset.Interface @@ -69,12 +71,13 @@ func ParseFlags() (*Configuration, error) { argGrpcPort = pflag.Uint32("grpc-port", DefaultBGPGrpcPort, "The port for grpc to listen, default:50051") argClusterAs = pflag.Uint32("cluster-as", DefaultBGPClusterAs, "The as number of container network, default 65000") argRouterID = pflag.String("router-id", "", "The address for the speaker to use as router id, default the node ip") - argNeighborAddress = pflag.String("neighbor-address", "", "The router address the speaker connects to.") - argNeighborIPv6Address = pflag.String("neighbor-ipv6-address", "", "The router address the speaker connects to.") + argNeighborAddress = pflag.String("neighbor-address", "", "Comma separated IPv4 router addresses the speaker connects to.") + argNeighborIPv6Address = pflag.String("neighbor-ipv6-address", "", "Comma separated IPv6 router addresses the speaker connects to.") argNeighborAs = pflag.Uint32("neighbor-as", DefaultBGPNeighborAs, "The router as number, default 65001") argAuthPassword = pflag.String("auth-password", "", "bgp peer auth password") argHoldTime = pflag.Duration("holdtime", DefaultBGPHoldtime, "ovn-speaker goes down abnormally, the local saving time of BGP route will be affected.Holdtime must be in the range 3s to 65536s. (default 90s)") argPprofPort = pflag.Uint32("pprof-port", DefaultPprofPort, "The port to get profiling data, default: 10667") + argNodeName = pflag.String("node-name", os.Getenv(util.HostnameEnv), "Name of the node on which the speaker is running on.") argKubeConfigFile = pflag.String("kubeconfig", "", "Path to kubeconfig file with authorization and master location information. If not set use the inCluster token.") argPassiveMode = pflag.BoolP("passivemode", "", false, "Set BGP Speaker to passive model,do not actively initiate connections to peers ") argEbgpMultihopTTL = pflag.Uint8("ebgp-multihop", DefaultEbgpMultiHop, "The TTL value of EBGP peer, default: 1") @@ -104,12 +107,6 @@ func ParseFlags() (*Configuration, error) { if *argRouterID != "" && net.ParseIP(*argRouterID) == nil { return nil, fmt.Errorf("invalid router-id format: %s", *argRouterID) } - if *argNeighborAddress != "" && net.ParseIP(*argNeighborAddress).To4() == nil { - return nil, fmt.Errorf("invalid neighbor-address format: %s", *argNeighborAddress) - } - if *argNeighborIPv6Address != "" && net.ParseIP(*argNeighborIPv6Address).To16() == nil { - return nil, fmt.Errorf("invalid neighbor-ipv6-address format: %s", *argNeighborIPv6Address) - } if *argEbgpMultihopTTL < 1 || *argEbgpMultihopTTL > 255 { return nil, errors.New("the bgp MultihopTtl must be in the range 1 to 255") } @@ -120,12 +117,11 @@ func ParseFlags() (*Configuration, error) { GrpcPort: *argGrpcPort, ClusterAs: *argClusterAs, RouterID: *argRouterID, - NeighborAddress: *argNeighborAddress, - NeighborIPv6Address: *argNeighborIPv6Address, NeighborAs: *argNeighborAs, AuthPassword: *argAuthPassword, HoldTime: ht, PprofPort: *argPprofPort, + NodeName: strings.ToLower(*argNodeName), KubeConfigFile: *argKubeConfigFile, GracefulRestart: *argGracefulRestart, GracefulRestartDeferralTime: *argGracefulRestartDeferralTime, @@ -134,6 +130,23 @@ func ParseFlags() (*Configuration, error) { EbgpMultihopTTL: *argEbgpMultihopTTL, } + if *argNeighborAddress != "" { + config.NeighborAddresses = strings.Split(*argNeighborAddress, ",") + for _, addr := range config.NeighborAddresses { + if ip := net.ParseIP(addr); ip == nil || ip.To4() == nil { + return nil, fmt.Errorf("invalid neighbor-address format: %s", *argNeighborAddress) + } + } + } + if *argNeighborIPv6Address != "" { + config.NeighborIPv6Addresses = strings.Split(*argNeighborIPv6Address, ",") + for _, addr := range config.NeighborIPv6Addresses { + if ip := net.ParseIP(addr); ip == nil || ip.To16() == nil { + return nil, fmt.Errorf("invalid neighbor-ipv6-address format: %s", *argNeighborIPv6Address) + } + } + } + if config.RouterID == "" { config.RouterID = os.Getenv("POD_IP") if config.RouterID == "" { @@ -203,7 +216,6 @@ func (config *Configuration) checkGracefulRestartOptions() error { func (config *Configuration) initBgpServer() error { maxSize := 256 << 20 - peersMap := make(map[api.Family_Afi]string) var listenPort int32 = -1 grpcOpts := []grpc.ServerOption{grpc.MaxRecvMsgSize(maxSize), grpc.MaxSendMsgSize(maxSize)} s := gobgp.NewBgpServer( @@ -211,11 +223,9 @@ func (config *Configuration) initBgpServer() error { gobgp.GrpcOption(grpcOpts)) go s.Serve() - if config.NeighborAddress != "" { - peersMap[api.Family_AFI_IP] = config.NeighborAddress - } - if config.NeighborIPv6Address != "" { - peersMap[api.Family_AFI_IP6] = config.NeighborIPv6Address + peersMap := map[api.Family_Afi][]string{ + api.Family_AFI_IP: config.NeighborAddresses, + api.Family_AFI_IP6: config.NeighborIPv6Addresses, } if config.PassiveMode { @@ -231,56 +241,57 @@ func (config *Configuration) initBgpServer() error { }); err != nil { return err } - for ipFamily, address := range peersMap { - peer := &api.Peer{ - Timers: &api.Timers{Config: &api.TimersConfig{HoldTime: uint64(config.HoldTime)}}, - Conf: &api.PeerConf{ - NeighborAddress: address, - PeerAsn: config.NeighborAs, - }, - Transport: &api.Transport{ - PassiveMode: config.PassiveMode, - }, - } - if config.EbgpMultihopTTL != DefaultEbgpMultiHop { - peer.EbgpMultihop = &api.EbgpMultihop{ - Enabled: true, - MultihopTtl: uint32(config.EbgpMultihopTTL), + for ipFamily, addresses := range peersMap { + for _, addr := range addresses { + peer := &api.Peer{ + Timers: &api.Timers{Config: &api.TimersConfig{HoldTime: uint64(config.HoldTime)}}, + Conf: &api.PeerConf{ + NeighborAddress: addr, + PeerAsn: config.NeighborAs, + }, + Transport: &api.Transport{ + PassiveMode: config.PassiveMode, + }, } - } - if config.AuthPassword != "" { - peer.Conf.AuthPassword = config.AuthPassword - } - if config.GracefulRestart { - - if err := config.checkGracefulRestartOptions(); err != nil { - return err + if config.EbgpMultihopTTL != DefaultEbgpMultiHop { + peer.EbgpMultihop = &api.EbgpMultihop{ + Enabled: true, + MultihopTtl: uint32(config.EbgpMultihopTTL), + } } - peer.GracefulRestart = &api.GracefulRestart{ - Enabled: true, - RestartTime: uint32(config.GracefulRestartTime.Seconds()), - DeferralTime: uint32(config.GracefulRestartDeferralTime.Seconds()), - LocalRestarting: true, + if config.AuthPassword != "" { + peer.Conf.AuthPassword = config.AuthPassword } - peer.AfiSafis = []*api.AfiSafi{ - { - Config: &api.AfiSafiConfig{ - Family: &api.Family{Afi: ipFamily, Safi: api.Family_SAFI_UNICAST}, - Enabled: true, - }, - MpGracefulRestart: &api.MpGracefulRestart{ - Config: &api.MpGracefulRestartConfig{ + if config.GracefulRestart { + if err := config.checkGracefulRestartOptions(); err != nil { + return err + } + peer.GracefulRestart = &api.GracefulRestart{ + Enabled: true, + RestartTime: uint32(config.GracefulRestartTime.Seconds()), + DeferralTime: uint32(config.GracefulRestartDeferralTime.Seconds()), + LocalRestarting: true, + } + peer.AfiSafis = []*api.AfiSafi{ + { + Config: &api.AfiSafiConfig{ + Family: &api.Family{Afi: ipFamily, Safi: api.Family_SAFI_UNICAST}, Enabled: true, }, + MpGracefulRestart: &api.MpGracefulRestart{ + Config: &api.MpGracefulRestartConfig{ + Enabled: true, + }, + }, }, - }, + } } - } - if err := s.AddPeer(context.Background(), &api.AddPeerRequest{ - Peer: peer, - }); err != nil { - return err + if err := s.AddPeer(context.Background(), &api.AddPeerRequest{ + Peer: peer, + }); err != nil { + return err + } } } diff --git a/pkg/speaker/subnet.go b/pkg/speaker/subnet.go index 33c2e8b96a9f..23ec2522f7c0 100644 --- a/pkg/speaker/subnet.go +++ b/pkg/speaker/subnet.go @@ -22,6 +22,12 @@ import ( "github.com/kubeovn/kube-ovn/pkg/util" ) +const ( + // "cluster" is the default policy + announcePolicyCluster = "cluster" + announcePolicyLocal = "local" +) + func isPodAlive(p *v1.Pod) bool { if p.Status.Phase == v1.PodSucceeded && p.Spec.RestartPolicy != v1.RestartPolicyAlways { return false @@ -74,24 +80,68 @@ func (c *Controller) syncSubnetRoutes() { } } + localSubnets := make(map[string]string, 2) for _, subnet := range subnets { - if subnet.Status.IsReady() && subnet.Annotations != nil && subnet.Annotations[util.BgpAnnotation] == "true" { + if subnet.Status.IsReady() && subnet.Annotations != nil { ips := strings.Split(subnet.Spec.CIDRBlock, ",") - for _, cidr := range ips { - ipFamily := util.CheckProtocol(cidr) - bgpExpected[ipFamily] = append(bgpExpected[ipFamily], cidr) + policy := subnet.Annotations[util.BgpAnnotation] + if policy == "" { + continue + } + + switch policy { + case "true": + fallthrough + case announcePolicyCluster: + for _, cidr := range ips { + ipFamily := util.CheckProtocol(cidr) + bgpExpected[ipFamily] = append(bgpExpected[ipFamily], cidr) + } + case announcePolicyLocal: + localSubnets[subnet.Name] = subnet.Spec.CIDRBlock + default: + klog.Warningf("invalid subnet annotation %s=%s", util.BgpAnnotation, policy) } } } for _, pod := range pods { - if isPodAlive(pod) && !pod.Spec.HostNetwork && pod.Annotations[util.BgpAnnotation] == "true" && pod.Status.PodIP != "" { - podIps := pod.Status.PodIPs - for _, podIP := range podIps { - ipFamily := util.CheckProtocol(podIP.IP) - bgpExpected[ipFamily] = append(bgpExpected[ipFamily], fmt.Sprintf("%s/%d", podIP.IP, maskMap[ipFamily])) + if pod.Spec.HostNetwork || pod.Status.PodIP == "" || len(pod.Annotations) == 0 || !isPodAlive(pod) { + continue + } + + ips := make(map[string]string, 2) + if policy := pod.Annotations[util.BgpAnnotation]; policy != "" { + switch policy { + case "true": + fallthrough + case announcePolicyCluster: + for _, podIP := range pod.Status.PodIPs { + ips[util.CheckProtocol(podIP.IP)] = podIP.IP + } + case announcePolicyLocal: + if pod.Spec.NodeName == c.config.NodeName { + for _, podIP := range pod.Status.PodIPs { + ips[util.CheckProtocol(podIP.IP)] = podIP.IP + } + } + default: + klog.Warningf("invalid pod annotation %s=%s", util.BgpAnnotation, policy) + } + } else if pod.Spec.NodeName == c.config.NodeName { + cidrBlock := localSubnets[pod.Annotations[util.LogicalSwitchAnnotation]] + if cidrBlock != "" { + for _, podIP := range pod.Status.PodIPs { + if util.CIDRContainIP(cidrBlock, podIP.IP) { + ips[util.CheckProtocol(podIP.IP)] = podIP.IP + } + } } } + + for ipFamily, ip := range ips { + bgpExpected[ipFamily] = append(bgpExpected[ipFamily], fmt.Sprintf("%s/%d", ip, maskMap[ipFamily])) + } } klog.V(5).Infof("expected announce ipv4 routes: %v, ipv6 routes: %v", bgpExpected[kubeovnv1.ProtocolIPv4], bgpExpected[kubeovnv1.ProtocolIPv6]) @@ -110,7 +160,7 @@ func (c *Controller) syncSubnetRoutes() { } } - if c.config.NeighborAddress != "" { + if len(c.config.NeighborAddresses) != 0 { listPathRequest := &bgpapi.ListPathRequest{ TableType: bgpapi.TableType_GLOBAL, Family: &bgpapi.Family{Afi: bgpapi.Family_AFI_IP, Safi: bgpapi.Family_SAFI_UNICAST}, @@ -137,8 +187,7 @@ func (c *Controller) syncSubnetRoutes() { } } - if c.config.NeighborIPv6Address != "" { - + if len(c.config.NeighborIPv6Addresses) != 0 { listIPv6PathRequest := &bgpapi.ListPathRequest{ TableType: bgpapi.TableType_GLOBAL, Family: &bgpapi.Family{Afi: bgpapi.Family_AFI_IP6, Safi: bgpapi.Family_SAFI_UNICAST}, @@ -216,24 +265,26 @@ func (c *Controller) addRoute(route string) error { if err != nil { return err } - _, err = c.config.BgpServer.AddPath(context.Background(), &bgpapi.AddPathRequest{ - Path: &bgpapi.Path{ - Family: &bgpapi.Family{Afi: routeAfi, Safi: bgpapi.Family_SAFI_UNICAST}, - Nlri: nlri, - Pattrs: attrs, - }, - }) - if err != nil { - klog.Errorf("add path failed, %v", err) - return err + for _, attr := range attrs { + _, err = c.config.BgpServer.AddPath(context.Background(), &bgpapi.AddPathRequest{ + Path: &bgpapi.Path{ + Family: &bgpapi.Family{Afi: routeAfi, Safi: bgpapi.Family_SAFI_UNICAST}, + Nlri: nlri, + Pattrs: attr, + }, + }) + if err != nil { + klog.Errorf("add path failed, %v", err) + return err + } } return nil } -func (c *Controller) getNlriAndAttrs(route string) (*anypb.Any, []*anypb.Any, error) { - neighborAddr := c.config.NeighborAddress +func (c *Controller) getNlriAndAttrs(route string) (*anypb.Any, [][]*anypb.Any, error) { + neighborAddresses := c.config.NeighborAddresses if util.CheckProtocol(route) == kubeovnv1.ProtocolIPv6 { - neighborAddr = c.config.NeighborIPv6Address + neighborAddresses = c.config.NeighborIPv6Addresses } prefix, prefixLen, err := parseRoute(route) @@ -244,13 +295,18 @@ func (c *Controller) getNlriAndAttrs(route string) (*anypb.Any, []*anypb.Any, er Prefix: prefix, PrefixLen: prefixLen, }) - a1, _ := anypb.New(&bgpapi.OriginAttribute{ - Origin: 0, - }) - a2, _ := anypb.New(&bgpapi.NextHopAttribute{ - NextHop: getNextHopAttribute(neighborAddr, c.config.RouterID), - }) - attrs := []*anypb.Any{a1, a2} + + attrs := make([][]*anypb.Any, 0, len(neighborAddresses)) + for _, addr := range neighborAddresses { + a1, _ := anypb.New(&bgpapi.OriginAttribute{ + Origin: 0, + }) + a2, _ := anypb.New(&bgpapi.NextHopAttribute{ + NextHop: getNextHopAttribute(addr, c.config.RouterID), + }) + attrs = append(attrs, []*anypb.Any{a1, a2}) + } + return nlri, attrs, err } @@ -264,16 +320,18 @@ func (c *Controller) delRoute(route string) error { if err != nil { return err } - err = c.config.BgpServer.DeletePath(context.Background(), &bgpapi.DeletePathRequest{ - Path: &bgpapi.Path{ - Family: &bgpapi.Family{Afi: routeAfi, Safi: bgpapi.Family_SAFI_UNICAST}, - Nlri: nlri, - Pattrs: attrs, - }, - }) - if err != nil { - klog.Errorf("del path failed, %v", err) - return err + for _, attr := range attrs { + err = c.config.BgpServer.DeletePath(context.Background(), &bgpapi.DeletePathRequest{ + Path: &bgpapi.Path{ + Family: &bgpapi.Family{Afi: routeAfi, Safi: bgpapi.Family_SAFI_UNICAST}, + Nlri: nlri, + Pattrs: attr, + }, + }) + if err != nil { + klog.Errorf("del path failed, %v", err) + return err + } } return nil } diff --git a/yamls/clab-bgp-ha.yaml.j2 b/yamls/clab-bgp-ha.yaml.j2 new file mode 100644 index 000000000000..f225fc9919e9 --- /dev/null +++ b/yamls/clab-bgp-ha.yaml.j2 @@ -0,0 +1,98 @@ +name: bgp +topology: + kinds: + linux: + image: kubeovn/kube-ovn:{{ kube_ovn_version }} + cmd: bash + + nodes: + switch: + kind: linux + exec: + - ip link add br0 type bridge + - ip link set net1 master br0 + - ip link set net2 master br0 + - ip link set net3 master br0 + - ip link set net4 master br0 + - ip link set net5 master br0 + - ip link set net6 master br0 + - ip link set net7 master br0 + - ip link set br0 up + router-1: + kind: linux + image: frrouting/frr:v8.4.1 + labels: + app: frr + exec: + - ip link delete eth0 + - ip address add 10.0.1.1/24 dev net1 + - ip address add 10.0.2.1/24 dev net2 + - touch /etc/frr/vtysh.conf + - sed -i -e 's/bgpd=no/bgpd=yes/g' /etc/frr/daemons + - /usr/lib/frr/frrinit.sh start + - >- + vtysh -c 'conf t' + -c 'frr defaults datacenter' + -c 'router bgp 65001' + -c ' bgp router-id 10.0.1.1' + -c ' no bgp ebgp-requires-policy' + -c ' neighbor SERVERS peer-group' + -c ' neighbor SERVERS remote-as external' + -c ' neighbor 10.0.1.101 peer-group SERVERS' + -c ' neighbor 10.0.1.102 peer-group SERVERS' + -c ' address-family ipv4 unicast' + -c ' redistribute connected' + -c ' exit-address-family' + -c '!' + router-2: + kind: linux + image: frrouting/frr:v8.4.1 + labels: + app: frr + exec: + - ip link delete eth0 + - ip address add 10.0.1.2/24 dev net1 + - ip address add 10.0.2.2/24 dev net2 + - touch /etc/frr/vtysh.conf + - sed -i -e 's/bgpd=no/bgpd=yes/g' /etc/frr/daemons + - /usr/lib/frr/frrinit.sh start + - >- + vtysh -c 'conf t' + -c 'frr defaults datacenter' + -c 'router bgp 65001' + -c ' bgp router-id 10.0.1.2' + -c ' no bgp ebgp-requires-policy' + -c ' neighbor SERVERS peer-group' + -c ' neighbor SERVERS remote-as external' + -c ' neighbor 10.0.1.101 peer-group SERVERS' + -c ' neighbor 10.0.1.102 peer-group SERVERS' + -c ' address-family ipv4 unicast' + -c ' redistribute connected' + -c ' exit-address-family' + -c '!' + k8s-master: + kind: linux + network-mode: container:kube-ovn-control-plane + exec: + - ip address add 10.0.1.101/24 dev net1 + - ip route add 10.0.0.0/16 via 10.0.1.1 + k8s-worker: + kind: linux + network-mode: container:kube-ovn-worker + exec: + - ip address add 10.0.1.102/24 dev net1 + - ip route add 10.0.0.0/16 via 10.0.1.1 + ext: + kind: linux + exec: + - ip address add 10.0.2.101/24 dev net1 + - ip route replace default nexthop via 10.0.2.1 weight 1 nexthop via 10.0.2.2 weight 1 + + links: + - endpoints: ["switch:net1", "router-1:net1"] + - endpoints: ["switch:net2", "router-1:net2"] + - endpoints: ["switch:net3", "router-2:net1"] + - endpoints: ["switch:net4", "router-2:net2"] + - endpoints: ["switch:net5", "k8s-master:net1"] + - endpoints: ["switch:net6", "k8s-worker:net1"] + - endpoints: ["switch:net7", "ext:net1"] diff --git a/yamls/clab-bgp.yaml.j2 b/yamls/clab-bgp.yaml.j2 new file mode 100644 index 000000000000..54d791cb5446 --- /dev/null +++ b/yamls/clab-bgp.yaml.j2 @@ -0,0 +1,60 @@ +name: bgp +topology: + kinds: + linux: + image: kubeovn/kube-ovn:{{ kube_ovn_version }} + cmd: bash + + nodes: + router: + kind: linux + image: frrouting/frr:v8.4.1 + labels: + app: frr + exec: + - ip link delete eth0 + - ip link add br0 type bridge + - ip link set net1 master br0 + - ip link set net2 master br0 + - ip link set br0 up + - ip address add 10.0.1.1/24 dev br0 + - ip address add 10.0.2.1/24 dev net3 + - touch /etc/frr/vtysh.conf + - sed -i -e 's/bgpd=no/bgpd=yes/g' /etc/frr/daemons + - /usr/lib/frr/frrinit.sh start + - >- + vtysh -c 'conf t' + -c 'frr defaults datacenter' + -c 'router bgp 65001' + -c ' bgp router-id 10.0.1.1' + -c ' no bgp ebgp-requires-policy' + -c ' neighbor SERVERS peer-group' + -c ' neighbor SERVERS remote-as external' + -c ' neighbor 10.0.1.2 peer-group SERVERS' + -c ' neighbor 10.0.1.3 peer-group SERVERS' + -c ' address-family ipv4 unicast' + -c ' redistribute connected' + -c ' exit-address-family' + -c '!' + k8s-master: + kind: linux + network-mode: container:kube-ovn-control-plane + exec: + - ip address add 10.0.1.2/24 dev net1 + - ip route add 10.0.0.0/16 via 10.0.1.1 + k8s-worker: + kind: linux + network-mode: container:kube-ovn-worker + exec: + - ip address add 10.0.1.3/24 dev net1 + - ip route add 10.0.0.0/16 via 10.0.1.1 + ext: + kind: linux + exec: + - ip address add 10.0.2.2/24 dev net1 + - ip route replace default via 10.0.2.1 + + links: + - endpoints: ["router:net1", "k8s-master:net1"] + - endpoints: ["router:net2", "k8s-worker:net1"] + - endpoints: ["router:net3", "ext:net1"] diff --git a/yamls/speaker.yaml b/yamls/speaker.yaml index 4c63bc1e9b90..6ad768c0aa7d 100644 --- a/yamls/speaker.yaml +++ b/yamls/speaker.yaml @@ -38,6 +38,10 @@ spec: - --neighbor-as=65030 - --cluster-as=65000 env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName - name: POD_IP valueFrom: fieldRef: