diff --git a/cmd/yurthub/app/options/options.go b/cmd/yurthub/app/options/options.go index 9ee56055e23..d49b54b108c 100644 --- a/cmd/yurthub/app/options/options.go +++ b/cmd/yurthub/app/options/options.go @@ -72,6 +72,8 @@ type YurtHubOptions struct { EnableIptables bool HubAgentDummyIfIP string HubAgentDummyIfName string + CloudProviderIP string + CloudProviderName string DiskCachePath string EnableResourceFilter bool DisabledResourceFilters []string @@ -212,7 +214,7 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) { fs.BoolVar(&o.EnableResourceFilter, "enable-resource-filter", o.EnableResourceFilter, "enable to filter response that comes back from reverse proxy") fs.StringSliceVar(&o.DisabledResourceFilters, "disabled-resource-filters", o.DisabledResourceFilters, "disable resource filters to handle response") fs.StringVar(&o.NodePoolName, "nodepool-name", o.NodePoolName, "the name of node pool that runs hub agent") - fs.StringVar(&o.WorkingMode, "working-mode", o.WorkingMode, "the working mode of yurthub(edge, cloud).") + fs.StringVar(&o.WorkingMode, "working-mode", o.WorkingMode, "the working mode of yurthub(edge, cloud, local).") fs.DurationVar(&o.KubeletHealthGracePeriod, "kubelet-health-grace-period", o.KubeletHealthGracePeriod, "the amount of time which we allow kubelet to be unresponsive before stop renew node lease") fs.BoolVar(&o.EnableNodePool, "enable-node-pool", o.EnableNodePool, "enable list/watch nodepools resource or not for filters(only used for testing)") fs.MarkDeprecated("enable-node-pool", "It is planned to be removed from OpenYurt in the future version, please use --enable-pool-service-topology instead") diff --git a/cmd/yurthub/app/start.go b/cmd/yurthub/app/start.go index 4b2b7f7e4a4..96a2ea267e2 100644 --- a/cmd/yurthub/app/start.go +++ b/cmd/yurthub/app/start.go @@ -39,6 +39,7 @@ import ( "github.com/openyurtio/openyurt/pkg/yurthub/gc" "github.com/openyurtio/openyurt/pkg/yurthub/healthchecker" hubrest "github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/rest" + "github.com/openyurtio/openyurt/pkg/yurthub/locallb" "github.com/openyurtio/openyurt/pkg/yurthub/proxy" "github.com/openyurtio/openyurt/pkg/yurthub/server" "github.com/openyurtio/openyurt/pkg/yurthub/tenant" @@ -117,7 +118,7 @@ func Run(ctx context.Context, cfg *config.YurtHubConfiguration) error { return fmt.Errorf("could not new cloud health checker, %w", err) } } else { - klog.Infof("%d. disable health checker for node %s because it is a cloud node", trace, cfg.NodeName) + klog.Infof("%d. disable health checker for node %s because it is a cloud or local node", trace, cfg.NodeName) // In cloud mode, cloud health checker is not needed. // This fake checker will always report that the cloud is healthy and yurt coordinator is unhealthy. cloudHealthChecker = healthchecker.NewFakeChecker(true, make(map[string]int)) @@ -136,7 +137,7 @@ func Run(ctx context.Context, cfg *config.YurtHubConfiguration) error { klog.Infof("%d. new cache manager with storage wrapper and serializer manager", trace) cacheMgr = cachemanager.NewCacheManager(cfg.StorageWrapper, cfg.SerializerManager, cfg.RESTMapperManager, cfg.SharedFactory) } else { - klog.Infof("%d. disable cache manager for node %s because it is a cloud node", trace, cfg.NodeName) + klog.Infof("%d. disable cache manager for node %s because it is a cloud or local node", trace, cfg.NodeName) } trace++ @@ -148,7 +149,7 @@ func Run(ctx context.Context, cfg *config.YurtHubConfiguration) error { } gcMgr.Run() } else { - klog.Infof("%d. disable gc manager for node %s because it is a cloud node", trace, cfg.NodeName) + klog.Infof("%d. disable gc manager for node %s because it is a cloud or local node", trace, cfg.NodeName) } trace++ @@ -199,6 +200,18 @@ func Run(ctx context.Context, cfg *config.YurtHubConfiguration) error { } trace++ + if cfg.WorkingMode == util.WorkingModeLocal { + klog.Infof("%d. new locallb manager for node %s ", trace, cfg.NodeName) + locallbMgr, err := locallb.NewLocalLBManager(cfg.SharedFactory) + if err != nil { + return fmt.Errorf("could not new locallb manager, %w", err) + } + locallbMgr.Run() + } else { + klog.Infof("%d. disable locallb manager for node %s because it is not a local node", trace, cfg.NodeName) + } + trace++ + if cfg.NetworkMgr != nil { cfg.NetworkMgr.Run(ctx.Done()) } diff --git a/go.mod b/go.mod index 91446e6c0f0..089bdacc062 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/openyurtio/openyurt -go 1.20 +go 1.21 + +toolchain go1.21.5 require ( github.com/aliyun/alibaba-cloud-sdk-go v1.62.156 @@ -46,6 +48,7 @@ require ( k8s.io/component-base v0.28.9 k8s.io/component-helpers v0.28.9 k8s.io/controller-manager v0.28.9 + k8s.io/klog v1.0.0 k8s.io/klog/v2 v2.100.1 k8s.io/kube-controller-manager v0.28.9 k8s.io/kubectl v0.28.9 @@ -83,6 +86,7 @@ require ( github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect + github.com/coreos/go-iptables v0.8.0 github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect diff --git a/go.sum b/go.sum index 1a532381c73..e3ef65b5316 100644 --- a/go.sum +++ b/go.sum @@ -638,6 +638,7 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -677,6 +678,8 @@ github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/coreos/go-iptables v0.8.0 h1:MPc2P89IhuVpLI7ETL/2tx3XZ61VeICZjYqDEgNsPRc= +github.com/coreos/go-iptables v0.8.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= @@ -694,6 +697,7 @@ github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4Kfc github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/edgexfoundry/go-mod-core-contracts/v3 v3.0.0 h1:xjwCI34DLM31cSl1q9XmYgXS3JqXufQJMgohnLLLDx0= github.com/edgexfoundry/go-mod-core-contracts/v3 v3.0.0/go.mod h1:zzzWGWij6wAqm1go9TLs++TFMIsBqBb1eRnIj4mRxGw= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -777,6 +781,7 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= +github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= @@ -793,6 +798,7 @@ github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+ github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= @@ -814,6 +820,7 @@ github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptG github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= @@ -940,6 +947,7 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= @@ -971,6 +979,7 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -1027,6 +1036,7 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= @@ -1034,6 +1044,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/maxatome/go-testdeep v1.12.0 h1:Ql7Go8Tg0C1D/uMMX59LAoYK7LffeJQ6X2T04nTH68g= +github.com/maxatome/go-testdeep v1.12.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= @@ -1170,11 +1181,14 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= @@ -1206,6 +1220,7 @@ github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= +github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= @@ -1220,6 +1235,7 @@ github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcY github.com/xhit/go-str2duration v1.2.0/go.mod h1:3cPSlfZlUHVlneIVfePFWcJZsuwf+P1v2SRTV4cUmp4= github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1232,16 +1248,21 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= +go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.etcd.io/etcd/api/v3 v3.5.9 h1:4wSsluwyTbGGmyjJktOf3wFQoTBIURXHnq9n/G/JQHs= go.etcd.io/etcd/api/v3 v3.5.9/go.mod h1:uyAal843mC8uUVSLWz6eHa/d971iDGnCRpmKd2Z+X8k= go.etcd.io/etcd/client/pkg/v3 v3.5.9 h1:oidDC4+YEuSIQbsR94rY9gur91UPL6DnxDCIYd2IGsE= go.etcd.io/etcd/client/pkg/v3 v3.5.9/go.mod h1:y+CzeSmkMpWN2Jyu1npecjB9BBnABxGM4pN8cGuJeL4= go.etcd.io/etcd/client/v2 v2.305.9 h1:YZ2OLi0OvR0H75AcgSUajjd5uqKDKocQUqROTG11jIo= +go.etcd.io/etcd/client/v2 v2.305.9/go.mod h1:0NBdNx9wbxtEQLwAQtrDHwx58m02vXpDcgSYI2seohQ= go.etcd.io/etcd/client/v3 v3.5.9 h1:r5xghnU7CwbUxD/fbUtRyJGaYNfDun8sp/gTr1hew6E= go.etcd.io/etcd/client/v3 v3.5.9/go.mod h1:i/Eo5LrZ5IKqpbtpPDuaUnDOUv471oDg8cjQaUr2MbA= go.etcd.io/etcd/pkg/v3 v3.5.9 h1:6R2jg/aWd/zB9+9JxmijDKStGJAPFsX3e6BeJkMi6eQ= +go.etcd.io/etcd/pkg/v3 v3.5.9/go.mod h1:BZl0SAShQFk0IpLWR78T/+pyt8AruMHhTNNX73hkNVY= go.etcd.io/etcd/raft/v3 v3.5.9 h1:ZZ1GIHoUlHsn0QVqiRysAm3/81Xx7+i2d7nSdWxlOiI= +go.etcd.io/etcd/raft/v3 v3.5.9/go.mod h1:WnFkqzFdZua4LVlVXQEGhmooLeyS7mqzS4Pf4BCVqXg= go.etcd.io/etcd/server/v3 v3.5.9 h1:vomEmmxeztLtS5OEH7d0hBAg4cjVIu9wXuNzUZx2ZA0= +go.etcd.io/etcd/server/v3 v3.5.9/go.mod h1:GgI1fQClQCFIzuVjlvdbMxNbnISt90gdfYyqiAIt65g= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -2072,6 +2093,8 @@ k8s.io/component-helpers v0.28.9/go.mod h1:TdAkLbywEDE2CB5h8LbM/W03T3k8wvqAaoPcE k8s.io/controller-manager v0.28.9 h1:muAtmO2mDN7pDkAJQMknvWy+WQhkvvi/jK1V82+qbLw= k8s.io/controller-manager v0.28.9/go.mod h1:RYP65K6GWLRWYZR7PRRaStfvgeXkhCGZwJsxRPuaDV0= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= +k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kms v0.28.9 h1:ApCWJulBl+uFRTr2jtTpG1lffmqqMuLnOH/RUbtO4UY= diff --git a/pkg/yurthub/locallb/iptables.go b/pkg/yurthub/locallb/iptables.go new file mode 100644 index 00000000000..092dcc9bb56 --- /dev/null +++ b/pkg/yurthub/locallb/iptables.go @@ -0,0 +1,22 @@ +/* +Copyright 2024 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package locallb + +import ( + "github.com/coreos/go-iptables/iptables" +) + diff --git a/pkg/yurthub/locallb/locallb.go b/pkg/yurthub/locallb/locallb.go new file mode 100644 index 00000000000..0067171fd88 --- /dev/null +++ b/pkg/yurthub/locallb/locallb.go @@ -0,0 +1,119 @@ +/* +Copyright 2024 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package locallb + +import ( + "strconv" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/tools/cache" + "k8s.io/klog" +) + +type locallbManager struct { + ips []string + iptablesManager *IptablesManager +} + +func NewLocalLBManager(informerFactory informers.SharedInformerFactory) (*locallbManager, error) { + m := &locallbManager{ + ips: make([]string, 0), + iptablesManager: NewIptablesManager(options.HubAgentDummyIfIP, strconv.Itoa(options.YurtHubProxyPort)), + } + podInformer := informerFactory.Core().V1().Pods() + informer := podInformer.Informer() + informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: m.addPod, + UpdateFunc: m.updatePod, + DeleteFunc: m.deletePod, + }) + + m.watchIP(informerFactory) + return m, nil +} + +func (m *locallbManager) Run() { + m.loadbalancing() +} + +func (m *locallbManager) loadbalancing() error { + m.configureNetwork() + return nil +} + +func (m *locallbManager) configureNetwork() error { + // m.fetchIP() + return nil +} + +func (m *locallbManager) watchIP(informerFactory informers.SharedInformerFactory) { + stopCh := make(chan struct{}) + defer close(stopCh) + informerFactory.Start(stopCh) + informerFactory.WaitForCacheSync(stopCh) + <-stopCh +} + +func (m *locallbManager) addPod(obj interface{}) { + pod, ok := obj.(*corev1.Pod) + if !ok { + klog.Errorf("could not convert to *corev1.Pod") + return + } + + klog.Infof("pod added %s", pod.GetName()) + m.ips = append(m.ips, pod.Status.PodIP) +} + +func (m *locallbManager) updatePod(oldObj, newObj interface{}) { + oldPod, ok := oldObj.(*corev1.Pod) + if !ok { + klog.Errorf("could not convert to *corev1.Pod") + return + } + newPod, ok := newObj.(*corev1.Pod) + if !ok { + klog.Errorf("could not convert to *corev1.Pod") + return + } + + klog.Infof("pod updated from %s to %s", oldPod.GetName(), newPod.GetName()) + for i, ip := range m.ips { + if ip == oldPod.Status.PodIP { + m.ips[i] = newPod.Status.PodIP + return + } + } +} + +func (m *locallbManager) deletePod(obj interface{}) { + pod, ok := obj.(*corev1.Pod) + if !ok { + klog.Errorf("could not convert to *corev1.Pod") + return + } + + klog.Infof("pod deleted %s", pod.GetName()) + var updatedIPs []string + for _, existingIP := range m.ips { + if existingIP != pod.Status.PodIP { + updatedIPs = append(updatedIPs, existingIP) + } + } + m.ips = updatedIPs +} diff --git a/pkg/yurthub/locallb/network_unused.go b/pkg/yurthub/locallb/network_unused.go new file mode 100644 index 00000000000..ee632498016 --- /dev/null +++ b/pkg/yurthub/locallb/network_unused.go @@ -0,0 +1,107 @@ +/* +Copyright 2024 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package locallb + +import ( + "net" + "time" + + "k8s.io/klog/v2" + + "github.com/openyurtio/openyurt/cmd/yurthub/app/options" + "github.com/openyurtio/openyurt/pkg/yurthub/network" +) + +const ( + SyncNetworkPeriod = 60 +) + +type NetworkManager struct { + ifController network.DummyInterfaceController + iptablesManager *network.IptablesManager + dummyIfIPs []net.IP + dummyIfNames []string + enableIptables bool +} + +func NewNetworkManager(options *options.YurtHubOptions) (*NetworkManager, error) { + m := &NetworkManager{ + ifController: network.NewDummyInterfaceController(), + iptablesManager: network.NewIptablesManager("255.255.255.255", "255"), + dummyIfIPs: []net.IP{net.ParseIP("255.255.255.255")}, + dummyIfNames: []string{"test255"}, + enableIptables: options.EnableIptables, + } + if err := m.configureNetwork(); err != nil { + return nil, err + } + + return m, nil +} + +func (m *NetworkManager) Run(stopCh <-chan struct{}) { + go func() { + ticker := time.NewTicker(SyncNetworkPeriod * time.Second) + defer ticker.Stop() + + for { + select { + case <-stopCh: + klog.Infof("exit network manager run goroutine normally") + if m.enableIptables { + if err := m.iptablesManager.CleanUpIptablesRules(); err != nil { + klog.Errorf("could not cleanup iptables, %v", err) + } + } + for _, dummyIfName := range m.dummyIfNames { + err := m.ifController.DeleteDummyInterface(dummyIfName) + if err != nil { + klog.Errorf("could not delete dummy interface %s, %v", dummyIfName, err) + } else { + klog.Infof("remove dummy interface %s successfully", dummyIfName) + } + } + return + case <-ticker.C: + if err := m.configureNetwork(); err != nil { + // do nothing here + klog.Warningf("could not configure network, %v", err) + } + } + } + }() +} + +func (m *NetworkManager) configureNetwork() error { + for i := 0; i < len(m.dummyIfIPs); i++ { + err := m.ifController.EnsureDummyInterface(m.dummyIfNames[i], m.dummyIfIPs[i]) + if err != nil { + klog.Errorf("ensure dummy interface failed, %v", err) + return err + } + } + + if m.enableIptables { + err := m.iptablesManager.EnsureIptablesRules() + if err != nil { + klog.Errorf("ensure iptables for dummy interface failed, %v", err) + return err + } + } + + return nil +} diff --git a/pkg/yurthub/network/iptables.go b/pkg/yurthub/network/iptables.go index 2657976b4f8..0b6a779ad56 100644 --- a/pkg/yurthub/network/iptables.go +++ b/pkg/yurthub/network/iptables.go @@ -64,6 +64,7 @@ func makeupIptablesRules(ifIP, ifPort string) []iptablesRule { } } +// 检查os底层iptable，是否和im.rules数据结构预期一样？ func (im *IptablesManager) EnsureIptablesRules() error { var errs []error for _, rule := range im.rules { @@ -77,6 +78,7 @@ func (im *IptablesManager) EnsureIptablesRules() error { return utilerrors.NewAggregate(errs) } +// 把os底层的iptable清空，为什么im.rules数据结构不清空？ func (im *IptablesManager) CleanUpIptablesRules() error { var errs []error for _, rule := range im.rules { diff --git a/pkg/yurthub/util/util.go b/pkg/yurthub/util/util.go index 460b4d421a6..b0f3f4ed26b 100644 --- a/pkg/yurthub/util/util.go +++ b/pkg/yurthub/util/util.go @@ -53,6 +53,8 @@ const ( WorkingModeCloud WorkingMode = "cloud" // WorkingModeEdge represents yurthub is working in edge mode, which means yurthub is deployed on the edge side. WorkingModeEdge WorkingMode = "edge" + // WorkingModeLocal represents yurthub is working in local mode, which means yurthub is deployed on the local side. + WorkingModeLocal WorkingMode = "local" // ProxyReqContentType represents request content type context key ProxyReqContentType ProxyKeyType = iota @@ -358,7 +360,7 @@ func IsSupportedLBMode(lbMode string) bool { // IsSupportedWorkingMode check working mode is supported or not func IsSupportedWorkingMode(workingMode WorkingMode) bool { switch workingMode { - case WorkingModeCloud, WorkingModeEdge: + case WorkingModeCloud, WorkingModeEdge, WorkingModeLocal: return true } diff --git a/pkg/yurthub/util/util_test.go b/pkg/yurthub/util/util_test.go index c963f179836..d9ce84df9fe 100644 --- a/pkg/yurthub/util/util_test.go +++ b/pkg/yurthub/util/util_test.go @@ -280,6 +280,7 @@ func TestIsSupportedWorkingMode(t *testing.T) { }{ {"working mode cloud", args{WorkingModeCloud}, true}, {"working mode edge", args{WorkingModeEdge}, true}, + {"working mode local", args{WorkingModeLocal}, true}, {"no working mode", args{}, false}, } for _, tt := range tests {