Skip to content

Commit

Permalink
1. add nodelifecycle controller
Browse files Browse the repository at this point in the history
2. don't mark pods not ready when node has pod binding annotation
  • Loading branch information
rambohe-ch committed Oct 24, 2023
1 parent 0c1c982 commit 4cf3e47
Show file tree
Hide file tree
Showing 49 changed files with 10,564 additions and 299 deletions.
24 changes: 2 additions & 22 deletions charts/yurt-manager/templates/yurt-manager-auto-generated.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -572,7 +572,7 @@ webhooks:
name: yurt-manager-webhook-service
namespace: {{ .Release.Namespace }}
path: /mutate-core-openyurt-io-v1-node
failurePolicy: Fail
failurePolicy: Ignore
name: mutate.core.v1.node.openyurt.io
rules:
- apiGroups:
Expand Down Expand Up @@ -743,7 +743,7 @@ webhooks:
name: yurt-manager-webhook-service
namespace: {{ .Release.Namespace }}
path: /validate-core-openyurt-io-v1-node
failurePolicy: Fail
failurePolicy: Ignore
name: validate.core.v1.node.openyurt.io
rules:
- apiGroups:
Expand Down Expand Up @@ -797,26 +797,6 @@ webhooks:
resources:
- platformadmins
sideEffects: None
- admissionReviewVersions:
- v1
- v1beta1
clientConfig:
service:
name: yurt-manager-webhook-service
namespace: {{ .Release.Namespace }}
path: /validate-core-openyurt-io-v1-pod
failurePolicy: Fail
name: validate.core.v1.pod.openyurt.io
rules:
- apiGroups:
- ""
apiVersions:
- v1
operations:
- DELETE
resources:
- pods
sideEffects: None
- admissionReviewVersions:
- v1
- v1beta1
Expand Down
35 changes: 23 additions & 12 deletions cmd/yurt-manager/app/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/cli/globalflag"
"k8s.io/component-base/term"
Expand Down Expand Up @@ -146,11 +147,17 @@ func PrintFlags(flags *pflag.FlagSet) {

// Run runs the KubeControllerManagerOptions. This should never exit.
func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error {

ctrl.SetLogger(klogr.New())

ctx := ctrl.SetupSignalHandler()
cfg := ctrl.GetConfigOrDie()
if len(c.ComponentConfig.Generic.Kubeconfig) != 0 {
config, err := clientcmd.BuildConfigFromFlags("", c.ComponentConfig.Generic.Kubeconfig)
if err != nil {
klog.Infof("could not build rest config, %v", err)
return err
}
cfg = config
}
setRestConfig(cfg, c)

mgr, err := ctrl.NewManager(cfg, ctrl.Options{
Expand All @@ -173,7 +180,7 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error {
}

setupLog.Info("setup controllers")
if err = controller.SetupWithManager(c, mgr); err != nil {
if err = controller.SetupWithManager(ctx, c, mgr); err != nil {
setupLog.Error(err, "unable to setup controllers")
os.Exit(1)
}
Expand All @@ -184,16 +191,20 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error {
os.Exit(1)
}

// +kubebuilder:scaffold:builder
setupLog.Info("initialize webhook")
if err := webhook.Initialize(ctx, c, mgr.GetConfig()); err != nil {
setupLog.Error(err, "unable to initialize webhook")
os.Exit(1)
}
if len(webhook.WebhookHandlerPath) != 0 {
// +kubebuilder:scaffold:builder
setupLog.Info("initialize webhook")
if err := webhook.Initialize(ctx, c, mgr.GetConfig()); err != nil {
setupLog.Error(err, "unable to initialize webhook")
os.Exit(1)
}

if err := mgr.AddReadyzCheck("webhook-ready", mgr.GetWebhookServer().StartedChecker()); err != nil {
setupLog.Error(err, "unable to add readyz check")
os.Exit(1)
if err := mgr.AddReadyzCheck("webhook-ready", mgr.GetWebhookServer().StartedChecker()); err != nil {
setupLog.Error(err, "unable to add readyz check")
os.Exit(1)
}
} else {
klog.Infof("no webhook is registered, so skip webhook setup")
}

if err := mgr.AddHealthzCheck("health", healthz.Ping); err != nil {
Expand Down
3 changes: 2 additions & 1 deletion cmd/yurt-manager/app/options/generic.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ func (o *GenericOptions) ApplyTo(cfg *config.GenericConfiguration, controllerAli
cfg.RestConfigQPS = o.RestConfigQPS
cfg.RestConfigBurst = o.RestConfigBurst
cfg.WorkingNamespace = o.WorkingNamespace
cfg.Kubeconfig = o.Kubeconfig

cfg.Controllers = make([]string, len(o.Controllers))
for i, initialName := range o.Controllers {
Expand Down Expand Up @@ -135,6 +136,6 @@ func (o *GenericOptions) AddFlags(fs *pflag.FlagSet, allControllers, disabledByD
strings.Join(allControllers, ", "), strings.Join(disabledByDefaultControllers, ", ")))
fs.StringSliceVar(&o.DisabledWebhooks, "disable-independent-webhooks", o.DisabledWebhooks, "A list of webhooks to disable. "+
"'*' disables all independent webhooks, 'foo' disables the independent webhook named 'foo'.")

fs.StringVar(&o.Kubeconfig, "kubeconfig", o.Kubeconfig, "Path to kubeconfig file with authorization and master location information")
features.DefaultMutableFeatureGate.AddFlag(fs)
}
87 changes: 87 additions & 0 deletions cmd/yurt-manager/app/options/nodelifecyclecontroller.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package options

import (
"fmt"
"time"

"github.com/spf13/pflag"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kube-controller-manager/config/v1alpha1"

"github.com/openyurtio/openyurt/cmd/yurt-manager/names"
)

// NodeLifecycleControllerOptions holds the NodeLifecycleController options.
type NodeLifecycleControllerOptions struct {
*v1alpha1.NodeLifecycleControllerConfiguration
}

func NewNodeLifecycleControllerOptions() *NodeLifecycleControllerOptions {
return &NodeLifecycleControllerOptions{
NodeLifecycleControllerConfiguration: &v1alpha1.NodeLifecycleControllerConfiguration{
PodEvictionTimeout: metav1.Duration{Duration: 5 * time.Minute},
NodeMonitorGracePeriod: metav1.Duration{Duration: 40 * time.Second},
NodeStartupGracePeriod: metav1.Duration{Duration: 60 * time.Second},
},
}
}

// AddFlags adds flags related to NodeLifecycleController for controller manager to the specified FlagSet.
func (o *NodeLifecycleControllerOptions) AddFlags(fs *pflag.FlagSet) {
if o == nil {
return
}

fs.DurationVar(&o.NodeStartupGracePeriod.Duration, "node-startup-grace-period", o.NodeStartupGracePeriod.Duration,
"Amount of time which we allow starting Node to be unresponsive before marking it unhealthy.")
fs.DurationVar(&o.NodeMonitorGracePeriod.Duration, "node-monitor-grace-period", o.NodeMonitorGracePeriod.Duration,
"Amount of time which we allow running Node to be unresponsive before marking it unhealthy. "+
"Must be N times more than kubelet's nodeStatusUpdateFrequency, "+
"where N means number of retries allowed for kubelet to post node status.")
fs.Float32Var(&o.NodeEvictionRate, "node-eviction-rate", 0.1, "Number of nodes per second on which pods are deleted in case of node failure when a zone is healthy (see --unhealthy-zone-threshold for definition of healthy/unhealthy). Zone refers to entire cluster in non-multizone clusters.")
fs.Float32Var(&o.SecondaryNodeEvictionRate, "secondary-node-eviction-rate", 0.01, "Number of nodes per second on which pods are deleted in case of node failure when a zone is unhealthy (see --unhealthy-zone-threshold for definition of healthy/unhealthy). Zone refers to entire cluster in non-multizone clusters. This value is implicitly overridden to 0 if the cluster size is smaller than --large-cluster-size-threshold.")
fs.Int32Var(&o.LargeClusterSizeThreshold, "large-cluster-size-threshold", 50, fmt.Sprintf("Number of nodes from which %s treats the cluster as large for the eviction logic purposes. --secondary-node-eviction-rate is implicitly overridden to 0 for clusters this size or smaller.", names.NodeLifeCycleController))
fs.Float32Var(&o.UnhealthyZoneThreshold, "unhealthy-zone-threshold", 0.55, "Fraction of Nodes in a zone which needs to be not Ready (minimum 3) for zone to be treated as unhealthy. ")
}

// ApplyTo fills up NodeLifecycleController config with options.
func (o *NodeLifecycleControllerOptions) ApplyTo(cfg *v1alpha1.NodeLifecycleControllerConfiguration) error {
if o == nil {
return nil
}

cfg.NodeStartupGracePeriod = o.NodeStartupGracePeriod
cfg.NodeMonitorGracePeriod = o.NodeMonitorGracePeriod
cfg.NodeEvictionRate = o.NodeEvictionRate
cfg.SecondaryNodeEvictionRate = o.SecondaryNodeEvictionRate
cfg.LargeClusterSizeThreshold = o.LargeClusterSizeThreshold
cfg.UnhealthyZoneThreshold = o.UnhealthyZoneThreshold

return nil
}

// Validate checks validation of NodeLifecycleControllerOptions.
func (o *NodeLifecycleControllerOptions) Validate() []error {
if o == nil {
return nil
}

errs := []error{}
return errs
}
8 changes: 7 additions & 1 deletion cmd/yurt-manager/app/options/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ type YurtManagerOptions struct {
YurtAppDaemonController *YurtAppDaemonControllerOptions
PlatformAdminController *PlatformAdminControllerOptions
YurtAppOverriderController *YurtAppOverriderControllerOptions
NodeLifeCycleController *NodeLifecycleControllerOptions
}

// NewYurtManagerOptions creates a new YurtManagerOptions with a default config.
Expand All @@ -47,6 +48,7 @@ func NewYurtManagerOptions() (*YurtManagerOptions, error) {
YurtAppDaemonController: NewYurtAppDaemonControllerOptions(),
PlatformAdminController: NewPlatformAdminControllerOptions(),
YurtAppOverriderController: NewYurtAppOverriderControllerOptions(),
NodeLifeCycleController: NewNodeLifecycleControllerOptions(),
}

return &s, nil
Expand All @@ -61,7 +63,7 @@ func (y *YurtManagerOptions) Flags(allControllers, disabledByDefaultControllers
y.YurtAppDaemonController.AddFlags(fss.FlagSet("yurtappdaemon controller"))
y.PlatformAdminController.AddFlags(fss.FlagSet("iot controller"))
y.YurtAppOverriderController.AddFlags(fss.FlagSet("yurtappoverrider controller"))
// Please Add Other controller flags @kadisi
y.NodeLifeCycleController.AddFlags(fss.FlagSet("nodelifecycle controller"))

return fss
}
Expand All @@ -76,6 +78,7 @@ func (y *YurtManagerOptions) Validate(allControllers []string, controllerAliases
errs = append(errs, y.YurtAppDaemonController.Validate()...)
errs = append(errs, y.PlatformAdminController.Validate()...)
errs = append(errs, y.YurtAppOverriderController.Validate()...)
errs = append(errs, y.NodeLifeCycleController.Validate()...)
return utilerrors.NewAggregate(errs)
}

Expand All @@ -102,6 +105,9 @@ func (y *YurtManagerOptions) ApplyTo(c *config.Config, controllerAliases map[str
if err := y.GatewayPickupController.ApplyTo(&c.ComponentConfig.GatewayPickupController); err != nil {
return err
}
if err := y.NodeLifeCycleController.ApplyTo(&c.ComponentConfig.NodeLifeCycleController); err != nil {
return err
}
return nil
}

Expand Down
2 changes: 2 additions & 0 deletions cmd/yurt-manager/names/controller_names.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ const (
GatewayInternalServiceController = "gateway-internal-service-controller"
GatewayPublicServiceController = "gateway-public-service"
GatewayDNSController = "gateway-dns-controller"
NodeLifeCycleController = "node-life-cycle-controller"
)

func YurtManagerControllerAliases() map[string]string {
Expand All @@ -56,5 +57,6 @@ func YurtManagerControllerAliases() map[string]string {
"gatewayinternalservice": GatewayInternalServiceController,
"gatewaypublicservice": GatewayPublicServiceController,
"gatewaydns": GatewayDNSController,
"nodelifecycle": NodeLifeCycleController,
}
}
5 changes: 4 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ require (
github.com/evanphx/json-patch v5.6.0+incompatible
github.com/go-resty/resty/v2 v2.7.0
github.com/golang-jwt/jwt v3.2.2+incompatible
github.com/google/go-cmp v0.5.9
github.com/google/uuid v1.3.0
github.com/gorilla/mux v1.8.0
github.com/hashicorp/go-version v1.6.0
Expand Down Expand Up @@ -45,7 +46,9 @@ require (
k8s.io/component-helpers v0.22.3
k8s.io/controller-manager v0.22.3
k8s.io/klog/v2 v2.9.0
k8s.io/kube-controller-manager v0.0.0
k8s.io/kubectl v0.22.3
k8s.io/kubelet v0.0.0
k8s.io/kubernetes v1.22.3
k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b
sigs.k8s.io/apiserver-network-proxy v0.0.15
Expand Down Expand Up @@ -103,7 +106,6 @@ require (
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/googleapis/gnostic v0.5.5 // indirect
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
Expand Down Expand Up @@ -166,6 +168,7 @@ require (
gopkg.in/ini.v1 v1.66.2 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
k8s.io/cloud-provider v0.22.3 // indirect
k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect
Expand Down
3 changes: 3 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1165,6 +1165,7 @@ k8s.io/cli-runtime v0.22.3 h1:AeOgaDpb/k36amWsjyyIU+FLpLzzdmoLD5gn38c5fio=
k8s.io/cli-runtime v0.22.3/go.mod h1:um6JvCxV9Hrhq0zCUxcqYoY7/wF64g6IYgOViI8sg6Q=
k8s.io/client-go v0.22.3 h1:6onkOSc+YNdwq5zXE0wFXicq64rrym+mXwHu/CPVGO4=
k8s.io/client-go v0.22.3/go.mod h1:ElDjYf8gvZsKDYexmsmnMQ0DYO8W9RwBjfQ1PI53yow=
k8s.io/cloud-provider v0.22.3 h1:ZsWdB0WmyjKlE901EM14BuSvnN+QPGrCGjcfDc+b5NI=
k8s.io/cloud-provider v0.22.3/go.mod h1:GsKMR5EnNH4zcfkEvOxBPEZVuRvadVRkZvGqYxxBvO4=
k8s.io/cluster-bootstrap v0.22.3 h1:uTrzquwoXsstQ6PCea0dYbKWcPCetMp4MZEkZbT+Ei0=
k8s.io/cluster-bootstrap v0.22.3/go.mod h1:FVBAeGJ/T6QbNgGb7DX98FCjExJnNLsRXtGRMjEQ26I=
Expand All @@ -1183,6 +1184,7 @@ k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAE
k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM=
k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
k8s.io/kube-aggregator v0.22.3/go.mod h1:TIpLq1HvR/S4y75i3y+4q9ik3ZvgyaDz72CBfDS0A6E=
k8s.io/kube-controller-manager v0.22.3 h1:DatYcgMKAn28e2A7MiMULoRoft3SaCV/qVk+FoGTUw0=
k8s.io/kube-controller-manager v0.22.3/go.mod h1:7biFk6Azf7xD+pzTScw7X9M5vGScqYp4J4wOT61QL1s=
k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 h1:E3J9oCLlaobFUqsjG9DfKbP2BmgwBL2p7pn0A3dG9W4=
Expand All @@ -1191,6 +1193,7 @@ k8s.io/kube-proxy v0.22.3/go.mod h1:9ta1U8GKKo6by981sN/L6MhFJzPWxMdfh7plVPH1I2s=
k8s.io/kube-scheduler v0.22.3/go.mod h1:jVLHSttd8cSejBLOeiWE+g8etA6XdOBGiR8tI577OhU=
k8s.io/kubectl v0.22.3 h1:xziSHHyFHg2nt9vE6A0XqW5dOePNSlzxG8z3z+IY63E=
k8s.io/kubectl v0.22.3/go.mod h1:gcpQHPOx+Jke9Og6Li7YxR/ZuaOtFUeJw7xHH617tHs=
k8s.io/kubelet v0.22.3 h1:C21Kg66Zzvc21uJITEPg4stGMcSZsR1JB+7+6Uwm8zs=
k8s.io/kubelet v0.22.3/go.mod h1:9nUZNGUigU2uAIm7kgf8BsvYDI9KjIE5nt9+yI1+p7w=
k8s.io/legacy-cloud-providers v0.22.3/go.mod h1:eEOOaRtP2PuCVkjZvuTPa6ZgyPpzJkCVqpE3YtuArLQ=
k8s.io/metrics v0.22.3/go.mod h1:HbLFLRKtXzoC/6tHLQAlO9AeOBXZp2eB6SsgkbujoNI=
Expand Down
Loading

0 comments on commit 4cf3e47

Please sign in to comment.