diff --git a/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml b/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml index 6cb167d5c95..b87fc6c5c98 100644 --- a/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml +++ b/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml @@ -1438,6 +1438,25 @@ webhooks: resources: - endpoints sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: yurt-manager-webhook-service + namespace: {{ .Release.Namespace }} + path: /mutate-core-openyurt-io-v1-endpointslice + failurePolicy: Ignore + name: mutate.core.v1.endpointslice.openyurt.io + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - UPDATE + resources: + - endpointslices + sideEffects: None - admissionReviewVersions: - v1 - v1beta1 diff --git a/pkg/yurtmanager/webhook/endpointslice/v1/endpointslice_default.go b/pkg/yurtmanager/webhook/endpointslice/v1/endpointslice_default.go new file mode 100644 index 00000000000..505388b0982 --- /dev/null +++ b/pkg/yurtmanager/webhook/endpointslice/v1/endpointslice_default.go @@ -0,0 +1,115 @@ +/* +Copyright 2024 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + discovery "k8s.io/api/discovery/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + nodeutil "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/node" + podutil "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/pod" +) + +// Default satisfies the defaulting webhook interface. +func (webhook *EndpointSliceHandler) Default(ctx context.Context, obj runtime.Object) error { + endpoints, ok := obj.(*discovery.EndpointSlice) + if !ok { + apierrors.NewBadRequest(fmt.Sprintf("expected an EndpointSlice object but got %T", obj)) + } + + return remapAutonomyEndpoints(ctx, webhook.Client, endpoints) +} + +// isNodeAutonomous checks if the node has autonomy annotations +// and returns true if it does, false otherwise. +func isNodeAutonomous(ctx context.Context, c client.Client, nodeName string) (bool, error) { + node := &corev1.Node{} + err := c.Get(ctx, client.ObjectKey{Name: nodeName}, node) + if err != nil { + // If node doesn't exist, it doesn't have autonomy + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + + return nodeutil.IsPodBoundenToNode(node), nil +} + +// isPodCrashLoopBackOff checks if the pod is crashloopbackoff +// and returns true if it is, false otherwise. +func isPodCrashLoopBackOff(ctx context.Context, c client.Client, podName, namespace string) (bool, error) { + pod := &corev1.Pod{} + err := c.Get(ctx, client.ObjectKey{Name: podName, Namespace: namespace}, pod) + if err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + + return podutil.IsPodCrashLoopBackOff(pod.Status), nil +} + +func remapAutonomyEndpoints(ctx context.Context, client client.Client, slice *discovery.EndpointSlice) error { + if slice == nil || len(slice.Endpoints) == 0 { + return nil + } + + for _, e := range slice.Endpoints { + // If the endpoint is ready, skip + if e.Conditions.Ready != nil && *e.Conditions.Ready { + continue + } + + // If the endpoint doesn't have a node name or target ref, skip + if e.NodeName == nil || *e.NodeName == "" || e.TargetRef == nil { + continue + } + + isAutonomous, err := isNodeAutonomous(ctx, client, *e.NodeName) + if err != nil { + return err + } + + // If the node doesn't have autonomy, skip + if !isAutonomous { + continue + } + + isPodCrashLoopBackOff, err := isPodCrashLoopBackOff(ctx, client, e.TargetRef.Name, e.TargetRef.Namespace) + if err != nil { + return err + } + + // If the pod is in crashloopbackoff, skip + if isPodCrashLoopBackOff { + continue + } + + // Set not ready addresses to ready + *e.Conditions.Ready = true + } + + return nil +} diff --git a/pkg/yurtmanager/webhook/endpointslice/v1/endpointslice_default_test.go b/pkg/yurtmanager/webhook/endpointslice/v1/endpointslice_default_test.go new file mode 100644 index 00000000000..3d0681622fa --- /dev/null +++ b/pkg/yurtmanager/webhook/endpointslice/v1/endpointslice_default_test.go @@ -0,0 +1,506 @@ +/* +Copyright 2024 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1_test + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + discovery "k8s.io/api/discovery/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/openyurtio/openyurt/pkg/apis" + "github.com/openyurtio/openyurt/pkg/projectinfo" + nodeutils "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/node" + v1 "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/endpointslice/v1" + "github.com/stretchr/testify/require" + "k8s.io/utils/ptr" + + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestDefault_AutonomyAnnotations(t *testing.T) { + // Fix the pod to ready for the test + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "default", + }, + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodReady, + Status: corev1.ConditionTrue, + }, + }, + }, + } + + // Test cases for Default + tests := []struct { + name string + node *corev1.Node + inputObj runtime.Object + expectedObj *discovery.EndpointSlice + expectErr bool + }{ + { + name: "Node autonomy duration annotation", + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Annotations: map[string]string{ + projectinfo.GetNodeAutonomyDurationAnnotation(): "10m", + }, + }, + }, + inputObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{endpoint1(true), endpoint2(false)}, + }, + expectedObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{endpoint1(true), endpoint2(true)}, + }, + expectErr: false, + }, + { + name: "Node autonomy duration annotation empty", + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Annotations: map[string]string{ + projectinfo.GetNodeAutonomyDurationAnnotation(): "", // empty + }, + }, + }, + inputObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{endpoint1(false), endpoint2(false)}, + }, + expectedObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{endpoint1(false), endpoint2(false)}, // not updated to Ready + }, + expectErr: false, + }, + { + name: "Autonomy annotation true", + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Annotations: map[string]string{ + projectinfo.GetAutonomyAnnotation(): "true", + }, + }, + }, + inputObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{endpoint1(false), endpoint2(false)}, + }, + expectedObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{endpoint1(true), endpoint2(true)}, + }, + expectErr: false, + }, + { + name: "Autonomy annotation false", + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Annotations: map[string]string{ + projectinfo.GetAutonomyAnnotation(): "false", + }, + }, + }, + inputObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{endpoint1(true), endpoint2(false)}, + }, + expectedObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{endpoint1(true), endpoint2(false)}, // not updated to Ready + }, + expectErr: false, + }, + { + name: "Pod binding annotation true", + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Annotations: map[string]string{ + nodeutils.PodBindingAnnotation: "true", + }, + }, + }, + inputObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{endpoint1(false), endpoint2(false)}, + }, + expectedObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{endpoint1(true), endpoint2(true)}, + }, + expectErr: false, + }, + { + name: "Pod binding annotation false", + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Annotations: map[string]string{ + nodeutils.PodBindingAnnotation: "false", + }, + }, + }, + inputObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{endpoint1(false), endpoint2(false)}, + }, + expectedObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{endpoint1(false), endpoint2(false)}, // not updated to Ready + }, + expectErr: false, + }, + { + name: "Node has no annotations", + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Annotations: map[string]string{}, // Nothing + }, + }, + inputObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{endpoint1(false)}, + }, + expectedObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{endpoint1(false)}, + }, + expectErr: false, + }, + { + name: "Other node", + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node3", // Other + Annotations: map[string]string{ + projectinfo.GetNodeAutonomyDurationAnnotation(): "10m", + }, + }, + }, + inputObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{endpoint1(false)}, + }, + expectedObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{endpoint1(false)}, + }, + expectErr: false, + }, + { + name: "Node name is empty", + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "", + Annotations: map[string]string{ + projectinfo.GetNodeAutonomyDurationAnnotation(): "", // empty + }, + }, + }, + inputObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{endpoint1(false), endpoint2(false)}, + }, + expectedObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{endpoint1(false), endpoint2(false)}, // not updated to Ready + }, + expectErr: false, + }, + { + name: "Node name and target ref nil", + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + projectinfo.GetNodeAutonomyDurationAnnotation(): "", // empty + }, + }, + }, + inputObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{{ + Addresses: []string{"172.16.0.17", "172.16.0.18"}, + Conditions: discovery.EndpointConditions{ + Ready: ptr.To(false), + }, + }}, + }, + expectedObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{{ + Addresses: []string{"172.16.0.17", "172.16.0.18"}, + Conditions: discovery.EndpointConditions{ + Ready: ptr.To(false), + }, + }}, // not updated to Ready + }, + expectErr: false, + }, + { + name: "Endpoint slice is empty", + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + projectinfo.GetNodeAutonomyDurationAnnotation(): "", // empty + }, + }, + }, + inputObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{}, + }, + expectedObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{}, + }, + expectErr: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + scheme := runtime.NewScheme() + err := clientgoscheme.AddToScheme(scheme) + require.NoError(t, err, "Fail to add kubernetes clint-go custom resource") + + apis.AddToScheme(scheme) + + // Build client + clientBuilder := fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(pod) + if tc.node != nil { + clientBuilder = clientBuilder.WithObjects(tc.node) + } + + // Invoke Default + w := &v1.EndpointSliceHandler{Client: clientBuilder.Build()} + err = w.Default(context.TODO(), tc.inputObj) + if tc.expectErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + // Check the result + require.Equal(t, tc.expectedObj, tc.inputObj) + }) + } +} + +func TestDefault_PodCrashLoopBack(t *testing.T) { + // Fix the node annotation to autonomy duration + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Annotations: map[string]string{ + projectinfo.GetNodeAutonomyDurationAnnotation(): "10m", + }, + }, + } + + // Test cases for Default + tests := []struct { + name string + inputObj runtime.Object + pod *corev1.Pod + expectedObj *discovery.EndpointSlice + expectErr bool + }{ + { + name: "Pod not crashloopback", + inputObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{endpoint1(false), endpoint2Pod2(false)}, + }, + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "default", + }, + Status: corev1.PodStatus{ + ContainerStatuses: []corev1.ContainerStatus{ + { + State: corev1.ContainerState{ + Running: &corev1.ContainerStateRunning{}, + }, + }, + }, + }, + }, + expectedObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{endpoint1(true), endpoint2Pod2(true)}, /// updates regardless of matching pod + }, + expectErr: false, + }, + { + name: "Pod is crashloopbackoff", + inputObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{endpoint2Pod2(false)}, + }, + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "default", + }, + Status: corev1.PodStatus{ + ContainerStatuses: []corev1.ContainerStatus{ + { + State: corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{ + Reason: "CrashLoopBackOff", + }, + }, + }, + }, + }, + }, + expectedObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{endpoint2Pod2(false)}, + }, + expectErr: false, + }, + { + name: "Pod no container states", + inputObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{endpoint1(false)}, + }, + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "default", + }, + Status: corev1.PodStatus{}, + }, + expectedObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{endpoint1(true)}, + }, + expectErr: false, + }, + { + name: "Pod multiple container statuses", + inputObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{endpoint1(false), endpoint2Pod2(false)}, + }, + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "default", + }, + Status: corev1.PodStatus{ + ContainerStatuses: []corev1.ContainerStatus{ + { + State: corev1.ContainerState{ + Running: &corev1.ContainerStateRunning{}, + }, + }, + { + State: corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{ + Reason: "CrashLoopBackOff", + }, + }, + }, + }, + }, + }, + expectedObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{endpoint1(true), endpoint2Pod2(false)}, + }, + expectErr: false, + }, + { + name: "Pod is empty", + inputObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{endpoint1(false), endpoint2Pod2(false)}, + }, + pod: &corev1.Pod{}, // Empty pod + expectedObj: &discovery.EndpointSlice{ + Endpoints: []discovery.Endpoint{endpoint1(true), endpoint2Pod2(true)}, + }, + expectErr: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + scheme := runtime.NewScheme() + err := clientgoscheme.AddToScheme(scheme) + require.NoError(t, err, "Fail to add kubernetes clint-go custom resource") + + apis.AddToScheme(scheme) + + // Build client + clientBuilder := fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(node) + + // Pod + if tc.pod != nil { + clientBuilder = clientBuilder.WithObjects(tc.pod) + } + + // Invoke Default + w := &v1.EndpointSliceHandler{Client: clientBuilder.Build()} + err = w.Default(context.TODO(), tc.inputObj) + if tc.expectErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + // Check the result + require.Equal(t, tc.expectedObj, tc.inputObj) + }) + } +} + +func endpoint1(ready bool) discovery.Endpoint { + return discovery.Endpoint{ + Addresses: []string{"172.16.0.17", "172.16.0.18"}, + Conditions: discovery.EndpointConditions{ + Ready: ptr.To(ready), + }, + NodeName: ptr.To("node1"), + TargetRef: &corev1.ObjectReference{ + Kind: "Pod", + Name: "pod1", + Namespace: "default", + }, + } +} + +func endpoint2(ready bool) discovery.Endpoint { + return discovery.Endpoint{ + Addresses: []string{"10.244.1.2", "10.244.1.3"}, + Conditions: discovery.EndpointConditions{ + Ready: ptr.To(ready), + }, + NodeName: ptr.To("node1"), + TargetRef: &corev1.ObjectReference{ + Kind: "Pod", + Name: "pod1", + Namespace: "default", + }, + } +} + +func endpoint2Pod2(ready bool) discovery.Endpoint { + return discovery.Endpoint{ + Addresses: []string{"10.244.1.2", "10.244.1.3"}, + Conditions: discovery.EndpointConditions{ + Ready: ptr.To(ready), + }, + NodeName: ptr.To("node1"), + TargetRef: &corev1.ObjectReference{ + Kind: "Pod", + Name: "pod2", + Namespace: "default", + }, + } +} diff --git a/pkg/yurtmanager/webhook/endpointslice/v1/endpointslice_handler.go b/pkg/yurtmanager/webhook/endpointslice/v1/endpointslice_handler.go new file mode 100644 index 00000000000..bd3e8162a82 --- /dev/null +++ b/pkg/yurtmanager/webhook/endpointslice/v1/endpointslice_handler.go @@ -0,0 +1,49 @@ +/* +Copyright 2024 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + v1 "k8s.io/api/discovery/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + yurtClient "github.com/openyurtio/openyurt/cmd/yurt-manager/app/client" + "github.com/openyurtio/openyurt/cmd/yurt-manager/names" + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util" +) + +const ( + WebhookName = "endpointslice" +) + +// EndpointSliceHandler implements a defaulting webhook for EndpointSlice. +type EndpointSliceHandler struct { + Client client.Client +} + +// SetupWebhookWithManager sets up EndpointSlice webhooks. +func (webhook *EndpointSliceHandler) SetupWebhookWithManager(mgr ctrl.Manager) (string, string, error) { + // init + webhook.Client = yurtClient.GetClientByControllerNameOrDie(mgr, names.NodeLifeCycleController) + + return util.RegisterWebhook(mgr, &v1.EndpointSlice{}, webhook) +} + +// +kubebuilder:webhook:path=/mutate-core-openyurt-io-v1-endpointslice,mutating=true,failurePolicy=ignore,sideEffects=None,admissionReviewVersions=v1,groups="",resources=endpointslices,verbs=update,versions=v1,name=mutate.core.v1.endpointslice.openyurt.io + +var _ webhook.CustomDefaulter = &EndpointSliceHandler{} diff --git a/pkg/yurtmanager/webhook/server.go b/pkg/yurtmanager/webhook/server.go index fe19979c326..80d24479e75 100644 --- a/pkg/yurtmanager/webhook/server.go +++ b/pkg/yurtmanager/webhook/server.go @@ -32,6 +32,7 @@ import ( controller "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/base" v1alpha1deploymentrender "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/deploymentrender/v1alpha1" v1endpoints "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/endpoints/v1" + v1endpointslice "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/endpointslice/v1" v1beta1gateway "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/gateway/v1beta1" v1node "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/node/v1" v1beta1nodepool "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/nodepool/v1beta1" @@ -84,7 +85,7 @@ func init() { independentWebhooks[v1node.WebhookName] = &v1node.NodeHandler{} independentWebhooks[v1alpha1pod.WebhookName] = &v1alpha1pod.PodHandler{} independentWebhooks[v1endpoints.WebhookName] = &v1endpoints.EndpointsHandler{} - + independentWebhooks[v1endpointslice.WebhookName] = &v1endpointslice.EndpointSliceHandler{} } // Note !!! @kadisi