diff --git a/api/v1/ytsaurus_types.go b/api/v1/ytsaurus_types.go index 5333844c..e0286595 100644 --- a/api/v1/ytsaurus_types.go +++ b/api/v1/ytsaurus_types.go @@ -20,6 +20,8 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/ytsaurus/ytsaurus-k8s-operator/pkg/consts" ) // EmbeddedPersistentVolumeClaim is an embedded version of k8s.io/api/core/v1.PersistentVolumeClaim. @@ -645,11 +647,14 @@ type YtsaurusSpec struct { //+optional EnableFullUpdate bool `json:"enableFullUpdate"` //+optional - //+kubebuilder:validation:Enum={"","Nothing","MasterOnly","DataNodesOnly","TabletNodesOnly","ExecNodesOnly","StatelessOnly","Everything"} - // UpdateSelector is an experimental field. Behaviour may change. - // If UpdateSelector is not empty EnableFullUpdate is ignored. + //+optional + // Deprecated: UpdateSelector is an experimental field. Behaviour may change. UpdateSelector UpdateSelector `json:"updateSelector"` + //+optional + // Controls the components that should be updated during the update process. + UpdateSelectors []ComponentUpdateSelector `json:"updateSelectors,omitempty"` + NodeSelector map[string]string `json:"nodeSelector,omitempty"` Tolerations []corev1.Toleration `json:"tolerations,omitempty"` @@ -727,26 +732,14 @@ type TabletCellBundleInfo struct { type UpdateSelector string -const ( - // UpdateSelectorUnspecified means that selector is disabled and would be ignored completely. - UpdateSelectorUnspecified UpdateSelector = "" - // UpdateSelectorNothing means that no component could be updated. - UpdateSelectorNothing UpdateSelector = "Nothing" - // UpdateSelectorMasterOnly means that only master could be updated. - UpdateSelectorMasterOnly UpdateSelector = "MasterOnly" - // UpdateSelectorTabletNodesOnly means that only data nodes could be updated - UpdateSelectorDataNodesOnly UpdateSelector = "DataNodesOnly" - // UpdateSelectorTabletNodesOnly means that only tablet nodes could be updated - UpdateSelectorTabletNodesOnly UpdateSelector = "TabletNodesOnly" - // UpdateSelectorExecNodesOnly means that only tablet nodes could be updated - UpdateSelectorExecNodesOnly UpdateSelector = "ExecNodesOnly" - // UpdateSelectorStatelessOnly means that only stateless components (everything but master, data nodes, and tablet nodes) - // could be updated. - UpdateSelectorStatelessOnly UpdateSelector = "StatelessOnly" - // UpdateSelectorEverything means that all components could be updated. - // With this setting and if master or tablet nodes need update all the components would be updated. - UpdateSelectorEverything UpdateSelector = "Everything" -) +type ComponentUpdateSelector struct { + //+optional + Component consts.ComponentType `json:"componentKind,omitempty"` + //+optional + ComponentGroup consts.ComponentGroup `json:"componentGroup,omitempty"` + + // TODO(#325): Add name field for specific sts and rolling options +} type UpdateFlow string diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index cf994d1c..a5b6df15 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -404,6 +404,21 @@ func (in *CommonSpec) DeepCopy() *CommonSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ComponentUpdateSelector) DeepCopyInto(out *ComponentUpdateSelector) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ComponentUpdateSelector. +func (in *ComponentUpdateSelector) DeepCopy() *ComponentUpdateSelector { + if in == nil { + return nil + } + out := new(ComponentUpdateSelector) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ControllerAgentsSpec) DeepCopyInto(out *ControllerAgentsSpec) { *out = *in @@ -1938,6 +1953,11 @@ func (in *YtsaurusSpec) DeepCopyInto(out *YtsaurusSpec) { *out = new(OauthServiceSpec) (*in).DeepCopyInto(*out) } + if in.UpdateSelectors != nil { + in, out := &in.UpdateSelectors, &out.UpdateSelectors + *out = make([]ComponentUpdateSelector, len(*in)) + copy(*out, *in) + } if in.NodeSelector != nil { in, out := &in.NodeSelector, &out.NodeSelector *out = make(map[string]string, len(*in)) diff --git a/config/crd/bases/cluster.ytsaurus.tech_ytsaurus.yaml b/config/crd/bases/cluster.ytsaurus.tech_ytsaurus.yaml index 9b353fec..bef3e6b5 100644 --- a/config/crd/bases/cluster.ytsaurus.tech_ytsaurus.yaml +++ b/config/crd/bases/cluster.ytsaurus.tech_ytsaurus.yaml @@ -34657,18 +34657,20 @@ spec: uiImage: type: string updateSelector: - description: UpdateSelector is an experimental field. Behaviour may - change. - enum: - - "" - - Nothing - - MasterOnly - - DataNodesOnly - - TabletNodesOnly - - ExecNodesOnly - - StatelessOnly - - Everything + description: 'Deprecated: UpdateSelector is an experimental field. + Behaviour may change.' type: string + updateSelectors: + description: Controls the components that should be updated during + the update process. + items: + properties: + componentGroup: + type: string + componentKind: + type: string + type: object + type: array useIpv4: default: false type: boolean diff --git a/controllers/sync.go b/controllers/sync.go index 0928151c..118804c7 100644 --- a/controllers/sync.go +++ b/controllers/sync.go @@ -433,103 +433,115 @@ type updateMeta struct { componentNames []string } -func canUpdateComponent(selector ytv1.UpdateSelector, component consts.ComponentType) bool { - switch selector { - case ytv1.UpdateSelectorNothing: - return false - case ytv1.UpdateSelectorMasterOnly: - return component == consts.MasterType - case ytv1.UpdateSelectorDataNodesOnly: - return component == consts.DataNodeType - case ytv1.UpdateSelectorTabletNodesOnly: - return component == consts.TabletNodeType - case ytv1.UpdateSelectorExecNodesOnly: - return component == consts.ExecNodeType - case ytv1.UpdateSelectorStatelessOnly: - switch component { - case consts.MasterType: - return false - case consts.DataNodeType: - return false - case consts.TabletNodeType: - return false - } - return true - case ytv1.UpdateSelectorEverything: - return true - default: - return false +func getFlowFromComponent(component consts.ComponentType) ytv1.UpdateFlow { + if component == consts.MasterType { + return ytv1.UpdateFlowMaster } + if component == consts.TabletNodeType { + return ytv1.UpdateFlowTabletNodes + } + if component == consts.DataNodeType || component == consts.ExecNodeType { + return ytv1.UpdateFlowFull + } + return ytv1.UpdateFlowStateless +} + +func canUpdateComponent(selectors []ytv1.ComponentUpdateSelector, component consts.ComponentType) (bool, error) { + for _, selector := range selectors { + if selector.Component != "" { + if selector.Component == component { + return true, nil + } + } else { + switch selector.ComponentGroup { + case consts.ComponentGroupEverything: + return true, nil + case consts.ComponentGroupNone: + return false, nil + case consts.ComponentGroupStateful: + if component == consts.DataNodeType || component == consts.TabletNodeType { + return true, nil + } + case consts.ComponentGroupStateless: + if component != consts.DataNodeType && component != consts.TabletNodeType && component != consts.MasterType { + return true, nil + } + default: + return false, fmt.Errorf("unexpected component group %s", selector.ComponentGroup) + } + } + } + return false, nil } // chooseUpdateFlow considers spec and decides if operator should proceed with update or block. // Block case is indicated with non-empty blockMsg. // If update is not blocked, updateMeta containing a chosen flow and the component names to update returned. -func chooseUpdateFlow(spec ytv1.YtsaurusSpec, needUpdate []components.Component) (meta updateMeta, blockMsg string) { - configuredSelector := spec.UpdateSelector - if configuredSelector == ytv1.UpdateSelectorUnspecified { - if spec.EnableFullUpdate { - configuredSelector = ytv1.UpdateSelectorEverything - } else { - configuredSelector = ytv1.UpdateSelectorStatelessOnly - } +func chooseUpdateFlow(ctx context.Context, spec ytv1.YtsaurusSpec, needUpdate []components.Component) (meta updateMeta, blockMsg string) { + logger := log.FromContext(ctx) + configuredSelectors := spec.UpdateSelectors + if len(configuredSelectors) == 0 && spec.EnableFullUpdate { + configuredSelectors = []ytv1.ComponentUpdateSelector{{ComponentGroup: consts.ComponentGroupEverything}} } + for _, selector := range configuredSelectors { + logger.Info("NEW LOG: Configured selector", "component", selector.Component, "group", selector.ComponentGroup) + } + logger.Info("NEW LOG: Spec", "enableFullUpdate", spec.EnableFullUpdate) + needFullUpdate := false var canUpdate []string var cannotUpdate []string - needFullUpdate := false + flows := make(map[ytv1.UpdateFlow]struct{}) for _, comp := range needUpdate { componentType := comp.GetType() componentName := comp.GetName() - if canUpdateComponent(configuredSelector, componentType) { + upd, err := canUpdateComponent(configuredSelectors, componentType) + if err != nil { + return updateMeta{}, err.Error() + } + if upd { canUpdate = append(canUpdate, componentName) + flows[getFlowFromComponent(componentType)] = struct{}{} + logger.Info("NEW LOG: Can update", "component", componentName, "flow", getFlowFromComponent(componentType), "name", comp.GetName(), "type", comp.GetType()) } else { cannotUpdate = append(cannotUpdate, componentName) + logger.Info("NEW LOG: Can't update", "component", componentName) } - if !canUpdateComponent(ytv1.UpdateSelectorStatelessOnly, componentType) && componentType != consts.DataNodeType { + + statelessCheck, err := canUpdateComponent([]ytv1.ComponentUpdateSelector{{ComponentGroup: consts.ComponentGroupStateless}}, componentType) + if !statelessCheck && componentType != consts.DataNodeType { + logger.Info("NEW LOG: Need full update", "component", componentName) needFullUpdate = true } } - if len(canUpdate) == 0 { + if len(canUpdate) == 0 { // & len(spec.UpdateSelectors) == 0 if len(cannotUpdate) != 0 { return updateMeta{}, fmt.Sprintf("All components allowed by updateSelector are uptodate, update of {%s} is not allowed", strings.Join(cannotUpdate, ", ")) } return updateMeta{}, "All components are uptodate" } - switch configuredSelector { - case ytv1.UpdateSelectorEverything: + if len(configuredSelectors) == 1 && configuredSelectors[0].ComponentGroup == consts.ComponentGroupEverything { if needFullUpdate { - return updateMeta{ - flow: ytv1.UpdateFlowFull, - componentNames: nil, - }, "" + return updateMeta{flow: ytv1.UpdateFlowFull, componentNames: nil}, "" } else { - return updateMeta{ - flow: ytv1.UpdateFlowStateless, - componentNames: canUpdate, - }, "" - } - case ytv1.UpdateSelectorMasterOnly: - return updateMeta{ - flow: ytv1.UpdateFlowMaster, - componentNames: canUpdate, - }, "" - case ytv1.UpdateSelectorTabletNodesOnly: - return updateMeta{ - flow: ytv1.UpdateFlowTabletNodes, - componentNames: canUpdate, - }, "" - case ytv1.UpdateSelectorDataNodesOnly, ytv1.UpdateSelectorExecNodesOnly, ytv1.UpdateSelectorStatelessOnly: - return updateMeta{ - flow: ytv1.UpdateFlowStateless, - componentNames: canUpdate, - }, "" - default: - return updateMeta{}, fmt.Sprintf("Unexpected update selector %s", configuredSelector) + return updateMeta{flow: ytv1.UpdateFlowStateless, componentNames: canUpdate}, "" + } + } + + if len(flows) == 0 { + return updateMeta{}, fmt.Sprintf("Unexpected state: no flows for components {%s}", strings.Join(canUpdate, ", ")) + } + + if len(flows) == 1 { + for flow := range flows { + return updateMeta{flow: flow, componentNames: canUpdate}, "" + } } + // If more than one flow is possible, we choose to follow full update flow. + return updateMeta{flow: ytv1.UpdateFlowFull, componentNames: canUpdate}, "" } func (r *YtsaurusReconciler) Sync(ctx context.Context, resource *ytv1.Ytsaurus) (ctrl.Result, error) { @@ -584,7 +596,7 @@ func (r *YtsaurusReconciler) Sync(ctx context.Context, resource *ytv1.Ytsaurus) needUpdateNames = append(needUpdateNames, c.GetName()) } logger = logger.WithValues("componentsForUpdateAll", needUpdateNames) - meta, blockMsg := chooseUpdateFlow(ytsaurus.GetResource().Spec, needUpdate) + meta, blockMsg := chooseUpdateFlow(ctx, ytsaurus.GetResource().Spec, needUpdate) if blockMsg != "" { logger.Info(blockMsg) return ctrl.Result{Requeue: true}, nil diff --git a/docs/api.md b/docs/api.md index 255ba065..996b6437 100644 --- a/docs/api.md +++ b/docs/api.md @@ -327,6 +327,23 @@ _Appears in:_ | `imagePullSecrets` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#localobjectreference-v1-core) array_ | | | | +#### ComponentUpdateSelector + + + + + + + +_Appears in:_ +- [YtsaurusSpec](#ytsaurusspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `componentKind` _[ComponentType](#componenttype)_ | | | | +| `componentGroup` _[ComponentGroup](#componentgroup)_ | | | | + + #### ControllerAgentsSpec @@ -2048,16 +2065,6 @@ _Underlying type:_ _string_ _Appears in:_ - [YtsaurusSpec](#ytsaurusspec) -| Field | Description | -| --- | --- | -| `` | UpdateSelectorUnspecified means that selector is disabled and would be ignored completely.
| -| `Nothing` | UpdateSelectorNothing means that no component could be updated.
| -| `MasterOnly` | UpdateSelectorMasterOnly means that only master could be updated.
| -| `DataNodesOnly` | UpdateSelectorTabletNodesOnly means that only data nodes could be updated
| -| `TabletNodesOnly` | UpdateSelectorTabletNodesOnly means that only tablet nodes could be updated
| -| `ExecNodesOnly` | UpdateSelectorExecNodesOnly means that only tablet nodes could be updated
| -| `StatelessOnly` | UpdateSelectorStatelessOnly means that only stateless components (everything but master, data nodes, and tablet nodes)
could be updated.
| -| `Everything` | UpdateSelectorEverything means that all components could be updated.
With this setting and if master or tablet nodes need update all the components would be updated.
| #### UpdateState @@ -2204,7 +2211,8 @@ _Appears in:_ | `oauthService` _[OauthServiceSpec](#oauthservicespec)_ | | | | | `isManaged` _boolean_ | | true | | | `enableFullUpdate` _boolean_ | | true | | -| `updateSelector` _[UpdateSelector](#updateselector)_ | UpdateSelector is an experimental field. Behaviour may change.
If UpdateSelector is not empty EnableFullUpdate is ignored. | | Enum: [ Nothing MasterOnly DataNodesOnly TabletNodesOnly ExecNodesOnly StatelessOnly Everything]
| +| `updateSelector` _[UpdateSelector](#updateselector)_ | Deprecated: UpdateSelector is an experimental field. Behaviour may change. | | | +| `updateSelectors` _[ComponentUpdateSelector](#componentupdateselector) array_ | Controls the components that should be updated during the update process. | | | | `nodeSelector` _object (keys:string, values:string)_ | | | | | `tolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#toleration-v1-core) array_ | | | | | `bootstrap` _[BootstrapSpec](#bootstrapspec)_ | | | | diff --git a/pkg/consts/types.go b/pkg/consts/types.go index 702a8e00..cfe66f7f 100644 --- a/pkg/consts/types.go +++ b/pkg/consts/types.go @@ -23,3 +23,12 @@ const ( ChytType ComponentType = "CHYT" SpytType ComponentType = "SPYT" ) + +type ComponentGroup string + +const ( + ComponentGroupStateless ComponentGroup = "Stateless" + ComponentGroupStateful ComponentGroup = "Stateful" + ComponentGroupEverything ComponentGroup = "Everything" + ComponentGroupNone ComponentGroup = "None" +) diff --git a/test/e2e/ytsaurus_controller_test.go b/test/e2e/ytsaurus_controller_test.go index 314f3745..a3e707ff 100644 --- a/test/e2e/ytsaurus_controller_test.go +++ b/test/e2e/ytsaurus_controller_test.go @@ -43,8 +43,8 @@ import ( const ( pollInterval = time.Millisecond * 250 reactionTimeout = time.Second * 150 - bootstrapTimeout = time.Minute * 3 - upgradeTimeout = time.Minute * 7 + bootstrapTimeout = time.Minute * 10 + upgradeTimeout = time.Minute * 10 ) var getYtClient = getYtHTTPClient @@ -306,7 +306,7 @@ var _ = Describe("Basic e2e test for Ytsaurus controller", Label("e2e"), func() It("Should be updated according to UpdateSelector=Everything", func(ctx context.Context) { By("Run cluster update with selector: nothing") - ytsaurus.Spec.UpdateSelector = ytv1.UpdateSelectorNothing + ytsaurus.Spec.UpdateSelectors = []ytv1.ComponentUpdateSelector{{ComponentGroup: consts.ComponentGroupNone}} // We want change in all yson configs, new discovery instance will trigger that. ytsaurus.Spec.Discovery.InstanceCount += 1 UpdateObject(ctx, ytsaurus) @@ -320,7 +320,7 @@ var _ = Describe("Basic e2e test for Ytsaurus controller", Label("e2e"), func() ) By("Update cluster update with strategy full") - ytsaurus.Spec.UpdateSelector = ytv1.UpdateSelectorEverything + ytsaurus.Spec.UpdateSelectors = []ytv1.ComponentUpdateSelector{{ComponentGroup: consts.ComponentGroupEverything}} ytsaurus.Spec.Discovery.InstanceCount += 1 UpdateObject(ctx, ytsaurus) @@ -340,7 +340,7 @@ var _ = Describe("Basic e2e test for Ytsaurus controller", Label("e2e"), func() It("Should be updated according to UpdateSelector=TabletNodesOnly,ExecNodesOnly", func(ctx context.Context) { By("Run cluster update with selector:ExecNodesOnly") - ytsaurus.Spec.UpdateSelector = ytv1.UpdateSelectorExecNodesOnly + ytsaurus.Spec.UpdateSelectors = []ytv1.ComponentUpdateSelector{{Component: consts.ExecNodeType}} ytsaurus.Spec.Discovery.InstanceCount += 1 UpdateObject(ctx, ytsaurus) @@ -358,7 +358,7 @@ var _ = Describe("Basic e2e test for Ytsaurus controller", Label("e2e"), func() Expect(pods.Updated).To(ConsistOf("end-0"), "updated") By("Run cluster update with selector:TabletNodesOnly") - ytsaurus.Spec.UpdateSelector = ytv1.UpdateSelectorTabletNodesOnly + ytsaurus.Spec.UpdateSelectors = []ytv1.ComponentUpdateSelector{{Component: consts.TabletNodeType}} ytsaurus.Spec.Discovery.InstanceCount += 1 UpdateObject(ctx, ytsaurus) @@ -379,7 +379,7 @@ var _ = Describe("Basic e2e test for Ytsaurus controller", Label("e2e"), func() It("Should be updated according to UpdateSelector=MasterOnly,StatelessOnly", Label("basic"), func(ctx context.Context) { By("Run cluster update with selector:MasterOnly") - ytsaurus.Spec.UpdateSelector = ytv1.UpdateSelectorMasterOnly + ytsaurus.Spec.UpdateSelectors = []ytv1.ComponentUpdateSelector{{Component: consts.MasterType}} ytsaurus.Spec.Discovery.InstanceCount += 1 UpdateObject(ctx, ytsaurus) @@ -396,7 +396,7 @@ var _ = Describe("Basic e2e test for Ytsaurus controller", Label("e2e"), func() Expect(pods.Updated).To(ConsistOf("ms-0"), "updated") By("Run cluster update with selector:StatelessOnly") - ytsaurus.Spec.UpdateSelector = ytv1.UpdateSelectorStatelessOnly + ytsaurus.Spec.UpdateSelectors = []ytv1.ComponentUpdateSelector{{ComponentGroup: consts.ComponentGroupStateless}} ytsaurus.Spec.Discovery.InstanceCount += 1 UpdateObject(ctx, ytsaurus) diff --git a/ytop-chart/templates/crds/ytsaurus.cluster.ytsaurus.tech.yaml b/ytop-chart/templates/crds/ytsaurus.cluster.ytsaurus.tech.yaml index 435ba563..4c5c5ea3 100644 --- a/ytop-chart/templates/crds/ytsaurus.cluster.ytsaurus.tech.yaml +++ b/ytop-chart/templates/crds/ytsaurus.cluster.ytsaurus.tech.yaml @@ -34668,18 +34668,20 @@ spec: uiImage: type: string updateSelector: - description: UpdateSelector is an experimental field. Behaviour may - change. - enum: - - "" - - Nothing - - MasterOnly - - DataNodesOnly - - TabletNodesOnly - - ExecNodesOnly - - StatelessOnly - - Everything + description: 'Deprecated: UpdateSelector is an experimental field. + Behaviour may change.' type: string + updateSelectors: + description: Controls the components that should be updated during + the update process. + items: + properties: + componentGroup: + type: string + componentKind: + type: string + type: object + type: array useIpv4: default: false type: boolean