From d2e95a8ee8588de0f170a94a431fbb46aee43a30 Mon Sep 17 00:00:00 2001 From: hugoShaka Date: Tue, 3 Dec 2024 18:00:34 -0500 Subject: [PATCH] autoupdate: implement time-based strategy This commit implements the time-based rollout strategy describen in RFD 184. The autoupdate_agent_rollout controller will make the groups active based on their start days, start hour, and maintenance duration. Once the maintenance window is over, the group becomes DONE. In the DONE state, new agents will instalkl the target version but existing agents will no longer be told to actively update. --- lib/autoupdate/rollout/strategy_timebased.go | 110 ++++++ .../rollout/strategy_timebased_test.go | 314 ++++++++++++++++++ 2 files changed, 424 insertions(+) create mode 100644 lib/autoupdate/rollout/strategy_timebased.go create mode 100644 lib/autoupdate/rollout/strategy_timebased_test.go diff --git a/lib/autoupdate/rollout/strategy_timebased.go b/lib/autoupdate/rollout/strategy_timebased.go new file mode 100644 index 0000000000000..c5abc34be5588 --- /dev/null +++ b/lib/autoupdate/rollout/strategy_timebased.go @@ -0,0 +1,110 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package rollout + +import ( + "context" + "log/slog" + + "github.com/gravitational/trace" + "github.com/jonboulle/clockwork" + + "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" + update "github.com/gravitational/teleport/api/types/autoupdate" +) + +const ( + updateReasonInWindow = "in_window" + updateReasonOutsideWindow = "outside_window" +) + +type timeBasedStrategy struct { + log *slog.Logger + clock clockwork.Clock +} + +func (h *timeBasedStrategy) name() string { + return update.AgentsStrategyTimeBased +} + +func newTimeBasedStrategy(log *slog.Logger, clock clockwork.Clock) (rolloutStrategy, error) { + if log == nil { + return nil, trace.BadParameter("missing log") + } + if clock == nil { + return nil, trace.BadParameter("missing clock") + } + return &timeBasedStrategy{ + log: log.With("strategy", update.AgentsStrategyTimeBased), + clock: clock, + }, nil +} + +func (h *timeBasedStrategy) progressRollout(ctx context.Context, groups []*autoupdate.AutoUpdateAgentRolloutStatusGroup) error { + now := h.clock.Now() + // We always process every group regardless of the order. + var errs []error + for _, group := range groups { + switch group.State { + case autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE: + // We start any group unstarted group in window. + // Done groups can transition back to active if they enter their maintenance window again. + // Some agents might have missed the previous windows and might expected to try again. + shouldBeActive, err := inWindow(group, now) + if err != nil { + // In time-based rollouts, groups are not dependent. + // Failing to transition a group should affect other groups. + // We reflect that something went wrong in the status and go to the next group. + setGroupState(group, group.State, updateReasonReconcilerError, now) + errs = append(errs, err) + continue + } + if shouldBeActive { + setGroupState(group, autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, updateReasonInWindow, now) + } else { + setGroupState(group, group.State, updateReasonOutsideWindow, now) + } + case autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK: + // We don't touch any group that was manually rolled back. + // Something happened and we should not try to update again. + case autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE: + // The group is currently being updated. We check if the maintenance + // is over and if we should transition it to the done state + shouldBeActive, err := inWindow(group, now) + if err != nil { + // In time-based rollouts, groups are not dependent. + // Failing to transition a group should affect other groups. + // We reflect that something went wrong in the status and go to the next group. + setGroupState(group, group.State, updateReasonReconcilerError, now) + errs = append(errs, err) + continue + } + + if shouldBeActive { + setGroupState(group, autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, updateReasonInWindow, now) + } else { + setGroupState(group, autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE, updateReasonOutsideWindow, now) + } + default: + return trace.BadParameter("unknown autoupdate group state: %v", group.State) + } + } + return trace.NewAggregate(errs...) +} diff --git a/lib/autoupdate/rollout/strategy_timebased_test.go b/lib/autoupdate/rollout/strategy_timebased_test.go new file mode 100644 index 0000000000000..91db29d42e469 --- /dev/null +++ b/lib/autoupdate/rollout/strategy_timebased_test.go @@ -0,0 +1,314 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package rollout + +import ( + "context" + "testing" + "time" + + "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" + "github.com/gravitational/teleport/lib/utils" +) + +func Test_progressGroupsTimeBased(t *testing.T) { + clock := clockwork.NewFakeClockAt(testSunday) + log := utils.NewSlogLoggerForTests() + strategy, err := newTimeBasedStrategy(log, clock) + require.NoError(t, err) + + groupName := "test-group" + canStartToday := everyWeekday + cannotStartToday := everyWeekdayButSunday + lastUpdate := timestamppb.New(clock.Now().Add(-5 * time.Minute)) + ctx := context.Background() + + tests := []struct { + name string + initialState []*autoupdate.AutoUpdateAgentRolloutStatusGroup + expectedState []*autoupdate.AutoUpdateAgentRolloutStatusGroup + }{ + { + name: "unstarted -> unstarted", + initialState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: lastUpdate, + LastUpdateReason: updateReasonCreated, + ConfigDays: cannotStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + expectedState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonOutsideWindow, + ConfigDays: cannotStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + }, + { + name: "unstarted -> active", + initialState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: lastUpdate, + LastUpdateReason: updateReasonCreated, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + expectedState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + StartTime: timestamppb.New(clock.Now()), + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonInWindow, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + }, + { + name: "done -> done", + initialState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE, + LastUpdateTime: lastUpdate, + LastUpdateReason: updateReasonOutsideWindow, + ConfigDays: cannotStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + expectedState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE, + LastUpdateTime: lastUpdate, + LastUpdateReason: updateReasonOutsideWindow, + ConfigDays: cannotStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + }, + { + name: "done -> active", + initialState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE, + LastUpdateTime: lastUpdate, + StartTime: lastUpdate, + LastUpdateReason: updateReasonOutsideWindow, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + expectedState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + StartTime: timestamppb.New(clock.Now()), + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonInWindow, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + }, + { + name: "active -> active", + initialState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + StartTime: lastUpdate, + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonInWindow, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + expectedState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + StartTime: lastUpdate, + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonInWindow, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + }, + { + name: "active -> done", + initialState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + StartTime: lastUpdate, + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonInWindow, + ConfigDays: cannotStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + expectedState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE, + StartTime: lastUpdate, + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonOutsideWindow, + ConfigDays: cannotStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + }, + { + name: "rolledback is a dead end", + initialState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName + "-in-maintenance", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK, + LastUpdateTime: lastUpdate, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + { + Name: groupName + "-out-of-maintenance", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK, + LastUpdateTime: lastUpdate, + ConfigDays: cannotStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + expectedState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName + "-in-maintenance", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK, + LastUpdateTime: lastUpdate, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + { + Name: groupName + "-out-of-maintenance", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK, + LastUpdateTime: lastUpdate, + ConfigDays: cannotStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + }, + { + name: "mix of everything", + initialState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: "new group should start", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: lastUpdate, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + { + Name: "done group should start", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE, + LastUpdateTime: lastUpdate, + StartTime: lastUpdate, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + { + Name: "rolledback group should do nothing", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK, + LastUpdateTime: lastUpdate, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + { + Name: "old group should stop", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + LastUpdateTime: lastUpdate, + StartTime: lastUpdate, + ConfigDays: cannotStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + expectedState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: "new group should start", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + StartTime: timestamppb.New(clock.Now()), + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonInWindow, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + { + Name: "done group should start", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + StartTime: timestamppb.New(clock.Now()), + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonInWindow, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + { + Name: "rolledback group should do nothing", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK, + LastUpdateTime: lastUpdate, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + { + Name: "old group should stop", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE, + StartTime: lastUpdate, + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonOutsideWindow, + ConfigDays: cannotStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := strategy.progressRollout(ctx, tt.initialState) + require.NoError(t, err) + // We use require.Equal instead of Elements match because group order matters. + // It's not super important for time-based, but is crucial for halt-on-error. + // So it's better to be more conservative and validate order never changes for + // both strategies. + require.Equal(t, tt.expectedState, tt.initialState) + }) + } +}