From 1ea5f2e5ea21f7d36486dd102dcc37ac2b4bcd5f Mon Sep 17 00:00:00 2001 From: David Symons Date: Fri, 2 Jul 2021 13:09:27 +1000 Subject: [PATCH] Add ability to exclude pods based on time via annotations --- chaoskube/chaoskube.go | 124 ++++++++++++++++++++++++++++-------- chaoskube/chaoskube_test.go | 87 +++++++++++++++++++++---- util/util.go | 28 ++++++++ 3 files changed, 201 insertions(+), 38 deletions(-) diff --git a/chaoskube/chaoskube.go b/chaoskube/chaoskube.go index 0bced9b4..fa884320 100644 --- a/chaoskube/chaoskube.go +++ b/chaoskube/chaoskube.go @@ -6,7 +6,6 @@ import ( "fmt" "math/rand" "regexp" - "strings" "time" multierror "github.com/hashicorp/go-multierror" @@ -94,6 +93,8 @@ var ( msgTimeOfDayExcluded = "time of day excluded" // msgDayOfYearExcluded is the log message when termination is suspended due to the day of year filter msgDayOfYearExcluded = "day of year excluded" + // msgFailedToParseAnnotation is the log message when a filter fails to parse a pod annotation + msgFailedToParseAnnotation = "failed to parse annotation, '%v', excluding from candidates" ) // New returns a new instance of Chaoskube. It expects: @@ -181,7 +182,7 @@ func (c *Chaoskube) TerminateVictims(ctx context.Context) error { } } - victims, err := c.Victims(ctx) + victims, err := c.Victims(ctx, now) if err == errPodNotFound { c.Logger.Debug(msgVictimNotFound) return nil @@ -200,8 +201,8 @@ func (c *Chaoskube) TerminateVictims(ctx context.Context) error { } // Victims returns up to N pods as configured by MaxKill flag -func (c *Chaoskube) Victims(ctx context.Context) ([]v1.Pod, error) { - pods, err := c.Candidates(ctx) +func (c *Chaoskube) Victims(ctx context.Context, now time.Time) ([]v1.Pod, error) { + pods, err := c.Candidates(ctx, now) if err != nil { return []v1.Pod{}, err } @@ -220,7 +221,7 @@ func (c *Chaoskube) Victims(ctx context.Context) ([]v1.Pod, error) { // Candidates returns the list of pods that are available for termination. // It returns all pods that match the configured label, annotation and namespace selectors. -func (c *Chaoskube) Candidates(ctx context.Context) ([]v1.Pod, error) { +func (c *Chaoskube) Candidates(ctx context.Context, now time.Time) ([]v1.Pod, error) { listOptions := metav1.ListOptions{LabelSelector: c.Labels.String()} podList, err := c.Client.CoreV1().Pods(v1.NamespaceAll).List(ctx, listOptions) @@ -248,24 +249,11 @@ func (c *Chaoskube) Candidates(ctx context.Context) ([]v1.Pod, error) { pods = filterTerminatingPods(pods) pods = filterByPodName(pods, c.IncludedPodNames, c.ExcludedPodNames) - pods = filterByMinimumAge( - pods, - strings.Join([]string{c.ConfigAnnotationPrefix, "minimum-age"}, "/"), - c.MinimumAge, - c.Now(), - c.Logger, - ) - - pods = filterByFrequency( - pods, - strings.Join([]string{c.ConfigAnnotationPrefix, "frequency"}, "/"), - c.DefaultFrequency, - c.Interval, - c.Logger, - ) + pods = filterByMinimumAge(pods, c.ConfigAnnotationPrefix, c.MinimumAge, c.Now(), c.Logger) + pods = filterByFrequency(pods, c.ConfigAnnotationPrefix, c.DefaultFrequency, c.Interval, c.Logger) + pods = filterByTime(pods, c.ConfigAnnotationPrefix, now, c.Logger) pods = filterByOwnerReference(pods) - return pods, nil } @@ -498,11 +486,13 @@ func filterTerminatingPods(pods []v1.Pod) []v1.Pod { // filterByMinimumAge filters pods by creation time. Only pods // older than minimumAge are returned -func filterByMinimumAge(pods []v1.Pod, annotation string, minimumAge time.Duration, now time.Time, logger log.FieldLogger) []v1.Pod { - if annotation == "" && minimumAge <= time.Duration(0) { +func filterByMinimumAge(pods []v1.Pod, annotationPrefix string, minimumAge time.Duration, now time.Time, logger log.FieldLogger) []v1.Pod { + if annotationPrefix == "" && minimumAge <= time.Duration(0) { return pods } + annotation := util.FormatAnnotation(annotationPrefix, "minimum-age") + defaultCreationTime := now.Add(-minimumAge) filteredList := []v1.Pod{} @@ -514,7 +504,7 @@ func filterByMinimumAge(pods []v1.Pod, annotation string, minimumAge time.Durati if ok { minimumAgeOverride, err := time.ParseDuration(text) if err != nil { - logger.WithField("err", err).Warn("failed to parse frequency annotation, excluding from candidates") + logger.WithField("err", err).Warnf(msgFailedToParseAnnotation, annotation) continue } @@ -575,11 +565,13 @@ func filterByOwnerReference(pods []v1.Pod) []v1.Pod { return filteredList } -func filterByFrequency(pods []v1.Pod, annotation string, defaultFrequency string, interval time.Duration, logger log.FieldLogger) []v1.Pod { - if annotation == "" && defaultFrequency == "" { +func filterByFrequency(pods []v1.Pod, annotationPrefix string, defaultFrequency string, interval time.Duration, logger log.FieldLogger) []v1.Pod { + if annotationPrefix == "" && defaultFrequency == "" { return pods } + annotation := util.FormatAnnotation(annotationPrefix, "frequency") + filteredList := []v1.Pod{} for _, pod := range pods { text, ok := pod.Annotations[annotation] @@ -596,7 +588,7 @@ func filterByFrequency(pods []v1.Pod, annotation string, defaultFrequency string chance, err := util.ParseFrequency(text, interval) if err != nil { - logger.WithField("err", err).Warn("failed to parse frequency annotation, excluding from candidates") + logger.WithField("err", err).Warnf(msgFailedToParseAnnotation, annotation) continue } @@ -607,3 +599,81 @@ func filterByFrequency(pods []v1.Pod, annotation string, defaultFrequency string return filteredList } + +func filterByTime(pods []v1.Pod, annotationPrefix string, now time.Time, logger log.FieldLogger) []v1.Pod { + if annotationPrefix == "" { + return pods + } + + timezoneAnnotation := util.FormatAnnotation(annotationPrefix, "timezone") + weekdaysAnnotation := util.FormatAnnotation(annotationPrefix, "excluded-weekdays") + timesOfDayAnnotation := util.FormatAnnotation(annotationPrefix, "excluded-times-of-day") + daysOfYearAnnotation := util.FormatAnnotation(annotationPrefix, "excluded-days-of-year") + + filteredList := []v1.Pod{} + +checkingPods: + for _, pod := range pods { + localNow := now + + text, ok := pod.Annotations[timezoneAnnotation] + if ok { + location, err := time.LoadLocation(text) + if err != nil { + logger.WithField("err", err).WithField("pod-name", pod.Name).WithField("pod-namespace", pod.Namespace). + Warnf(msgFailedToParseAnnotation, timezoneAnnotation) + + continue checkingPods + } + + localNow = localNow.In(location) + } + + // Weekdays + text, ok = pod.Annotations[weekdaysAnnotation] + if ok { + days := util.ParseWeekdays(text) + for _, wd := range days { + if wd == localNow.Weekday() { + continue checkingPods + } + } + } + + // Times of day + text, ok = pod.Annotations[timesOfDayAnnotation] + if ok { + periods, err := util.ParseTimePeriods(text) + if err != nil { + logger.WithField("err", err).Warnf(msgFailedToParseAnnotation, timesOfDayAnnotation) + continue checkingPods + } + + for _, tp := range periods { + if tp.Includes(localNow) { + continue checkingPods + } + } + } + + // Days of year + text, ok = pod.Annotations[daysOfYearAnnotation] + if ok { + days, err := util.ParseDays(text) + if err != nil { + logger.WithField("err", err).Warnf(msgFailedToParseAnnotation, daysOfYearAnnotation) + continue checkingPods + } + + for _, d := range days { + if d.Day() == localNow.Day() && d.Month() == localNow.Month() { + continue checkingPods + } + } + } + + filteredList = append(filteredList, pod) + } + + return filteredList +} diff --git a/chaoskube/chaoskube_test.go b/chaoskube/chaoskube_test.go index 4351ce35..551491d1 100644 --- a/chaoskube/chaoskube_test.go +++ b/chaoskube/chaoskube_test.go @@ -5,7 +5,6 @@ import ( "math/rand" "regexp" "sort" - "strings" "testing" "time" @@ -419,7 +418,7 @@ func (suite *Suite) TestNoVictimReturnsError() { 1, ) - _, err := chaoskube.Victims(context.Background()) + _, err := chaoskube.Victims(context.Background(), time.Now()) suite.Equal(err, errPodNotFound) suite.EqualError(err, "pod not found") } @@ -727,7 +726,7 @@ func (suite *Suite) TestTerminateVictim() { err := chaoskube.TerminateVictims(context.Background()) suite.Require().NoError(err) - pods, err := chaoskube.Candidates(context.Background()) + pods, err := chaoskube.Candidates(context.Background(), time.Now()) suite.Require().NoError(err) suite.Len(pods, tt.remainingPodCount) @@ -766,14 +765,14 @@ func (suite *Suite) TestTerminateNoVictimLogsInfo() { // helper functions func (suite *Suite) assertCandidates(chaoskube *Chaoskube, expected []map[string]string) { - pods, err := chaoskube.Candidates(context.Background()) + pods, err := chaoskube.Candidates(context.Background(), time.Now()) suite.Require().NoError(err) suite.AssertPods(pods, expected) } func (suite *Suite) assertVictims(chaoskube *Chaoskube, expected []map[string]string) { - victims, err := chaoskube.Victims(context.Background()) + victims, err := chaoskube.Victims(context.Background(), time.Now()) suite.Require().NoError(err) for i, victim := range victims { @@ -940,11 +939,11 @@ func (suite *Suite) TestMinimumAge() { []v1.Pod{overriddenMinAge}, }, } { - annotation := strings.Join([]string{util.DefaultBaseAnnotation, "minimum-age"}, "/") - pods := filterByMinimumAge(tt.pods, annotation, + pods := filterByMinimumAge(tt.pods, util.DefaultBaseAnnotation, tt.minimumAge, now, logger) - suite.Assert().ElementsMatch(tt.expected, pods) + suite.Assert().ElementsMatch(tt.expected, pods, + "minimum-age: %v", tt.minimumAge) } } @@ -1136,11 +1135,77 @@ func (suite *Suite) TestFilterByFrequency() { } { rand.Seed(tt.seed) - annotation := strings.Join([]string{util.DefaultBaseAnnotation, "frequency"}, "/") - results := filterByFrequency(pods, annotation, + results := filterByFrequency(pods, util.DefaultBaseAnnotation, tt.defaultFrequency, interval, logger) - suite.Assert().ElementsMatch(tt.expected, results) + suite.Assert().ElementsMatch(tt.expected, results, + "seed: %v, default: %v", tt.seed, tt.defaultFrequency) + } +} + +func (suite *Suite) TestFilterByWeekdays() { + logger, _ := test.NewNullLogger() + now := ThankGodItsFriday{}.Now() + + brisbane := "Australia/Brisbane" + brisbaneTimezone, _ := time.LoadLocation(brisbane) + + noExcludes := util.NewPodBuilder("default", "no-excludes").Build() + + neverFriday := util.NewPodBuilder("default", "never-friday"). + WithExcludedWeekdays("Fri").Build() + neverBeforeFridayBrisbane := util.NewPodBuilder("default", "never-before-friday"). + WithExcludedWeekdays("Mon,Tue,Wed,Thu").WithTimezone(brisbane).Build() + + neverAt3pm := util.NewPodBuilder("default", "never-at-3pm"). + WithExcludedTimesOfDay("15:00-16:00").Build() + neverAt8amBrisbane := util.NewPodBuilder("default", "never-at-8am-brisbane"). + WithExcludedTimesOfDay("08:00-09:00").WithTimezone(brisbane).Build() + + neverOnSept24th := util.NewPodBuilder("default", "never-on-sept-24"). + WithExcludedDaysOfYear("Sep24").Build() + neverOnSept28thBrisbane := util.NewPodBuilder("default", "never-on-sept-28-brisbane"). + WithExcludedDaysOfYear("Sep28").WithTimezone(brisbane).Build() + + pods := []v1.Pod{ + noExcludes, + neverFriday, + neverBeforeFridayBrisbane, + neverAt3pm, + neverAt8amBrisbane, + neverOnSept24th, + neverOnSept28thBrisbane, + } + + for _, tt := range []struct { + now time.Time + expected []v1.Pod + }{ + { + now: now, + expected: []v1.Pod{noExcludes, neverBeforeFridayBrisbane, neverAt8amBrisbane, neverOnSept28thBrisbane}, + }, + { + now: now.Add(2 * time.Hour), + expected: []v1.Pod{noExcludes, neverBeforeFridayBrisbane, neverAt3pm, neverAt8amBrisbane, neverOnSept28thBrisbane}, + }, + { + now: now.Add(7 * time.Hour), + expected: []v1.Pod{noExcludes, neverBeforeFridayBrisbane, neverAt3pm, neverOnSept28thBrisbane}, + }, + { + now: now.AddDate(0, 0, 1), + expected: []v1.Pod{noExcludes, neverFriday, neverBeforeFridayBrisbane, neverAt8amBrisbane, neverOnSept24th, neverOnSept28thBrisbane}, + }, + { + now: now.AddDate(0, 0, 4), + expected: []v1.Pod{noExcludes, neverFriday, neverAt8amBrisbane, neverOnSept24th, neverOnSept28thBrisbane}, + }, + } { + results := filterByTime(pods, util.DefaultBaseAnnotation, tt.now, logger) + + suite.Assert().ElementsMatch(tt.expected, results, + "now: %v, now (brisbane): %v, weekday; %v", tt.now, tt.now.In(brisbaneTimezone), tt.now.Weekday().String()) } } diff --git a/util/util.go b/util/util.go index ceed4171..2e1b79de 100644 --- a/util/util.go +++ b/util/util.go @@ -167,6 +167,10 @@ func FormatDays(days []time.Time) []string { return formattedDays } +func FormatAnnotation(prefix, name string) string { + return strings.Join([]string{prefix, name}, "/") +} + // NewNamespace returns a new namespace instance for testing purposes. func NewNamespace(name string) v1.Namespace { return v1.Namespace{ @@ -282,3 +286,27 @@ func (b PodBuilder) WithMinimumAge(text string) PodBuilder { b.Annotations[annotation] = text return b } +func (b PodBuilder) WithTimezone(text string) PodBuilder { + annotation := strings.Join([]string{DefaultBaseAnnotation, "timezone"}, "/") + + b.Annotations[annotation] = text + return b +} +func (b PodBuilder) WithExcludedWeekdays(text string) PodBuilder { + annotation := strings.Join([]string{DefaultBaseAnnotation, "excluded-weekdays"}, "/") + + b.Annotations[annotation] = text + return b +} +func (b PodBuilder) WithExcludedTimesOfDay(text string) PodBuilder { + annotation := strings.Join([]string{DefaultBaseAnnotation, "excluded-times-of-day"}, "/") + + b.Annotations[annotation] = text + return b +} +func (b PodBuilder) WithExcludedDaysOfYear(text string) PodBuilder { + annotation := strings.Join([]string{DefaultBaseAnnotation, "excluded-days-of-year"}, "/") + + b.Annotations[annotation] = text + return b +}