Skip to content

Commit

Permalink
Add ability to exclude pods based on time via annotations
Browse files Browse the repository at this point in the history
  • Loading branch information
@multimac
multimac committed Jul 2, 2021
1 parent 98c4db6 commit 1ea5f2e
Show file tree
Hide file tree
Showing 3 changed files with 201 additions and 38 deletions.
124 changes: 97 additions & 27 deletions chaoskube/chaoskube.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"fmt"
"math/rand"
"regexp"
"strings"
"time"

multierror "github.com/hashicorp/go-multierror"
Expand Down Expand Up @@ -94,6 +93,8 @@ var (
msgTimeOfDayExcluded = "time of day excluded"
// msgDayOfYearExcluded is the log message when termination is suspended due to the day of year filter
msgDayOfYearExcluded = "day of year excluded"
// msgFailedToParseAnnotation is the log message when a filter fails to parse a pod annotation
msgFailedToParseAnnotation = "failed to parse annotation, '%v', excluding from candidates"
)

// New returns a new instance of Chaoskube. It expects:
Expand Down Expand Up @@ -181,7 +182,7 @@ func (c *Chaoskube) TerminateVictims(ctx context.Context) error {
}
}

victims, err := c.Victims(ctx)
victims, err := c.Victims(ctx, now)
if err == errPodNotFound {
c.Logger.Debug(msgVictimNotFound)
return nil
Expand All @@ -200,8 +201,8 @@ func (c *Chaoskube) TerminateVictims(ctx context.Context) error {
}

// Victims returns up to N pods as configured by MaxKill flag
func (c *Chaoskube) Victims(ctx context.Context) ([]v1.Pod, error) {
pods, err := c.Candidates(ctx)
func (c *Chaoskube) Victims(ctx context.Context, now time.Time) ([]v1.Pod, error) {
pods, err := c.Candidates(ctx, now)
if err != nil {
return []v1.Pod{}, err
}
Expand All @@ -220,7 +221,7 @@ func (c *Chaoskube) Victims(ctx context.Context) ([]v1.Pod, error) {

// Candidates returns the list of pods that are available for termination.
// It returns all pods that match the configured label, annotation and namespace selectors.
func (c *Chaoskube) Candidates(ctx context.Context) ([]v1.Pod, error) {
func (c *Chaoskube) Candidates(ctx context.Context, now time.Time) ([]v1.Pod, error) {
listOptions := metav1.ListOptions{LabelSelector: c.Labels.String()}

podList, err := c.Client.CoreV1().Pods(v1.NamespaceAll).List(ctx, listOptions)
Expand Down Expand Up @@ -248,24 +249,11 @@ func (c *Chaoskube) Candidates(ctx context.Context) ([]v1.Pod, error) {
pods = filterTerminatingPods(pods)
pods = filterByPodName(pods, c.IncludedPodNames, c.ExcludedPodNames)

pods = filterByMinimumAge(
pods,
strings.Join([]string{c.ConfigAnnotationPrefix, "minimum-age"}, "/"),
c.MinimumAge,
c.Now(),
c.Logger,
)

pods = filterByFrequency(
pods,
strings.Join([]string{c.ConfigAnnotationPrefix, "frequency"}, "/"),
c.DefaultFrequency,
c.Interval,
c.Logger,
)
pods = filterByMinimumAge(pods, c.ConfigAnnotationPrefix, c.MinimumAge, c.Now(), c.Logger)
pods = filterByFrequency(pods, c.ConfigAnnotationPrefix, c.DefaultFrequency, c.Interval, c.Logger)
pods = filterByTime(pods, c.ConfigAnnotationPrefix, now, c.Logger)

pods = filterByOwnerReference(pods)

return pods, nil
}

Expand Down Expand Up @@ -498,11 +486,13 @@ func filterTerminatingPods(pods []v1.Pod) []v1.Pod {

// filterByMinimumAge filters pods by creation time. Only pods
// older than minimumAge are returned
func filterByMinimumAge(pods []v1.Pod, annotation string, minimumAge time.Duration, now time.Time, logger log.FieldLogger) []v1.Pod {
if annotation == "" && minimumAge <= time.Duration(0) {
func filterByMinimumAge(pods []v1.Pod, annotationPrefix string, minimumAge time.Duration, now time.Time, logger log.FieldLogger) []v1.Pod {
if annotationPrefix == "" && minimumAge <= time.Duration(0) {
return pods
}

annotation := util.FormatAnnotation(annotationPrefix, "minimum-age")

defaultCreationTime := now.Add(-minimumAge)
filteredList := []v1.Pod{}

Expand All @@ -514,7 +504,7 @@ func filterByMinimumAge(pods []v1.Pod, annotation string, minimumAge time.Durati
if ok {
minimumAgeOverride, err := time.ParseDuration(text)
if err != nil {
logger.WithField("err", err).Warn("failed to parse frequency annotation, excluding from candidates")
logger.WithField("err", err).Warnf(msgFailedToParseAnnotation, annotation)
continue
}

Expand Down Expand Up @@ -575,11 +565,13 @@ func filterByOwnerReference(pods []v1.Pod) []v1.Pod {
return filteredList
}

func filterByFrequency(pods []v1.Pod, annotation string, defaultFrequency string, interval time.Duration, logger log.FieldLogger) []v1.Pod {
if annotation == "" && defaultFrequency == "" {
func filterByFrequency(pods []v1.Pod, annotationPrefix string, defaultFrequency string, interval time.Duration, logger log.FieldLogger) []v1.Pod {
if annotationPrefix == "" && defaultFrequency == "" {
return pods
}

annotation := util.FormatAnnotation(annotationPrefix, "frequency")

filteredList := []v1.Pod{}
for _, pod := range pods {
text, ok := pod.Annotations[annotation]
Expand All @@ -596,7 +588,7 @@ func filterByFrequency(pods []v1.Pod, annotation string, defaultFrequency string

chance, err := util.ParseFrequency(text, interval)
if err != nil {
logger.WithField("err", err).Warn("failed to parse frequency annotation, excluding from candidates")
logger.WithField("err", err).Warnf(msgFailedToParseAnnotation, annotation)
continue
}

Expand All @@ -607,3 +599,81 @@ func filterByFrequency(pods []v1.Pod, annotation string, defaultFrequency string

return filteredList
}

func filterByTime(pods []v1.Pod, annotationPrefix string, now time.Time, logger log.FieldLogger) []v1.Pod {
if annotationPrefix == "" {
return pods
}

timezoneAnnotation := util.FormatAnnotation(annotationPrefix, "timezone")
weekdaysAnnotation := util.FormatAnnotation(annotationPrefix, "excluded-weekdays")
timesOfDayAnnotation := util.FormatAnnotation(annotationPrefix, "excluded-times-of-day")
daysOfYearAnnotation := util.FormatAnnotation(annotationPrefix, "excluded-days-of-year")

filteredList := []v1.Pod{}

checkingPods:
for _, pod := range pods {
localNow := now

text, ok := pod.Annotations[timezoneAnnotation]
if ok {
location, err := time.LoadLocation(text)
if err != nil {
logger.WithField("err", err).WithField("pod-name", pod.Name).WithField("pod-namespace", pod.Namespace).
Warnf(msgFailedToParseAnnotation, timezoneAnnotation)

continue checkingPods
}

localNow = localNow.In(location)
}

// Weekdays
text, ok = pod.Annotations[weekdaysAnnotation]
if ok {
days := util.ParseWeekdays(text)
for _, wd := range days {
if wd == localNow.Weekday() {
continue checkingPods
}
}
}

// Times of day
text, ok = pod.Annotations[timesOfDayAnnotation]
if ok {
periods, err := util.ParseTimePeriods(text)
if err != nil {
logger.WithField("err", err).Warnf(msgFailedToParseAnnotation, timesOfDayAnnotation)
continue checkingPods
}

for _, tp := range periods {
if tp.Includes(localNow) {
continue checkingPods
}
}
}

// Days of year
text, ok = pod.Annotations[daysOfYearAnnotation]
if ok {
days, err := util.ParseDays(text)
if err != nil {
logger.WithField("err", err).Warnf(msgFailedToParseAnnotation, daysOfYearAnnotation)
continue checkingPods
}

for _, d := range days {
if d.Day() == localNow.Day() && d.Month() == localNow.Month() {
continue checkingPods
}
}
}

filteredList = append(filteredList, pod)
}

return filteredList
}
87 changes: 76 additions & 11 deletions chaoskube/chaoskube_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"math/rand"
"regexp"
"sort"
"strings"
"testing"
"time"

Expand Down Expand Up @@ -419,7 +418,7 @@ func (suite *Suite) TestNoVictimReturnsError() {
1,
)

_, err := chaoskube.Victims(context.Background())
_, err := chaoskube.Victims(context.Background(), time.Now())
suite.Equal(err, errPodNotFound)
suite.EqualError(err, "pod not found")
}
Expand Down Expand Up @@ -727,7 +726,7 @@ func (suite *Suite) TestTerminateVictim() {
err := chaoskube.TerminateVictims(context.Background())
suite.Require().NoError(err)

pods, err := chaoskube.Candidates(context.Background())
pods, err := chaoskube.Candidates(context.Background(), time.Now())
suite.Require().NoError(err)

suite.Len(pods, tt.remainingPodCount)
Expand Down Expand Up @@ -766,14 +765,14 @@ func (suite *Suite) TestTerminateNoVictimLogsInfo() {
// helper functions

func (suite *Suite) assertCandidates(chaoskube *Chaoskube, expected []map[string]string) {
pods, err := chaoskube.Candidates(context.Background())
pods, err := chaoskube.Candidates(context.Background(), time.Now())
suite.Require().NoError(err)

suite.AssertPods(pods, expected)
}

func (suite *Suite) assertVictims(chaoskube *Chaoskube, expected []map[string]string) {
victims, err := chaoskube.Victims(context.Background())
victims, err := chaoskube.Victims(context.Background(), time.Now())
suite.Require().NoError(err)

for i, victim := range victims {
Expand Down Expand Up @@ -940,11 +939,11 @@ func (suite *Suite) TestMinimumAge() {
[]v1.Pod{overriddenMinAge},
},
} {
annotation := strings.Join([]string{util.DefaultBaseAnnotation, "minimum-age"}, "/")
pods := filterByMinimumAge(tt.pods, annotation,
pods := filterByMinimumAge(tt.pods, util.DefaultBaseAnnotation,
tt.minimumAge, now, logger)

suite.Assert().ElementsMatch(tt.expected, pods)
suite.Assert().ElementsMatch(tt.expected, pods,
"minimum-age: %v", tt.minimumAge)
}
}

Expand Down Expand Up @@ -1136,11 +1135,77 @@ func (suite *Suite) TestFilterByFrequency() {
} {
rand.Seed(tt.seed)

annotation := strings.Join([]string{util.DefaultBaseAnnotation, "frequency"}, "/")
results := filterByFrequency(pods, annotation,
results := filterByFrequency(pods, util.DefaultBaseAnnotation,
tt.defaultFrequency, interval, logger)

suite.Assert().ElementsMatch(tt.expected, results)
suite.Assert().ElementsMatch(tt.expected, results,
"seed: %v, default: %v", tt.seed, tt.defaultFrequency)
}
}

func (suite *Suite) TestFilterByWeekdays() {
logger, _ := test.NewNullLogger()
now := ThankGodItsFriday{}.Now()

brisbane := "Australia/Brisbane"
brisbaneTimezone, _ := time.LoadLocation(brisbane)

noExcludes := util.NewPodBuilder("default", "no-excludes").Build()

neverFriday := util.NewPodBuilder("default", "never-friday").
WithExcludedWeekdays("Fri").Build()
neverBeforeFridayBrisbane := util.NewPodBuilder("default", "never-before-friday").
WithExcludedWeekdays("Mon,Tue,Wed,Thu").WithTimezone(brisbane).Build()

neverAt3pm := util.NewPodBuilder("default", "never-at-3pm").
WithExcludedTimesOfDay("15:00-16:00").Build()
neverAt8amBrisbane := util.NewPodBuilder("default", "never-at-8am-brisbane").
WithExcludedTimesOfDay("08:00-09:00").WithTimezone(brisbane).Build()

neverOnSept24th := util.NewPodBuilder("default", "never-on-sept-24").
WithExcludedDaysOfYear("Sep24").Build()
neverOnSept28thBrisbane := util.NewPodBuilder("default", "never-on-sept-28-brisbane").
WithExcludedDaysOfYear("Sep28").WithTimezone(brisbane).Build()

pods := []v1.Pod{
noExcludes,
neverFriday,
neverBeforeFridayBrisbane,
neverAt3pm,
neverAt8amBrisbane,
neverOnSept24th,
neverOnSept28thBrisbane,
}

for _, tt := range []struct {
now time.Time
expected []v1.Pod
}{
{
now: now,
expected: []v1.Pod{noExcludes, neverBeforeFridayBrisbane, neverAt8amBrisbane, neverOnSept28thBrisbane},
},
{
now: now.Add(2 * time.Hour),
expected: []v1.Pod{noExcludes, neverBeforeFridayBrisbane, neverAt3pm, neverAt8amBrisbane, neverOnSept28thBrisbane},
},
{
now: now.Add(7 * time.Hour),
expected: []v1.Pod{noExcludes, neverBeforeFridayBrisbane, neverAt3pm, neverOnSept28thBrisbane},
},
{
now: now.AddDate(0, 0, 1),
expected: []v1.Pod{noExcludes, neverFriday, neverBeforeFridayBrisbane, neverAt8amBrisbane, neverOnSept24th, neverOnSept28thBrisbane},
},
{
now: now.AddDate(0, 0, 4),
expected: []v1.Pod{noExcludes, neverFriday, neverAt8amBrisbane, neverOnSept24th, neverOnSept28thBrisbane},
},
} {
results := filterByTime(pods, util.DefaultBaseAnnotation, tt.now, logger)

suite.Assert().ElementsMatch(tt.expected, results,
"now: %v, now (brisbane): %v, weekday; %v", tt.now, tt.now.In(brisbaneTimezone), tt.now.Weekday().String())
}
}

Expand Down
28 changes: 28 additions & 0 deletions util/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,10 @@ func FormatDays(days []time.Time) []string {
return formattedDays
}

func FormatAnnotation(prefix, name string) string {
return strings.Join([]string{prefix, name}, "/")
}

// NewNamespace returns a new namespace instance for testing purposes.
func NewNamespace(name string) v1.Namespace {
return v1.Namespace{
Expand Down Expand Up @@ -282,3 +286,27 @@ func (b PodBuilder) WithMinimumAge(text string) PodBuilder {
b.Annotations[annotation] = text
return b
}
func (b PodBuilder) WithTimezone(text string) PodBuilder {
annotation := strings.Join([]string{DefaultBaseAnnotation, "timezone"}, "/")

b.Annotations[annotation] = text
return b
}
func (b PodBuilder) WithExcludedWeekdays(text string) PodBuilder {
annotation := strings.Join([]string{DefaultBaseAnnotation, "excluded-weekdays"}, "/")

b.Annotations[annotation] = text
return b
}
func (b PodBuilder) WithExcludedTimesOfDay(text string) PodBuilder {
annotation := strings.Join([]string{DefaultBaseAnnotation, "excluded-times-of-day"}, "/")

b.Annotations[annotation] = text
return b
}
func (b PodBuilder) WithExcludedDaysOfYear(text string) PodBuilder {
annotation := strings.Join([]string{DefaultBaseAnnotation, "excluded-days-of-year"}, "/")

b.Annotations[annotation] = text
return b
}

0 comments on commit 1ea5f2e

Please sign in to comment.