From f291f9e840d4c6da4a4048a3d96e951f14241494 Mon Sep 17 00:00:00 2001 From: GaruGaru Date: Sat, 9 Nov 2019 19:47:09 +0100 Subject: [PATCH 01/10] Added notifier feature, Added slack webhook notifier integration --- chaoskube/chaoskube.go | 13 ++++++-- chaoskube/chaoskube_test.go | 32 ++++++++++++++++++ internal/testutil/notifier.go | 18 ++++++++++ main.go | 27 +++++++++++++++ notifier/noop.go | 11 ++++++ notifier/notifier.go | 7 ++++ notifier/slack.go | 63 +++++++++++++++++++++++++++++++++++ 7 files changed, 169 insertions(+), 2 deletions(-) create mode 100644 internal/testutil/notifier.go create mode 100644 notifier/noop.go create mode 100644 notifier/notifier.go create mode 100644 notifier/slack.go diff --git a/chaoskube/chaoskube.go b/chaoskube/chaoskube.go index eaae805a..2e994a06 100644 --- a/chaoskube/chaoskube.go +++ b/chaoskube/chaoskube.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/linki/chaoskube/notifier" "regexp" "time" @@ -67,6 +68,9 @@ type Chaoskube struct { Now func() time.Time MaxKill int + + // chaos events notifier + Notifier notifier.Notifier } var ( @@ -90,7 +94,7 @@ var ( // * a logger implementing logrus.FieldLogger to send log output to // * what specific terminator to use to imbue chaos on victim pods // * whether to enable/disable dry-run mode -func New(client kubernetes.Interface, labels, annotations, namespaces, namespaceLabels labels.Selector, includedPodNames, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator, maxKill int) *Chaoskube { +func New(client kubernetes.Interface, labels, annotations, namespaces, namespaceLabels labels.Selector, includedPodNames, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator, maxKill int, notifier notifier.Notifier) *Chaoskube { broadcaster := record.NewBroadcaster() broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: client.CoreV1().Events(v1.NamespaceAll)}) recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "chaoskube"}) @@ -114,6 +118,7 @@ func New(client kubernetes.Interface, labels, annotations, namespaces, namespace EventRecorder: recorder, Now: time.Now, MaxKill: maxKill, + Notifier: notifier, } } @@ -175,7 +180,6 @@ func (c *Chaoskube) TerminateVictims() error { for _, victim := range victims { err = c.DeletePod(victim) result = multierror.Append(result, err) - } return result.ErrorOrNil() @@ -257,6 +261,11 @@ func (c *Chaoskube) DeletePod(victim v1.Pod) error { c.EventRecorder.Event(ref, v1.EventTypeNormal, "Killing", "Pod was terminated by chaoskube to introduce chaos.") + err = c.Notifier.NotifyTermination(victim) + if err != nil { + c.Logger.Warn("unable to notify pod termination", err) + } + return nil } diff --git a/chaoskube/chaoskube_test.go b/chaoskube/chaoskube_test.go index 970421fb..d2a3eea7 100644 --- a/chaoskube/chaoskube_test.go +++ b/chaoskube/chaoskube_test.go @@ -35,6 +35,7 @@ type podInfo struct { var ( logger, logOutput = test.NewNullLogger() + testNotifier = testutil.NewTestNotifier() ) func (suite *Suite) SetupTest() { @@ -59,6 +60,7 @@ func (suite *Suite) TestNew() { dryRun = true terminator = terminator.NewDeletePodTerminator(client, logger, 10*time.Second) maxKill = 1 + notifier = testNotifier ) chaoskube := New( @@ -78,6 +80,7 @@ func (suite *Suite) TestNew() { dryRun, terminator, maxKill, + notifier, ) suite.Require().NotNil(chaoskube) @@ -723,6 +726,10 @@ func (suite *Suite) assertVictim(chaoskube *Chaoskube, expected map[string]strin suite.assertVictims(chaoskube, []map[string]string{expected}) } +func (suite *Suite) assertNotified(notifier *testutil.TestNotifier) { + suite.Assert().Greater(notifier.Calls,0 ) +} + func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration) *Chaoskube { chaoskube := suite.setup( labelSelector, @@ -797,6 +804,7 @@ func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Sele dryRun, terminator.NewDeletePodTerminator(client, nullLogger, gracePeriod), maxKill, + testNotifier, ) } @@ -971,3 +979,27 @@ func (suite *Suite) TestFilterByOwnerReference() { } } } + +func (suite *Suite) TestNotifierCall() { + chaoskube := suite.setupWithPods( + labels.Everything(), + labels.Everything(), + labels.Everything(), + labels.Everything(), + ®exp.Regexp{}, + ®exp.Regexp{}, + []time.Weekday{}, + []util.TimePeriod{}, + []time.Time{}, + time.UTC, + time.Duration(0), + false, + 10, + ) + + victim := util.NewPod("default", "foo", v1.PodRunning) + err := chaoskube.DeletePod(victim) + + suite.Require().NoError(err) + suite.assertNotified(testNotifier) +} diff --git a/internal/testutil/notifier.go b/internal/testutil/notifier.go new file mode 100644 index 00000000..d656e86b --- /dev/null +++ b/internal/testutil/notifier.go @@ -0,0 +1,18 @@ +package testutil + +import ( + "k8s.io/api/core/v1" +) + +type TestNotifier struct { + Calls int +} + +func NewTestNotifier() *TestNotifier { + return &TestNotifier{} +} + +func (t *TestNotifier) NotifyTermination(victim v1.Pod) error { + t.Calls++ + return nil +} diff --git a/main.go b/main.go index 3cf2b45c..dc1a7f3b 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "github.com/linki/chaoskube/notifier" "io/ioutil" "math/rand" "net/http" @@ -56,6 +57,8 @@ var ( gracePeriod time.Duration logFormat string logCaller bool + notifierType string + notifierWebhook string ) func init() { @@ -83,6 +86,8 @@ func init() { kingpin.Flag("grace-period", "Grace period to terminate Pods. Negative values will use the Pod's grace period.").Default("-1s").DurationVar(&gracePeriod) kingpin.Flag("log-format", "Specify the format of the log messages. Options are text and json. Defaults to text.").Default("text").EnumVar(&logFormat, "text", "json") kingpin.Flag("log-caller", "Include the calling function name and location in the log messages.").BoolVar(&logCaller) + kingpin.Flag("notifier", "Notifier sink for pod termination supported [noop, slack]").Default(notifier.NotifierNoop).StringVar(¬ifierType) + kingpin.Flag("notifier-webhook", "The address of the webhook for notifications").StringVar(¬ifierWebhook) } func main() { @@ -123,6 +128,8 @@ func main() { "metricsAddress": metricsAddress, "gracePeriod": gracePeriod, "logFormat": logFormat, + "notifier": notifierType, + "notifierWebhook": notifierWebhook, }).Debug("reading config") log.WithFields(log.Fields{ @@ -191,6 +198,8 @@ func main() { "offset": offset / int(time.Hour/time.Second), }).Info("setting timezone") + terminationNotifier := createNotifier(notifierType, notifierWebhook) + chaoskube := chaoskube.New( client, labelSelector, @@ -208,6 +217,7 @@ func main() { dryRun, terminator.NewDeletePodTerminator(client, log.StandardLogger(), gracePeriod), maxKill, + terminationNotifier, ) if metricsAddress != "" { @@ -277,6 +287,23 @@ func parseSelector(str string) labels.Selector { return selector } +func createNotifier(notifierType string, webhook string) notifier.Notifier { + if notifierType == "" || notifierType == notifier.NotifierNoop { + return notifier.NoopNotifier{} + } + + if notifierType == notifier.NotifierSlack { + return notifier.NewSlackNotifier(webhook) + } + + log.WithFields(log.Fields{ + "notifier": notifierType, + "webhook": webhook, + }).Warn("failed to parse notifier type, falling back to default (noop)") + + return notifier.NoopNotifier{} +} + func serveMetrics() { http.Handle("/metrics", promhttp.Handler()) http.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { diff --git a/notifier/noop.go b/notifier/noop.go new file mode 100644 index 00000000..e98c6f9d --- /dev/null +++ b/notifier/noop.go @@ -0,0 +1,11 @@ +package notifier + +import "k8s.io/api/core/v1" + +const NotifierNoop = "noop" + +type NoopNotifier struct{} + +func (n NoopNotifier) NotifyTermination(victim v1.Pod) error { + return nil +} diff --git a/notifier/notifier.go b/notifier/notifier.go new file mode 100644 index 00000000..26690a8a --- /dev/null +++ b/notifier/notifier.go @@ -0,0 +1,7 @@ +package notifier + +import v1 "k8s.io/api/core/v1" + +type Notifier interface { + NotifyTermination(victim v1.Pod) error +} diff --git a/notifier/slack.go b/notifier/slack.go new file mode 100644 index 00000000..0fc1d7d1 --- /dev/null +++ b/notifier/slack.go @@ -0,0 +1,63 @@ +package notifier + +import ( + "bytes" + "encoding/json" + "fmt" + "k8s.io/api/core/v1" + "net/http" + "time" +) + +const NotifierSlack = "slack" + +var DefaultTimeout time.Duration = 15 * time.Second + +type Slack struct { + Webhook string + Client *http.Client +} + +type request struct { + Message string `json:"text"` +} + +func NewSlackNotifier(webhook string) *Slack { + return &Slack{ + Webhook: webhook, + Client: &http.Client{ + Timeout: DefaultTimeout, + }, + } +} + +func (s Slack) NotifyTermination(victim v1.Pod) error { + message := request{ + Message: fmt.Sprintf("pod %s/%s is begin terminated.", victim.Namespace, victim.Name), + } + messageBody, err := json.Marshal(message) + + if err != nil { + return err + } + + req, err := http.NewRequest(http.MethodPost, s.Webhook, bytes.NewBuffer(messageBody)) + + if err != nil { + return err + } + + req.Header.Add("Content-Type", "application/json") + + res, err := s.Client.Do(req) + + if err != nil { + return err + } + + if res.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected status code %d from slack webhook %s", res.StatusCode, s.Webhook) + } + + return nil +} From 635db647321dd20e2a25f21979e86d5540f96983 Mon Sep 17 00:00:00 2001 From: GaruGaru Date: Sun, 10 Nov 2019 11:36:42 +0100 Subject: [PATCH 02/10] Added attachment to slack notification --- notifier/slack.go | 55 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 4 deletions(-) diff --git a/notifier/slack.go b/notifier/slack.go index 0fc1d7d1..bf3faf88 100644 --- a/notifier/slack.go +++ b/notifier/slack.go @@ -19,7 +19,28 @@ type Slack struct { } type request struct { - Message string `json:"text"` + Message string `json:"text"` + Attachments []attachment `json:"attachments"` +} +type SlackField struct { + Title string `yaml:"title,omitempty" json:"title,omitempty"` + Value string `yaml:"value,omitempty" json:"value,omitempty"` + Short *bool `yaml:"short,omitempty" json:"short,omitempty"` +} + +type attachment struct { + Title string `json:"title,omitempty"` + TitleLink string `json:"title_link,omitempty"` + Pretext string `json:"pretext,omitempty"` + Text string `json:"text"` + Fallback string `json:"fallback"` + CallbackID string `json:"callback_id"` + Fields []SlackField `json:"fields,omitempty"` + ImageURL string `json:"image_url,omitempty"` + ThumbURL string `json:"thumb_url,omitempty"` + Footer string `json:"footer"` + Color string `json:"color,omitempty"` + MrkdwnIn []string `json:"mrkdwn_in,omitempty"` } func NewSlackNotifier(webhook string) *Slack { @@ -31,10 +52,36 @@ func NewSlackNotifier(webhook string) *Slack { } } -func (s Slack) NotifyTermination(victim v1.Pod) error { - message := request{ - Message: fmt.Sprintf("pod %s/%s is begin terminated.", victim.Namespace, victim.Name), +func createSlackRequest(victim v1.Pod) request { + attach := attachment{ + Title: "Chaos event - Pod termination", + Text: fmt.Sprintf("pod %s has been selected by chaos-kube for termination", victim.Name), + Footer: "chaos-kube", + Color: "#F35A00", + } + + short := len(victim.Namespace) < 20 && len(victim.Name) < 20 + + attach.Fields = []SlackField{ + { + Title: "namespace", + Value: victim.Namespace, + Short: &short, + }, + { + Title: "pod", + Value: victim.Name, + Short: &short, + }, + } + + return request{ + Attachments: []attachment{attach}, } +} +func (s Slack) NotifyTermination(victim v1.Pod) error { + message := createSlackRequest(victim) + messageBody, err := json.Marshal(message) if err != nil { From 41371190910ac222dba006f2576cfa0a0f9210ca Mon Sep 17 00:00:00 2001 From: Tommaso Garuglieri Date: Tue, 12 Nov 2019 09:55:55 +0100 Subject: [PATCH 03/10] Fixed tests for notifier, added slack --- chaoskube/chaoskube.go | 6 +++- chaoskube/chaoskube_test.go | 7 +++-- internal/testutil/notifier.go | 18 ----------- main.go | 31 ++++++------------ notifier/noop.go | 9 +++--- notifier/notifier.go | 30 ++++++++++++++++-- notifier/notifier_test.go | 48 ++++++++++++++++++++++++++++ notifier/slack.go | 59 +++++++++++++++++------------------ notifier/slack_test.go | 53 +++++++++++++++++++++++++++++++ 9 files changed, 180 insertions(+), 81 deletions(-) delete mode 100644 internal/testutil/notifier.go create mode 100644 notifier/notifier_test.go create mode 100644 notifier/slack_test.go diff --git a/chaoskube/chaoskube.go b/chaoskube/chaoskube.go index 2e994a06..d9669175 100644 --- a/chaoskube/chaoskube.go +++ b/chaoskube/chaoskube.go @@ -261,7 +261,11 @@ func (c *Chaoskube) DeletePod(victim v1.Pod) error { c.EventRecorder.Event(ref, v1.EventTypeNormal, "Killing", "Pod was terminated by chaoskube to introduce chaos.") - err = c.Notifier.NotifyTermination(victim) + err = c.Notifier.NotifyTermination(notifier.Termination{ + Pod: victim.Name, + Namespace: victim.Namespace, + }) + if err != nil { c.Logger.Warn("unable to notify pod termination", err) } diff --git a/chaoskube/chaoskube_test.go b/chaoskube/chaoskube_test.go index d2a3eea7..6ab2c291 100644 --- a/chaoskube/chaoskube_test.go +++ b/chaoskube/chaoskube_test.go @@ -2,6 +2,7 @@ package chaoskube import ( "context" + "github.com/linki/chaoskube/notifier" "math/rand" "regexp" "testing" @@ -35,7 +36,7 @@ type podInfo struct { var ( logger, logOutput = test.NewNullLogger() - testNotifier = testutil.NewTestNotifier() + testNotifier = ¬ifier.Noop{} ) func (suite *Suite) SetupTest() { @@ -726,8 +727,8 @@ func (suite *Suite) assertVictim(chaoskube *Chaoskube, expected map[string]strin suite.assertVictims(chaoskube, []map[string]string{expected}) } -func (suite *Suite) assertNotified(notifier *testutil.TestNotifier) { - suite.Assert().Greater(notifier.Calls,0 ) +func (suite *Suite) assertNotified(notifier *notifier.Noop) { + suite.Assert().Greater(notifier.Calls, 0) } func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration) *Chaoskube { diff --git a/internal/testutil/notifier.go b/internal/testutil/notifier.go deleted file mode 100644 index d656e86b..00000000 --- a/internal/testutil/notifier.go +++ /dev/null @@ -1,18 +0,0 @@ -package testutil - -import ( - "k8s.io/api/core/v1" -) - -type TestNotifier struct { - Calls int -} - -func NewTestNotifier() *TestNotifier { - return &TestNotifier{} -} - -func (t *TestNotifier) NotifyTermination(victim v1.Pod) error { - t.Calls++ - return nil -} diff --git a/main.go b/main.go index dc1a7f3b..20e8fe88 100644 --- a/main.go +++ b/main.go @@ -57,8 +57,7 @@ var ( gracePeriod time.Duration logFormat string logCaller bool - notifierType string - notifierWebhook string + slackWebhook string ) func init() { @@ -86,8 +85,7 @@ func init() { kingpin.Flag("grace-period", "Grace period to terminate Pods. Negative values will use the Pod's grace period.").Default("-1s").DurationVar(&gracePeriod) kingpin.Flag("log-format", "Specify the format of the log messages. Options are text and json. Defaults to text.").Default("text").EnumVar(&logFormat, "text", "json") kingpin.Flag("log-caller", "Include the calling function name and location in the log messages.").BoolVar(&logCaller) - kingpin.Flag("notifier", "Notifier sink for pod termination supported [noop, slack]").Default(notifier.NotifierNoop).StringVar(¬ifierType) - kingpin.Flag("notifier-webhook", "The address of the webhook for notifications").StringVar(¬ifierWebhook) + kingpin.Flag("slack-webhook", "The address of the slack webhook for notifications").StringVar(&slackWebhook) } func main() { @@ -128,8 +126,7 @@ func main() { "metricsAddress": metricsAddress, "gracePeriod": gracePeriod, "logFormat": logFormat, - "notifier": notifierType, - "notifierWebhook": notifierWebhook, + "slackWebhook": slackWebhook, }).Debug("reading config") log.WithFields(log.Fields{ @@ -198,7 +195,7 @@ func main() { "offset": offset / int(time.Hour/time.Second), }).Info("setting timezone") - terminationNotifier := createNotifier(notifierType, notifierWebhook) + notifiers := createNotifier() chaoskube := chaoskube.New( client, @@ -217,7 +214,7 @@ func main() { dryRun, terminator.NewDeletePodTerminator(client, log.StandardLogger(), gracePeriod), maxKill, - terminationNotifier, + notifiers, ) if metricsAddress != "" { @@ -287,21 +284,13 @@ func parseSelector(str string) labels.Selector { return selector } -func createNotifier(notifierType string, webhook string) notifier.Notifier { - if notifierType == "" || notifierType == notifier.NotifierNoop { - return notifier.NoopNotifier{} +func createNotifier() notifier.Notifier { + notifiers := notifier.New() + if slackWebhook != "" { + notifiers.Add(notifier.NewSlackNotifier(slackWebhook)) } - if notifierType == notifier.NotifierSlack { - return notifier.NewSlackNotifier(webhook) - } - - log.WithFields(log.Fields{ - "notifier": notifierType, - "webhook": webhook, - }).Warn("failed to parse notifier type, falling back to default (noop)") - - return notifier.NoopNotifier{} + return notifiers } func serveMetrics() { diff --git a/notifier/noop.go b/notifier/noop.go index e98c6f9d..a5101e4c 100644 --- a/notifier/noop.go +++ b/notifier/noop.go @@ -1,11 +1,12 @@ package notifier -import "k8s.io/api/core/v1" - const NotifierNoop = "noop" -type NoopNotifier struct{} +type Noop struct { + Calls int +} -func (n NoopNotifier) NotifyTermination(victim v1.Pod) error { +func (t *Noop) NotifyTermination(termination Termination) error { + t.Calls++ return nil } diff --git a/notifier/notifier.go b/notifier/notifier.go index 26690a8a..a49c5ac6 100644 --- a/notifier/notifier.go +++ b/notifier/notifier.go @@ -1,7 +1,31 @@ package notifier -import v1 "k8s.io/api/core/v1" - type Notifier interface { - NotifyTermination(victim v1.Pod) error + NotifyTermination(term Termination) error +} + +type Termination struct { + Pod string + Namespace string +} + +type Notifiers struct { + notifiers []Notifier +} + +func New() *Notifiers { + return &Notifiers{notifiers: make([]Notifier, 0)} +} + +func (m *Notifiers) NotifyTermination(term Termination) error { + for _, n := range m.notifiers { + if err := n.NotifyTermination(term); err != nil { + return err + } + } + return nil +} + +func (m *Notifiers) Add(notifier Notifier) { + m.notifiers = append(m.notifiers, notifier) } diff --git a/notifier/notifier_test.go b/notifier/notifier_test.go new file mode 100644 index 00000000..fd75030c --- /dev/null +++ b/notifier/notifier_test.go @@ -0,0 +1,48 @@ +package notifier + +import ( + "testing" +) + +func TestMultiNotifierWithoutNotifiers(t *testing.T) { + manager := New() + err := manager.NotifyTermination(Termination{}) + if err != nil { + t.Fatal(err) + } +} + +func TestMultiNotifierWithNotifier(t *testing.T) { + manager := New() + n := Noop{} + manager.Add(&n) + err := manager.NotifyTermination(Termination{}) + if err != nil { + t.Fatal(err) + } + + if n.Calls != 1 { + t.Errorf("expected %d calls to notifier but got %d", 1, n.Calls) + } +} + +func TestMultiNotifierWithMultipleNotifier(t *testing.T) { + manager := New() + n1 := Noop{} + n2 := Noop{} + manager.Add(&n1) + manager.Add(&n2) + + err := manager.NotifyTermination(Termination{}) + if err != nil { + t.Fatal(err) + } + + if n1.Calls != 1 { + t.Errorf("expected %d calls to notifier n1 but got %d", 1, n1.Calls) + } + + if n2.Calls != 1 { + t.Errorf("expected %d calls to notifier n2 but got %d", 1, n2.Calls) + } +} diff --git a/notifier/slack.go b/notifier/slack.go index bf3faf88..e72a9439 100644 --- a/notifier/slack.go +++ b/notifier/slack.go @@ -4,25 +4,26 @@ import ( "bytes" "encoding/json" "fmt" - "k8s.io/api/core/v1" "net/http" "time" ) const NotifierSlack = "slack" -var DefaultTimeout time.Duration = 15 * time.Second +var NotificationColor = "#F35A00" +var DefaultTimeout = 10 * time.Second type Slack struct { Webhook string Client *http.Client } -type request struct { +type slackMessage struct { Message string `json:"text"` Attachments []attachment `json:"attachments"` } -type SlackField struct { + +type slackField struct { Title string `yaml:"title,omitempty" json:"title,omitempty"` Value string `yaml:"value,omitempty" json:"value,omitempty"` Short *bool `yaml:"short,omitempty" json:"short,omitempty"` @@ -35,7 +36,7 @@ type attachment struct { Text string `json:"text"` Fallback string `json:"fallback"` CallbackID string `json:"callback_id"` - Fields []SlackField `json:"fields,omitempty"` + Fields []slackField `json:"fields,omitempty"` ImageURL string `json:"image_url,omitempty"` ThumbURL string `json:"thumb_url,omitempty"` Footer string `json:"footer"` @@ -46,62 +47,58 @@ type attachment struct { func NewSlackNotifier(webhook string) *Slack { return &Slack{ Webhook: webhook, - Client: &http.Client{ - Timeout: DefaultTimeout, - }, + Client: &http.Client{Timeout: DefaultTimeout}, } } -func createSlackRequest(victim v1.Pod) request { - attach := attachment{ - Title: "Chaos event - Pod termination", - Text: fmt.Sprintf("pod %s has been selected by chaos-kube for termination", victim.Name), - Footer: "chaos-kube", - Color: "#F35A00", - } - - short := len(victim.Namespace) < 20 && len(victim.Name) < 20 +func (s Slack) NotifyTermination(term Termination) error { + title := "Chaos event - Pod termination" + text := fmt.Sprintf("pod %s has been selected by chaos-kube for termination", term.Pod) - attach.Fields = []SlackField{ + short := len(term.Namespace) < 20 && len(term.Pod) < 20 + fields := []slackField{ { Title: "namespace", - Value: victim.Namespace, + Value: term.Namespace, Short: &short, }, { Title: "pod", - Value: victim.Name, + Value: term.Pod, Short: &short, }, } - return request{ - Attachments: []attachment{attach}, + message := createSlackRequest(title, text, fields) + return s.sendSlackMessage(message) +} + +func createSlackRequest(title string, text string, fields []slackField) slackMessage { + return slackMessage{ + Attachments: []attachment{{ + Title: title, + Text: text, + Footer: "chaos-kube", + Color: NotificationColor, + Fields: fields, + }}, } } -func (s Slack) NotifyTermination(victim v1.Pod) error { - message := createSlackRequest(victim) +func (s Slack) sendSlackMessage(message slackMessage) error { messageBody, err := json.Marshal(message) - if err != nil { return err } - req, err := http.NewRequest(http.MethodPost, s.Webhook, bytes.NewBuffer(messageBody)) - if err != nil { return err } - req.Header.Add("Content-Type", "application/json") - res, err := s.Client.Do(req) - if err != nil { return err } - if res.StatusCode != http.StatusOK { return fmt.Errorf("unexpected status code %d from slack webhook %s", res.StatusCode, s.Webhook) } diff --git a/notifier/slack_test.go b/notifier/slack_test.go new file mode 100644 index 00000000..a65e4465 --- /dev/null +++ b/notifier/slack_test.go @@ -0,0 +1,53 @@ +package notifier + +import ( + "github.com/stretchr/testify/assert" + "net/http" + "net/http/httptest" + "testing" +) + +func TestSlackNotificationForTerminationStatusOk(t *testing.T) { + webhookPath := "/services/T07M5HUDA/BQ1U5VDGA/yhpIczRK0cZ3jDLK1U8qD634" + + testServer := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { + assert.Equal(t, req.URL.Path, webhookPath) + res.WriteHeader(200) + res.Write([]byte("ok")) + })) + + defer testServer.Close() + + slack := NewSlackNotifier(testServer.URL + webhookPath) + err := slack.NotifyTermination(Termination{ + Pod: "chaos-57df4db6b-h9ktj", + Namespace: "chaos", + }) + + if err != nil { + t.Fatal(err) + } +} + +func TestSlackNotificationForTerminationStatus500(t *testing.T) { + webhookPath := "/services/T07M5HUDA/BQ1U5VDGA/yhpIczRK0cZ3jDLK1U8qD634" + + testServer := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { + assert.Equal(t, req.URL.Path, webhookPath) + res.WriteHeader(500) + if _, err := res.Write([]byte("ok")); err != nil { + t.Fatal(err) + } + })) + defer testServer.Close() + + slack := NewSlackNotifier(testServer.URL + webhookPath) + err := slack.NotifyTermination(Termination{ + Pod: "chaos-57df4db6b-h9ktj", + Namespace: "chaos", + }) + + if err == nil { + t.Fatal("expected error on status code 500") + } +} From bf9a3e0c6cfb7f5e5a17a711e180e5044004c99e Mon Sep 17 00:00:00 2001 From: GaruGaru Date: Sat, 9 Nov 2019 19:47:09 +0100 Subject: [PATCH 04/10] Added notifier feature, Added slack webhook notifier integration Added attachment to slack notification Fixed tests for notifier, added slack --- chaoskube/chaoskube.go | 17 +++++- chaoskube/chaoskube_test.go | 33 +++++++++++ main.go | 16 ++++++ notifier/noop.go | 12 ++++ notifier/notifier.go | 31 +++++++++++ notifier/notifier_test.go | 48 ++++++++++++++++ notifier/slack.go | 107 ++++++++++++++++++++++++++++++++++++ notifier/slack_test.go | 53 ++++++++++++++++++ 8 files changed, 315 insertions(+), 2 deletions(-) create mode 100644 notifier/noop.go create mode 100644 notifier/notifier.go create mode 100644 notifier/notifier_test.go create mode 100644 notifier/slack.go create mode 100644 notifier/slack_test.go diff --git a/chaoskube/chaoskube.go b/chaoskube/chaoskube.go index eaae805a..d9669175 100644 --- a/chaoskube/chaoskube.go +++ b/chaoskube/chaoskube.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/linki/chaoskube/notifier" "regexp" "time" @@ -67,6 +68,9 @@ type Chaoskube struct { Now func() time.Time MaxKill int + + // chaos events notifier + Notifier notifier.Notifier } var ( @@ -90,7 +94,7 @@ var ( // * a logger implementing logrus.FieldLogger to send log output to // * what specific terminator to use to imbue chaos on victim pods // * whether to enable/disable dry-run mode -func New(client kubernetes.Interface, labels, annotations, namespaces, namespaceLabels labels.Selector, includedPodNames, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator, maxKill int) *Chaoskube { +func New(client kubernetes.Interface, labels, annotations, namespaces, namespaceLabels labels.Selector, includedPodNames, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator, maxKill int, notifier notifier.Notifier) *Chaoskube { broadcaster := record.NewBroadcaster() broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: client.CoreV1().Events(v1.NamespaceAll)}) recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "chaoskube"}) @@ -114,6 +118,7 @@ func New(client kubernetes.Interface, labels, annotations, namespaces, namespace EventRecorder: recorder, Now: time.Now, MaxKill: maxKill, + Notifier: notifier, } } @@ -175,7 +180,6 @@ func (c *Chaoskube) TerminateVictims() error { for _, victim := range victims { err = c.DeletePod(victim) result = multierror.Append(result, err) - } return result.ErrorOrNil() @@ -257,6 +261,15 @@ func (c *Chaoskube) DeletePod(victim v1.Pod) error { c.EventRecorder.Event(ref, v1.EventTypeNormal, "Killing", "Pod was terminated by chaoskube to introduce chaos.") + err = c.Notifier.NotifyTermination(notifier.Termination{ + Pod: victim.Name, + Namespace: victim.Namespace, + }) + + if err != nil { + c.Logger.Warn("unable to notify pod termination", err) + } + return nil } diff --git a/chaoskube/chaoskube_test.go b/chaoskube/chaoskube_test.go index 970421fb..6ab2c291 100644 --- a/chaoskube/chaoskube_test.go +++ b/chaoskube/chaoskube_test.go @@ -2,6 +2,7 @@ package chaoskube import ( "context" + "github.com/linki/chaoskube/notifier" "math/rand" "regexp" "testing" @@ -35,6 +36,7 @@ type podInfo struct { var ( logger, logOutput = test.NewNullLogger() + testNotifier = ¬ifier.Noop{} ) func (suite *Suite) SetupTest() { @@ -59,6 +61,7 @@ func (suite *Suite) TestNew() { dryRun = true terminator = terminator.NewDeletePodTerminator(client, logger, 10*time.Second) maxKill = 1 + notifier = testNotifier ) chaoskube := New( @@ -78,6 +81,7 @@ func (suite *Suite) TestNew() { dryRun, terminator, maxKill, + notifier, ) suite.Require().NotNil(chaoskube) @@ -723,6 +727,10 @@ func (suite *Suite) assertVictim(chaoskube *Chaoskube, expected map[string]strin suite.assertVictims(chaoskube, []map[string]string{expected}) } +func (suite *Suite) assertNotified(notifier *notifier.Noop) { + suite.Assert().Greater(notifier.Calls, 0) +} + func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration) *Chaoskube { chaoskube := suite.setup( labelSelector, @@ -797,6 +805,7 @@ func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Sele dryRun, terminator.NewDeletePodTerminator(client, nullLogger, gracePeriod), maxKill, + testNotifier, ) } @@ -971,3 +980,27 @@ func (suite *Suite) TestFilterByOwnerReference() { } } } + +func (suite *Suite) TestNotifierCall() { + chaoskube := suite.setupWithPods( + labels.Everything(), + labels.Everything(), + labels.Everything(), + labels.Everything(), + ®exp.Regexp{}, + ®exp.Regexp{}, + []time.Weekday{}, + []util.TimePeriod{}, + []time.Time{}, + time.UTC, + time.Duration(0), + false, + 10, + ) + + victim := util.NewPod("default", "foo", v1.PodRunning) + err := chaoskube.DeletePod(victim) + + suite.Require().NoError(err) + suite.assertNotified(testNotifier) +} diff --git a/main.go b/main.go index 3cf2b45c..20e8fe88 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "github.com/linki/chaoskube/notifier" "io/ioutil" "math/rand" "net/http" @@ -56,6 +57,7 @@ var ( gracePeriod time.Duration logFormat string logCaller bool + slackWebhook string ) func init() { @@ -83,6 +85,7 @@ func init() { kingpin.Flag("grace-period", "Grace period to terminate Pods. Negative values will use the Pod's grace period.").Default("-1s").DurationVar(&gracePeriod) kingpin.Flag("log-format", "Specify the format of the log messages. Options are text and json. Defaults to text.").Default("text").EnumVar(&logFormat, "text", "json") kingpin.Flag("log-caller", "Include the calling function name and location in the log messages.").BoolVar(&logCaller) + kingpin.Flag("slack-webhook", "The address of the slack webhook for notifications").StringVar(&slackWebhook) } func main() { @@ -123,6 +126,7 @@ func main() { "metricsAddress": metricsAddress, "gracePeriod": gracePeriod, "logFormat": logFormat, + "slackWebhook": slackWebhook, }).Debug("reading config") log.WithFields(log.Fields{ @@ -191,6 +195,8 @@ func main() { "offset": offset / int(time.Hour/time.Second), }).Info("setting timezone") + notifiers := createNotifier() + chaoskube := chaoskube.New( client, labelSelector, @@ -208,6 +214,7 @@ func main() { dryRun, terminator.NewDeletePodTerminator(client, log.StandardLogger(), gracePeriod), maxKill, + notifiers, ) if metricsAddress != "" { @@ -277,6 +284,15 @@ func parseSelector(str string) labels.Selector { return selector } +func createNotifier() notifier.Notifier { + notifiers := notifier.New() + if slackWebhook != "" { + notifiers.Add(notifier.NewSlackNotifier(slackWebhook)) + } + + return notifiers +} + func serveMetrics() { http.Handle("/metrics", promhttp.Handler()) http.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { diff --git a/notifier/noop.go b/notifier/noop.go new file mode 100644 index 00000000..a5101e4c --- /dev/null +++ b/notifier/noop.go @@ -0,0 +1,12 @@ +package notifier + +const NotifierNoop = "noop" + +type Noop struct { + Calls int +} + +func (t *Noop) NotifyTermination(termination Termination) error { + t.Calls++ + return nil +} diff --git a/notifier/notifier.go b/notifier/notifier.go new file mode 100644 index 00000000..a49c5ac6 --- /dev/null +++ b/notifier/notifier.go @@ -0,0 +1,31 @@ +package notifier + +type Notifier interface { + NotifyTermination(term Termination) error +} + +type Termination struct { + Pod string + Namespace string +} + +type Notifiers struct { + notifiers []Notifier +} + +func New() *Notifiers { + return &Notifiers{notifiers: make([]Notifier, 0)} +} + +func (m *Notifiers) NotifyTermination(term Termination) error { + for _, n := range m.notifiers { + if err := n.NotifyTermination(term); err != nil { + return err + } + } + return nil +} + +func (m *Notifiers) Add(notifier Notifier) { + m.notifiers = append(m.notifiers, notifier) +} diff --git a/notifier/notifier_test.go b/notifier/notifier_test.go new file mode 100644 index 00000000..fd75030c --- /dev/null +++ b/notifier/notifier_test.go @@ -0,0 +1,48 @@ +package notifier + +import ( + "testing" +) + +func TestMultiNotifierWithoutNotifiers(t *testing.T) { + manager := New() + err := manager.NotifyTermination(Termination{}) + if err != nil { + t.Fatal(err) + } +} + +func TestMultiNotifierWithNotifier(t *testing.T) { + manager := New() + n := Noop{} + manager.Add(&n) + err := manager.NotifyTermination(Termination{}) + if err != nil { + t.Fatal(err) + } + + if n.Calls != 1 { + t.Errorf("expected %d calls to notifier but got %d", 1, n.Calls) + } +} + +func TestMultiNotifierWithMultipleNotifier(t *testing.T) { + manager := New() + n1 := Noop{} + n2 := Noop{} + manager.Add(&n1) + manager.Add(&n2) + + err := manager.NotifyTermination(Termination{}) + if err != nil { + t.Fatal(err) + } + + if n1.Calls != 1 { + t.Errorf("expected %d calls to notifier n1 but got %d", 1, n1.Calls) + } + + if n2.Calls != 1 { + t.Errorf("expected %d calls to notifier n2 but got %d", 1, n2.Calls) + } +} diff --git a/notifier/slack.go b/notifier/slack.go new file mode 100644 index 00000000..e72a9439 --- /dev/null +++ b/notifier/slack.go @@ -0,0 +1,107 @@ +package notifier + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "time" +) + +const NotifierSlack = "slack" + +var NotificationColor = "#F35A00" +var DefaultTimeout = 10 * time.Second + +type Slack struct { + Webhook string + Client *http.Client +} + +type slackMessage struct { + Message string `json:"text"` + Attachments []attachment `json:"attachments"` +} + +type slackField struct { + Title string `yaml:"title,omitempty" json:"title,omitempty"` + Value string `yaml:"value,omitempty" json:"value,omitempty"` + Short *bool `yaml:"short,omitempty" json:"short,omitempty"` +} + +type attachment struct { + Title string `json:"title,omitempty"` + TitleLink string `json:"title_link,omitempty"` + Pretext string `json:"pretext,omitempty"` + Text string `json:"text"` + Fallback string `json:"fallback"` + CallbackID string `json:"callback_id"` + Fields []slackField `json:"fields,omitempty"` + ImageURL string `json:"image_url,omitempty"` + ThumbURL string `json:"thumb_url,omitempty"` + Footer string `json:"footer"` + Color string `json:"color,omitempty"` + MrkdwnIn []string `json:"mrkdwn_in,omitempty"` +} + +func NewSlackNotifier(webhook string) *Slack { + return &Slack{ + Webhook: webhook, + Client: &http.Client{Timeout: DefaultTimeout}, + } +} + +func (s Slack) NotifyTermination(term Termination) error { + title := "Chaos event - Pod termination" + text := fmt.Sprintf("pod %s has been selected by chaos-kube for termination", term.Pod) + + short := len(term.Namespace) < 20 && len(term.Pod) < 20 + fields := []slackField{ + { + Title: "namespace", + Value: term.Namespace, + Short: &short, + }, + { + Title: "pod", + Value: term.Pod, + Short: &short, + }, + } + + message := createSlackRequest(title, text, fields) + return s.sendSlackMessage(message) +} + +func createSlackRequest(title string, text string, fields []slackField) slackMessage { + return slackMessage{ + Attachments: []attachment{{ + Title: title, + Text: text, + Footer: "chaos-kube", + Color: NotificationColor, + Fields: fields, + }}, + } +} + +func (s Slack) sendSlackMessage(message slackMessage) error { + messageBody, err := json.Marshal(message) + if err != nil { + return err + } + req, err := http.NewRequest(http.MethodPost, s.Webhook, bytes.NewBuffer(messageBody)) + if err != nil { + return err + } + req.Header.Add("Content-Type", "application/json") + res, err := s.Client.Do(req) + if err != nil { + return err + } + if res.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected status code %d from slack webhook %s", res.StatusCode, s.Webhook) + } + + return nil +} diff --git a/notifier/slack_test.go b/notifier/slack_test.go new file mode 100644 index 00000000..a65e4465 --- /dev/null +++ b/notifier/slack_test.go @@ -0,0 +1,53 @@ +package notifier + +import ( + "github.com/stretchr/testify/assert" + "net/http" + "net/http/httptest" + "testing" +) + +func TestSlackNotificationForTerminationStatusOk(t *testing.T) { + webhookPath := "/services/T07M5HUDA/BQ1U5VDGA/yhpIczRK0cZ3jDLK1U8qD634" + + testServer := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { + assert.Equal(t, req.URL.Path, webhookPath) + res.WriteHeader(200) + res.Write([]byte("ok")) + })) + + defer testServer.Close() + + slack := NewSlackNotifier(testServer.URL + webhookPath) + err := slack.NotifyTermination(Termination{ + Pod: "chaos-57df4db6b-h9ktj", + Namespace: "chaos", + }) + + if err != nil { + t.Fatal(err) + } +} + +func TestSlackNotificationForTerminationStatus500(t *testing.T) { + webhookPath := "/services/T07M5HUDA/BQ1U5VDGA/yhpIczRK0cZ3jDLK1U8qD634" + + testServer := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { + assert.Equal(t, req.URL.Path, webhookPath) + res.WriteHeader(500) + if _, err := res.Write([]byte("ok")); err != nil { + t.Fatal(err) + } + })) + defer testServer.Close() + + slack := NewSlackNotifier(testServer.URL + webhookPath) + err := slack.NotifyTermination(Termination{ + Pod: "chaos-57df4db6b-h9ktj", + Namespace: "chaos", + }) + + if err == nil { + t.Fatal("expected error on status code 500") + } +} From 4b8b3e90401769d1d35d0dddf7d8539e5e43e12c Mon Sep 17 00:00:00 2001 From: Tommaso Garuglieri Date: Tue, 12 Nov 2019 10:19:01 +0100 Subject: [PATCH 05/10] handled error on http write in slack_test.go --- notifier/slack_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/notifier/slack_test.go b/notifier/slack_test.go index a65e4465..5215220c 100644 --- a/notifier/slack_test.go +++ b/notifier/slack_test.go @@ -13,7 +13,9 @@ func TestSlackNotificationForTerminationStatusOk(t *testing.T) { testServer := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { assert.Equal(t, req.URL.Path, webhookPath) res.WriteHeader(200) - res.Write([]byte("ok")) + if _, err := res.Write([]byte("ok")); err != nil { + t.Fatal(err) + } })) defer testServer.Close() From 45b01a48b9b9479212f04192476e6ff1c0f4cd97 Mon Sep 17 00:00:00 2001 From: Martin Linkhorst Date: Wed, 27 Nov 2019 15:32:10 +0100 Subject: [PATCH 06/10] chore: refactor termination notification code a bit --- chaoskube/chaoskube.go | 12 ++------ notifier/noop.go | 6 +++- notifier/notifier.go | 15 +++++----- notifier/notifier_test.go | 51 ++++++++++++++++---------------- notifier/slack.go | 12 ++++---- notifier/slack_test.go | 61 +++++++++++++++++++++------------------ 6 files changed, 81 insertions(+), 76 deletions(-) diff --git a/chaoskube/chaoskube.go b/chaoskube/chaoskube.go index d9669175..1fcd9c0e 100644 --- a/chaoskube/chaoskube.go +++ b/chaoskube/chaoskube.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "github.com/linki/chaoskube/notifier" "regexp" "time" @@ -24,6 +23,7 @@ import ( "k8s.io/client-go/tools/reference" "github.com/linki/chaoskube/metrics" + "github.com/linki/chaoskube/notifier" "github.com/linki/chaoskube/terminator" "github.com/linki/chaoskube/util" ) @@ -68,7 +68,6 @@ type Chaoskube struct { Now func() time.Time MaxKill int - // chaos events notifier Notifier notifier.Notifier } @@ -261,13 +260,8 @@ func (c *Chaoskube) DeletePod(victim v1.Pod) error { c.EventRecorder.Event(ref, v1.EventTypeNormal, "Killing", "Pod was terminated by chaoskube to introduce chaos.") - err = c.Notifier.NotifyTermination(notifier.Termination{ - Pod: victim.Name, - Namespace: victim.Namespace, - }) - - if err != nil { - c.Logger.Warn("unable to notify pod termination", err) + if err := c.Notifier.NotifyTermination(victim); err != nil { + c.Logger.WithField("err", err).Warn("failed to notify pod termination") } return nil diff --git a/notifier/noop.go b/notifier/noop.go index a5101e4c..dc6f3b92 100644 --- a/notifier/noop.go +++ b/notifier/noop.go @@ -1,12 +1,16 @@ package notifier +import ( + v1 "k8s.io/api/core/v1" +) + const NotifierNoop = "noop" type Noop struct { Calls int } -func (t *Noop) NotifyTermination(termination Termination) error { +func (t *Noop) NotifyTermination(pod v1.Pod) error { t.Calls++ return nil } diff --git a/notifier/notifier.go b/notifier/notifier.go index a49c5ac6..abfdc852 100644 --- a/notifier/notifier.go +++ b/notifier/notifier.go @@ -1,12 +1,11 @@ package notifier -type Notifier interface { - NotifyTermination(term Termination) error -} +import ( + v1 "k8s.io/api/core/v1" +) -type Termination struct { - Pod string - Namespace string +type Notifier interface { + NotifyTermination(pod v1.Pod) error } type Notifiers struct { @@ -17,9 +16,9 @@ func New() *Notifiers { return &Notifiers{notifiers: make([]Notifier, 0)} } -func (m *Notifiers) NotifyTermination(term Termination) error { +func (m *Notifiers) NotifyTermination(pod v1.Pod) error { for _, n := range m.notifiers { - if err := n.NotifyTermination(term); err != nil { + if err := n.NotifyTermination(pod); err != nil { return err } } diff --git a/notifier/notifier_test.go b/notifier/notifier_test.go index fd75030c..a862bda0 100644 --- a/notifier/notifier_test.go +++ b/notifier/notifier_test.go @@ -2,47 +2,48 @@ package notifier import ( "testing" + + v1 "k8s.io/api/core/v1" + + "github.com/linki/chaoskube/internal/testutil" + + "github.com/stretchr/testify/suite" ) -func TestMultiNotifierWithoutNotifiers(t *testing.T) { +type NotifierSuite struct { + testutil.TestSuite +} + +func (suite *NotifierSuite) TestMultiNotifierWithoutNotifiers() { manager := New() - err := manager.NotifyTermination(Termination{}) - if err != nil { - t.Fatal(err) - } + err := manager.NotifyTermination(v1.Pod{}) + suite.NoError(err) } -func TestMultiNotifierWithNotifier(t *testing.T) { +func (suite *NotifierSuite) TestMultiNotifierWithNotifier() { manager := New() n := Noop{} manager.Add(&n) - err := manager.NotifyTermination(Termination{}) - if err != nil { - t.Fatal(err) - } - - if n.Calls != 1 { - t.Errorf("expected %d calls to notifier but got %d", 1, n.Calls) - } + err := manager.NotifyTermination(v1.Pod{}) + suite.Require().NoError(err) + + suite.Equal(1, n.Calls) } -func TestMultiNotifierWithMultipleNotifier(t *testing.T) { +func (suite *NotifierSuite) TestMultiNotifierWithMultipleNotifier() { manager := New() n1 := Noop{} n2 := Noop{} manager.Add(&n1) manager.Add(&n2) - err := manager.NotifyTermination(Termination{}) - if err != nil { - t.Fatal(err) - } + err := manager.NotifyTermination(v1.Pod{}) + suite.Require().NoError(err) - if n1.Calls != 1 { - t.Errorf("expected %d calls to notifier n1 but got %d", 1, n1.Calls) - } + suite.Equal(1, n1.Calls) + suite.Equal(1, n2.Calls) +} - if n2.Calls != 1 { - t.Errorf("expected %d calls to notifier n2 but got %d", 1, n2.Calls) - } +func TestNotifierSuite(t *testing.T) { + suite.Run(t, new(NotifierSuite)) } diff --git a/notifier/slack.go b/notifier/slack.go index e72a9439..ab1f259d 100644 --- a/notifier/slack.go +++ b/notifier/slack.go @@ -6,6 +6,8 @@ import ( "fmt" "net/http" "time" + + v1 "k8s.io/api/core/v1" ) const NotifierSlack = "slack" @@ -51,20 +53,20 @@ func NewSlackNotifier(webhook string) *Slack { } } -func (s Slack) NotifyTermination(term Termination) error { +func (s Slack) NotifyTermination(pod v1.Pod) error { title := "Chaos event - Pod termination" - text := fmt.Sprintf("pod %s has been selected by chaos-kube for termination", term.Pod) + text := fmt.Sprintf("pod %s has been selected by chaos-kube for termination", pod.Name) - short := len(term.Namespace) < 20 && len(term.Pod) < 20 + short := len(pod.Namespace) < 20 && len(pod.Name) < 20 fields := []slackField{ { Title: "namespace", - Value: term.Namespace, + Value: pod.Namespace, Short: &short, }, { Title: "pod", - Value: term.Pod, + Value: pod.Name, Short: &short, }, } diff --git a/notifier/slack_test.go b/notifier/slack_test.go index 5215220c..1eb1a7db 100644 --- a/notifier/slack_test.go +++ b/notifier/slack_test.go @@ -1,55 +1,60 @@ package notifier import ( - "github.com/stretchr/testify/assert" "net/http" "net/http/httptest" "testing" + + v1 "k8s.io/api/core/v1" + + "github.com/linki/chaoskube/internal/testutil" + "github.com/linki/chaoskube/util" + + "github.com/stretchr/testify/suite" ) -func TestSlackNotificationForTerminationStatusOk(t *testing.T) { +type SlackSuite struct { + testutil.TestSuite +} + +func (suite *SlackSuite) TestSlackNotificationForTerminationStatusOk() { webhookPath := "/services/T07M5HUDA/BQ1U5VDGA/yhpIczRK0cZ3jDLK1U8qD634" testServer := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { - assert.Equal(t, req.URL.Path, webhookPath) + suite.Require().Equal(webhookPath, req.URL.Path) res.WriteHeader(200) - if _, err := res.Write([]byte("ok")); err != nil { - t.Fatal(err) - } + _, err := res.Write([]byte("ok")) + suite.Require().NoError(err) })) - defer testServer.Close() + testPod := util.NewPod("chaos", "chaos-57df4db6b-h9ktj", v1.PodRunning) + slack := NewSlackNotifier(testServer.URL + webhookPath) - err := slack.NotifyTermination(Termination{ - Pod: "chaos-57df4db6b-h9ktj", - Namespace: "chaos", - }) - - if err != nil { - t.Fatal(err) - } + err := slack.NotifyTermination(testPod) + + suite.NoError(err) } -func TestSlackNotificationForTerminationStatus500(t *testing.T) { +func (suite *SlackSuite) TestSlackNotificationForTerminationStatus500() { webhookPath := "/services/T07M5HUDA/BQ1U5VDGA/yhpIczRK0cZ3jDLK1U8qD634" testServer := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { - assert.Equal(t, req.URL.Path, webhookPath) + suite.Require().Equal(webhookPath, req.URL.Path) res.WriteHeader(500) - if _, err := res.Write([]byte("ok")); err != nil { - t.Fatal(err) - } + _, err := res.Write([]byte("ok")) + suite.Require().NoError(err) })) defer testServer.Close() + testPod := util.NewPod("chaos", "chaos-57df4db6b-h9ktj", v1.PodRunning) + slack := NewSlackNotifier(testServer.URL + webhookPath) - err := slack.NotifyTermination(Termination{ - Pod: "chaos-57df4db6b-h9ktj", - Namespace: "chaos", - }) - - if err == nil { - t.Fatal("expected error on status code 500") - } + err := slack.NotifyTermination(testPod) + + suite.Error(err) +} + +func TestSlackSuite(t *testing.T) { + suite.Run(t, new(SlackSuite)) } From 4cccb9df4a1183c2f7cf36668ef18a82f3936818 Mon Sep 17 00:00:00 2001 From: Martin Linkhorst Date: Wed, 27 Nov 2019 15:35:27 +0100 Subject: [PATCH 07/10] chore: rearrange imports --- chaoskube/chaoskube_test.go | 2 +- main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/chaoskube/chaoskube_test.go b/chaoskube/chaoskube_test.go index 6ab2c291..c0fda563 100644 --- a/chaoskube/chaoskube_test.go +++ b/chaoskube/chaoskube_test.go @@ -2,7 +2,6 @@ package chaoskube import ( "context" - "github.com/linki/chaoskube/notifier" "math/rand" "regexp" "testing" @@ -18,6 +17,7 @@ import ( "k8s.io/client-go/kubernetes/fake" "github.com/linki/chaoskube/internal/testutil" + "github.com/linki/chaoskube/notifier" "github.com/linki/chaoskube/terminator" "github.com/linki/chaoskube/util" diff --git a/main.go b/main.go index fb9d4e06..20333c4e 100644 --- a/main.go +++ b/main.go @@ -3,7 +3,6 @@ package main import ( "context" "fmt" - "github.com/linki/chaoskube/notifier" "io/ioutil" "math/rand" "net/http" @@ -27,6 +26,7 @@ import ( "k8s.io/klog" "github.com/linki/chaoskube/chaoskube" + "github.com/linki/chaoskube/notifier" "github.com/linki/chaoskube/terminator" "github.com/linki/chaoskube/util" ) From b3dd7f31d20bc6d78b034231257128f66e2bd411 Mon Sep 17 00:00:00 2001 From: Tommaso Garuglieri Date: Fri, 29 Nov 2019 17:27:05 +0100 Subject: [PATCH 08/10] rename Notifier NotifyTermination to NotifyPodTermination --- chaoskube/chaoskube.go | 2 +- notifier/noop.go | 2 +- notifier/notifier.go | 6 +++--- notifier/notifier_test.go | 6 +++--- notifier/slack.go | 2 +- notifier/slack_test.go | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/chaoskube/chaoskube.go b/chaoskube/chaoskube.go index 1fcd9c0e..ce302d65 100644 --- a/chaoskube/chaoskube.go +++ b/chaoskube/chaoskube.go @@ -260,7 +260,7 @@ func (c *Chaoskube) DeletePod(victim v1.Pod) error { c.EventRecorder.Event(ref, v1.EventTypeNormal, "Killing", "Pod was terminated by chaoskube to introduce chaos.") - if err := c.Notifier.NotifyTermination(victim); err != nil { + if err := c.Notifier.NotifyPodTermination(victim); err != nil { c.Logger.WithField("err", err).Warn("failed to notify pod termination") } diff --git a/notifier/noop.go b/notifier/noop.go index dc6f3b92..4438abdf 100644 --- a/notifier/noop.go +++ b/notifier/noop.go @@ -10,7 +10,7 @@ type Noop struct { Calls int } -func (t *Noop) NotifyTermination(pod v1.Pod) error { +func (t *Noop) NotifyPodTermination(pod v1.Pod) error { t.Calls++ return nil } diff --git a/notifier/notifier.go b/notifier/notifier.go index abfdc852..90b28322 100644 --- a/notifier/notifier.go +++ b/notifier/notifier.go @@ -5,7 +5,7 @@ import ( ) type Notifier interface { - NotifyTermination(pod v1.Pod) error + NotifyPodTermination(pod v1.Pod) error } type Notifiers struct { @@ -16,9 +16,9 @@ func New() *Notifiers { return &Notifiers{notifiers: make([]Notifier, 0)} } -func (m *Notifiers) NotifyTermination(pod v1.Pod) error { +func (m *Notifiers) NotifyPodTermination(pod v1.Pod) error { for _, n := range m.notifiers { - if err := n.NotifyTermination(pod); err != nil { + if err := n.NotifyPodTermination(pod); err != nil { return err } } diff --git a/notifier/notifier_test.go b/notifier/notifier_test.go index a862bda0..3c640bf1 100644 --- a/notifier/notifier_test.go +++ b/notifier/notifier_test.go @@ -16,7 +16,7 @@ type NotifierSuite struct { func (suite *NotifierSuite) TestMultiNotifierWithoutNotifiers() { manager := New() - err := manager.NotifyTermination(v1.Pod{}) + err := manager.NotifyPodTermination(v1.Pod{}) suite.NoError(err) } @@ -24,7 +24,7 @@ func (suite *NotifierSuite) TestMultiNotifierWithNotifier() { manager := New() n := Noop{} manager.Add(&n) - err := manager.NotifyTermination(v1.Pod{}) + err := manager.NotifyPodTermination(v1.Pod{}) suite.Require().NoError(err) suite.Equal(1, n.Calls) @@ -37,7 +37,7 @@ func (suite *NotifierSuite) TestMultiNotifierWithMultipleNotifier() { manager.Add(&n1) manager.Add(&n2) - err := manager.NotifyTermination(v1.Pod{}) + err := manager.NotifyPodTermination(v1.Pod{}) suite.Require().NoError(err) suite.Equal(1, n1.Calls) diff --git a/notifier/slack.go b/notifier/slack.go index ab1f259d..76649540 100644 --- a/notifier/slack.go +++ b/notifier/slack.go @@ -53,7 +53,7 @@ func NewSlackNotifier(webhook string) *Slack { } } -func (s Slack) NotifyTermination(pod v1.Pod) error { +func (s Slack) NotifyPodTermination(pod v1.Pod) error { title := "Chaos event - Pod termination" text := fmt.Sprintf("pod %s has been selected by chaos-kube for termination", pod.Name) diff --git a/notifier/slack_test.go b/notifier/slack_test.go index 1eb1a7db..a001b7bb 100644 --- a/notifier/slack_test.go +++ b/notifier/slack_test.go @@ -31,7 +31,7 @@ func (suite *SlackSuite) TestSlackNotificationForTerminationStatusOk() { testPod := util.NewPod("chaos", "chaos-57df4db6b-h9ktj", v1.PodRunning) slack := NewSlackNotifier(testServer.URL + webhookPath) - err := slack.NotifyTermination(testPod) + err := slack.NotifyPodTermination(testPod) suite.NoError(err) } @@ -50,7 +50,7 @@ func (suite *SlackSuite) TestSlackNotificationForTerminationStatus500() { testPod := util.NewPod("chaos", "chaos-57df4db6b-h9ktj", v1.PodRunning) slack := NewSlackNotifier(testServer.URL + webhookPath) - err := slack.NotifyTermination(testPod) + err := slack.NotifyPodTermination(testPod) suite.Error(err) } From 42996203b80ac99e97c301f90aa9470ce55c683e Mon Sep 17 00:00:00 2001 From: Martin Linkhorst Date: Mon, 2 Dec 2019 14:06:47 +0100 Subject: [PATCH 09/10] chore: add slack notification to changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c99b295..a851efc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ + * [#158](https://github.com/linki/chaoskube/pull/158) Support for sending Slack notifications @GaruGaru + ## v0.16.0 - 2019-11-08 Features: From f6d6d5c2c16b61e01bb75d4fd69aa7d378d701e1 Mon Sep 17 00:00:00 2001 From: GaruGaru Date: Sat, 7 Dec 2019 15:09:26 +0100 Subject: [PATCH 10/10] accumulate errors instead of failfast on multiple notifiers --- notifier/notifier.go | 6 ++++-- notifier/notifier_test.go | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/notifier/notifier.go b/notifier/notifier.go index 90b28322..015f9fb6 100644 --- a/notifier/notifier.go +++ b/notifier/notifier.go @@ -1,6 +1,7 @@ package notifier import ( + multierror "github.com/hashicorp/go-multierror" v1 "k8s.io/api/core/v1" ) @@ -17,12 +18,13 @@ func New() *Notifiers { } func (m *Notifiers) NotifyPodTermination(pod v1.Pod) error { + var result error for _, n := range m.notifiers { if err := n.NotifyPodTermination(pod); err != nil { - return err + result = multierror.Append(result, err) } } - return nil + return result } func (m *Notifiers) Add(notifier Notifier) { diff --git a/notifier/notifier_test.go b/notifier/notifier_test.go index 3c640bf1..c3db18dd 100644 --- a/notifier/notifier_test.go +++ b/notifier/notifier_test.go @@ -1,6 +1,8 @@ package notifier import ( + "fmt" + "github.com/hashicorp/go-multierror" "testing" v1 "k8s.io/api/core/v1" @@ -14,6 +16,12 @@ type NotifierSuite struct { testutil.TestSuite } +type FailingNotifier struct{} + +func (f FailingNotifier) NotifyPodTermination(pod v1.Pod) error { + return fmt.Errorf("notify error") +} + func (suite *NotifierSuite) TestMultiNotifierWithoutNotifiers() { manager := New() err := manager.NotifyPodTermination(v1.Pod{}) @@ -44,6 +52,36 @@ func (suite *NotifierSuite) TestMultiNotifierWithMultipleNotifier() { suite.Equal(1, n2.Calls) } +func (suite *NotifierSuite) TestMultiNotifierWithNotifierError() { + manager := New() + f := FailingNotifier{} + manager.Add(&f) + err := manager.NotifyPodTermination(v1.Pod{}) + suite.Require().Error(err) +} + +func (suite *NotifierSuite) TestMultiNotifierWithNotifierMultipleError() { + manager := New() + f0 := FailingNotifier{} + f1 := FailingNotifier{} + manager.Add(&f0) + manager.Add(&f1) + err := manager.NotifyPodTermination(v1.Pod{}).(*multierror.Error) + suite.Require().Error(err) + suite.Require().Len(err.Errors, 2) +} + +func (suite *NotifierSuite) TestMultiNotifierWithOneFailingNotifier() { + manager := New() + f := FailingNotifier{} + n := Noop{} + manager.Add(&n) + manager.Add(&f) + err := manager.NotifyPodTermination(v1.Pod{}).(*multierror.Error) + suite.Require().Error(err) + suite.Require().Len(err.Errors, 1) +} + func TestNotifierSuite(t *testing.T) { suite.Run(t, new(NotifierSuite)) }