Feature/disable subscriptions

metrics/notifier.go

@@ -3,6 +3,7 @@ package metrics
 // NotifierMetrics is a collection of metrics used in notifier
 type NotifierMetrics struct {
 	SubsMalformed          Meter
+	SubsBrokenDisabled     Meter
 	EventsReceived         Meter
 	EventsMalformed        Meter
 	EventsProcessingFailed Meter
@@ -16,6 +17,7 @@ type NotifierMetrics struct {
 func ConfigureNotifierMetrics(registry Registry, prefix string) *NotifierMetrics {
 	return &NotifierMetrics{
 		SubsMalformed:          registry.NewMeter("subs", "malformed"),
+		SubsBrokenDisabled:     registry.NewMeter("subs", "broken_disabled"),
 		EventsReceived:         registry.NewMeter("events", "received"),
 		EventsMalformed:        registry.NewMeter("events", "malformed"),
 		EventsProcessingFailed: registry.NewMeter("events", "failed"),

notifier/notifier.go

@@ -195,6 +195,12 @@ func (notifier *StandardNotifier) runSender(sender moira.Sender, ch chan Notific
 			switch e := err.(type) {
 			case moira.SenderBrokenContactError:
 				log.Errorf("Cannot send to broken contact: %s", e.Error())
+				if count, errorDisabling := disableBrokenContactSubscriptions(&pkg.Contact, notifier.database, log); errorDisabling != nil {
+					log.Errorf("Failed to disable broken subscriptions: ", errorDisabling)
+				} else {
+					notifier.metrics.SubsBrokenDisabled.Mark(int64(count))
+					log.Infof("Disabled %d broken subscriptions", count)
+				}
 
 			default:
 				log.Errorf("Cannot send notification: %s", err.Error())
@@ -203,3 +209,40 @@ func (notifier *StandardNotifier) runSender(sender moira.Sender, ch chan Notific
 		}
 	}
 }
+
+func disableBrokenContactSubscriptions(brokenContact *moira.ContactData, database moira.Database, log moira.Logger) (
+	disabledCount int, e error) {
+	subsIDs, err := database.GetUserSubscriptionIDs(brokenContact.User)
+	if err != nil {
+		return 0, err
+	}
+	if len(subsIDs) == 0 {
+		return 0, nil
+	}
+
+	subs, e := database.GetSubscriptions(subsIDs)
+	if e != nil {
+		return 0, e
+	}
+
+	disableSubs := []*moira.SubscriptionData{}
+	for _, s := range subs {
+		if len(s.Contacts) == 1 {
+			if s.Contacts[0] == brokenContact.ID {
+				s.Enabled = false
+				disableSubs = append(disableSubs, s)
+			}
+		} else {
+			log.Warningf("Skipped disable subscription with broken contact, "+
+				"contacts more than 1, contacts = %v", s.Contacts)
+		}
+	}
+	disableCount := len(disableSubs)
+	if disableCount > 0 {
+		if err := database.SaveSubscriptions(disableSubs); err != nil {
+			return 0, err
+		}
+		return disableCount, nil
+	}
+	return 0, nil
+}

notifier/notifier_test.go

@@ -131,10 +131,12 @@ func TestNoResendForSendToBrokenContact(t *testing.T) {
 		Events: eventsData,
 		Contact: moira.ContactData{
 			Type: "test",
+			User: "someuser",
 		},
 	}
 	sender.EXPECT().SendEvents(eventsData, pkg.Contact, pkg.Trigger, plots, pkg.Throttled).
 		Return(moira.NewSenderBrokenContactError(fmt.Errorf("some sender reason")))
+	dataBase.EXPECT().GetUserSubscriptionIDs(pkg.Contact.User).Return([]string{}, nil)
 
 	var wg sync.WaitGroup
 	notif.Send(&pkg, &wg)
@@ -236,7 +238,8 @@ func afterTest() {
 var subID = "SubscriptionID-000000000000001"
 
 var event = moira.NotificationEvent{
-	Metric:         "generate.event.1",
+	Metric: "generate.event.1",
+
 	State:          moira.StateOK,
 	OldState:       moira.StateWARN,
 	TriggerID:      "triggerID-0000000000001",