Address PR comments

Signed-off-by: Emmanuel Lodovice <[email protected]>

alertobserver/alertobserver.go

@@ -18,14 +18,15 @@ import (
 )
 
 const (
-	EventAlertReceived          string = "received"
-	EventAlertRejected          string = "rejected"
-	EventAlertAddedToAggrGroup  string = "addedAggrGroup"
-	EventAlertPipelineStart     string = "pipelineStart"
-	EventAlertPipelinePassStage string = "pipelinePassStage"
-	EventAlertMuted             string = "muted"
-	EventAlertSent              string = "sent"
-	EventAlertSendFailed        string = "sendFailed"
+	EventAlertReceived             string = "received"
+	EventAlertRejected             string = "rejected"
+	EventAlertAddedToAggrGroup     string = "addedAggrGroup"
+	EventAlertFailedAddToAggrGroup string = "failedAddAggrGroup"
+	EventAlertPipelineStart        string = "pipelineStart"
+	EventAlertPipelinePassStage    string = "pipelinePassStage"
+	EventAlertMuted                string = "muted"
+	EventAlertSent                 string = "sent"
+	EventAlertSendFailed           string = "sendFailed"
 )
 
 type AlertEventMeta map[string]interface{}

dispatch/dispatch.go

@@ -332,7 +332,11 @@ func (d *Dispatcher) processAlert(alert *types.Alert, route *Route) {
 	// If the group does not exist, create it. But check the limit first.
 	if limit := d.limits.MaxNumberOfAggregationGroups(); limit > 0 && d.aggrGroupsNum >= limit {
 		d.metrics.aggrGroupLimitReached.Inc()
-		level.Error(d.logger).Log("msg", "Too many aggregation groups, cannot create new group for alert", "groups", d.aggrGroupsNum, "limit", limit, "alert", alert.Name())
+		errMsg := "Too many aggregation groups, cannot create new group for alert"
+		level.Error(d.logger).Log("msg", errMsg, "groups", d.aggrGroupsNum, "limit", limit, "alert", alert.Name())
+		if d.alertLCObserver != nil {
+			d.alertLCObserver.Observe(alertobserver.EventAlertFailedAddToAggrGroup, []*types.Alert{alert}, alertobserver.AlertEventMeta{"msg": errMsg})
+		}
 		return
 	}
 

dispatch/dispatch_test.go

@@ -596,15 +596,15 @@ route:
 	recorder := &recordStage{alerts: make(map[string]map[model.Fingerprint]*types.Alert)}
 	m := NewDispatcherMetrics(true, prometheus.NewRegistry())
 	observer := alertobserver.NewFakeLifeCycleObserver()
-	dispatcher := NewDispatcher(alerts, route, recorder, marker, timeout, nil, logger, m, observer)
+	lim := limits{groups: 1}
+	dispatcher := NewDispatcher(alerts, route, recorder, marker, timeout, lim, logger, m, observer)
 	go dispatcher.Run()
 	defer dispatcher.Stop()
 
 	// Create alerts. the dispatcher will automatically create the groups.
-	inputAlerts := []*types.Alert{
-		newAlert(model.LabelSet{"alertname": "OtherAlert", "cluster": "cc", "service": "dd"}),
-	}
-	err = alerts.Put(inputAlerts...)
+	alert1 := newAlert(model.LabelSet{"alertname": "OtherAlert", "cluster": "cc", "service": "dd"})
+	alert2 := newAlert(model.LabelSet{"alertname": "YetAnotherAlert", "cluster": "cc", "service": "db"})
+	err = alerts.Put(alert1, alert2)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -613,10 +613,13 @@ route:
 		time.Sleep(200 * time.Millisecond)
 	}
 	require.Equal(t, 1, len(recorder.Alerts()))
-	require.Equal(t, inputAlerts[0].Fingerprint(), observer.AlertsPerEvent[alertobserver.EventAlertAddedToAggrGroup][0].Fingerprint())
-	groupFp := getGroupLabels(inputAlerts[0], route).Fingerprint()
+	require.Equal(t, alert1.Fingerprint(), observer.AlertsPerEvent[alertobserver.EventAlertAddedToAggrGroup][0].Fingerprint())
+	groupFp := getGroupLabels(alert1, route).Fingerprint()
 	groupKey := dispatcher.aggrGroupsPerRoute[route][groupFp].GroupKey()
 	require.Equal(t, groupKey, observer.MetaPerEvent[alertobserver.EventAlertAddedToAggrGroup][0]["groupKey"].(string))
+
+	require.Equal(t, 1, len(observer.AlertsPerEvent[alertobserver.EventAlertFailedAddToAggrGroup]))
+	require.Equal(t, alert2.Fingerprint(), observer.AlertsPerEvent[alertobserver.EventAlertFailedAddToAggrGroup][0].Fingerprint())
 }
 
 type recordStage struct {