diff --git a/database/redis/metric_test.go b/database/redis/metric_test.go index 339932b0e..d133c9af8 100644 --- a/database/redis/metric_test.go +++ b/database/redis/metric_test.go @@ -1,11 +1,14 @@ package redis import ( + "context" "errors" "fmt" + "sync/atomic" "testing" "time" + "github.com/go-redis/redis/v8" "github.com/moira-alert/moira" logging "github.com/moira-alert/moira/logging/zerolog_adapter" "github.com/patrickmn/go-cache" @@ -507,6 +510,15 @@ func TestCleanupOutdatedMetrics(t *testing.T) { dataBase.Flush() defer dataBase.Flush() + Convey("When clean up metrics with wrong value of duration was called (positive number)", t, func() { + err := dataBase.CleanUpOutdatedMetrics(time.Hour) + So( + err, + ShouldResemble, + errors.New("clean up duration value must be less than zero, otherwise all metrics will be removed"), + ) + }) + Convey("Given 2 metrics with 2 values older then 1 minute and 2 values younger then 1 minute", t, func() { const ( metric1 = "my.test.super.metric" @@ -620,15 +632,6 @@ func TestCleanupOutdatedMetrics(t *testing.T) { }, }) - Convey("When clean up metrics with wrong value of duration was called (positive number)", func() { - err = dataBase.CleanUpOutdatedMetrics(time.Hour) - So( - err, - ShouldResemble, - errors.New("clean up duration value must be less than zero, otherwise all metrics will be removed"), - ) - }) - Convey("When clean up metrics older then 1 minute was called", func() { err = dataBase.CleanUpOutdatedMetrics(-time.Minute) So(err, ShouldBeNil) @@ -869,7 +872,7 @@ func TestRemoveMetricsByPrefix(t *testing.T) { client := *dataBase.client const pattern = "my.test.*.metric*" - Convey("Given metrics with pattern my.test.*", t, func() { + Convey("Given metrics with pattern my.test.*", t, func(c C) { for i := 1; i <= 10; i++ { err := dataBase.SaveMetrics( map[string]*moira.MatchedMetric{ @@ -898,29 +901,110 @@ func TestRemoveMetricsByPrefix(t *testing.T) { So(err, ShouldBeNil) } - result := client.Keys(dataBase.context, "moira-metric-data:my.test*").Val() - So(len(result), ShouldResemble, 20) - result = client.Keys(dataBase.context, "moira-metric-retention:my.test*").Val() - So(len(result), ShouldResemble, 20) + var metricsCount int32 + incrementMetric := func(ctx context.Context, shard redis.UniversalClient) { + result := len(shard.Keys(ctx, "moira-metric-data:my.test*").Val()) + atomic.AddInt32(&metricsCount, int32(result)) + } + switch cl := client.(type) { + case *redis.ClusterClient: + err := cl.ForEachMaster(dataBase.context, func(ctx context.Context, shard *redis.Client) error { + incrementMetric(ctx, shard) + return nil + }) + So(err, ShouldBeNil) + default: + incrementMetric(dataBase.context, cl) + } + So(atomic.LoadInt32(&metricsCount), ShouldEqual, 20) + + var retentionsCount int32 + incrementRetention := func(ctx context.Context, shard redis.UniversalClient) { + result := len(shard.Keys(ctx, "moira-metric-retention:my.test*").Val()) + atomic.AddInt32(&retentionsCount, int32(result)) + } + switch cl := client.(type) { + case *redis.ClusterClient: + err := cl.ForEachMaster(dataBase.context, func(ctx context.Context, shard *redis.Client) error { + incrementRetention(ctx, shard) + return nil + }) + So(err, ShouldBeNil) + default: + incrementRetention(dataBase.context, cl) + } + So(atomic.LoadInt32(&retentionsCount), ShouldEqual, 20) patternMetricsCount := client.SCard(dataBase.context, "moira-pattern-metrics:my.test.*.metric*").Val() - So(patternMetricsCount, ShouldResemble, int64(20)) + So(patternMetricsCount, ShouldEqual, int64(20)) Convey("When remove metrics by prefix my.test.super. was called", func() { err := dataBase.RemoveMetricsByPrefix("my.test.super.") So(err, ShouldBeNil) - Convey("No metric data for metrics with this prefix should not exist", func() { - result = client.Keys(dataBase.context, "moira-metric-data:my.test*").Val() - So(len(result), ShouldResemble, 10) - result = client.Keys(dataBase.context, "moira-metric-retention:my.test*").Val() - So(len(result), ShouldResemble, 10) - result = client.Keys(dataBase.context, "moira-metric-data:my.test.mega.*").Val() - So(len(result), ShouldResemble, 10) - result = client.Keys(dataBase.context, "moira-metric-retention:my.test.mega.*").Val() - So(len(result), ShouldResemble, 10) + Convey("No metric data for metrics with this prefix should exist", func() { + var allMetricsCount int + switch cl := client.(type) { + case *redis.ClusterClient: + err = cl.ForEachMaster(dataBase.context, func(ctx context.Context, shard *redis.Client) error { + result := shard.Keys(ctx, "moira-metric-data:my.test*").Val() + allMetricsCount += len(result) + return nil + }) + So(err, ShouldBeNil) + default: + result := client.Keys(dataBase.context, "moira-metric-data:my.test*").Val() + allMetricsCount += len(result) + } + So(allMetricsCount, ShouldEqual, 10) + + var allRetentionsCount int + switch cl := client.(type) { + case *redis.ClusterClient: + err = cl.ForEachMaster(dataBase.context, func(ctx context.Context, shard *redis.Client) error { + result := shard.Keys(ctx, "moira-metric-retention:my.test*").Val() + allRetentionsCount += len(result) + return nil + }) + So(err, ShouldBeNil) + default: + result := client.Keys(dataBase.context, "moira-metric-retention:my.test*").Val() + allRetentionsCount += len(result) + } + So(allRetentionsCount, ShouldEqual, 10) + + var megaMetricsCount int + switch cl := client.(type) { + case *redis.ClusterClient: + err = cl.ForEachMaster(dataBase.context, func(ctx context.Context, shard *redis.Client) error { + result := shard.Keys(ctx, "moira-metric-data:my.test.mega.*").Val() + megaMetricsCount += len(result) + return nil + }) + So(err, ShouldBeNil) + default: + result := client.Keys(dataBase.context, "moira-metric-data:my.test.mega.*").Val() + megaMetricsCount += len(result) + } + So(megaMetricsCount, ShouldEqual, 10) + + var megaRetentionsCount int + switch cl := client.(type) { + case *redis.ClusterClient: + err = cl.ForEachMaster(dataBase.context, func(ctx context.Context, shard *redis.Client) error { + result := shard.Keys(ctx, "moira-metric-retention:my.test.mega.*").Val() + megaRetentionsCount += len(result) + return nil + }) + So(err, ShouldBeNil) + default: + result := client.Keys(dataBase.context, "moira-metric-retention:my.test.mega.*").Val() + megaRetentionsCount += len(result) + } + So(megaRetentionsCount, ShouldEqual, 10) + patternMetricsCount := client.SCard(dataBase.context, "moira-pattern-metrics:my.test.*.metric*").Val() - So(patternMetricsCount, ShouldResemble, int64(10)) + So(patternMetricsCount, ShouldEqual, int64(10)) }) }) }) @@ -934,7 +1018,7 @@ func TestRemoveAllMetrics(t *testing.T) { client := *dataBase.client const pattern = "my.test.*.metric*" - Convey("Given metrics with pattern my.test.*", t, func() { + Convey("Given metrics with pattern my.test.*", t, func(c C) { for i := 1; i <= 10; i++ { err := dataBase.SaveMetrics( map[string]*moira.MatchedMetric{ @@ -963,25 +1047,92 @@ func TestRemoveAllMetrics(t *testing.T) { So(err, ShouldBeNil) } - result := client.Keys(dataBase.context, "moira-metric-data:my.test*").Val() - So(len(result), ShouldResemble, 20) - result = client.Keys(dataBase.context, "moira-metric-retention:my.test*").Val() - So(len(result), ShouldResemble, 20) + var metricsCount int32 + incrementMetric := func(ctx context.Context, shard redis.UniversalClient) { + result := len(shard.Keys(ctx, "moira-metric-data:my.test*").Val()) + atomic.AddInt32(&metricsCount, int32(result)) + } + switch cl := client.(type) { + case *redis.ClusterClient: + err := cl.ForEachMaster(dataBase.context, func(ctx context.Context, shard *redis.Client) error { + incrementMetric(ctx, shard) + return nil + }) + So(err, ShouldBeNil) + default: + incrementMetric(dataBase.context, cl) + } + So(atomic.LoadInt32(&metricsCount), ShouldEqual, 20) + + var retentionsCount int32 + incrementRetention := func(ctx context.Context, shard redis.UniversalClient) { + result := len(shard.Keys(ctx, "moira-metric-retention:my.test*").Val()) + atomic.AddInt32(&retentionsCount, int32(result)) + } + switch cl := client.(type) { + case *redis.ClusterClient: + err := cl.ForEachMaster(dataBase.context, func(ctx context.Context, shard *redis.Client) error { + incrementRetention(ctx, shard) + return nil + }) + So(err, ShouldBeNil) + default: + incrementRetention(dataBase.context, cl) + } + So(atomic.LoadInt32(&retentionsCount), ShouldEqual, 20) patternMetricsCount := client.SCard(dataBase.context, "moira-pattern-metrics:my.test.*.metric*").Val() - So(patternMetricsCount, ShouldResemble, int64(20)) + So(patternMetricsCount, ShouldEqual, int64(20)) Convey("When remove all metrics was called", func() { err := dataBase.RemoveAllMetrics() So(err, ShouldBeNil) Convey("No metric data should not exist", func() { - result = client.Keys(dataBase.context, "moira-metric-data:*").Val() - So(len(result), ShouldResemble, 0) - result = client.Keys(dataBase.context, "moira-metric-retention:*").Val() - So(len(result), ShouldResemble, 0) - result = client.Keys(dataBase.context, "moira-pattern-metrics*").Val() - So(len(result), ShouldResemble, 0) + var metricsCount int + switch cl := client.(type) { + case *redis.ClusterClient: + err = cl.ForEachMaster(dataBase.context, func(ctx context.Context, shard *redis.Client) error { + result := shard.Keys(ctx, "moira-metric-data:*").Val() + metricsCount += len(result) + return nil + }) + So(err, ShouldBeNil) + default: + result := client.Keys(dataBase.context, "moira-metric-data:*").Val() + metricsCount += len(result) + } + So(metricsCount, ShouldEqual, 0) + + var retentionsCount int + switch cl := client.(type) { + case *redis.ClusterClient: + err = cl.ForEachMaster(dataBase.context, func(ctx context.Context, shard *redis.Client) error { + result := shard.Keys(ctx, "moira-metric-retention:*").Val() + retentionsCount += len(result) + return nil + }) + So(err, ShouldBeNil) + default: + result := client.Keys(dataBase.context, "moira-metric-retention:*").Val() + retentionsCount += len(result) + } + So(retentionsCount, ShouldEqual, 0) + + var patternsCount int + switch cl := client.(type) { + case *redis.ClusterClient: + err = cl.ForEachMaster(dataBase.context, func(ctx context.Context, shard *redis.Client) error { + result := shard.Keys(ctx, "moira-pattern-metrics*").Val() + patternsCount += len(result) + return nil + }) + So(err, ShouldBeNil) + default: + result := client.Keys(dataBase.context, "moira-pattern-metrics*").Val() + patternsCount += len(result) + } + So(patternsCount, ShouldEqual, 0) }) }) })