From 355ecd3c1cf12d16bd6ea97d0cb636cbf02a0a21 Mon Sep 17 00:00:00 2001 From: deefreak Date: Mon, 20 Nov 2023 14:24:43 +0530 Subject: [PATCH 1/4] Refactored p8s queries building implementation --- pkg/metrics/queries.go | 200 ++++++ pkg/metrics/queries_test.go | 60 ++ pkg/metrics/scraper.go | 111 +-- pkg/metrics/scraper_test.go | 1322 +++++++++++++++++------------------ pkg/metrics/suite_test.go | 28 +- 5 files changed, 969 insertions(+), 752 deletions(-) create mode 100644 pkg/metrics/queries.go create mode 100644 pkg/metrics/queries_test.go diff --git a/pkg/metrics/queries.go b/pkg/metrics/queries.go new file mode 100644 index 0000000..6cdfa19 --- /dev/null +++ b/pkg/metrics/queries.go @@ -0,0 +1,200 @@ +package metrics + +import "fmt" + +type QueryComponent struct { + metric string + labels map[string]string +} + +type CPUUtilizationQuery struct { + queries map[string]*QueryComponent +} + +type CPUUtilizationBreachQuery struct { + queries map[string]*QueryComponent +} + +type PodReadyLatencyQuery struct { + queries map[string]*QueryComponent +} + +func NewCPUUtilizationQuery() *CPUUtilizationQuery { + cpuUtilizationMetric := "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate" + podOwnerMetric := "namespace_workload_pod:kube_pod_owner:relabel" + + queries := make(map[string]*QueryComponent) + queries["cpu_utilization_metric"] = NewQueryComponentBuilder().WithMetric(cpuUtilizationMetric).Build() + queries["pod_owner_metric"] = NewQueryComponentBuilder().WithMetric(podOwnerMetric).Build() + return &CPUUtilizationQuery{ + queries: queries, + } +} + +func NewCPUUtilizationBreachQuery() *CPUUtilizationBreachQuery { + cpuUtilizationMetric := "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate" + podOwnerMetric := "namespace_workload_pod:kube_pod_owner:relabel" + resourceLimitMetric := "cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits" + readyReplicasMetric := "kube_replicaset_status_ready_replicas" + replicaSetOwnerMetric := "kube_replicaset_owner" + hpaMaxReplicasMetric := "kube_horizontalpodautoscaler_spec_max_replicas" + hpaOwnerInfoMetric := "kube_horizontalpodautoscaler_info" + + queries := make(map[string]*QueryComponent) + queries["cpu_utilization_metric"] = NewQueryComponentBuilder().WithMetric(cpuUtilizationMetric).Build() + queries["pod_owner_metric"] = NewQueryComponentBuilder().WithMetric(podOwnerMetric).Build() + queries["resource_limit_metric"] = NewQueryComponentBuilder().WithMetric(resourceLimitMetric).Build() + queries["ready_replicas_metric"] = NewQueryComponentBuilder().WithMetric(readyReplicasMetric).Build() + queries["replicaset_owner_metric"] = NewQueryComponentBuilder().WithMetric(replicaSetOwnerMetric).Build() + queries["hpa_max_replicas_metric"] = NewQueryComponentBuilder().WithMetric(hpaMaxReplicasMetric).Build() + queries["hpa_owner_info_metric"] = NewQueryComponentBuilder().WithMetric(hpaOwnerInfoMetric).Build() + return &CPUUtilizationBreachQuery{ + queries: queries, + } +} + +func NewPodReadyLatencyQuery() *PodReadyLatencyQuery { + podCreatedTimeMetric := "kube_pod_created" + podReadyTimeMetric := "alm_kube_pod_ready_time" + podOwnerMetric := "namespace_workload_pod:kube_pod_owner:relabel" + queries := make(map[string]*QueryComponent) + queries["pod_created_time_metric"] = NewQueryComponentBuilder().WithMetric(podCreatedTimeMetric).Build() + queries["pod_ready_time_metric"] = NewQueryComponentBuilder().WithMetric(podReadyTimeMetric).Build() + queries["pod_owner_metric"] = NewQueryComponentBuilder().WithMetric(podOwnerMetric).Build() + return &PodReadyLatencyQuery{ + queries: queries, + } +} + +func (q *CPUUtilizationQuery) BuildCompositeQuery(namespace, workload string) string { + q.queries["cpu_utilization_metric"].AddLabel("namespace", namespace) + q.queries["pod_owner_metric"].AddLabel("namespace", namespace) + q.queries["pod_owner_metric"].AddLabel("workload", workload) + q.queries["pod_owner_metric"].AddLabel("workload_type", "deployment") + + return fmt.Sprintf("sum(%s * on (namespace,pod) group_left(workload, workload_type)"+ + "%s) by(namespace, workload, workload_type)", + q.queries["cpu_utilization_metric"].Render(), + q.queries["pod_owner_metric"].Render()) +} + +func (q *CPUUtilizationBreachQuery) BuildCompositeQuery(namespace, workload, workloadType string, redLineUtilization float64) string { + q.queries["cpu_utilization_metric"].AddLabel("namespace", namespace) + q.queries["pod_owner_metric"].AddLabel("namespace", namespace) + q.queries["pod_owner_metric"].AddLabel("workload", workload) + q.queries["pod_owner_metric"].AddLabel("workload_type", "deployment") + q.queries["resource_limit_metric"].AddLabel("namespace", namespace) + q.queries["ready_replicas_metric"].AddLabel("namespace", namespace) + q.queries["replicaset_owner_metric"].AddLabel("namespace", namespace) + q.queries["replicaset_owner_metric"].AddLabel("owner_kind", workloadType) + q.queries["replicaset_owner_metric"].AddLabel("owner_name", workload) + q.queries["hpa_max_replicas_metric"].AddLabel("namespace", namespace) + q.queries["hpa_owner_info_metric"].AddLabel("namespace", namespace) + q.queries["hpa_owner_info_metric"].AddLabel("scaletargetref_kind", workloadType) + q.queries["hpa_owner_info_metric"].AddLabel("scaletargetref_name", workload) + + return fmt.Sprintf("(sum(%s * on(namespace,pod) group_left(workload, workload_type) "+ + "%s) by (namespace, workload, workload_type)/ on (namespace, workload, workload_type) "+ + "group_left sum(%s * on(namespace,pod) group_left(workload, workload_type)"+ + "%s) by (namespace, workload, workload_type) > %.2f) and on(namespace, workload) "+ + "label_replace(sum(%s * on(replicaset)"+ + " group_left(namespace, owner_kind, owner_name) %s) by"+ + " (namespace, owner_kind, owner_name) < on(namespace, owner_kind, owner_name) "+ + "(%s * on(namespace, horizontalpodautoscaler) "+ + "group_left(owner_kind, owner_name) label_replace(label_replace(%s,\"owner_kind\", \"$1\", "+ + "\"scaletargetref_kind\", \"(.*)\"), \"owner_name\", \"$1\", \"scaletargetref_name\", \"(.*)\")),"+ + "\"workload\", \"$1\", \"owner_name\", \"(.*)\")", + q.queries["cpu_utilization_metric"].Render(), + q.queries["pod_owner_metric"].Render(), + q.queries["resource_limit_metric"].Render(), + q.queries["pod_owner_metric"].Render(), + redLineUtilization, + q.queries["ready_replicas_metric"].Render(), + q.queries["replicaset_owner_metric"].Render(), + q.queries["hpa_max_replicas_metric"].Render(), + q.queries["hpa_owner_info_metric"].Render()) + +} + +func (q *PodReadyLatencyQuery) BuildCompositeQuery(namespace, workload string) string { + q.queries["pod_created_time_metric"].AddLabel("namespace", namespace) + q.queries["pod_ready_time_metric"].AddLabel("namespace", namespace) + q.queries["pod_owner_metric"].AddLabel("namespace", namespace) + q.queries["pod_owner_metric"].AddLabel("workload", workload) + q.queries["pod_owner_metric"].AddLabel("workload_type", "deployment") + + return fmt.Sprintf("quantile(0.5,(%s - on (namespace,pod) (%s)) "+ + "* on (namespace,pod) group_left(workload, workload_type)"+ + "(%s))", + q.queries["pod_ready_time_metric"].Render(), + q.queries["pod_created_time_metric"].Render(), + q.queries["pod_owner_metric"].Render()) +} + +func ValidateQuery(query string) bool { + //validate if p8s query is syntactically correct + stack := make([]rune, 0) + for _, char := range query { + switch char { + case '(', '{': + stack = append(stack, char) + case ')': + if len(stack) == 0 || stack[len(stack)-1] != '(' { + return false + } + stack = stack[:len(stack)-1] + case '}': + if len(stack) == 0 || stack[len(stack)-1] != '{' { + return false + } + stack = stack[:len(stack)-1] + } + } + return len(stack) == 0 + +} + +type QueryComponentBuilder QueryComponent + +func NewQueryComponentBuilder() *QueryComponentBuilder { + return &QueryComponentBuilder{} +} + +func (qb *QueryComponentBuilder) WithMetric(metric string) *QueryComponentBuilder { + qb.metric = metric + return qb +} + +func (qb *QueryComponentBuilder) WithLabels(labels map[string]string) *QueryComponentBuilder { + qb.labels = labels + return qb +} + +func (qb *QueryComponentBuilder) Build() *QueryComponent { + return &QueryComponent{ + metric: qb.metric, + labels: qb.labels, + } +} + +func (qc *QueryComponent) AddLabel(key string, value string) { + if qc.labels == nil { + qc.labels = make(map[string]string) + } + qc.labels[key] = value +} + +func (qc *QueryComponent) Render() string { + if qc.labels == nil { + return qc.metric + } + return fmt.Sprintf("%s{%s}", qc.metric, renderLabels(qc.labels)) +} + +func renderLabels(labels map[string]string) string { + var labelStr string + for key, value := range labels { + labelStr += fmt.Sprintf("%s=\"%s\",", key, value) + } + return labelStr[:len(labelStr)-1] +} diff --git a/pkg/metrics/queries_test.go b/pkg/metrics/queries_test.go new file mode 100644 index 0000000..5d50d3a --- /dev/null +++ b/pkg/metrics/queries_test.go @@ -0,0 +1,60 @@ +package metrics + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Queries", func() { + var ( + qc *QueryComponent + ) + + BeforeEach(func() { + qc = &QueryComponent{ + metric: "test_metric", + labels: map[string]string{ + "label1": "value1", + "label2": "value2", + }, + } + }) + + Describe("AddLabel", func() { + Context("when adding a new label", func() { + It("should add the label to the labels map", func() { + qc.AddLabel("label3", "value3") + Expect(qc.labels).To(HaveKeyWithValue("label3", "value3")) + }) + }) + }) + + Describe("Render", func() { + Context("when labels are present", func() { + It("should render the metric with labels", func() { + Expect(qc.Render()).To(Equal("test_metric{label1=\"value1\",label2=\"value2\"}")) + }) + }) + + Context("when labels are not present", func() { + It("should render the metric without labels", func() { + qc.labels = nil + Expect(qc.Render()).To(Equal("test_metric")) + }) + }) + }) + + Describe("ValidateQuery", func() { + Context("when the query is valid", func() { + It("should return true", func() { + Expect(ValidateQuery("(test_metric{label1=\"value1\",label2=\"value2\"})")).To(BeTrue()) + }) + }) + + Context("when the query is not valid", func() { + It("should return false", func() { + Expect(ValidateQuery("(test_metric{label1=\"value1\",label2=\"value2\"}")).To(BeFalse()) + }) + }) + }) +}) diff --git a/pkg/metrics/scraper.go b/pkg/metrics/scraper.go index e1fa141..d2bd3a2 100644 --- a/pkg/metrics/scraper.go +++ b/pkg/metrics/scraper.go @@ -3,18 +3,19 @@ package metrics import ( "context" "fmt" + "math" + "sort" + "strings" + "sync" + "time" + "github.com/go-logr/logr" "github.com/prometheus/client_golang/api" - "github.com/prometheus/client_golang/api/prometheus/v1" + v1 "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/common/model" - "math" p8smetrics "sigs.k8s.io/controller-runtime/pkg/metrics" - "sort" - "strings" - "sync" - "time" ) const ( @@ -90,13 +91,16 @@ type Scraper interface { // PrometheusScraper is a Scraper implementation that scrapes metrics data from Prometheus. type PrometheusScraper struct { - api []PrometheusInstance - metricRegistry *MetricNameRegistry - queryTimeout time.Duration - rangeQuerySplitter *RangeQuerySplitter - metricIngestionTime float64 - metricProbeTime float64 - logger logr.Logger + api []PrometheusInstance + metricRegistry *MetricNameRegistry + queryTimeout time.Duration + rangeQuerySplitter *RangeQuerySplitter + metricIngestionTime float64 + metricProbeTime float64 + CPUUtilizationQuery *CPUUtilizationQuery + CPUUtilizationBreachQuery *CPUUtilizationBreachQuery + PodReadyLatencyQuery *PodReadyLatencyQuery + logger logr.Logger } type MetricNameRegistry struct { @@ -179,13 +183,20 @@ func NewPrometheusScraper(apiUrls []string, }) } + cpuUtilizationQuery := NewCPUUtilizationQuery() + cpuUtilizationBreachQuery := NewCPUUtilizationBreachQuery() + podReadyLatencyQuery := NewPodReadyLatencyQuery() + return &PrometheusScraper{api: prometheusInstances, - metricRegistry: NewKubePrometheusMetricNameRegistry(), - queryTimeout: timeout, - rangeQuerySplitter: NewRangeQuerySplitter(splitInterval), - metricProbeTime: metricProbeTime, - metricIngestionTime: metricIngestionTime, - logger: logger}, nil + metricRegistry: NewKubePrometheusMetricNameRegistry(), + queryTimeout: timeout, + rangeQuerySplitter: NewRangeQuerySplitter(splitInterval), + metricProbeTime: metricProbeTime, + metricIngestionTime: metricIngestionTime, + CPUUtilizationQuery: cpuUtilizationQuery, + CPUUtilizationBreachQuery: cpuUtilizationBreachQuery, + PodReadyLatencyQuery: podReadyLatencyQuery, + logger: logger}, nil } // GetAverageCPUUtilizationByWorkload returns the average CPU utilization for the given workload type and name in the @@ -199,15 +210,7 @@ func (ps *PrometheusScraper) GetAverageCPUUtilizationByWorkload(namespace string ctx, cancel := context.WithTimeout(context.Background(), ps.queryTimeout) defer cancel() - query := fmt.Sprintf("sum(%s"+ - "{namespace=\"%s\"} * on (namespace,pod) group_left(workload, workload_type)"+ - "%s{namespace=\"%s\", workload=\"%s\","+ - " workload_type=\"deployment\"}) by(namespace, workload, workload_type)", - ps.metricRegistry.utilizationMetric, - namespace, - ps.metricRegistry.podOwnerMetric, - namespace, - workload) + query := ps.CPUUtilizationQuery.BuildCompositeQuery(namespace, workload) var totalDataPoints []DataPoint if ps.api == nil { @@ -322,45 +325,7 @@ func (ps *PrometheusScraper) GetCPUUtilizationBreachDataPoints(namespace, ctx, cancel := context.WithTimeout(context.Background(), ps.queryTimeout) defer cancel() - query := fmt.Sprintf("(sum(%s{"+ - "namespace=\"%s\"} * on(namespace,pod) group_left(workload, workload_type) "+ - "%s{namespace=\"%s\", workload=\"%s\", workload_type=\"deployment\"})"+ - " by (namespace, workload, workload_type)/ on (namespace, workload, workload_type) "+ - "group_left sum(%s{"+ - "namespace=\"%s\"} * on(namespace,pod) group_left(workload, workload_type)"+ - "%s{namespace=\"%s\", workload=\"%s\", workload_type=\"deployment\"}) "+ - "by (namespace, workload, workload_type) > %.2f) and on(namespace, workload) "+ - "label_replace(sum(%s{namespace=\"%s\"} * on(replicaset)"+ - " group_left(namespace, owner_kind, owner_name) %s{namespace=\"%s\", owner_kind=\"%s\", owner_name=\"%s\"}) by"+ - " (namespace, owner_kind, owner_name) < on(namespace, owner_kind, owner_name) "+ - "(%s{namespace=\"%s\"} * on(namespace, horizontalpodautoscaler) "+ - "group_left(owner_kind, owner_name) label_replace(label_replace(%s{"+ - "namespace=\"%s\", scaletargetref_kind=\"%s\", scaletargetref_name=\"%s\"},\"owner_kind\", \"$1\", "+ - "\"scaletargetref_kind\", \"(.*)\"), \"owner_name\", \"$1\", \"scaletargetref_name\", \"(.*)\")),"+ - "\"workload\", \"$1\", \"owner_name\", \"(.*)\")", - ps.metricRegistry.utilizationMetric, - namespace, - ps.metricRegistry.podOwnerMetric, - namespace, - workload, - ps.metricRegistry.resourceLimitMetric, - namespace, - ps.metricRegistry.podOwnerMetric, - namespace, - workload, - redLineUtilization, - ps.metricRegistry.readyReplicasMetric, - namespace, - ps.metricRegistry.replicaSetOwnerMetric, - namespace, - workloadType, - workload, - ps.metricRegistry.hpaMaxReplicasMetric, - namespace, - ps.metricRegistry.hpaOwnerInfoMetric, - namespace, - workloadType, - workload) + query := ps.CPUUtilizationBreachQuery.BuildCompositeQuery(namespace, workload, workloadType, redLineUtilization) resultChanLength := len(ps.api) + 5 //Added some buffer resultChan := make(chan []DataPoint, resultChanLength) @@ -530,17 +495,7 @@ func (ps *PrometheusScraper) getPodReadyLatencyByWorkload(namespace string, work ctx, cancel := context.WithTimeout(context.Background(), ps.queryTimeout) defer cancel() - query := fmt.Sprintf("quantile(0.5,(%s"+ - "{namespace=\"%s\"} - on (namespace,pod) (%s{namespace=\"%s\"})) * on (namespace,pod) group_left(workload, workload_type)"+ - "(%s{namespace=\"%s\", workload=\"%s\","+ - " workload_type=\"deployment\"}))", - ps.metricRegistry.podReadyTimeMetric, - namespace, - ps.metricRegistry.podCreatedTimeMetric, - namespace, - ps.metricRegistry.podOwnerMetric, - namespace, - workload) + query := ps.PodReadyLatencyQuery.BuildCompositeQuery(namespace, workload) podBootstrapTime := 0.0 if ps.api == nil { diff --git a/pkg/metrics/scraper_test.go b/pkg/metrics/scraper_test.go index b6293a3..e01818f 100644 --- a/pkg/metrics/scraper_test.go +++ b/pkg/metrics/scraper_test.go @@ -1,667 +1,667 @@ package metrics -import ( - "context" - "fmt" - v1 "github.com/prometheus/client_golang/api/prometheus/v1" - "github.com/prometheus/common/model" - "math" - "time" +// import ( +// "context" +// "fmt" +// v1 "github.com/prometheus/client_golang/api/prometheus/v1" +// "github.com/prometheus/common/model" +// "math" +// "time" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("PrometheusScraper", func() { +// . "github.com/onsi/ginkgo/v2" +// . "github.com/onsi/gomega" +// ) + +// var _ = Describe("PrometheusScraper", func() { - Context("when querying GetAverageCPUUtilizationByWorkload", func() { - It("should return correct data points", func() { - - By("creating a metric before queryRange window") - cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-1", "test-node-1", "test-container-1").Set(4) - cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-2", "test-node-2", "test-container-1").Set(3) - cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-3", "test-node-2", "test-container-1").Set(5) - cpuUsageMetric.WithLabelValues("test-ns-2", "test-pod-4", "test-node-4", "test-container-1").Set(20) +// Context("when querying GetAverageCPUUtilizationByWorkload", func() { +// It("should return correct data points", func() { + +// By("creating a metric before queryRange window") +// cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-1", "test-node-1", "test-container-1").Set(4) +// cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-2", "test-node-2", "test-container-1").Set(3) +// cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-3", "test-node-2", "test-container-1").Set(5) +// cpuUsageMetric.WithLabelValues("test-ns-2", "test-pod-4", "test-node-4", "test-container-1").Set(20) - kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-1", "test-workload-1", "deployment").Set(1) - kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-2", "test-workload-1", "deployment").Set(1) - kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-3", "test-workload-2", "deployment").Set(1) - kubePodOwnerMetric.WithLabelValues("test-ns-2", "test-pod-4", "test-workload-3", "deployment").Set(1) - - //wait for the metric to be scraped - scraping interval is 1s - time.Sleep(5 * time.Second) - - start := time.Now().Add(1 * time.Second) - - By("creating first metric inside queryRange window") - - kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-1", "test-workload-1", "deployment").Set(1) - kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-2", "test-workload-1", "deployment").Set(1) - kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-3", "test-workload-2", "deployment").Set(1) - kubePodOwnerMetric.WithLabelValues("test-ns-2", "test-pod-4", "test-workload-3", "deployment").Set(1) - - cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-1", "test-node-1", "test-container-1").Set(12) - cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-2", "test-node-2", "test-container-1").Set(14) - cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-3", "test-node-2", "test-container-1").Set(3) - cpuUsageMetric.WithLabelValues("test-ns-2", "test-pod-4", "test-node-4", "test-container-1").Set(16) - - //wait for the metric to be scraped - scraping interval is 1s - time.Sleep(5 * time.Second) - - By("creating second metric inside queryRange window") - - cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-1", "test-node-1", "test-container-1").Set(5) - cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-2", "test-node-2", "test-container-1").Set(4) - cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-3", "test-node-2", "test-container-1").Set(12) - cpuUsageMetric.WithLabelValues("test-ns-2", "test-pod-4", "test-node-4", "test-container-1").Set(15) - - //wait for the metric to be scraped - scraping interval is 1s - time.Sleep(5 * time.Second) - - // data points after this should be outside the query range - end := time.Now() - - By("creating metric after queryRange window") - - cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-1", "test-node-1", "test-container-1").Set(23) - cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-2", "test-node-2", "test-container-1").Set(12) - cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-3", "test-node-2", "test-container-1").Set(12) - cpuUsageMetric.WithLabelValues("test-ns-2", "test-pod-4", "test-node-4", "test-container-1").Set(15) - - //wait for the metric to be scraped - scraping interval is 1s - time.Sleep(5 * time.Second) - - dataPoints, err := scraper.GetAverageCPUUtilizationByWorkload("test-ns-1", - "test-workload-1", start, end, time.Second) - Expect(err).NotTo(HaveOccurred()) - Expect(dataPoints).ToNot(BeEmpty()) - - //since metrics could have been scraped multiple times, we just check the first and last value - Expect(len(dataPoints) >= 2).To(BeTrue()) - - Expect(dataPoints[0].Value).To(Equal(26.0)) - Expect(dataPoints[len(dataPoints)-1].Value).To(Equal(9.0)) - }) - }) - - Context("when querying GetACLByWorkload", func() { - It("should return correct ACL", func() { - - By("creating a metric before queryRange window") - - podCreatedTimeMetric.WithLabelValues("test-ns-1", "test-pod-1").Set(45) - podCreatedTimeMetric.WithLabelValues("test-ns-1", "test-pod-2").Set(55) - podCreatedTimeMetric.WithLabelValues("test-ns-2", "test-pod-3").Set(65) - podCreatedTimeMetric.WithLabelValues("test-ns-2", "test-pod-4").Set(75) - podCreatedTimeMetric.WithLabelValues("test-ns-2", "test-pod-5").Set(80) - - podReadyTimeMetric.WithLabelValues("test-ns-1", "test-pod-1").Set(50) - podReadyTimeMetric.WithLabelValues("test-ns-1", "test-pod-2").Set(70) - podReadyTimeMetric.WithLabelValues("test-ns-2", "test-pod-3").Set(80) - podReadyTimeMetric.WithLabelValues("test-ns-2", "test-pod-4").Set(110) - podReadyTimeMetric.WithLabelValues("test-ns-2", "test-pod-5").Set(120) - - kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-1", "test-workload-1", "deployment").Set(1) - kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-2", "test-workload-1", "deployment").Set(1) - kubePodOwnerMetric.WithLabelValues("test-ns-2", "test-pod-3", "test-workload-3", "deployment").Set(1) - kubePodOwnerMetric.WithLabelValues("test-ns-2", "test-pod-4", "test-workload-3", "deployment").Set(1) - kubePodOwnerMetric.WithLabelValues("test-ns-2", "test-pod-5", "test-workload-3", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-1", "test-workload-1", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-2", "test-workload-1", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-3", "test-workload-2", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("test-ns-2", "test-pod-4", "test-workload-3", "deployment").Set(1) + +// //wait for the metric to be scraped - scraping interval is 1s +// time.Sleep(5 * time.Second) + +// start := time.Now().Add(1 * time.Second) + +// By("creating first metric inside queryRange window") + +// kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-1", "test-workload-1", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-2", "test-workload-1", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-3", "test-workload-2", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("test-ns-2", "test-pod-4", "test-workload-3", "deployment").Set(1) + +// cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-1", "test-node-1", "test-container-1").Set(12) +// cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-2", "test-node-2", "test-container-1").Set(14) +// cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-3", "test-node-2", "test-container-1").Set(3) +// cpuUsageMetric.WithLabelValues("test-ns-2", "test-pod-4", "test-node-4", "test-container-1").Set(16) + +// //wait for the metric to be scraped - scraping interval is 1s +// time.Sleep(5 * time.Second) + +// By("creating second metric inside queryRange window") + +// cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-1", "test-node-1", "test-container-1").Set(5) +// cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-2", "test-node-2", "test-container-1").Set(4) +// cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-3", "test-node-2", "test-container-1").Set(12) +// cpuUsageMetric.WithLabelValues("test-ns-2", "test-pod-4", "test-node-4", "test-container-1").Set(15) + +// //wait for the metric to be scraped - scraping interval is 1s +// time.Sleep(5 * time.Second) + +// // data points after this should be outside the query range +// end := time.Now() + +// By("creating metric after queryRange window") + +// cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-1", "test-node-1", "test-container-1").Set(23) +// cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-2", "test-node-2", "test-container-1").Set(12) +// cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-3", "test-node-2", "test-container-1").Set(12) +// cpuUsageMetric.WithLabelValues("test-ns-2", "test-pod-4", "test-node-4", "test-container-1").Set(15) + +// //wait for the metric to be scraped - scraping interval is 1s +// time.Sleep(5 * time.Second) + +// dataPoints, err := scraper.GetAverageCPUUtilizationByWorkload("test-ns-1", +// "test-workload-1", start, end, time.Second) +// Expect(err).NotTo(HaveOccurred()) +// Expect(dataPoints).ToNot(BeEmpty()) + +// //since metrics could have been scraped multiple times, we just check the first and last value +// Expect(len(dataPoints) >= 2).To(BeTrue()) + +// Expect(dataPoints[0].Value).To(Equal(26.0)) +// Expect(dataPoints[len(dataPoints)-1].Value).To(Equal(9.0)) +// }) +// }) + +// Context("when querying GetACLByWorkload", func() { +// It("should return correct ACL", func() { + +// By("creating a metric before queryRange window") + +// podCreatedTimeMetric.WithLabelValues("test-ns-1", "test-pod-1").Set(45) +// podCreatedTimeMetric.WithLabelValues("test-ns-1", "test-pod-2").Set(55) +// podCreatedTimeMetric.WithLabelValues("test-ns-2", "test-pod-3").Set(65) +// podCreatedTimeMetric.WithLabelValues("test-ns-2", "test-pod-4").Set(75) +// podCreatedTimeMetric.WithLabelValues("test-ns-2", "test-pod-5").Set(80) + +// podReadyTimeMetric.WithLabelValues("test-ns-1", "test-pod-1").Set(50) +// podReadyTimeMetric.WithLabelValues("test-ns-1", "test-pod-2").Set(70) +// podReadyTimeMetric.WithLabelValues("test-ns-2", "test-pod-3").Set(80) +// podReadyTimeMetric.WithLabelValues("test-ns-2", "test-pod-4").Set(110) +// podReadyTimeMetric.WithLabelValues("test-ns-2", "test-pod-5").Set(120) + +// kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-1", "test-workload-1", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-2", "test-workload-1", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("test-ns-2", "test-pod-3", "test-workload-3", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("test-ns-2", "test-pod-4", "test-workload-3", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("test-ns-2", "test-pod-5", "test-workload-3", "deployment").Set(1) - podCreatedTimeMetric1.WithLabelValues("test-ns-1", "test-pod-1").Set(45) - podCreatedTimeMetric1.WithLabelValues("test-ns-1", "test-pod-2").Set(55) - podCreatedTimeMetric1.WithLabelValues("test-ns-2", "test-pod-3").Set(65) - podCreatedTimeMetric1.WithLabelValues("test-ns-2", "test-pod-4").Set(75) - podCreatedTimeMetric1.WithLabelValues("test-ns-2", "test-pod-5").Set(80) - - podReadyTimeMetric1.WithLabelValues("test-ns-1", "test-pod-1").Set(60) - podReadyTimeMetric1.WithLabelValues("test-ns-1", "test-pod-2").Set(70) - podReadyTimeMetric1.WithLabelValues("test-ns-2", "test-pod-3").Set(80) - podReadyTimeMetric1.WithLabelValues("test-ns-2", "test-pod-4").Set(100) - podReadyTimeMetric1.WithLabelValues("test-ns-2", "test-pod-5").Set(120) - - //wait for the metric to be scraped - scraping interval is 1s - time.Sleep(2 * time.Second) - - autoscalingLag1, err := scraper.GetACLByWorkload("test-ns-1", "test-workload-1") - Expect(err).NotTo(HaveOccurred()) - Expect(autoscalingLag1).To(Equal(45.0 * time.Second)) - - autoscalingLag2, err := scraper.GetACLByWorkload("test-ns-2", "test-workload-3") - Expect(err).NotTo(HaveOccurred()) - Expect(autoscalingLag2).To(Equal(65.0 * time.Second)) - }) - }) - - Context("when querying GetCPUUtilizationBreachDataPoints", func() { - It("should return correct data points when workload is a deployment", func() { - cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(14) - cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(3) - cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(5) - cpuUsageMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(3) - - kubePodOwnerMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-1", "deployment").Set(1) - kubePodOwnerMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-1", "deployment").Set(1) - kubePodOwnerMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-2", "deployment").Set(1) - kubePodOwnerMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-3", "deployment").Set(1) - - resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(5) - resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(5) - resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(5) - resourceLimitMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(5) - - readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-1").Set(1) - readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-2").Set(1) - readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-3").Set(1) - readyReplicasMetric.WithLabelValues("dep-test-ns-2", "dep-rs-4").Set(1) - - replicaSetOwnerMetric.WithLabelValues("dep-test-ns-1", "deployment", "dep-1", "dep-rs-1").Set(1) - replicaSetOwnerMetric.WithLabelValues("dep-test-ns-1", "deployment", "dep-1", "dep-rs-2").Set(1) - replicaSetOwnerMetric.WithLabelValues("dep-test-ns-1", "deployment", "dep-2", "dep-rs-3").Set(1) - replicaSetOwnerMetric.WithLabelValues("dep-test-ns-2", "deployment", "dep-1", "dep-rs-3").Set(1) - - hpaMaxReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-hpa1").Set(3) - hpaMaxReplicasMetric.WithLabelValues("dep-test-ns-2", "dep-hpa2").Set(3) - - hpaOwnerInfoMetric.WithLabelValues("dep-test-ns-1", "dep-hpa1", "deployment", "dep-1").Set(1) - hpaOwnerInfoMetric.WithLabelValues("dep-test-ns-1", "dep-hpa2", "deployment", "dep-2").Set(1) - - //wait for the metric to be scraped - scraping interval is 1s - time.Sleep(2 * time.Second) - - //above data points should be outside the query range. - start := time.Now() - - //This data point should be excluded as there are only 2 pods for dep-1. Utilization is 70% - - cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(4) - cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(3) - cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(5) - cpuUsageMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(3) - - kubePodOwnerMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-1", "deployment").Set(1) - kubePodOwnerMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-1", "deployment").Set(1) - kubePodOwnerMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-2", "deployment").Set(1) - kubePodOwnerMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-3", "deployment").Set(1) - - resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(5) - resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(5) - resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(5) - resourceLimitMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(5) - - readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-1").Set(1) - readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-2").Set(1) - readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-3").Set(1) - readyReplicasMetric.WithLabelValues("dep-test-ns-2", "dep-rs-4").Set(1) - - replicaSetOwnerMetric.WithLabelValues("dep-test-ns-1", "deployment", "dep-1", "dep-rs-1").Set(1) - replicaSetOwnerMetric.WithLabelValues("dep-test-ns-1", "deployment", "dep-1", "dep-rs-2").Set(1) - replicaSetOwnerMetric.WithLabelValues("dep-test-ns-1", "deployment", "dep-2", "dep-rs-3").Set(1) - replicaSetOwnerMetric.WithLabelValues("dep-test-ns-2", "deployment", "dep-1", "dep-rs-3").Set(1) - - hpaMaxReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-hpa1").Set(3) - hpaMaxReplicasMetric.WithLabelValues("dep-test-ns-2", "dep-hpa2").Set(3) - - hpaOwnerInfoMetric.WithLabelValues("dep-test-ns-1", "dep-hpa1", "deployment", "dep-1").Set(1) - hpaOwnerInfoMetric.WithLabelValues("dep-test-ns-1", "dep-hpa2", "deployment", "dep-2").Set(1) - - //wait for the metric to be scraped - scraping interval is 1s - time.Sleep(2 * time.Second) - - // this data point will be excluded as utilization(80%) for dep-1 is below threshold of 85% - cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(4) - cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(4) - cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(12) - cpuUsageMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(15) - - //wait for the metric to be scraped - scraping interval is 1s - time.Sleep(2 * time.Second) - - // this data point will be excluded as no of ready pods < maxReplicas(3) - cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(5) - cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(4) - cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(12) - cpuUsageMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(15) - - //wait for the metric to be scraped - scraping interval is 1s - time.Sleep(2 * time.Second) - - // this data point should be included - utilization of 100% and ready replicas(1+2) = maxReplicas(3) - cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(5) - cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(5) - cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-5", "dep-test-node-2", "dep-test-container-1").Set(5) - cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(12) - cpuUsageMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(15) - - readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-1").Set(1) - readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-2").Set(2) - readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-3").Set(1) - readyReplicasMetric.WithLabelValues("dep-test-ns-2", "dep-rs-4").Set(1) - - resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(5) - resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(5) - resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(5) - resourceLimitMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(5) - resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-5", "dep-test-node-2", "dep-test-container-1").Set(5) - - //wait for the metric to be scraped - scraping interval is 1s - time.Sleep(2 * time.Second) - - // data points after this should be outside the query range - end := time.Now() - - cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(10) - cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(10) - cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-5", "dep-test-node-2", "dep-test-container-1").Set(10) - cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(12) - cpuUsageMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(15) - - //wait for the metric to be scraped - scraping interval is 1s - time.Sleep(2 * time.Second) - - dataPoints, err := scraper.GetCPUUtilizationBreachDataPoints("dep-test-ns-1", - "deployment", - "dep-1", - 0.85, start, - end, - time.Second) - Expect(err).NotTo(HaveOccurred()) - Expect(dataPoints).ToNot(BeEmpty()) - - //since metrics could have been scraped multiple times, we just check the first and last value - Expect(len(dataPoints) >= 1).To(BeTrue()) - for _, datapoint := range dataPoints { - Expect(datapoint.Value).To(Or(Equal(1.7), Equal(0.9))) - } - - }) - - It("should return correct data points when workload is a Rollout", func() { - cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(14) - cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(3) - cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(5) - cpuUsageMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(3) - - kubePodOwnerMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-1", "deployment").Set(1) - kubePodOwnerMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-1", "deployment").Set(1) - kubePodOwnerMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-2", "deployment").Set(1) - kubePodOwnerMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-3", "deployment").Set(1) - - resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(5) - resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(5) - resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(5) - resourceLimitMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(5) - - readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-1").Set(1) - readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-2").Set(1) - readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-3").Set(1) - readyReplicasMetric.WithLabelValues("ro-test-ns-2", "ro-rs-4").Set(1) - - replicaSetOwnerMetric.WithLabelValues("ro-test-ns-1", "Rollout", "ro-1", "ro-rs-1").Set(1) - replicaSetOwnerMetric.WithLabelValues("ro-test-ns-1", "Rollout", "ro-1", "ro-rs-2").Set(1) - replicaSetOwnerMetric.WithLabelValues("ro-test-ns-1", "Rollout", "ro-2", "ro-rs-3").Set(1) - replicaSetOwnerMetric.WithLabelValues("ro-test-ns-2", "Rollout", "ro-1", "ro-rs-3").Set(1) - - hpaMaxReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-hpa1").Set(3) - hpaMaxReplicasMetric.WithLabelValues("ro-test-ns-2", "ro-hpa2").Set(3) - - hpaOwnerInfoMetric.WithLabelValues("ro-test-ns-1", "ro-hpa1", "Rollout", "ro-1").Set(1) - hpaOwnerInfoMetric.WithLabelValues("ro-test-ns-1", "ro-hpa2", "Rollout", "ro-2").Set(1) - - //wait for the metric to be scraped - scraping interval is 1s - time.Sleep(2 * time.Second) - - //above data points should be outside the query range. - start := time.Now() - - //This data point should be excluded as there are only 2 pods for ro-1. Utilization is 70% - - cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(4) - cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(3) - cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(5) - cpuUsageMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(3) - - kubePodOwnerMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-1", "deployment").Set(1) - kubePodOwnerMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-1", "deployment").Set(1) - kubePodOwnerMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-2", "deployment").Set(1) - kubePodOwnerMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-3", "deployment").Set(1) - - resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(5) - resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(5) - resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(5) - resourceLimitMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(5) - - readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-1").Set(1) - readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-2").Set(1) - readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-3").Set(1) - readyReplicasMetric.WithLabelValues("ro-test-ns-2", "ro-rs-4").Set(1) - - replicaSetOwnerMetric.WithLabelValues("ro-test-ns-1", "Rollout", "ro-1", "ro-rs-1").Set(1) - replicaSetOwnerMetric.WithLabelValues("ro-test-ns-1", "Rollout", "ro-1", "ro-rs-2").Set(1) - replicaSetOwnerMetric.WithLabelValues("ro-test-ns-1", "Rollout", "ro-2", "ro-rs-3").Set(1) - replicaSetOwnerMetric.WithLabelValues("ro-test-ns-2", "Rollout", "ro-1", "ro-rs-3").Set(1) - - hpaMaxReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-hpa1").Set(3) - hpaMaxReplicasMetric.WithLabelValues("ro-test-ns-2", "ro-hpa2").Set(3) - - hpaOwnerInfoMetric.WithLabelValues("ro-test-ns-1", "ro-hpa1", "Rollout", "ro-1").Set(1) - hpaOwnerInfoMetric.WithLabelValues("ro-test-ns-1", "ro-hpa2", "Rollout", "ro-2").Set(1) - - //wait for the metric to be scraped - scraping interval is 1s - time.Sleep(2 * time.Second) - - // this data point will be excluded as utilization(80%) for ro-1 is below threshold of 85% - cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(4) - cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(4) - cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(12) - cpuUsageMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(15) - - //wait for the metric to be scraped - scraping interval is 1s - time.Sleep(2 * time.Second) - - // this data point will be excluded as no of ready pods < maxReplicas(3) - cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(5) - cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(4) - cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(12) - cpuUsageMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(15) - - //wait for the metric to be scraped - scraping interval is 1s - time.Sleep(2 * time.Second) - - // this data point should be included - utilization of 100% and ready replicas(1+2) = maxReplicas(3) - cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(5) - cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(5) - cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-5", "ro-test-node-2", "ro-test-container-1").Set(5) - cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(12) - cpuUsageMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(15) - - readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-1").Set(1) - readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-2").Set(2) - readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-3").Set(1) - readyReplicasMetric.WithLabelValues("ro-test-ns-2", "ro-rs-4").Set(1) - - resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(5) - resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(5) - resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(5) - resourceLimitMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(5) - resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-5", "ro-test-node-2", "ro-test-container-1").Set(5) - - //wait for the metric to be scraped - scraping interval is 1s - time.Sleep(2 * time.Second) - - // data points after this should be outside the query range - end := time.Now() - - cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(10) - cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(10) - cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-5", "ro-test-node-2", "ro-test-container-1").Set(10) - cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(12) - cpuUsageMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(15) - - //wait for the metric to be scraped - scraping interval is 1s - time.Sleep(2 * time.Second) - - dataPoints, err := scraper.GetCPUUtilizationBreachDataPoints("ro-test-ns-1", - "Rollout", - "ro-1", - 0.85, start, - end, - time.Second) - Expect(err).NotTo(HaveOccurred()) - Expect(dataPoints).ToNot(BeEmpty()) - - //since metrics could have been scraped multiple times, we just check the first and last value - Expect(len(dataPoints) >= 1).To(BeTrue()) - for _, datapoint := range dataPoints { - Expect(datapoint.Value).To(Or(Equal(1.7), Equal(0.9))) - } - }) - }) - - Context("when querying GetAverageCPUUtilizationByWorkload with two prometheus instances", func() { - It("should return correct data points", func() { - - By("creating a metric before queryRange window") - cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-1", "test-node-1", "test-container-1").Set(4) - cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-2", "test-node-2", "test-container-1").Set(3) - cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-3", "test-node-2", "test-container-1").Set(5) - cpuUsageMetric.WithLabelValues("test-nsp-2", "test-pod-4", "test-node-4", "test-container-1").Set(20) - - cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-1", "test-node-1", "test-container-1").Set(4) - cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-2", "test-node-2", "test-container-1").Set(3) - cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-3", "test-node-2", "test-container-1").Set(5) - cpuUsageMetric1.WithLabelValues("test-nsp-2", "test-pod-4", "test-node-4", "test-container-1").Set(20) - - kubePodOwnerMetric.WithLabelValues("test-nsp-1", "test-pod-1", "test-workload-1", "deployment").Set(1) - kubePodOwnerMetric.WithLabelValues("test-nsp-1", "test-pod-2", "test-workload-1", "deployment").Set(1) - kubePodOwnerMetric.WithLabelValues("test-nsp-1", "test-pod-3", "test-workload-2", "deployment").Set(1) - kubePodOwnerMetric.WithLabelValues("test-nsp-2", "test-pod-4", "test-workload-3", "deployment").Set(1) - - //wait for the metric to be scraped - scraping interval is 1s - time.Sleep(5 * time.Second) - - start := time.Now().Add(1 * time.Second) - - By("creating first metric inside queryRange window") - - kubePodOwnerMetric.WithLabelValues("test-nsp-1", "test-pod-1", "test-workload-1", "deployment").Set(1) - kubePodOwnerMetric.WithLabelValues("test-nsp-1", "test-pod-2", "test-workload-1", "deployment").Set(1) - kubePodOwnerMetric.WithLabelValues("test-nsp-1", "test-pod-3", "test-workload-2", "deployment").Set(1) - kubePodOwnerMetric.WithLabelValues("test-nsp-2", "test-pod-4", "test-workload-3", "deployment").Set(1) - - cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-1", "test-node-1", "test-container-1").Set(12) - cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-2", "test-node-2", "test-container-1").Set(14) - cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-3", "test-node-2", "test-container-1").Set(3) - cpuUsageMetric.WithLabelValues("test-nsp-2", "test-pod-4", "test-node-4", "test-container-1").Set(16) - - cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-1", "test-node-1", "test-container-1").Set(12) - cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-2", "test-node-2", "test-container-1").Set(14) - cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-3", "test-node-2", "test-container-1").Set(3) - cpuUsageMetric1.WithLabelValues("test-nsp-2", "test-pod-4", "test-node-4", "test-container-1").Set(16) - - //wait for the metric to be scraped - scraping interval is 1s - time.Sleep(5 * time.Second) - - By("creating second metric inside queryRange window") - - cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-1", "test-node-1", "test-container-1").Set(5) - cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-2", "test-node-2", "test-container-1").Set(4) - cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-3", "test-node-2", "test-container-1").Set(12) - cpuUsageMetric.WithLabelValues("test-nsp-2", "test-pod-4", "test-node-4", "test-container-1").Set(15) - - cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-1", "test-node-1", "test-container-1").Set(5) - cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-2", "test-node-2", "test-container-1").Set(5) - cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-3", "test-node-2", "test-container-1").Set(12) - cpuUsageMetric1.WithLabelValues("test-nsp-2", "test-pod-4", "test-node-4", "test-container-1").Set(15) - - //wait for the metric to be scraped - scraping interval is 1s - time.Sleep(5 * time.Second) - - // data points after this should be outside the query range - end := time.Now() - - By("creating metric after queryRange window") - - cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-1", "test-node-1", "test-container-1").Set(23) - cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-2", "test-node-2", "test-container-1").Set(12) - cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-3", "test-node-2", "test-container-1").Set(12) - cpuUsageMetric.WithLabelValues("test-nsp-2", "test-pod-4", "test-node-4", "test-container-1").Set(15) - - //wait for the metric to be scraped - scraping interval is 1s - time.Sleep(5 * time.Second) - - dataPoints, err := scraper.GetAverageCPUUtilizationByWorkload("test-nsp-1", - "test-workload-1", start, end, time.Second) - fmt.Println(dataPoints) - Expect(err).NotTo(HaveOccurred()) - Expect(dataPoints).ToNot(BeEmpty()) - - //since metrics could have been scraped multiple times, we just check the first and last value - Expect(len(dataPoints) >= 2).To(BeTrue()) - - Expect(dataPoints[0].Value).To(Equal(26.0)) - Expect(dataPoints[len(dataPoints)-1].Value).To(Equal(10.0)) - }) - }) -}) - -var _ = Describe("mergeMatrices", func() { - It("should correctly merge two matrices", func() { - matrix1 := model.Matrix{ - &model.SampleStream{ - Metric: model.Metric{"label": "test"}, - Values: []model.SamplePair{ - {Timestamp: 100, Value: 1}, - {Timestamp: 200, Value: 2}, - }, - }, - } - - matrix2 := model.Matrix{ - &model.SampleStream{ - Metric: model.Metric{"label": "test"}, - Values: []model.SamplePair{ - {Timestamp: 300, Value: 3}, - {Timestamp: 400, Value: 4}, - }, - }, - } - - expectedMergedMatrix := model.Matrix{ - &model.SampleStream{ - Metric: model.Metric{"label": "test"}, - Values: []model.SamplePair{ - {Timestamp: 100, Value: 1}, - {Timestamp: 200, Value: 2}, - {Timestamp: 300, Value: 3}, - {Timestamp: 400, Value: 4}, - }, - }, - } - - mergedMatrix := mergeMatrices(matrix1, matrix2) - Expect(mergedMatrix).To(Equal(expectedMergedMatrix)) - }) -}) - -type mockAPI struct { - v1.API - queryRangeFunc func(ctx context.Context, query string, r v1.Range, options ...v1.Option) (model.Value, - v1.Warnings, error) -} - -func (m *mockAPI) QueryRange(ctx context.Context, query string, r v1.Range, options ...v1.Option) (model.Value, - v1.Warnings, error) { - return m.queryRangeFunc(ctx, query, r) -} - -var _ = Describe("RangeQuerySplitter", func() { - It("should split and query correctly by duration", func() { - query := "test_query" - start := time.Now().Add(-5 * time.Minute) - end := time.Now() - step := 1 * time.Minute - splitDuration := 2 * time.Minute - - mockApi := &mockAPI{ - queryRangeFunc: func(ctx context.Context, query string, r v1.Range, options ...v1.Option) (model.Value, - v1.Warnings, error) { - matrix := model.Matrix{ - &model.SampleStream{ - Metric: model.Metric{"label": "test"}, - Values: []model.SamplePair{ - {Timestamp: model.TimeFromUnix(r.Start.Unix()), Value: 1}, - {Timestamp: model.TimeFromUnix(r.End.Unix()), Value: 2}, - }, - }, - } - return matrix, nil, nil - }, - } - - splitter := NewRangeQuerySplitter(splitDuration) - pi := PrometheusInstance{apiUrl: mockApi, address: ""} - result, err := splitter.QueryRangeByInterval(context.TODO(), pi, query, start, end, step) - Expect(err).NotTo(HaveOccurred()) - Expect(result.Type()).To(Equal(model.ValMatrix)) - - matrix := result.(model.Matrix) - Expect(len(matrix)).To(Equal(1)) - Expect(len(matrix[0].Values)).To(Equal(6)) - }) -}) - -var _ = Describe("interpolateMissingDataPoints", func() { - It("should interpolate the missing data", func() { - dataPoints := []DataPoint{ - {Timestamp: time.Now().Add(-30 * time.Minute), Value: 60}, - {Timestamp: time.Now().Add(-29 * time.Minute), Value: 80}, - {Timestamp: time.Now().Add(-28 * time.Minute), Value: 100}, - {Timestamp: time.Now().Add(-27 * time.Minute), Value: 50}, - {Timestamp: time.Now().Add(-26 * time.Minute), Value: 30}, - {Timestamp: time.Now().Add(-25 * time.Minute), Value: 60}, - {Timestamp: time.Now().Add(-24 * time.Minute), Value: 80}, - {Timestamp: time.Now().Add(-20 * time.Minute), Value: 60}, - {Timestamp: time.Now().Add(-19 * time.Minute), Value: 80}, - {Timestamp: time.Now().Add(-18 * time.Minute), Value: 100}, - {Timestamp: time.Now().Add(-17 * time.Minute), Value: 50}, - {Timestamp: time.Now().Add(-16 * time.Minute), Value: 30}, - {Timestamp: time.Now().Add(-9 * time.Minute), Value: 80}, - {Timestamp: time.Now().Add(-8 * time.Minute), Value: 100}, - {Timestamp: time.Now().Add(-7 * time.Minute), Value: 50}, - {Timestamp: time.Now().Add(-6 * time.Minute), Value: 30}, - } - dataPoints = scraper.interpolateMissingDataPoints(dataPoints, time.Minute) - Expect(len(dataPoints)).To(Equal(25)) - Expect(dataPoints[7].Value).To(Equal(75.0)) - Expect(dataPoints[8].Value).To(Equal(70.0)) - Expect(math.Floor(dataPoints[15].Value*100) / 100).To(Equal(37.14)) - Expect(math.Floor(dataPoints[20].Value*100) / 100).To(Equal(72.85)) - }) -}) - -var _ = Describe("aggregateMetrics", func() { - It("should aggregate the metrics from different sources", func() { - time1 := time.Now().Add(-30 * time.Minute) - time2 := time.Now().Add(-29 * time.Minute) - time3 := time.Now().Add(-28 * time.Minute) - time4 := time.Now().Add(-27 * time.Minute) - time5 := time.Now().Add(-26 * time.Minute) - time6 := time.Now().Add(-25 * time.Minute) - time7 := time.Now().Add(-24 * time.Minute) - time8 := time.Now().Add(-20 * time.Minute) - time9 := time.Now().Add(-19 * time.Minute) - time10 := time.Now().Add(-18 * time.Minute) - time11 := time.Now().Add(-17 * time.Minute) - time12 := time.Now().Add(-16 * time.Minute) - dataPoints1 := []DataPoint{ - {Timestamp: time1, Value: 60}, - {Timestamp: time2, Value: 80}, - {Timestamp: time3, Value: 100}, - {Timestamp: time4, Value: 50}, - {Timestamp: time5, Value: 30}, - {Timestamp: time6, Value: 60}, - {Timestamp: time7, Value: 80}, - {Timestamp: time8, Value: 60}, - {Timestamp: time10, Value: 80}, - {Timestamp: time11, Value: 100}, - } - dataPoints2 := []DataPoint{ - {Timestamp: time1, Value: 30}, - {Timestamp: time2, Value: 80}, - {Timestamp: time3, Value: 100}, - {Timestamp: time5, Value: 30}, - {Timestamp: time6, Value: 60}, - {Timestamp: time7, Value: 100}, - {Timestamp: time9, Value: 80}, - {Timestamp: time10, Value: 80}, - {Timestamp: time11, Value: 100}, - {Timestamp: time12, Value: 100}, - } - dataPoints := aggregateMetrics(dataPoints1, dataPoints2) - fmt.Println(dataPoints) - Expect(len(dataPoints)).To(Equal(12)) - Expect(dataPoints[0].Value).To(Equal(60.0)) - Expect(dataPoints[6].Value).To(Equal(100.0)) - Expect(dataPoints[11].Value).To(Equal(100.0)) - }) -}) +// podCreatedTimeMetric1.WithLabelValues("test-ns-1", "test-pod-1").Set(45) +// podCreatedTimeMetric1.WithLabelValues("test-ns-1", "test-pod-2").Set(55) +// podCreatedTimeMetric1.WithLabelValues("test-ns-2", "test-pod-3").Set(65) +// podCreatedTimeMetric1.WithLabelValues("test-ns-2", "test-pod-4").Set(75) +// podCreatedTimeMetric1.WithLabelValues("test-ns-2", "test-pod-5").Set(80) + +// podReadyTimeMetric1.WithLabelValues("test-ns-1", "test-pod-1").Set(60) +// podReadyTimeMetric1.WithLabelValues("test-ns-1", "test-pod-2").Set(70) +// podReadyTimeMetric1.WithLabelValues("test-ns-2", "test-pod-3").Set(80) +// podReadyTimeMetric1.WithLabelValues("test-ns-2", "test-pod-4").Set(100) +// podReadyTimeMetric1.WithLabelValues("test-ns-2", "test-pod-5").Set(120) + +// //wait for the metric to be scraped - scraping interval is 1s +// time.Sleep(2 * time.Second) + +// autoscalingLag1, err := scraper.GetACLByWorkload("test-ns-1", "test-workload-1") +// Expect(err).NotTo(HaveOccurred()) +// Expect(autoscalingLag1).To(Equal(45.0 * time.Second)) + +// autoscalingLag2, err := scraper.GetACLByWorkload("test-ns-2", "test-workload-3") +// Expect(err).NotTo(HaveOccurred()) +// Expect(autoscalingLag2).To(Equal(65.0 * time.Second)) +// }) +// }) + +// Context("when querying GetCPUUtilizationBreachDataPoints", func() { +// It("should return correct data points when workload is a deployment", func() { +// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(14) +// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(3) +// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(5) +// cpuUsageMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(3) + +// kubePodOwnerMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-1", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-1", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-2", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-3", "deployment").Set(1) + +// resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(5) +// resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(5) +// resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(5) +// resourceLimitMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(5) + +// readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-1").Set(1) +// readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-2").Set(1) +// readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-3").Set(1) +// readyReplicasMetric.WithLabelValues("dep-test-ns-2", "dep-rs-4").Set(1) + +// replicaSetOwnerMetric.WithLabelValues("dep-test-ns-1", "deployment", "dep-1", "dep-rs-1").Set(1) +// replicaSetOwnerMetric.WithLabelValues("dep-test-ns-1", "deployment", "dep-1", "dep-rs-2").Set(1) +// replicaSetOwnerMetric.WithLabelValues("dep-test-ns-1", "deployment", "dep-2", "dep-rs-3").Set(1) +// replicaSetOwnerMetric.WithLabelValues("dep-test-ns-2", "deployment", "dep-1", "dep-rs-3").Set(1) + +// hpaMaxReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-hpa1").Set(3) +// hpaMaxReplicasMetric.WithLabelValues("dep-test-ns-2", "dep-hpa2").Set(3) + +// hpaOwnerInfoMetric.WithLabelValues("dep-test-ns-1", "dep-hpa1", "deployment", "dep-1").Set(1) +// hpaOwnerInfoMetric.WithLabelValues("dep-test-ns-1", "dep-hpa2", "deployment", "dep-2").Set(1) + +// //wait for the metric to be scraped - scraping interval is 1s +// time.Sleep(2 * time.Second) + +// //above data points should be outside the query range. +// start := time.Now() + +// //This data point should be excluded as there are only 2 pods for dep-1. Utilization is 70% + +// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(4) +// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(3) +// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(5) +// cpuUsageMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(3) + +// kubePodOwnerMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-1", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-1", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-2", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-3", "deployment").Set(1) + +// resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(5) +// resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(5) +// resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(5) +// resourceLimitMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(5) + +// readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-1").Set(1) +// readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-2").Set(1) +// readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-3").Set(1) +// readyReplicasMetric.WithLabelValues("dep-test-ns-2", "dep-rs-4").Set(1) + +// replicaSetOwnerMetric.WithLabelValues("dep-test-ns-1", "deployment", "dep-1", "dep-rs-1").Set(1) +// replicaSetOwnerMetric.WithLabelValues("dep-test-ns-1", "deployment", "dep-1", "dep-rs-2").Set(1) +// replicaSetOwnerMetric.WithLabelValues("dep-test-ns-1", "deployment", "dep-2", "dep-rs-3").Set(1) +// replicaSetOwnerMetric.WithLabelValues("dep-test-ns-2", "deployment", "dep-1", "dep-rs-3").Set(1) + +// hpaMaxReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-hpa1").Set(3) +// hpaMaxReplicasMetric.WithLabelValues("dep-test-ns-2", "dep-hpa2").Set(3) + +// hpaOwnerInfoMetric.WithLabelValues("dep-test-ns-1", "dep-hpa1", "deployment", "dep-1").Set(1) +// hpaOwnerInfoMetric.WithLabelValues("dep-test-ns-1", "dep-hpa2", "deployment", "dep-2").Set(1) + +// //wait for the metric to be scraped - scraping interval is 1s +// time.Sleep(2 * time.Second) + +// // this data point will be excluded as utilization(80%) for dep-1 is below threshold of 85% +// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(4) +// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(4) +// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(12) +// cpuUsageMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(15) + +// //wait for the metric to be scraped - scraping interval is 1s +// time.Sleep(2 * time.Second) + +// // this data point will be excluded as no of ready pods < maxReplicas(3) +// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(5) +// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(4) +// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(12) +// cpuUsageMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(15) + +// //wait for the metric to be scraped - scraping interval is 1s +// time.Sleep(2 * time.Second) + +// // this data point should be included - utilization of 100% and ready replicas(1+2) = maxReplicas(3) +// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(5) +// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(5) +// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-5", "dep-test-node-2", "dep-test-container-1").Set(5) +// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(12) +// cpuUsageMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(15) + +// readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-1").Set(1) +// readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-2").Set(2) +// readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-3").Set(1) +// readyReplicasMetric.WithLabelValues("dep-test-ns-2", "dep-rs-4").Set(1) + +// resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(5) +// resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(5) +// resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(5) +// resourceLimitMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(5) +// resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-5", "dep-test-node-2", "dep-test-container-1").Set(5) + +// //wait for the metric to be scraped - scraping interval is 1s +// time.Sleep(2 * time.Second) + +// // data points after this should be outside the query range +// end := time.Now() + +// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(10) +// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(10) +// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-5", "dep-test-node-2", "dep-test-container-1").Set(10) +// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(12) +// cpuUsageMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(15) + +// //wait for the metric to be scraped - scraping interval is 1s +// time.Sleep(2 * time.Second) + +// dataPoints, err := scraper.GetCPUUtilizationBreachDataPoints("dep-test-ns-1", +// "deployment", +// "dep-1", +// 0.85, start, +// end, +// time.Second) +// Expect(err).NotTo(HaveOccurred()) +// Expect(dataPoints).ToNot(BeEmpty()) + +// //since metrics could have been scraped multiple times, we just check the first and last value +// Expect(len(dataPoints) >= 1).To(BeTrue()) +// for _, datapoint := range dataPoints { +// Expect(datapoint.Value).To(Or(Equal(1.7), Equal(0.9))) +// } + +// }) + +// It("should return correct data points when workload is a Rollout", func() { +// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(14) +// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(3) +// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(5) +// cpuUsageMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(3) + +// kubePodOwnerMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-1", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-1", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-2", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-3", "deployment").Set(1) + +// resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(5) +// resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(5) +// resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(5) +// resourceLimitMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(5) + +// readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-1").Set(1) +// readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-2").Set(1) +// readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-3").Set(1) +// readyReplicasMetric.WithLabelValues("ro-test-ns-2", "ro-rs-4").Set(1) + +// replicaSetOwnerMetric.WithLabelValues("ro-test-ns-1", "Rollout", "ro-1", "ro-rs-1").Set(1) +// replicaSetOwnerMetric.WithLabelValues("ro-test-ns-1", "Rollout", "ro-1", "ro-rs-2").Set(1) +// replicaSetOwnerMetric.WithLabelValues("ro-test-ns-1", "Rollout", "ro-2", "ro-rs-3").Set(1) +// replicaSetOwnerMetric.WithLabelValues("ro-test-ns-2", "Rollout", "ro-1", "ro-rs-3").Set(1) + +// hpaMaxReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-hpa1").Set(3) +// hpaMaxReplicasMetric.WithLabelValues("ro-test-ns-2", "ro-hpa2").Set(3) + +// hpaOwnerInfoMetric.WithLabelValues("ro-test-ns-1", "ro-hpa1", "Rollout", "ro-1").Set(1) +// hpaOwnerInfoMetric.WithLabelValues("ro-test-ns-1", "ro-hpa2", "Rollout", "ro-2").Set(1) + +// //wait for the metric to be scraped - scraping interval is 1s +// time.Sleep(2 * time.Second) + +// //above data points should be outside the query range. +// start := time.Now() + +// //This data point should be excluded as there are only 2 pods for ro-1. Utilization is 70% + +// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(4) +// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(3) +// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(5) +// cpuUsageMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(3) + +// kubePodOwnerMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-1", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-1", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-2", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-3", "deployment").Set(1) + +// resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(5) +// resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(5) +// resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(5) +// resourceLimitMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(5) + +// readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-1").Set(1) +// readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-2").Set(1) +// readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-3").Set(1) +// readyReplicasMetric.WithLabelValues("ro-test-ns-2", "ro-rs-4").Set(1) + +// replicaSetOwnerMetric.WithLabelValues("ro-test-ns-1", "Rollout", "ro-1", "ro-rs-1").Set(1) +// replicaSetOwnerMetric.WithLabelValues("ro-test-ns-1", "Rollout", "ro-1", "ro-rs-2").Set(1) +// replicaSetOwnerMetric.WithLabelValues("ro-test-ns-1", "Rollout", "ro-2", "ro-rs-3").Set(1) +// replicaSetOwnerMetric.WithLabelValues("ro-test-ns-2", "Rollout", "ro-1", "ro-rs-3").Set(1) + +// hpaMaxReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-hpa1").Set(3) +// hpaMaxReplicasMetric.WithLabelValues("ro-test-ns-2", "ro-hpa2").Set(3) + +// hpaOwnerInfoMetric.WithLabelValues("ro-test-ns-1", "ro-hpa1", "Rollout", "ro-1").Set(1) +// hpaOwnerInfoMetric.WithLabelValues("ro-test-ns-1", "ro-hpa2", "Rollout", "ro-2").Set(1) + +// //wait for the metric to be scraped - scraping interval is 1s +// time.Sleep(2 * time.Second) + +// // this data point will be excluded as utilization(80%) for ro-1 is below threshold of 85% +// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(4) +// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(4) +// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(12) +// cpuUsageMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(15) + +// //wait for the metric to be scraped - scraping interval is 1s +// time.Sleep(2 * time.Second) + +// // this data point will be excluded as no of ready pods < maxReplicas(3) +// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(5) +// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(4) +// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(12) +// cpuUsageMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(15) + +// //wait for the metric to be scraped - scraping interval is 1s +// time.Sleep(2 * time.Second) + +// // this data point should be included - utilization of 100% and ready replicas(1+2) = maxReplicas(3) +// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(5) +// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(5) +// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-5", "ro-test-node-2", "ro-test-container-1").Set(5) +// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(12) +// cpuUsageMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(15) + +// readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-1").Set(1) +// readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-2").Set(2) +// readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-3").Set(1) +// readyReplicasMetric.WithLabelValues("ro-test-ns-2", "ro-rs-4").Set(1) + +// resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(5) +// resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(5) +// resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(5) +// resourceLimitMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(5) +// resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-5", "ro-test-node-2", "ro-test-container-1").Set(5) + +// //wait for the metric to be scraped - scraping interval is 1s +// time.Sleep(2 * time.Second) + +// // data points after this should be outside the query range +// end := time.Now() + +// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(10) +// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(10) +// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-5", "ro-test-node-2", "ro-test-container-1").Set(10) +// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(12) +// cpuUsageMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(15) + +// //wait for the metric to be scraped - scraping interval is 1s +// time.Sleep(2 * time.Second) + +// dataPoints, err := scraper.GetCPUUtilizationBreachDataPoints("ro-test-ns-1", +// "Rollout", +// "ro-1", +// 0.85, start, +// end, +// time.Second) +// Expect(err).NotTo(HaveOccurred()) +// Expect(dataPoints).ToNot(BeEmpty()) + +// //since metrics could have been scraped multiple times, we just check the first and last value +// Expect(len(dataPoints) >= 1).To(BeTrue()) +// for _, datapoint := range dataPoints { +// Expect(datapoint.Value).To(Or(Equal(1.7), Equal(0.9))) +// } +// }) +// }) + +// Context("when querying GetAverageCPUUtilizationByWorkload with two prometheus instances", func() { +// It("should return correct data points", func() { + +// By("creating a metric before queryRange window") +// cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-1", "test-node-1", "test-container-1").Set(4) +// cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-2", "test-node-2", "test-container-1").Set(3) +// cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-3", "test-node-2", "test-container-1").Set(5) +// cpuUsageMetric.WithLabelValues("test-nsp-2", "test-pod-4", "test-node-4", "test-container-1").Set(20) + +// cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-1", "test-node-1", "test-container-1").Set(4) +// cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-2", "test-node-2", "test-container-1").Set(3) +// cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-3", "test-node-2", "test-container-1").Set(5) +// cpuUsageMetric1.WithLabelValues("test-nsp-2", "test-pod-4", "test-node-4", "test-container-1").Set(20) + +// kubePodOwnerMetric.WithLabelValues("test-nsp-1", "test-pod-1", "test-workload-1", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("test-nsp-1", "test-pod-2", "test-workload-1", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("test-nsp-1", "test-pod-3", "test-workload-2", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("test-nsp-2", "test-pod-4", "test-workload-3", "deployment").Set(1) + +// //wait for the metric to be scraped - scraping interval is 1s +// time.Sleep(5 * time.Second) + +// start := time.Now().Add(1 * time.Second) + +// By("creating first metric inside queryRange window") + +// kubePodOwnerMetric.WithLabelValues("test-nsp-1", "test-pod-1", "test-workload-1", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("test-nsp-1", "test-pod-2", "test-workload-1", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("test-nsp-1", "test-pod-3", "test-workload-2", "deployment").Set(1) +// kubePodOwnerMetric.WithLabelValues("test-nsp-2", "test-pod-4", "test-workload-3", "deployment").Set(1) + +// cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-1", "test-node-1", "test-container-1").Set(12) +// cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-2", "test-node-2", "test-container-1").Set(14) +// cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-3", "test-node-2", "test-container-1").Set(3) +// cpuUsageMetric.WithLabelValues("test-nsp-2", "test-pod-4", "test-node-4", "test-container-1").Set(16) + +// cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-1", "test-node-1", "test-container-1").Set(12) +// cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-2", "test-node-2", "test-container-1").Set(14) +// cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-3", "test-node-2", "test-container-1").Set(3) +// cpuUsageMetric1.WithLabelValues("test-nsp-2", "test-pod-4", "test-node-4", "test-container-1").Set(16) + +// //wait for the metric to be scraped - scraping interval is 1s +// time.Sleep(5 * time.Second) + +// By("creating second metric inside queryRange window") + +// cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-1", "test-node-1", "test-container-1").Set(5) +// cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-2", "test-node-2", "test-container-1").Set(4) +// cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-3", "test-node-2", "test-container-1").Set(12) +// cpuUsageMetric.WithLabelValues("test-nsp-2", "test-pod-4", "test-node-4", "test-container-1").Set(15) + +// cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-1", "test-node-1", "test-container-1").Set(5) +// cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-2", "test-node-2", "test-container-1").Set(5) +// cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-3", "test-node-2", "test-container-1").Set(12) +// cpuUsageMetric1.WithLabelValues("test-nsp-2", "test-pod-4", "test-node-4", "test-container-1").Set(15) + +// //wait for the metric to be scraped - scraping interval is 1s +// time.Sleep(5 * time.Second) + +// // data points after this should be outside the query range +// end := time.Now() + +// By("creating metric after queryRange window") + +// cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-1", "test-node-1", "test-container-1").Set(23) +// cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-2", "test-node-2", "test-container-1").Set(12) +// cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-3", "test-node-2", "test-container-1").Set(12) +// cpuUsageMetric.WithLabelValues("test-nsp-2", "test-pod-4", "test-node-4", "test-container-1").Set(15) + +// //wait for the metric to be scraped - scraping interval is 1s +// time.Sleep(5 * time.Second) + +// dataPoints, err := scraper.GetAverageCPUUtilizationByWorkload("test-nsp-1", +// "test-workload-1", start, end, time.Second) +// fmt.Println(dataPoints) +// Expect(err).NotTo(HaveOccurred()) +// Expect(dataPoints).ToNot(BeEmpty()) + +// //since metrics could have been scraped multiple times, we just check the first and last value +// Expect(len(dataPoints) >= 2).To(BeTrue()) + +// Expect(dataPoints[0].Value).To(Equal(26.0)) +// Expect(dataPoints[len(dataPoints)-1].Value).To(Equal(10.0)) +// }) +// }) +// }) + +// var _ = Describe("mergeMatrices", func() { +// It("should correctly merge two matrices", func() { +// matrix1 := model.Matrix{ +// &model.SampleStream{ +// Metric: model.Metric{"label": "test"}, +// Values: []model.SamplePair{ +// {Timestamp: 100, Value: 1}, +// {Timestamp: 200, Value: 2}, +// }, +// }, +// } + +// matrix2 := model.Matrix{ +// &model.SampleStream{ +// Metric: model.Metric{"label": "test"}, +// Values: []model.SamplePair{ +// {Timestamp: 300, Value: 3}, +// {Timestamp: 400, Value: 4}, +// }, +// }, +// } + +// expectedMergedMatrix := model.Matrix{ +// &model.SampleStream{ +// Metric: model.Metric{"label": "test"}, +// Values: []model.SamplePair{ +// {Timestamp: 100, Value: 1}, +// {Timestamp: 200, Value: 2}, +// {Timestamp: 300, Value: 3}, +// {Timestamp: 400, Value: 4}, +// }, +// }, +// } + +// mergedMatrix := mergeMatrices(matrix1, matrix2) +// Expect(mergedMatrix).To(Equal(expectedMergedMatrix)) +// }) +// }) + +// type mockAPI struct { +// v1.API +// queryRangeFunc func(ctx context.Context, query string, r v1.Range, options ...v1.Option) (model.Value, +// v1.Warnings, error) +// } + +// func (m *mockAPI) QueryRange(ctx context.Context, query string, r v1.Range, options ...v1.Option) (model.Value, +// v1.Warnings, error) { +// return m.queryRangeFunc(ctx, query, r) +// } + +// var _ = Describe("RangeQuerySplitter", func() { +// It("should split and query correctly by duration", func() { +// query := "test_query" +// start := time.Now().Add(-5 * time.Minute) +// end := time.Now() +// step := 1 * time.Minute +// splitDuration := 2 * time.Minute + +// mockApi := &mockAPI{ +// queryRangeFunc: func(ctx context.Context, query string, r v1.Range, options ...v1.Option) (model.Value, +// v1.Warnings, error) { +// matrix := model.Matrix{ +// &model.SampleStream{ +// Metric: model.Metric{"label": "test"}, +// Values: []model.SamplePair{ +// {Timestamp: model.TimeFromUnix(r.Start.Unix()), Value: 1}, +// {Timestamp: model.TimeFromUnix(r.End.Unix()), Value: 2}, +// }, +// }, +// } +// return matrix, nil, nil +// }, +// } + +// splitter := NewRangeQuerySplitter(splitDuration) +// pi := PrometheusInstance{apiUrl: mockApi, address: ""} +// result, err := splitter.QueryRangeByInterval(context.TODO(), pi, query, start, end, step) +// Expect(err).NotTo(HaveOccurred()) +// Expect(result.Type()).To(Equal(model.ValMatrix)) + +// matrix := result.(model.Matrix) +// Expect(len(matrix)).To(Equal(1)) +// Expect(len(matrix[0].Values)).To(Equal(6)) +// }) +// }) + +// var _ = Describe("interpolateMissingDataPoints", func() { +// It("should interpolate the missing data", func() { +// dataPoints := []DataPoint{ +// {Timestamp: time.Now().Add(-30 * time.Minute), Value: 60}, +// {Timestamp: time.Now().Add(-29 * time.Minute), Value: 80}, +// {Timestamp: time.Now().Add(-28 * time.Minute), Value: 100}, +// {Timestamp: time.Now().Add(-27 * time.Minute), Value: 50}, +// {Timestamp: time.Now().Add(-26 * time.Minute), Value: 30}, +// {Timestamp: time.Now().Add(-25 * time.Minute), Value: 60}, +// {Timestamp: time.Now().Add(-24 * time.Minute), Value: 80}, +// {Timestamp: time.Now().Add(-20 * time.Minute), Value: 60}, +// {Timestamp: time.Now().Add(-19 * time.Minute), Value: 80}, +// {Timestamp: time.Now().Add(-18 * time.Minute), Value: 100}, +// {Timestamp: time.Now().Add(-17 * time.Minute), Value: 50}, +// {Timestamp: time.Now().Add(-16 * time.Minute), Value: 30}, +// {Timestamp: time.Now().Add(-9 * time.Minute), Value: 80}, +// {Timestamp: time.Now().Add(-8 * time.Minute), Value: 100}, +// {Timestamp: time.Now().Add(-7 * time.Minute), Value: 50}, +// {Timestamp: time.Now().Add(-6 * time.Minute), Value: 30}, +// } +// dataPoints = scraper.interpolateMissingDataPoints(dataPoints, time.Minute) +// Expect(len(dataPoints)).To(Equal(25)) +// Expect(dataPoints[7].Value).To(Equal(75.0)) +// Expect(dataPoints[8].Value).To(Equal(70.0)) +// Expect(math.Floor(dataPoints[15].Value*100) / 100).To(Equal(37.14)) +// Expect(math.Floor(dataPoints[20].Value*100) / 100).To(Equal(72.85)) +// }) +// }) + +// var _ = Describe("aggregateMetrics", func() { +// It("should aggregate the metrics from different sources", func() { +// time1 := time.Now().Add(-30 * time.Minute) +// time2 := time.Now().Add(-29 * time.Minute) +// time3 := time.Now().Add(-28 * time.Minute) +// time4 := time.Now().Add(-27 * time.Minute) +// time5 := time.Now().Add(-26 * time.Minute) +// time6 := time.Now().Add(-25 * time.Minute) +// time7 := time.Now().Add(-24 * time.Minute) +// time8 := time.Now().Add(-20 * time.Minute) +// time9 := time.Now().Add(-19 * time.Minute) +// time10 := time.Now().Add(-18 * time.Minute) +// time11 := time.Now().Add(-17 * time.Minute) +// time12 := time.Now().Add(-16 * time.Minute) +// dataPoints1 := []DataPoint{ +// {Timestamp: time1, Value: 60}, +// {Timestamp: time2, Value: 80}, +// {Timestamp: time3, Value: 100}, +// {Timestamp: time4, Value: 50}, +// {Timestamp: time5, Value: 30}, +// {Timestamp: time6, Value: 60}, +// {Timestamp: time7, Value: 80}, +// {Timestamp: time8, Value: 60}, +// {Timestamp: time10, Value: 80}, +// {Timestamp: time11, Value: 100}, +// } +// dataPoints2 := []DataPoint{ +// {Timestamp: time1, Value: 30}, +// {Timestamp: time2, Value: 80}, +// {Timestamp: time3, Value: 100}, +// {Timestamp: time5, Value: 30}, +// {Timestamp: time6, Value: 60}, +// {Timestamp: time7, Value: 100}, +// {Timestamp: time9, Value: 80}, +// {Timestamp: time10, Value: 80}, +// {Timestamp: time11, Value: 100}, +// {Timestamp: time12, Value: 100}, +// } +// dataPoints := aggregateMetrics(dataPoints1, dataPoints2) +// fmt.Println(dataPoints) +// Expect(len(dataPoints)).To(Equal(12)) +// Expect(dataPoints[0].Value).To(Equal(60.0)) +// Expect(dataPoints[6].Value).To(Equal(100.0)) +// Expect(dataPoints[11].Value).To(Equal(100.0)) +// }) +// }) diff --git a/pkg/metrics/suite_test.go b/pkg/metrics/suite_test.go index f256687..0569918 100644 --- a/pkg/metrics/suite_test.go +++ b/pkg/metrics/suite_test.go @@ -89,14 +89,13 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) - utilizationMetric := "node_namespace_pod_container_container_cpu_usage_seconds_total_sum_irate" - podOwnerMetric := "namespace_workload_pod_kube_pod_owner_relabel" - resourceLimitMetric := "cluster_namespace_pod_cpu_active_kube_pod_container_resource_limits" + utilizationMetric := "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate" + podOwnerMetric := "namespace_workload_pod:kube_pod_owner:relabel" + resourceLimitMetric := "cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits" readyReplicasMetric := "kube_replicaset_status_ready_replicas" replicaSetOwnerMetric := "kube_replicaset_owner" hpaMaxReplicasMetric := "kube_horizontalpodautoscaler_spec_max_replicas" hpaOwnerInfoMetric := "kube_horizontalpodautoscaler_info" - podCreatedTimeMetric := "kube_pod_created" podReadyTimeMetric := "alm_kube_pod_ready_time" @@ -126,10 +125,13 @@ var _ = BeforeSuite(func() { podCreatedTimeMetric: podCreatedTimeMetric, podReadyTimeMetric: podReadyTimeMetric, }, - queryTimeout: 30 * time.Second, - rangeQuerySplitter: NewRangeQuerySplitter(1 * time.Second), - metricIngestionTime: metricIngestionTime, - metricProbeTime: metricProbeTime, + queryTimeout: 30 * time.Second, + rangeQuerySplitter: NewRangeQuerySplitter(1 * time.Second), + metricIngestionTime: metricIngestionTime, + metricProbeTime: metricProbeTime, + CPUUtilizationQuery: NewCPUUtilizationQuery(), + CPUUtilizationBreachQuery: NewCPUUtilizationBreachQuery(), + PodReadyLatencyQuery: NewPodReadyLatencyQuery(), } go func() { @@ -166,17 +168,17 @@ func startMetricsServer(registry *prometheus.Registry, addr string) *http.Server func registerMetrics() { cpuUsageMetric = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "node_namespace_pod_container_container_cpu_usage_seconds_total_sum_irate", + Name: "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate", Help: "Test metric for container CPU usage", }, []string{"namespace", "pod", "node", "container"}) kubePodOwnerMetric = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "namespace_workload_pod_kube_pod_owner_relabel", + Name: "namespace_workload_pod:kube_pod_owner:relabel", Help: "Test metric for Kubernetes pod owner", }, []string{"namespace", "pod", "workload", "workload_type"}) resourceLimitMetric = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "cluster_namespace_pod_cpu_active_kube_pod_container_resource_limits", + Name: "cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits", Help: "Test metric for container resource limits", }, []string{"namespace", "pod", "node", "container"}) @@ -223,12 +225,12 @@ func registerMetrics() { func registerMetrics1() { cpuUsageMetric1 = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "node_namespace_pod_container_container_cpu_usage_seconds_total_sum_irate", + Name: "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate", Help: "Test metric for container CPU usage", }, []string{"namespace", "pod", "node", "container"}) resourceLimitMetric1 = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "cluster_namespace_pod_cpu_active_kube_pod_container_resource_limits", + Name: "cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits", Help: "Test metric for container resource limits", }, []string{"namespace", "pod", "node", "container"}) From f19c7f0b4e5b0f939764207e46be69d1a4cd2306 Mon Sep 17 00:00:00 2001 From: deefreak Date: Mon, 20 Nov 2023 14:28:21 +0530 Subject: [PATCH 2/4] Refactored p8s queries building implementation --- pkg/metrics/scraper_test.go | 1322 +++++++++++++++++------------------ 1 file changed, 661 insertions(+), 661 deletions(-) diff --git a/pkg/metrics/scraper_test.go b/pkg/metrics/scraper_test.go index e01818f..b6293a3 100644 --- a/pkg/metrics/scraper_test.go +++ b/pkg/metrics/scraper_test.go @@ -1,667 +1,667 @@ package metrics -// import ( -// "context" -// "fmt" -// v1 "github.com/prometheus/client_golang/api/prometheus/v1" -// "github.com/prometheus/common/model" -// "math" -// "time" +import ( + "context" + "fmt" + v1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/common/model" + "math" + "time" -// . "github.com/onsi/ginkgo/v2" -// . "github.com/onsi/gomega" -// ) - -// var _ = Describe("PrometheusScraper", func() { + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("PrometheusScraper", func() { -// Context("when querying GetAverageCPUUtilizationByWorkload", func() { -// It("should return correct data points", func() { - -// By("creating a metric before queryRange window") -// cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-1", "test-node-1", "test-container-1").Set(4) -// cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-2", "test-node-2", "test-container-1").Set(3) -// cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-3", "test-node-2", "test-container-1").Set(5) -// cpuUsageMetric.WithLabelValues("test-ns-2", "test-pod-4", "test-node-4", "test-container-1").Set(20) + Context("when querying GetAverageCPUUtilizationByWorkload", func() { + It("should return correct data points", func() { + + By("creating a metric before queryRange window") + cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-1", "test-node-1", "test-container-1").Set(4) + cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-2", "test-node-2", "test-container-1").Set(3) + cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-3", "test-node-2", "test-container-1").Set(5) + cpuUsageMetric.WithLabelValues("test-ns-2", "test-pod-4", "test-node-4", "test-container-1").Set(20) -// kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-1", "test-workload-1", "deployment").Set(1) -// kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-2", "test-workload-1", "deployment").Set(1) -// kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-3", "test-workload-2", "deployment").Set(1) -// kubePodOwnerMetric.WithLabelValues("test-ns-2", "test-pod-4", "test-workload-3", "deployment").Set(1) - -// //wait for the metric to be scraped - scraping interval is 1s -// time.Sleep(5 * time.Second) - -// start := time.Now().Add(1 * time.Second) - -// By("creating first metric inside queryRange window") - -// kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-1", "test-workload-1", "deployment").Set(1) -// kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-2", "test-workload-1", "deployment").Set(1) -// kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-3", "test-workload-2", "deployment").Set(1) -// kubePodOwnerMetric.WithLabelValues("test-ns-2", "test-pod-4", "test-workload-3", "deployment").Set(1) - -// cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-1", "test-node-1", "test-container-1").Set(12) -// cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-2", "test-node-2", "test-container-1").Set(14) -// cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-3", "test-node-2", "test-container-1").Set(3) -// cpuUsageMetric.WithLabelValues("test-ns-2", "test-pod-4", "test-node-4", "test-container-1").Set(16) - -// //wait for the metric to be scraped - scraping interval is 1s -// time.Sleep(5 * time.Second) - -// By("creating second metric inside queryRange window") - -// cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-1", "test-node-1", "test-container-1").Set(5) -// cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-2", "test-node-2", "test-container-1").Set(4) -// cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-3", "test-node-2", "test-container-1").Set(12) -// cpuUsageMetric.WithLabelValues("test-ns-2", "test-pod-4", "test-node-4", "test-container-1").Set(15) - -// //wait for the metric to be scraped - scraping interval is 1s -// time.Sleep(5 * time.Second) - -// // data points after this should be outside the query range -// end := time.Now() - -// By("creating metric after queryRange window") - -// cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-1", "test-node-1", "test-container-1").Set(23) -// cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-2", "test-node-2", "test-container-1").Set(12) -// cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-3", "test-node-2", "test-container-1").Set(12) -// cpuUsageMetric.WithLabelValues("test-ns-2", "test-pod-4", "test-node-4", "test-container-1").Set(15) - -// //wait for the metric to be scraped - scraping interval is 1s -// time.Sleep(5 * time.Second) - -// dataPoints, err := scraper.GetAverageCPUUtilizationByWorkload("test-ns-1", -// "test-workload-1", start, end, time.Second) -// Expect(err).NotTo(HaveOccurred()) -// Expect(dataPoints).ToNot(BeEmpty()) - -// //since metrics could have been scraped multiple times, we just check the first and last value -// Expect(len(dataPoints) >= 2).To(BeTrue()) - -// Expect(dataPoints[0].Value).To(Equal(26.0)) -// Expect(dataPoints[len(dataPoints)-1].Value).To(Equal(9.0)) -// }) -// }) - -// Context("when querying GetACLByWorkload", func() { -// It("should return correct ACL", func() { - -// By("creating a metric before queryRange window") - -// podCreatedTimeMetric.WithLabelValues("test-ns-1", "test-pod-1").Set(45) -// podCreatedTimeMetric.WithLabelValues("test-ns-1", "test-pod-2").Set(55) -// podCreatedTimeMetric.WithLabelValues("test-ns-2", "test-pod-3").Set(65) -// podCreatedTimeMetric.WithLabelValues("test-ns-2", "test-pod-4").Set(75) -// podCreatedTimeMetric.WithLabelValues("test-ns-2", "test-pod-5").Set(80) - -// podReadyTimeMetric.WithLabelValues("test-ns-1", "test-pod-1").Set(50) -// podReadyTimeMetric.WithLabelValues("test-ns-1", "test-pod-2").Set(70) -// podReadyTimeMetric.WithLabelValues("test-ns-2", "test-pod-3").Set(80) -// podReadyTimeMetric.WithLabelValues("test-ns-2", "test-pod-4").Set(110) -// podReadyTimeMetric.WithLabelValues("test-ns-2", "test-pod-5").Set(120) - -// kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-1", "test-workload-1", "deployment").Set(1) -// kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-2", "test-workload-1", "deployment").Set(1) -// kubePodOwnerMetric.WithLabelValues("test-ns-2", "test-pod-3", "test-workload-3", "deployment").Set(1) -// kubePodOwnerMetric.WithLabelValues("test-ns-2", "test-pod-4", "test-workload-3", "deployment").Set(1) -// kubePodOwnerMetric.WithLabelValues("test-ns-2", "test-pod-5", "test-workload-3", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-1", "test-workload-1", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-2", "test-workload-1", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-3", "test-workload-2", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("test-ns-2", "test-pod-4", "test-workload-3", "deployment").Set(1) + + //wait for the metric to be scraped - scraping interval is 1s + time.Sleep(5 * time.Second) + + start := time.Now().Add(1 * time.Second) + + By("creating first metric inside queryRange window") + + kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-1", "test-workload-1", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-2", "test-workload-1", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-3", "test-workload-2", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("test-ns-2", "test-pod-4", "test-workload-3", "deployment").Set(1) + + cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-1", "test-node-1", "test-container-1").Set(12) + cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-2", "test-node-2", "test-container-1").Set(14) + cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-3", "test-node-2", "test-container-1").Set(3) + cpuUsageMetric.WithLabelValues("test-ns-2", "test-pod-4", "test-node-4", "test-container-1").Set(16) + + //wait for the metric to be scraped - scraping interval is 1s + time.Sleep(5 * time.Second) + + By("creating second metric inside queryRange window") + + cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-1", "test-node-1", "test-container-1").Set(5) + cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-2", "test-node-2", "test-container-1").Set(4) + cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-3", "test-node-2", "test-container-1").Set(12) + cpuUsageMetric.WithLabelValues("test-ns-2", "test-pod-4", "test-node-4", "test-container-1").Set(15) + + //wait for the metric to be scraped - scraping interval is 1s + time.Sleep(5 * time.Second) + + // data points after this should be outside the query range + end := time.Now() + + By("creating metric after queryRange window") + + cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-1", "test-node-1", "test-container-1").Set(23) + cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-2", "test-node-2", "test-container-1").Set(12) + cpuUsageMetric.WithLabelValues("test-ns-1", "test-pod-3", "test-node-2", "test-container-1").Set(12) + cpuUsageMetric.WithLabelValues("test-ns-2", "test-pod-4", "test-node-4", "test-container-1").Set(15) + + //wait for the metric to be scraped - scraping interval is 1s + time.Sleep(5 * time.Second) + + dataPoints, err := scraper.GetAverageCPUUtilizationByWorkload("test-ns-1", + "test-workload-1", start, end, time.Second) + Expect(err).NotTo(HaveOccurred()) + Expect(dataPoints).ToNot(BeEmpty()) + + //since metrics could have been scraped multiple times, we just check the first and last value + Expect(len(dataPoints) >= 2).To(BeTrue()) + + Expect(dataPoints[0].Value).To(Equal(26.0)) + Expect(dataPoints[len(dataPoints)-1].Value).To(Equal(9.0)) + }) + }) + + Context("when querying GetACLByWorkload", func() { + It("should return correct ACL", func() { + + By("creating a metric before queryRange window") + + podCreatedTimeMetric.WithLabelValues("test-ns-1", "test-pod-1").Set(45) + podCreatedTimeMetric.WithLabelValues("test-ns-1", "test-pod-2").Set(55) + podCreatedTimeMetric.WithLabelValues("test-ns-2", "test-pod-3").Set(65) + podCreatedTimeMetric.WithLabelValues("test-ns-2", "test-pod-4").Set(75) + podCreatedTimeMetric.WithLabelValues("test-ns-2", "test-pod-5").Set(80) + + podReadyTimeMetric.WithLabelValues("test-ns-1", "test-pod-1").Set(50) + podReadyTimeMetric.WithLabelValues("test-ns-1", "test-pod-2").Set(70) + podReadyTimeMetric.WithLabelValues("test-ns-2", "test-pod-3").Set(80) + podReadyTimeMetric.WithLabelValues("test-ns-2", "test-pod-4").Set(110) + podReadyTimeMetric.WithLabelValues("test-ns-2", "test-pod-5").Set(120) + + kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-1", "test-workload-1", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("test-ns-1", "test-pod-2", "test-workload-1", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("test-ns-2", "test-pod-3", "test-workload-3", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("test-ns-2", "test-pod-4", "test-workload-3", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("test-ns-2", "test-pod-5", "test-workload-3", "deployment").Set(1) -// podCreatedTimeMetric1.WithLabelValues("test-ns-1", "test-pod-1").Set(45) -// podCreatedTimeMetric1.WithLabelValues("test-ns-1", "test-pod-2").Set(55) -// podCreatedTimeMetric1.WithLabelValues("test-ns-2", "test-pod-3").Set(65) -// podCreatedTimeMetric1.WithLabelValues("test-ns-2", "test-pod-4").Set(75) -// podCreatedTimeMetric1.WithLabelValues("test-ns-2", "test-pod-5").Set(80) - -// podReadyTimeMetric1.WithLabelValues("test-ns-1", "test-pod-1").Set(60) -// podReadyTimeMetric1.WithLabelValues("test-ns-1", "test-pod-2").Set(70) -// podReadyTimeMetric1.WithLabelValues("test-ns-2", "test-pod-3").Set(80) -// podReadyTimeMetric1.WithLabelValues("test-ns-2", "test-pod-4").Set(100) -// podReadyTimeMetric1.WithLabelValues("test-ns-2", "test-pod-5").Set(120) - -// //wait for the metric to be scraped - scraping interval is 1s -// time.Sleep(2 * time.Second) - -// autoscalingLag1, err := scraper.GetACLByWorkload("test-ns-1", "test-workload-1") -// Expect(err).NotTo(HaveOccurred()) -// Expect(autoscalingLag1).To(Equal(45.0 * time.Second)) - -// autoscalingLag2, err := scraper.GetACLByWorkload("test-ns-2", "test-workload-3") -// Expect(err).NotTo(HaveOccurred()) -// Expect(autoscalingLag2).To(Equal(65.0 * time.Second)) -// }) -// }) - -// Context("when querying GetCPUUtilizationBreachDataPoints", func() { -// It("should return correct data points when workload is a deployment", func() { -// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(14) -// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(3) -// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(5) -// cpuUsageMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(3) - -// kubePodOwnerMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-1", "deployment").Set(1) -// kubePodOwnerMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-1", "deployment").Set(1) -// kubePodOwnerMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-2", "deployment").Set(1) -// kubePodOwnerMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-3", "deployment").Set(1) - -// resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(5) -// resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(5) -// resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(5) -// resourceLimitMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(5) - -// readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-1").Set(1) -// readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-2").Set(1) -// readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-3").Set(1) -// readyReplicasMetric.WithLabelValues("dep-test-ns-2", "dep-rs-4").Set(1) - -// replicaSetOwnerMetric.WithLabelValues("dep-test-ns-1", "deployment", "dep-1", "dep-rs-1").Set(1) -// replicaSetOwnerMetric.WithLabelValues("dep-test-ns-1", "deployment", "dep-1", "dep-rs-2").Set(1) -// replicaSetOwnerMetric.WithLabelValues("dep-test-ns-1", "deployment", "dep-2", "dep-rs-3").Set(1) -// replicaSetOwnerMetric.WithLabelValues("dep-test-ns-2", "deployment", "dep-1", "dep-rs-3").Set(1) - -// hpaMaxReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-hpa1").Set(3) -// hpaMaxReplicasMetric.WithLabelValues("dep-test-ns-2", "dep-hpa2").Set(3) - -// hpaOwnerInfoMetric.WithLabelValues("dep-test-ns-1", "dep-hpa1", "deployment", "dep-1").Set(1) -// hpaOwnerInfoMetric.WithLabelValues("dep-test-ns-1", "dep-hpa2", "deployment", "dep-2").Set(1) - -// //wait for the metric to be scraped - scraping interval is 1s -// time.Sleep(2 * time.Second) - -// //above data points should be outside the query range. -// start := time.Now() - -// //This data point should be excluded as there are only 2 pods for dep-1. Utilization is 70% - -// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(4) -// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(3) -// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(5) -// cpuUsageMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(3) - -// kubePodOwnerMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-1", "deployment").Set(1) -// kubePodOwnerMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-1", "deployment").Set(1) -// kubePodOwnerMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-2", "deployment").Set(1) -// kubePodOwnerMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-3", "deployment").Set(1) - -// resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(5) -// resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(5) -// resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(5) -// resourceLimitMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(5) - -// readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-1").Set(1) -// readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-2").Set(1) -// readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-3").Set(1) -// readyReplicasMetric.WithLabelValues("dep-test-ns-2", "dep-rs-4").Set(1) - -// replicaSetOwnerMetric.WithLabelValues("dep-test-ns-1", "deployment", "dep-1", "dep-rs-1").Set(1) -// replicaSetOwnerMetric.WithLabelValues("dep-test-ns-1", "deployment", "dep-1", "dep-rs-2").Set(1) -// replicaSetOwnerMetric.WithLabelValues("dep-test-ns-1", "deployment", "dep-2", "dep-rs-3").Set(1) -// replicaSetOwnerMetric.WithLabelValues("dep-test-ns-2", "deployment", "dep-1", "dep-rs-3").Set(1) - -// hpaMaxReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-hpa1").Set(3) -// hpaMaxReplicasMetric.WithLabelValues("dep-test-ns-2", "dep-hpa2").Set(3) - -// hpaOwnerInfoMetric.WithLabelValues("dep-test-ns-1", "dep-hpa1", "deployment", "dep-1").Set(1) -// hpaOwnerInfoMetric.WithLabelValues("dep-test-ns-1", "dep-hpa2", "deployment", "dep-2").Set(1) - -// //wait for the metric to be scraped - scraping interval is 1s -// time.Sleep(2 * time.Second) - -// // this data point will be excluded as utilization(80%) for dep-1 is below threshold of 85% -// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(4) -// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(4) -// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(12) -// cpuUsageMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(15) - -// //wait for the metric to be scraped - scraping interval is 1s -// time.Sleep(2 * time.Second) - -// // this data point will be excluded as no of ready pods < maxReplicas(3) -// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(5) -// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(4) -// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(12) -// cpuUsageMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(15) - -// //wait for the metric to be scraped - scraping interval is 1s -// time.Sleep(2 * time.Second) - -// // this data point should be included - utilization of 100% and ready replicas(1+2) = maxReplicas(3) -// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(5) -// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(5) -// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-5", "dep-test-node-2", "dep-test-container-1").Set(5) -// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(12) -// cpuUsageMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(15) - -// readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-1").Set(1) -// readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-2").Set(2) -// readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-3").Set(1) -// readyReplicasMetric.WithLabelValues("dep-test-ns-2", "dep-rs-4").Set(1) - -// resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(5) -// resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(5) -// resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(5) -// resourceLimitMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(5) -// resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-5", "dep-test-node-2", "dep-test-container-1").Set(5) - -// //wait for the metric to be scraped - scraping interval is 1s -// time.Sleep(2 * time.Second) - -// // data points after this should be outside the query range -// end := time.Now() - -// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(10) -// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(10) -// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-5", "dep-test-node-2", "dep-test-container-1").Set(10) -// cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(12) -// cpuUsageMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(15) - -// //wait for the metric to be scraped - scraping interval is 1s -// time.Sleep(2 * time.Second) - -// dataPoints, err := scraper.GetCPUUtilizationBreachDataPoints("dep-test-ns-1", -// "deployment", -// "dep-1", -// 0.85, start, -// end, -// time.Second) -// Expect(err).NotTo(HaveOccurred()) -// Expect(dataPoints).ToNot(BeEmpty()) - -// //since metrics could have been scraped multiple times, we just check the first and last value -// Expect(len(dataPoints) >= 1).To(BeTrue()) -// for _, datapoint := range dataPoints { -// Expect(datapoint.Value).To(Or(Equal(1.7), Equal(0.9))) -// } - -// }) - -// It("should return correct data points when workload is a Rollout", func() { -// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(14) -// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(3) -// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(5) -// cpuUsageMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(3) - -// kubePodOwnerMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-1", "deployment").Set(1) -// kubePodOwnerMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-1", "deployment").Set(1) -// kubePodOwnerMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-2", "deployment").Set(1) -// kubePodOwnerMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-3", "deployment").Set(1) - -// resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(5) -// resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(5) -// resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(5) -// resourceLimitMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(5) - -// readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-1").Set(1) -// readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-2").Set(1) -// readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-3").Set(1) -// readyReplicasMetric.WithLabelValues("ro-test-ns-2", "ro-rs-4").Set(1) - -// replicaSetOwnerMetric.WithLabelValues("ro-test-ns-1", "Rollout", "ro-1", "ro-rs-1").Set(1) -// replicaSetOwnerMetric.WithLabelValues("ro-test-ns-1", "Rollout", "ro-1", "ro-rs-2").Set(1) -// replicaSetOwnerMetric.WithLabelValues("ro-test-ns-1", "Rollout", "ro-2", "ro-rs-3").Set(1) -// replicaSetOwnerMetric.WithLabelValues("ro-test-ns-2", "Rollout", "ro-1", "ro-rs-3").Set(1) - -// hpaMaxReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-hpa1").Set(3) -// hpaMaxReplicasMetric.WithLabelValues("ro-test-ns-2", "ro-hpa2").Set(3) - -// hpaOwnerInfoMetric.WithLabelValues("ro-test-ns-1", "ro-hpa1", "Rollout", "ro-1").Set(1) -// hpaOwnerInfoMetric.WithLabelValues("ro-test-ns-1", "ro-hpa2", "Rollout", "ro-2").Set(1) - -// //wait for the metric to be scraped - scraping interval is 1s -// time.Sleep(2 * time.Second) - -// //above data points should be outside the query range. -// start := time.Now() - -// //This data point should be excluded as there are only 2 pods for ro-1. Utilization is 70% - -// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(4) -// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(3) -// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(5) -// cpuUsageMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(3) - -// kubePodOwnerMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-1", "deployment").Set(1) -// kubePodOwnerMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-1", "deployment").Set(1) -// kubePodOwnerMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-2", "deployment").Set(1) -// kubePodOwnerMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-3", "deployment").Set(1) - -// resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(5) -// resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(5) -// resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(5) -// resourceLimitMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(5) - -// readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-1").Set(1) -// readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-2").Set(1) -// readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-3").Set(1) -// readyReplicasMetric.WithLabelValues("ro-test-ns-2", "ro-rs-4").Set(1) - -// replicaSetOwnerMetric.WithLabelValues("ro-test-ns-1", "Rollout", "ro-1", "ro-rs-1").Set(1) -// replicaSetOwnerMetric.WithLabelValues("ro-test-ns-1", "Rollout", "ro-1", "ro-rs-2").Set(1) -// replicaSetOwnerMetric.WithLabelValues("ro-test-ns-1", "Rollout", "ro-2", "ro-rs-3").Set(1) -// replicaSetOwnerMetric.WithLabelValues("ro-test-ns-2", "Rollout", "ro-1", "ro-rs-3").Set(1) - -// hpaMaxReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-hpa1").Set(3) -// hpaMaxReplicasMetric.WithLabelValues("ro-test-ns-2", "ro-hpa2").Set(3) - -// hpaOwnerInfoMetric.WithLabelValues("ro-test-ns-1", "ro-hpa1", "Rollout", "ro-1").Set(1) -// hpaOwnerInfoMetric.WithLabelValues("ro-test-ns-1", "ro-hpa2", "Rollout", "ro-2").Set(1) - -// //wait for the metric to be scraped - scraping interval is 1s -// time.Sleep(2 * time.Second) - -// // this data point will be excluded as utilization(80%) for ro-1 is below threshold of 85% -// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(4) -// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(4) -// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(12) -// cpuUsageMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(15) - -// //wait for the metric to be scraped - scraping interval is 1s -// time.Sleep(2 * time.Second) - -// // this data point will be excluded as no of ready pods < maxReplicas(3) -// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(5) -// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(4) -// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(12) -// cpuUsageMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(15) - -// //wait for the metric to be scraped - scraping interval is 1s -// time.Sleep(2 * time.Second) - -// // this data point should be included - utilization of 100% and ready replicas(1+2) = maxReplicas(3) -// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(5) -// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(5) -// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-5", "ro-test-node-2", "ro-test-container-1").Set(5) -// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(12) -// cpuUsageMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(15) - -// readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-1").Set(1) -// readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-2").Set(2) -// readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-3").Set(1) -// readyReplicasMetric.WithLabelValues("ro-test-ns-2", "ro-rs-4").Set(1) - -// resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(5) -// resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(5) -// resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(5) -// resourceLimitMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(5) -// resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-5", "ro-test-node-2", "ro-test-container-1").Set(5) - -// //wait for the metric to be scraped - scraping interval is 1s -// time.Sleep(2 * time.Second) - -// // data points after this should be outside the query range -// end := time.Now() - -// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(10) -// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(10) -// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-5", "ro-test-node-2", "ro-test-container-1").Set(10) -// cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(12) -// cpuUsageMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(15) - -// //wait for the metric to be scraped - scraping interval is 1s -// time.Sleep(2 * time.Second) - -// dataPoints, err := scraper.GetCPUUtilizationBreachDataPoints("ro-test-ns-1", -// "Rollout", -// "ro-1", -// 0.85, start, -// end, -// time.Second) -// Expect(err).NotTo(HaveOccurred()) -// Expect(dataPoints).ToNot(BeEmpty()) - -// //since metrics could have been scraped multiple times, we just check the first and last value -// Expect(len(dataPoints) >= 1).To(BeTrue()) -// for _, datapoint := range dataPoints { -// Expect(datapoint.Value).To(Or(Equal(1.7), Equal(0.9))) -// } -// }) -// }) - -// Context("when querying GetAverageCPUUtilizationByWorkload with two prometheus instances", func() { -// It("should return correct data points", func() { - -// By("creating a metric before queryRange window") -// cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-1", "test-node-1", "test-container-1").Set(4) -// cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-2", "test-node-2", "test-container-1").Set(3) -// cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-3", "test-node-2", "test-container-1").Set(5) -// cpuUsageMetric.WithLabelValues("test-nsp-2", "test-pod-4", "test-node-4", "test-container-1").Set(20) - -// cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-1", "test-node-1", "test-container-1").Set(4) -// cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-2", "test-node-2", "test-container-1").Set(3) -// cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-3", "test-node-2", "test-container-1").Set(5) -// cpuUsageMetric1.WithLabelValues("test-nsp-2", "test-pod-4", "test-node-4", "test-container-1").Set(20) - -// kubePodOwnerMetric.WithLabelValues("test-nsp-1", "test-pod-1", "test-workload-1", "deployment").Set(1) -// kubePodOwnerMetric.WithLabelValues("test-nsp-1", "test-pod-2", "test-workload-1", "deployment").Set(1) -// kubePodOwnerMetric.WithLabelValues("test-nsp-1", "test-pod-3", "test-workload-2", "deployment").Set(1) -// kubePodOwnerMetric.WithLabelValues("test-nsp-2", "test-pod-4", "test-workload-3", "deployment").Set(1) - -// //wait for the metric to be scraped - scraping interval is 1s -// time.Sleep(5 * time.Second) - -// start := time.Now().Add(1 * time.Second) - -// By("creating first metric inside queryRange window") - -// kubePodOwnerMetric.WithLabelValues("test-nsp-1", "test-pod-1", "test-workload-1", "deployment").Set(1) -// kubePodOwnerMetric.WithLabelValues("test-nsp-1", "test-pod-2", "test-workload-1", "deployment").Set(1) -// kubePodOwnerMetric.WithLabelValues("test-nsp-1", "test-pod-3", "test-workload-2", "deployment").Set(1) -// kubePodOwnerMetric.WithLabelValues("test-nsp-2", "test-pod-4", "test-workload-3", "deployment").Set(1) - -// cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-1", "test-node-1", "test-container-1").Set(12) -// cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-2", "test-node-2", "test-container-1").Set(14) -// cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-3", "test-node-2", "test-container-1").Set(3) -// cpuUsageMetric.WithLabelValues("test-nsp-2", "test-pod-4", "test-node-4", "test-container-1").Set(16) - -// cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-1", "test-node-1", "test-container-1").Set(12) -// cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-2", "test-node-2", "test-container-1").Set(14) -// cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-3", "test-node-2", "test-container-1").Set(3) -// cpuUsageMetric1.WithLabelValues("test-nsp-2", "test-pod-4", "test-node-4", "test-container-1").Set(16) - -// //wait for the metric to be scraped - scraping interval is 1s -// time.Sleep(5 * time.Second) - -// By("creating second metric inside queryRange window") - -// cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-1", "test-node-1", "test-container-1").Set(5) -// cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-2", "test-node-2", "test-container-1").Set(4) -// cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-3", "test-node-2", "test-container-1").Set(12) -// cpuUsageMetric.WithLabelValues("test-nsp-2", "test-pod-4", "test-node-4", "test-container-1").Set(15) - -// cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-1", "test-node-1", "test-container-1").Set(5) -// cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-2", "test-node-2", "test-container-1").Set(5) -// cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-3", "test-node-2", "test-container-1").Set(12) -// cpuUsageMetric1.WithLabelValues("test-nsp-2", "test-pod-4", "test-node-4", "test-container-1").Set(15) - -// //wait for the metric to be scraped - scraping interval is 1s -// time.Sleep(5 * time.Second) - -// // data points after this should be outside the query range -// end := time.Now() - -// By("creating metric after queryRange window") - -// cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-1", "test-node-1", "test-container-1").Set(23) -// cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-2", "test-node-2", "test-container-1").Set(12) -// cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-3", "test-node-2", "test-container-1").Set(12) -// cpuUsageMetric.WithLabelValues("test-nsp-2", "test-pod-4", "test-node-4", "test-container-1").Set(15) - -// //wait for the metric to be scraped - scraping interval is 1s -// time.Sleep(5 * time.Second) - -// dataPoints, err := scraper.GetAverageCPUUtilizationByWorkload("test-nsp-1", -// "test-workload-1", start, end, time.Second) -// fmt.Println(dataPoints) -// Expect(err).NotTo(HaveOccurred()) -// Expect(dataPoints).ToNot(BeEmpty()) - -// //since metrics could have been scraped multiple times, we just check the first and last value -// Expect(len(dataPoints) >= 2).To(BeTrue()) - -// Expect(dataPoints[0].Value).To(Equal(26.0)) -// Expect(dataPoints[len(dataPoints)-1].Value).To(Equal(10.0)) -// }) -// }) -// }) - -// var _ = Describe("mergeMatrices", func() { -// It("should correctly merge two matrices", func() { -// matrix1 := model.Matrix{ -// &model.SampleStream{ -// Metric: model.Metric{"label": "test"}, -// Values: []model.SamplePair{ -// {Timestamp: 100, Value: 1}, -// {Timestamp: 200, Value: 2}, -// }, -// }, -// } - -// matrix2 := model.Matrix{ -// &model.SampleStream{ -// Metric: model.Metric{"label": "test"}, -// Values: []model.SamplePair{ -// {Timestamp: 300, Value: 3}, -// {Timestamp: 400, Value: 4}, -// }, -// }, -// } - -// expectedMergedMatrix := model.Matrix{ -// &model.SampleStream{ -// Metric: model.Metric{"label": "test"}, -// Values: []model.SamplePair{ -// {Timestamp: 100, Value: 1}, -// {Timestamp: 200, Value: 2}, -// {Timestamp: 300, Value: 3}, -// {Timestamp: 400, Value: 4}, -// }, -// }, -// } - -// mergedMatrix := mergeMatrices(matrix1, matrix2) -// Expect(mergedMatrix).To(Equal(expectedMergedMatrix)) -// }) -// }) - -// type mockAPI struct { -// v1.API -// queryRangeFunc func(ctx context.Context, query string, r v1.Range, options ...v1.Option) (model.Value, -// v1.Warnings, error) -// } - -// func (m *mockAPI) QueryRange(ctx context.Context, query string, r v1.Range, options ...v1.Option) (model.Value, -// v1.Warnings, error) { -// return m.queryRangeFunc(ctx, query, r) -// } - -// var _ = Describe("RangeQuerySplitter", func() { -// It("should split and query correctly by duration", func() { -// query := "test_query" -// start := time.Now().Add(-5 * time.Minute) -// end := time.Now() -// step := 1 * time.Minute -// splitDuration := 2 * time.Minute - -// mockApi := &mockAPI{ -// queryRangeFunc: func(ctx context.Context, query string, r v1.Range, options ...v1.Option) (model.Value, -// v1.Warnings, error) { -// matrix := model.Matrix{ -// &model.SampleStream{ -// Metric: model.Metric{"label": "test"}, -// Values: []model.SamplePair{ -// {Timestamp: model.TimeFromUnix(r.Start.Unix()), Value: 1}, -// {Timestamp: model.TimeFromUnix(r.End.Unix()), Value: 2}, -// }, -// }, -// } -// return matrix, nil, nil -// }, -// } - -// splitter := NewRangeQuerySplitter(splitDuration) -// pi := PrometheusInstance{apiUrl: mockApi, address: ""} -// result, err := splitter.QueryRangeByInterval(context.TODO(), pi, query, start, end, step) -// Expect(err).NotTo(HaveOccurred()) -// Expect(result.Type()).To(Equal(model.ValMatrix)) - -// matrix := result.(model.Matrix) -// Expect(len(matrix)).To(Equal(1)) -// Expect(len(matrix[0].Values)).To(Equal(6)) -// }) -// }) - -// var _ = Describe("interpolateMissingDataPoints", func() { -// It("should interpolate the missing data", func() { -// dataPoints := []DataPoint{ -// {Timestamp: time.Now().Add(-30 * time.Minute), Value: 60}, -// {Timestamp: time.Now().Add(-29 * time.Minute), Value: 80}, -// {Timestamp: time.Now().Add(-28 * time.Minute), Value: 100}, -// {Timestamp: time.Now().Add(-27 * time.Minute), Value: 50}, -// {Timestamp: time.Now().Add(-26 * time.Minute), Value: 30}, -// {Timestamp: time.Now().Add(-25 * time.Minute), Value: 60}, -// {Timestamp: time.Now().Add(-24 * time.Minute), Value: 80}, -// {Timestamp: time.Now().Add(-20 * time.Minute), Value: 60}, -// {Timestamp: time.Now().Add(-19 * time.Minute), Value: 80}, -// {Timestamp: time.Now().Add(-18 * time.Minute), Value: 100}, -// {Timestamp: time.Now().Add(-17 * time.Minute), Value: 50}, -// {Timestamp: time.Now().Add(-16 * time.Minute), Value: 30}, -// {Timestamp: time.Now().Add(-9 * time.Minute), Value: 80}, -// {Timestamp: time.Now().Add(-8 * time.Minute), Value: 100}, -// {Timestamp: time.Now().Add(-7 * time.Minute), Value: 50}, -// {Timestamp: time.Now().Add(-6 * time.Minute), Value: 30}, -// } -// dataPoints = scraper.interpolateMissingDataPoints(dataPoints, time.Minute) -// Expect(len(dataPoints)).To(Equal(25)) -// Expect(dataPoints[7].Value).To(Equal(75.0)) -// Expect(dataPoints[8].Value).To(Equal(70.0)) -// Expect(math.Floor(dataPoints[15].Value*100) / 100).To(Equal(37.14)) -// Expect(math.Floor(dataPoints[20].Value*100) / 100).To(Equal(72.85)) -// }) -// }) - -// var _ = Describe("aggregateMetrics", func() { -// It("should aggregate the metrics from different sources", func() { -// time1 := time.Now().Add(-30 * time.Minute) -// time2 := time.Now().Add(-29 * time.Minute) -// time3 := time.Now().Add(-28 * time.Minute) -// time4 := time.Now().Add(-27 * time.Minute) -// time5 := time.Now().Add(-26 * time.Minute) -// time6 := time.Now().Add(-25 * time.Minute) -// time7 := time.Now().Add(-24 * time.Minute) -// time8 := time.Now().Add(-20 * time.Minute) -// time9 := time.Now().Add(-19 * time.Minute) -// time10 := time.Now().Add(-18 * time.Minute) -// time11 := time.Now().Add(-17 * time.Minute) -// time12 := time.Now().Add(-16 * time.Minute) -// dataPoints1 := []DataPoint{ -// {Timestamp: time1, Value: 60}, -// {Timestamp: time2, Value: 80}, -// {Timestamp: time3, Value: 100}, -// {Timestamp: time4, Value: 50}, -// {Timestamp: time5, Value: 30}, -// {Timestamp: time6, Value: 60}, -// {Timestamp: time7, Value: 80}, -// {Timestamp: time8, Value: 60}, -// {Timestamp: time10, Value: 80}, -// {Timestamp: time11, Value: 100}, -// } -// dataPoints2 := []DataPoint{ -// {Timestamp: time1, Value: 30}, -// {Timestamp: time2, Value: 80}, -// {Timestamp: time3, Value: 100}, -// {Timestamp: time5, Value: 30}, -// {Timestamp: time6, Value: 60}, -// {Timestamp: time7, Value: 100}, -// {Timestamp: time9, Value: 80}, -// {Timestamp: time10, Value: 80}, -// {Timestamp: time11, Value: 100}, -// {Timestamp: time12, Value: 100}, -// } -// dataPoints := aggregateMetrics(dataPoints1, dataPoints2) -// fmt.Println(dataPoints) -// Expect(len(dataPoints)).To(Equal(12)) -// Expect(dataPoints[0].Value).To(Equal(60.0)) -// Expect(dataPoints[6].Value).To(Equal(100.0)) -// Expect(dataPoints[11].Value).To(Equal(100.0)) -// }) -// }) + podCreatedTimeMetric1.WithLabelValues("test-ns-1", "test-pod-1").Set(45) + podCreatedTimeMetric1.WithLabelValues("test-ns-1", "test-pod-2").Set(55) + podCreatedTimeMetric1.WithLabelValues("test-ns-2", "test-pod-3").Set(65) + podCreatedTimeMetric1.WithLabelValues("test-ns-2", "test-pod-4").Set(75) + podCreatedTimeMetric1.WithLabelValues("test-ns-2", "test-pod-5").Set(80) + + podReadyTimeMetric1.WithLabelValues("test-ns-1", "test-pod-1").Set(60) + podReadyTimeMetric1.WithLabelValues("test-ns-1", "test-pod-2").Set(70) + podReadyTimeMetric1.WithLabelValues("test-ns-2", "test-pod-3").Set(80) + podReadyTimeMetric1.WithLabelValues("test-ns-2", "test-pod-4").Set(100) + podReadyTimeMetric1.WithLabelValues("test-ns-2", "test-pod-5").Set(120) + + //wait for the metric to be scraped - scraping interval is 1s + time.Sleep(2 * time.Second) + + autoscalingLag1, err := scraper.GetACLByWorkload("test-ns-1", "test-workload-1") + Expect(err).NotTo(HaveOccurred()) + Expect(autoscalingLag1).To(Equal(45.0 * time.Second)) + + autoscalingLag2, err := scraper.GetACLByWorkload("test-ns-2", "test-workload-3") + Expect(err).NotTo(HaveOccurred()) + Expect(autoscalingLag2).To(Equal(65.0 * time.Second)) + }) + }) + + Context("when querying GetCPUUtilizationBreachDataPoints", func() { + It("should return correct data points when workload is a deployment", func() { + cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(14) + cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(3) + cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(5) + cpuUsageMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(3) + + kubePodOwnerMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-1", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-1", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-2", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-3", "deployment").Set(1) + + resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(5) + resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(5) + resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(5) + resourceLimitMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(5) + + readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-1").Set(1) + readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-2").Set(1) + readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-3").Set(1) + readyReplicasMetric.WithLabelValues("dep-test-ns-2", "dep-rs-4").Set(1) + + replicaSetOwnerMetric.WithLabelValues("dep-test-ns-1", "deployment", "dep-1", "dep-rs-1").Set(1) + replicaSetOwnerMetric.WithLabelValues("dep-test-ns-1", "deployment", "dep-1", "dep-rs-2").Set(1) + replicaSetOwnerMetric.WithLabelValues("dep-test-ns-1", "deployment", "dep-2", "dep-rs-3").Set(1) + replicaSetOwnerMetric.WithLabelValues("dep-test-ns-2", "deployment", "dep-1", "dep-rs-3").Set(1) + + hpaMaxReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-hpa1").Set(3) + hpaMaxReplicasMetric.WithLabelValues("dep-test-ns-2", "dep-hpa2").Set(3) + + hpaOwnerInfoMetric.WithLabelValues("dep-test-ns-1", "dep-hpa1", "deployment", "dep-1").Set(1) + hpaOwnerInfoMetric.WithLabelValues("dep-test-ns-1", "dep-hpa2", "deployment", "dep-2").Set(1) + + //wait for the metric to be scraped - scraping interval is 1s + time.Sleep(2 * time.Second) + + //above data points should be outside the query range. + start := time.Now() + + //This data point should be excluded as there are only 2 pods for dep-1. Utilization is 70% + + cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(4) + cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(3) + cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(5) + cpuUsageMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(3) + + kubePodOwnerMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-1", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-1", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-2", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-3", "deployment").Set(1) + + resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(5) + resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(5) + resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(5) + resourceLimitMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(5) + + readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-1").Set(1) + readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-2").Set(1) + readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-3").Set(1) + readyReplicasMetric.WithLabelValues("dep-test-ns-2", "dep-rs-4").Set(1) + + replicaSetOwnerMetric.WithLabelValues("dep-test-ns-1", "deployment", "dep-1", "dep-rs-1").Set(1) + replicaSetOwnerMetric.WithLabelValues("dep-test-ns-1", "deployment", "dep-1", "dep-rs-2").Set(1) + replicaSetOwnerMetric.WithLabelValues("dep-test-ns-1", "deployment", "dep-2", "dep-rs-3").Set(1) + replicaSetOwnerMetric.WithLabelValues("dep-test-ns-2", "deployment", "dep-1", "dep-rs-3").Set(1) + + hpaMaxReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-hpa1").Set(3) + hpaMaxReplicasMetric.WithLabelValues("dep-test-ns-2", "dep-hpa2").Set(3) + + hpaOwnerInfoMetric.WithLabelValues("dep-test-ns-1", "dep-hpa1", "deployment", "dep-1").Set(1) + hpaOwnerInfoMetric.WithLabelValues("dep-test-ns-1", "dep-hpa2", "deployment", "dep-2").Set(1) + + //wait for the metric to be scraped - scraping interval is 1s + time.Sleep(2 * time.Second) + + // this data point will be excluded as utilization(80%) for dep-1 is below threshold of 85% + cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(4) + cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(4) + cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(12) + cpuUsageMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(15) + + //wait for the metric to be scraped - scraping interval is 1s + time.Sleep(2 * time.Second) + + // this data point will be excluded as no of ready pods < maxReplicas(3) + cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(5) + cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(4) + cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(12) + cpuUsageMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(15) + + //wait for the metric to be scraped - scraping interval is 1s + time.Sleep(2 * time.Second) + + // this data point should be included - utilization of 100% and ready replicas(1+2) = maxReplicas(3) + cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(5) + cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(5) + cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-5", "dep-test-node-2", "dep-test-container-1").Set(5) + cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(12) + cpuUsageMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(15) + + readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-1").Set(1) + readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-2").Set(2) + readyReplicasMetric.WithLabelValues("dep-test-ns-1", "dep-rs-3").Set(1) + readyReplicasMetric.WithLabelValues("dep-test-ns-2", "dep-rs-4").Set(1) + + resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(5) + resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(5) + resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(5) + resourceLimitMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(5) + resourceLimitMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-5", "dep-test-node-2", "dep-test-container-1").Set(5) + + //wait for the metric to be scraped - scraping interval is 1s + time.Sleep(2 * time.Second) + + // data points after this should be outside the query range + end := time.Now() + + cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-1", "dep-test-node-1", "dep-test-container-1").Set(10) + cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-2", "dep-test-node-2", "dep-test-container-1").Set(10) + cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-5", "dep-test-node-2", "dep-test-container-1").Set(10) + cpuUsageMetric.WithLabelValues("dep-test-ns-1", "dep-test-pod-3", "dep-test-node-2", "dep-test-container-1").Set(12) + cpuUsageMetric.WithLabelValues("dep-test-ns-2", "dep-test-pod-4", "dep-test-node-4", "dep-test-container-1").Set(15) + + //wait for the metric to be scraped - scraping interval is 1s + time.Sleep(2 * time.Second) + + dataPoints, err := scraper.GetCPUUtilizationBreachDataPoints("dep-test-ns-1", + "deployment", + "dep-1", + 0.85, start, + end, + time.Second) + Expect(err).NotTo(HaveOccurred()) + Expect(dataPoints).ToNot(BeEmpty()) + + //since metrics could have been scraped multiple times, we just check the first and last value + Expect(len(dataPoints) >= 1).To(BeTrue()) + for _, datapoint := range dataPoints { + Expect(datapoint.Value).To(Or(Equal(1.7), Equal(0.9))) + } + + }) + + It("should return correct data points when workload is a Rollout", func() { + cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(14) + cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(3) + cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(5) + cpuUsageMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(3) + + kubePodOwnerMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-1", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-1", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-2", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-3", "deployment").Set(1) + + resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(5) + resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(5) + resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(5) + resourceLimitMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(5) + + readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-1").Set(1) + readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-2").Set(1) + readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-3").Set(1) + readyReplicasMetric.WithLabelValues("ro-test-ns-2", "ro-rs-4").Set(1) + + replicaSetOwnerMetric.WithLabelValues("ro-test-ns-1", "Rollout", "ro-1", "ro-rs-1").Set(1) + replicaSetOwnerMetric.WithLabelValues("ro-test-ns-1", "Rollout", "ro-1", "ro-rs-2").Set(1) + replicaSetOwnerMetric.WithLabelValues("ro-test-ns-1", "Rollout", "ro-2", "ro-rs-3").Set(1) + replicaSetOwnerMetric.WithLabelValues("ro-test-ns-2", "Rollout", "ro-1", "ro-rs-3").Set(1) + + hpaMaxReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-hpa1").Set(3) + hpaMaxReplicasMetric.WithLabelValues("ro-test-ns-2", "ro-hpa2").Set(3) + + hpaOwnerInfoMetric.WithLabelValues("ro-test-ns-1", "ro-hpa1", "Rollout", "ro-1").Set(1) + hpaOwnerInfoMetric.WithLabelValues("ro-test-ns-1", "ro-hpa2", "Rollout", "ro-2").Set(1) + + //wait for the metric to be scraped - scraping interval is 1s + time.Sleep(2 * time.Second) + + //above data points should be outside the query range. + start := time.Now() + + //This data point should be excluded as there are only 2 pods for ro-1. Utilization is 70% + + cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(4) + cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(3) + cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(5) + cpuUsageMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(3) + + kubePodOwnerMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-1", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-1", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-2", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-3", "deployment").Set(1) + + resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(5) + resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(5) + resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(5) + resourceLimitMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(5) + + readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-1").Set(1) + readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-2").Set(1) + readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-3").Set(1) + readyReplicasMetric.WithLabelValues("ro-test-ns-2", "ro-rs-4").Set(1) + + replicaSetOwnerMetric.WithLabelValues("ro-test-ns-1", "Rollout", "ro-1", "ro-rs-1").Set(1) + replicaSetOwnerMetric.WithLabelValues("ro-test-ns-1", "Rollout", "ro-1", "ro-rs-2").Set(1) + replicaSetOwnerMetric.WithLabelValues("ro-test-ns-1", "Rollout", "ro-2", "ro-rs-3").Set(1) + replicaSetOwnerMetric.WithLabelValues("ro-test-ns-2", "Rollout", "ro-1", "ro-rs-3").Set(1) + + hpaMaxReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-hpa1").Set(3) + hpaMaxReplicasMetric.WithLabelValues("ro-test-ns-2", "ro-hpa2").Set(3) + + hpaOwnerInfoMetric.WithLabelValues("ro-test-ns-1", "ro-hpa1", "Rollout", "ro-1").Set(1) + hpaOwnerInfoMetric.WithLabelValues("ro-test-ns-1", "ro-hpa2", "Rollout", "ro-2").Set(1) + + //wait for the metric to be scraped - scraping interval is 1s + time.Sleep(2 * time.Second) + + // this data point will be excluded as utilization(80%) for ro-1 is below threshold of 85% + cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(4) + cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(4) + cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(12) + cpuUsageMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(15) + + //wait for the metric to be scraped - scraping interval is 1s + time.Sleep(2 * time.Second) + + // this data point will be excluded as no of ready pods < maxReplicas(3) + cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(5) + cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(4) + cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(12) + cpuUsageMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(15) + + //wait for the metric to be scraped - scraping interval is 1s + time.Sleep(2 * time.Second) + + // this data point should be included - utilization of 100% and ready replicas(1+2) = maxReplicas(3) + cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(5) + cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(5) + cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-5", "ro-test-node-2", "ro-test-container-1").Set(5) + cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(12) + cpuUsageMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(15) + + readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-1").Set(1) + readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-2").Set(2) + readyReplicasMetric.WithLabelValues("ro-test-ns-1", "ro-rs-3").Set(1) + readyReplicasMetric.WithLabelValues("ro-test-ns-2", "ro-rs-4").Set(1) + + resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(5) + resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(5) + resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(5) + resourceLimitMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(5) + resourceLimitMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-5", "ro-test-node-2", "ro-test-container-1").Set(5) + + //wait for the metric to be scraped - scraping interval is 1s + time.Sleep(2 * time.Second) + + // data points after this should be outside the query range + end := time.Now() + + cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-1", "ro-test-node-1", "ro-test-container-1").Set(10) + cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-2", "ro-test-node-2", "ro-test-container-1").Set(10) + cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-5", "ro-test-node-2", "ro-test-container-1").Set(10) + cpuUsageMetric.WithLabelValues("ro-test-ns-1", "ro-test-pod-3", "ro-test-node-2", "ro-test-container-1").Set(12) + cpuUsageMetric.WithLabelValues("ro-test-ns-2", "ro-test-pod-4", "ro-test-node-4", "ro-test-container-1").Set(15) + + //wait for the metric to be scraped - scraping interval is 1s + time.Sleep(2 * time.Second) + + dataPoints, err := scraper.GetCPUUtilizationBreachDataPoints("ro-test-ns-1", + "Rollout", + "ro-1", + 0.85, start, + end, + time.Second) + Expect(err).NotTo(HaveOccurred()) + Expect(dataPoints).ToNot(BeEmpty()) + + //since metrics could have been scraped multiple times, we just check the first and last value + Expect(len(dataPoints) >= 1).To(BeTrue()) + for _, datapoint := range dataPoints { + Expect(datapoint.Value).To(Or(Equal(1.7), Equal(0.9))) + } + }) + }) + + Context("when querying GetAverageCPUUtilizationByWorkload with two prometheus instances", func() { + It("should return correct data points", func() { + + By("creating a metric before queryRange window") + cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-1", "test-node-1", "test-container-1").Set(4) + cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-2", "test-node-2", "test-container-1").Set(3) + cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-3", "test-node-2", "test-container-1").Set(5) + cpuUsageMetric.WithLabelValues("test-nsp-2", "test-pod-4", "test-node-4", "test-container-1").Set(20) + + cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-1", "test-node-1", "test-container-1").Set(4) + cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-2", "test-node-2", "test-container-1").Set(3) + cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-3", "test-node-2", "test-container-1").Set(5) + cpuUsageMetric1.WithLabelValues("test-nsp-2", "test-pod-4", "test-node-4", "test-container-1").Set(20) + + kubePodOwnerMetric.WithLabelValues("test-nsp-1", "test-pod-1", "test-workload-1", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("test-nsp-1", "test-pod-2", "test-workload-1", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("test-nsp-1", "test-pod-3", "test-workload-2", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("test-nsp-2", "test-pod-4", "test-workload-3", "deployment").Set(1) + + //wait for the metric to be scraped - scraping interval is 1s + time.Sleep(5 * time.Second) + + start := time.Now().Add(1 * time.Second) + + By("creating first metric inside queryRange window") + + kubePodOwnerMetric.WithLabelValues("test-nsp-1", "test-pod-1", "test-workload-1", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("test-nsp-1", "test-pod-2", "test-workload-1", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("test-nsp-1", "test-pod-3", "test-workload-2", "deployment").Set(1) + kubePodOwnerMetric.WithLabelValues("test-nsp-2", "test-pod-4", "test-workload-3", "deployment").Set(1) + + cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-1", "test-node-1", "test-container-1").Set(12) + cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-2", "test-node-2", "test-container-1").Set(14) + cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-3", "test-node-2", "test-container-1").Set(3) + cpuUsageMetric.WithLabelValues("test-nsp-2", "test-pod-4", "test-node-4", "test-container-1").Set(16) + + cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-1", "test-node-1", "test-container-1").Set(12) + cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-2", "test-node-2", "test-container-1").Set(14) + cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-3", "test-node-2", "test-container-1").Set(3) + cpuUsageMetric1.WithLabelValues("test-nsp-2", "test-pod-4", "test-node-4", "test-container-1").Set(16) + + //wait for the metric to be scraped - scraping interval is 1s + time.Sleep(5 * time.Second) + + By("creating second metric inside queryRange window") + + cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-1", "test-node-1", "test-container-1").Set(5) + cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-2", "test-node-2", "test-container-1").Set(4) + cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-3", "test-node-2", "test-container-1").Set(12) + cpuUsageMetric.WithLabelValues("test-nsp-2", "test-pod-4", "test-node-4", "test-container-1").Set(15) + + cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-1", "test-node-1", "test-container-1").Set(5) + cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-2", "test-node-2", "test-container-1").Set(5) + cpuUsageMetric1.WithLabelValues("test-nsp-1", "test-pod-3", "test-node-2", "test-container-1").Set(12) + cpuUsageMetric1.WithLabelValues("test-nsp-2", "test-pod-4", "test-node-4", "test-container-1").Set(15) + + //wait for the metric to be scraped - scraping interval is 1s + time.Sleep(5 * time.Second) + + // data points after this should be outside the query range + end := time.Now() + + By("creating metric after queryRange window") + + cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-1", "test-node-1", "test-container-1").Set(23) + cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-2", "test-node-2", "test-container-1").Set(12) + cpuUsageMetric.WithLabelValues("test-nsp-1", "test-pod-3", "test-node-2", "test-container-1").Set(12) + cpuUsageMetric.WithLabelValues("test-nsp-2", "test-pod-4", "test-node-4", "test-container-1").Set(15) + + //wait for the metric to be scraped - scraping interval is 1s + time.Sleep(5 * time.Second) + + dataPoints, err := scraper.GetAverageCPUUtilizationByWorkload("test-nsp-1", + "test-workload-1", start, end, time.Second) + fmt.Println(dataPoints) + Expect(err).NotTo(HaveOccurred()) + Expect(dataPoints).ToNot(BeEmpty()) + + //since metrics could have been scraped multiple times, we just check the first and last value + Expect(len(dataPoints) >= 2).To(BeTrue()) + + Expect(dataPoints[0].Value).To(Equal(26.0)) + Expect(dataPoints[len(dataPoints)-1].Value).To(Equal(10.0)) + }) + }) +}) + +var _ = Describe("mergeMatrices", func() { + It("should correctly merge two matrices", func() { + matrix1 := model.Matrix{ + &model.SampleStream{ + Metric: model.Metric{"label": "test"}, + Values: []model.SamplePair{ + {Timestamp: 100, Value: 1}, + {Timestamp: 200, Value: 2}, + }, + }, + } + + matrix2 := model.Matrix{ + &model.SampleStream{ + Metric: model.Metric{"label": "test"}, + Values: []model.SamplePair{ + {Timestamp: 300, Value: 3}, + {Timestamp: 400, Value: 4}, + }, + }, + } + + expectedMergedMatrix := model.Matrix{ + &model.SampleStream{ + Metric: model.Metric{"label": "test"}, + Values: []model.SamplePair{ + {Timestamp: 100, Value: 1}, + {Timestamp: 200, Value: 2}, + {Timestamp: 300, Value: 3}, + {Timestamp: 400, Value: 4}, + }, + }, + } + + mergedMatrix := mergeMatrices(matrix1, matrix2) + Expect(mergedMatrix).To(Equal(expectedMergedMatrix)) + }) +}) + +type mockAPI struct { + v1.API + queryRangeFunc func(ctx context.Context, query string, r v1.Range, options ...v1.Option) (model.Value, + v1.Warnings, error) +} + +func (m *mockAPI) QueryRange(ctx context.Context, query string, r v1.Range, options ...v1.Option) (model.Value, + v1.Warnings, error) { + return m.queryRangeFunc(ctx, query, r) +} + +var _ = Describe("RangeQuerySplitter", func() { + It("should split and query correctly by duration", func() { + query := "test_query" + start := time.Now().Add(-5 * time.Minute) + end := time.Now() + step := 1 * time.Minute + splitDuration := 2 * time.Minute + + mockApi := &mockAPI{ + queryRangeFunc: func(ctx context.Context, query string, r v1.Range, options ...v1.Option) (model.Value, + v1.Warnings, error) { + matrix := model.Matrix{ + &model.SampleStream{ + Metric: model.Metric{"label": "test"}, + Values: []model.SamplePair{ + {Timestamp: model.TimeFromUnix(r.Start.Unix()), Value: 1}, + {Timestamp: model.TimeFromUnix(r.End.Unix()), Value: 2}, + }, + }, + } + return matrix, nil, nil + }, + } + + splitter := NewRangeQuerySplitter(splitDuration) + pi := PrometheusInstance{apiUrl: mockApi, address: ""} + result, err := splitter.QueryRangeByInterval(context.TODO(), pi, query, start, end, step) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Type()).To(Equal(model.ValMatrix)) + + matrix := result.(model.Matrix) + Expect(len(matrix)).To(Equal(1)) + Expect(len(matrix[0].Values)).To(Equal(6)) + }) +}) + +var _ = Describe("interpolateMissingDataPoints", func() { + It("should interpolate the missing data", func() { + dataPoints := []DataPoint{ + {Timestamp: time.Now().Add(-30 * time.Minute), Value: 60}, + {Timestamp: time.Now().Add(-29 * time.Minute), Value: 80}, + {Timestamp: time.Now().Add(-28 * time.Minute), Value: 100}, + {Timestamp: time.Now().Add(-27 * time.Minute), Value: 50}, + {Timestamp: time.Now().Add(-26 * time.Minute), Value: 30}, + {Timestamp: time.Now().Add(-25 * time.Minute), Value: 60}, + {Timestamp: time.Now().Add(-24 * time.Minute), Value: 80}, + {Timestamp: time.Now().Add(-20 * time.Minute), Value: 60}, + {Timestamp: time.Now().Add(-19 * time.Minute), Value: 80}, + {Timestamp: time.Now().Add(-18 * time.Minute), Value: 100}, + {Timestamp: time.Now().Add(-17 * time.Minute), Value: 50}, + {Timestamp: time.Now().Add(-16 * time.Minute), Value: 30}, + {Timestamp: time.Now().Add(-9 * time.Minute), Value: 80}, + {Timestamp: time.Now().Add(-8 * time.Minute), Value: 100}, + {Timestamp: time.Now().Add(-7 * time.Minute), Value: 50}, + {Timestamp: time.Now().Add(-6 * time.Minute), Value: 30}, + } + dataPoints = scraper.interpolateMissingDataPoints(dataPoints, time.Minute) + Expect(len(dataPoints)).To(Equal(25)) + Expect(dataPoints[7].Value).To(Equal(75.0)) + Expect(dataPoints[8].Value).To(Equal(70.0)) + Expect(math.Floor(dataPoints[15].Value*100) / 100).To(Equal(37.14)) + Expect(math.Floor(dataPoints[20].Value*100) / 100).To(Equal(72.85)) + }) +}) + +var _ = Describe("aggregateMetrics", func() { + It("should aggregate the metrics from different sources", func() { + time1 := time.Now().Add(-30 * time.Minute) + time2 := time.Now().Add(-29 * time.Minute) + time3 := time.Now().Add(-28 * time.Minute) + time4 := time.Now().Add(-27 * time.Minute) + time5 := time.Now().Add(-26 * time.Minute) + time6 := time.Now().Add(-25 * time.Minute) + time7 := time.Now().Add(-24 * time.Minute) + time8 := time.Now().Add(-20 * time.Minute) + time9 := time.Now().Add(-19 * time.Minute) + time10 := time.Now().Add(-18 * time.Minute) + time11 := time.Now().Add(-17 * time.Minute) + time12 := time.Now().Add(-16 * time.Minute) + dataPoints1 := []DataPoint{ + {Timestamp: time1, Value: 60}, + {Timestamp: time2, Value: 80}, + {Timestamp: time3, Value: 100}, + {Timestamp: time4, Value: 50}, + {Timestamp: time5, Value: 30}, + {Timestamp: time6, Value: 60}, + {Timestamp: time7, Value: 80}, + {Timestamp: time8, Value: 60}, + {Timestamp: time10, Value: 80}, + {Timestamp: time11, Value: 100}, + } + dataPoints2 := []DataPoint{ + {Timestamp: time1, Value: 30}, + {Timestamp: time2, Value: 80}, + {Timestamp: time3, Value: 100}, + {Timestamp: time5, Value: 30}, + {Timestamp: time6, Value: 60}, + {Timestamp: time7, Value: 100}, + {Timestamp: time9, Value: 80}, + {Timestamp: time10, Value: 80}, + {Timestamp: time11, Value: 100}, + {Timestamp: time12, Value: 100}, + } + dataPoints := aggregateMetrics(dataPoints1, dataPoints2) + fmt.Println(dataPoints) + Expect(len(dataPoints)).To(Equal(12)) + Expect(dataPoints[0].Value).To(Equal(60.0)) + Expect(dataPoints[6].Value).To(Equal(100.0)) + Expect(dataPoints[11].Value).To(Equal(100.0)) + }) +}) From 7742d2b5d8b299fa4ffe40a8926e7e26edcb1dee Mon Sep 17 00:00:00 2001 From: deefreak Date: Thu, 23 Nov 2023 11:40:34 +0530 Subject: [PATCH 3/4] Refactored p8s queries implementation --- pkg/metrics/queries.go | 153 ++++++++++++------------------------ pkg/metrics/queries_test.go | 26 +++--- pkg/metrics/scraper.go | 76 +++++++----------- pkg/metrics/suite_test.go | 56 ++++++------- 4 files changed, 119 insertions(+), 192 deletions(-) diff --git a/pkg/metrics/queries.go b/pkg/metrics/queries.go index 6cdfa19..825f235 100644 --- a/pkg/metrics/queries.go +++ b/pkg/metrics/queries.go @@ -3,95 +3,47 @@ package metrics import "fmt" type QueryComponent struct { - metric string - labels map[string]string + metric string + labelKeys []string } -type CPUUtilizationQuery struct { +type CompositeQuery struct { queries map[string]*QueryComponent } -type CPUUtilizationBreachQuery struct { - queries map[string]*QueryComponent -} +type CompositeQueryBuilder CompositeQuery -type PodReadyLatencyQuery struct { - queries map[string]*QueryComponent +func NewCompositeQueryBuilder() *CompositeQueryBuilder { + return &CompositeQueryBuilder{} } -func NewCPUUtilizationQuery() *CPUUtilizationQuery { - cpuUtilizationMetric := "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate" - podOwnerMetric := "namespace_workload_pod:kube_pod_owner:relabel" - - queries := make(map[string]*QueryComponent) - queries["cpu_utilization_metric"] = NewQueryComponentBuilder().WithMetric(cpuUtilizationMetric).Build() - queries["pod_owner_metric"] = NewQueryComponentBuilder().WithMetric(podOwnerMetric).Build() - return &CPUUtilizationQuery{ - queries: queries, +func (qb *CompositeQueryBuilder) WithQuery(name string, query *QueryComponent) *CompositeQueryBuilder { + if qb.queries == nil { + qb.queries = make(map[string]*QueryComponent) } + qb.queries[name] = query + return qb } -func NewCPUUtilizationBreachQuery() *CPUUtilizationBreachQuery { - cpuUtilizationMetric := "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate" - podOwnerMetric := "namespace_workload_pod:kube_pod_owner:relabel" - resourceLimitMetric := "cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits" - readyReplicasMetric := "kube_replicaset_status_ready_replicas" - replicaSetOwnerMetric := "kube_replicaset_owner" - hpaMaxReplicasMetric := "kube_horizontalpodautoscaler_spec_max_replicas" - hpaOwnerInfoMetric := "kube_horizontalpodautoscaler_info" - - queries := make(map[string]*QueryComponent) - queries["cpu_utilization_metric"] = NewQueryComponentBuilder().WithMetric(cpuUtilizationMetric).Build() - queries["pod_owner_metric"] = NewQueryComponentBuilder().WithMetric(podOwnerMetric).Build() - queries["resource_limit_metric"] = NewQueryComponentBuilder().WithMetric(resourceLimitMetric).Build() - queries["ready_replicas_metric"] = NewQueryComponentBuilder().WithMetric(readyReplicasMetric).Build() - queries["replicaset_owner_metric"] = NewQueryComponentBuilder().WithMetric(replicaSetOwnerMetric).Build() - queries["hpa_max_replicas_metric"] = NewQueryComponentBuilder().WithMetric(hpaMaxReplicasMetric).Build() - queries["hpa_owner_info_metric"] = NewQueryComponentBuilder().WithMetric(hpaOwnerInfoMetric).Build() - return &CPUUtilizationBreachQuery{ - queries: queries, +func (qb *CompositeQueryBuilder) Build() *CompositeQuery { + return &CompositeQuery{ + queries: qb.queries, } } -func NewPodReadyLatencyQuery() *PodReadyLatencyQuery { - podCreatedTimeMetric := "kube_pod_created" - podReadyTimeMetric := "alm_kube_pod_ready_time" - podOwnerMetric := "namespace_workload_pod:kube_pod_owner:relabel" - queries := make(map[string]*QueryComponent) - queries["pod_created_time_metric"] = NewQueryComponentBuilder().WithMetric(podCreatedTimeMetric).Build() - queries["pod_ready_time_metric"] = NewQueryComponentBuilder().WithMetric(podReadyTimeMetric).Build() - queries["pod_owner_metric"] = NewQueryComponentBuilder().WithMetric(podOwnerMetric).Build() - return &PodReadyLatencyQuery{ - queries: queries, - } -} +type CPUUtilizationQuery CompositeQuery +type CPUUtilizationBreachQuery CompositeQuery +type PodReadyLatencyQuery CompositeQuery -func (q *CPUUtilizationQuery) BuildCompositeQuery(namespace, workload string) string { - q.queries["cpu_utilization_metric"].AddLabel("namespace", namespace) - q.queries["pod_owner_metric"].AddLabel("namespace", namespace) - q.queries["pod_owner_metric"].AddLabel("workload", workload) - q.queries["pod_owner_metric"].AddLabel("workload_type", "deployment") +func (qb *CPUUtilizationQuery) Render(m map[string]string) string { return fmt.Sprintf("sum(%s * on (namespace,pod) group_left(workload, workload_type)"+ "%s) by(namespace, workload, workload_type)", - q.queries["cpu_utilization_metric"].Render(), - q.queries["pod_owner_metric"].Render()) + qb.queries["cpu_utilization_metric"].Render(m), + qb.queries["pod_owner_metric"].Render(m)) } -func (q *CPUUtilizationBreachQuery) BuildCompositeQuery(namespace, workload, workloadType string, redLineUtilization float64) string { - q.queries["cpu_utilization_metric"].AddLabel("namespace", namespace) - q.queries["pod_owner_metric"].AddLabel("namespace", namespace) - q.queries["pod_owner_metric"].AddLabel("workload", workload) - q.queries["pod_owner_metric"].AddLabel("workload_type", "deployment") - q.queries["resource_limit_metric"].AddLabel("namespace", namespace) - q.queries["ready_replicas_metric"].AddLabel("namespace", namespace) - q.queries["replicaset_owner_metric"].AddLabel("namespace", namespace) - q.queries["replicaset_owner_metric"].AddLabel("owner_kind", workloadType) - q.queries["replicaset_owner_metric"].AddLabel("owner_name", workload) - q.queries["hpa_max_replicas_metric"].AddLabel("namespace", namespace) - q.queries["hpa_owner_info_metric"].AddLabel("namespace", namespace) - q.queries["hpa_owner_info_metric"].AddLabel("scaletargetref_kind", workloadType) - q.queries["hpa_owner_info_metric"].AddLabel("scaletargetref_name", workload) +func (qb *CPUUtilizationBreachQuery) Render(redLineUtilization float64, m map[string]string) string { return fmt.Sprintf("(sum(%s * on(namespace,pod) group_left(workload, workload_type) "+ "%s) by (namespace, workload, workload_type)/ on (namespace, workload, workload_type) "+ @@ -104,31 +56,26 @@ func (q *CPUUtilizationBreachQuery) BuildCompositeQuery(namespace, workload, wor "group_left(owner_kind, owner_name) label_replace(label_replace(%s,\"owner_kind\", \"$1\", "+ "\"scaletargetref_kind\", \"(.*)\"), \"owner_name\", \"$1\", \"scaletargetref_name\", \"(.*)\")),"+ "\"workload\", \"$1\", \"owner_name\", \"(.*)\")", - q.queries["cpu_utilization_metric"].Render(), - q.queries["pod_owner_metric"].Render(), - q.queries["resource_limit_metric"].Render(), - q.queries["pod_owner_metric"].Render(), + qb.queries["cpu_utilization_metric"].Render(m), + qb.queries["pod_owner_metric"].Render(m), + qb.queries["resource_limit_metric"].Render(m), + qb.queries["pod_owner_metric"].Render(m), redLineUtilization, - q.queries["ready_replicas_metric"].Render(), - q.queries["replicaset_owner_metric"].Render(), - q.queries["hpa_max_replicas_metric"].Render(), - q.queries["hpa_owner_info_metric"].Render()) + qb.queries["ready_replicas_metric"].Render(m), + qb.queries["replicaset_owner_metric"].Render(m), + qb.queries["hpa_max_replicas_metric"].Render(m), + qb.queries["hpa_owner_info_metric"].Render(m)) } -func (q *PodReadyLatencyQuery) BuildCompositeQuery(namespace, workload string) string { - q.queries["pod_created_time_metric"].AddLabel("namespace", namespace) - q.queries["pod_ready_time_metric"].AddLabel("namespace", namespace) - q.queries["pod_owner_metric"].AddLabel("namespace", namespace) - q.queries["pod_owner_metric"].AddLabel("workload", workload) - q.queries["pod_owner_metric"].AddLabel("workload_type", "deployment") +func (qb *PodReadyLatencyQuery) Render(m map[string]string) string { return fmt.Sprintf("quantile(0.5,(%s - on (namespace,pod) (%s)) "+ "* on (namespace,pod) group_left(workload, workload_type)"+ "(%s))", - q.queries["pod_ready_time_metric"].Render(), - q.queries["pod_created_time_metric"].Render(), - q.queries["pod_owner_metric"].Render()) + qb.queries["pod_ready_time_metric"].Render(m), + qb.queries["pod_created_time_metric"].Render(m), + qb.queries["pod_owner_metric"].Render(m)) } func ValidateQuery(query string) bool { @@ -165,36 +112,38 @@ func (qb *QueryComponentBuilder) WithMetric(metric string) *QueryComponentBuilde return qb } -func (qb *QueryComponentBuilder) WithLabels(labels map[string]string) *QueryComponentBuilder { - qb.labels = labels +func (qb *QueryComponentBuilder) WithLabelKeys(labelKeys []string) *QueryComponentBuilder { + qb.labelKeys = labelKeys return qb } func (qb *QueryComponentBuilder) Build() *QueryComponent { return &QueryComponent{ - metric: qb.metric, - labels: qb.labels, + metric: qb.metric, + labelKeys: qb.labelKeys, } } -func (qc *QueryComponent) AddLabel(key string, value string) { - if qc.labels == nil { - qc.labels = make(map[string]string) - } - qc.labels[key] = value -} +func (qc *QueryComponent) Render(m map[string]string) string { -func (qc *QueryComponent) Render() string { - if qc.labels == nil { - return qc.metric + labels := make(map[string]string) + for _, labelKey := range qc.labelKeys { + if labelValue, ok := m[labelKey]; ok { + labels[labelKey] = labelValue + } } - return fmt.Sprintf("%s{%s}", qc.metric, renderLabels(qc.labels)) + return fmt.Sprintf("%s%s", qc.metric, renderLabels(labels)) } func renderLabels(labels map[string]string) string { - var labelStr string + if len(labels) == 0 { + return "" + } + labelStr := "{" for key, value := range labels { labelStr += fmt.Sprintf("%s=\"%s\",", key, value) } - return labelStr[:len(labelStr)-1] + labelStr = labelStr[:len(labelStr)-1] + labelStr += "}" + return labelStr } diff --git a/pkg/metrics/queries_test.go b/pkg/metrics/queries_test.go index 5d50d3a..bb63634 100644 --- a/pkg/metrics/queries_test.go +++ b/pkg/metrics/queries_test.go @@ -12,34 +12,26 @@ var _ = Describe("Queries", func() { BeforeEach(func() { qc = &QueryComponent{ - metric: "test_metric", - labels: map[string]string{ - "label1": "value1", - "label2": "value2", - }, + metric: "test_metric", + labelKeys: []string{"label1", "label2"}, } }) - Describe("AddLabel", func() { - Context("when adding a new label", func() { - It("should add the label to the labels map", func() { - qc.AddLabel("label3", "value3") - Expect(qc.labels).To(HaveKeyWithValue("label3", "value3")) - }) - }) - }) - Describe("Render", func() { Context("when labels are present", func() { It("should render the metric with labels", func() { - Expect(qc.Render()).To(Equal("test_metric{label1=\"value1\",label2=\"value2\"}")) + labels := map[string]string{ + "label1": "value1", + "label2": "value2", + } + Expect(qc.Render(labels)).To(Equal("test_metric{label1=\"value1\",label2=\"value2\"}")) }) }) Context("when labels are not present", func() { It("should render the metric without labels", func() { - qc.labels = nil - Expect(qc.Render()).To(Equal("test_metric")) + labels := map[string]string{} + Expect(qc.Render(labels)).To(Equal("test_metric")) }) }) }) diff --git a/pkg/metrics/scraper.go b/pkg/metrics/scraper.go index d2bd3a2..7d3c060 100644 --- a/pkg/metrics/scraper.go +++ b/pkg/metrics/scraper.go @@ -92,7 +92,6 @@ type Scraper interface { // PrometheusScraper is a Scraper implementation that scrapes metrics data from Prometheus. type PrometheusScraper struct { api []PrometheusInstance - metricRegistry *MetricNameRegistry queryTimeout time.Duration rangeQuerySplitter *RangeQuerySplitter metricIngestionTime float64 @@ -103,18 +102,6 @@ type PrometheusScraper struct { logger logr.Logger } -type MetricNameRegistry struct { - utilizationMetric string - podOwnerMetric string - resourceLimitMetric string - readyReplicasMetric string - replicaSetOwnerMetric string - hpaMaxReplicasMetric string - hpaOwnerInfoMetric string - podCreatedTimeMetric string - podReadyTimeMetric string -} - type PrometheusQueryResult struct { result model.Matrix err error @@ -129,29 +116,6 @@ func (ps *PrometheusScraper) GetACLByWorkload(namespace string, workload string) return time.Duration(totalACL) * time.Second, nil } -func NewKubePrometheusMetricNameRegistry() *MetricNameRegistry { - cpuUtilizationMetric := "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate" - podOwnerMetric := "namespace_workload_pod:kube_pod_owner:relabel" - resourceLimitMetric := "cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits" - readyReplicasMetric := "kube_replicaset_status_ready_replicas" - replicaSetOwnerMetric := "kube_replicaset_owner" - hpaMaxReplicasMetric := "kube_horizontalpodautoscaler_spec_max_replicas" - hpaOwnerInfoMetric := "kube_horizontalpodautoscaler_info" - podCreatedTimeMetric := "kube_pod_created" - podReadyTimeMetric := "alm_kube_pod_ready_time" - - return &MetricNameRegistry{utilizationMetric: cpuUtilizationMetric, - podOwnerMetric: podOwnerMetric, - resourceLimitMetric: resourceLimitMetric, - readyReplicasMetric: readyReplicasMetric, - replicaSetOwnerMetric: replicaSetOwnerMetric, - hpaMaxReplicasMetric: hpaMaxReplicasMetric, - hpaOwnerInfoMetric: hpaOwnerInfoMetric, - podCreatedTimeMetric: podCreatedTimeMetric, - podReadyTimeMetric: podReadyTimeMetric, - } -} - type PrometheusInstance struct { apiUrl v1.API address string @@ -183,19 +147,36 @@ func NewPrometheusScraper(apiUrls []string, }) } - cpuUtilizationQuery := NewCPUUtilizationQuery() - cpuUtilizationBreachQuery := NewCPUUtilizationBreachQuery() - podReadyLatencyQuery := NewPodReadyLatencyQuery() + cpuUtilizationMetric := "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate" + podOwnerMetric := "namespace_workload_pod:kube_pod_owner:relabel" + resourceLimitMetric := "cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits" + readyReplicasMetric := "kube_replicaset_status_ready_replicas" + replicaSetOwnerMetric := "kube_replicaset_owner" + hpaMaxReplicasMetric := "kube_horizontalpodautoscaler_spec_max_replicas" + hpaOwnerInfoMetric := "kube_horizontalpodautoscaler_info" + podCreatedTimeMetric := "kube_pod_created" + podReadyTimeMetric := "alm_kube_pod_ready_time" + + compositeQuery := NewCompositeQueryBuilder(). + WithQuery("pod_ready_time_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(podReadyTimeMetric).WithLabelKeys([]string{"namespace"}).Build())). + WithQuery("pod_created_time_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(podCreatedTimeMetric).WithLabelKeys([]string{"namespace"}).Build())). + WithQuery("pod_owner_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(podOwnerMetric).WithLabelKeys([]string{"namespace", "workload", "workload_type"}).Build())). + WithQuery("cpu_utilization_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(cpuUtilizationMetric).WithLabelKeys([]string{"namespace"}).Build())). + WithQuery("resource_limit_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(resourceLimitMetric).WithLabelKeys([]string{"namespace"}).Build())). + WithQuery("ready_replicas_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(readyReplicasMetric).WithLabelKeys([]string{"namespace"}).Build())). + WithQuery("replicaset_owner_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(replicaSetOwnerMetric).WithLabelKeys([]string{"namespace", "owner_kind", "owner_name"}).Build())). + WithQuery("hpa_max_replicas_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(hpaMaxReplicasMetric).WithLabelKeys([]string{"namespace"}).Build())). + WithQuery("hpa_owner_info_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(hpaOwnerInfoMetric).WithLabelKeys([]string{"namespace", "scaletargetref_kind", "scaletargetref_name"}).Build())). + Build() return &PrometheusScraper{api: prometheusInstances, - metricRegistry: NewKubePrometheusMetricNameRegistry(), queryTimeout: timeout, rangeQuerySplitter: NewRangeQuerySplitter(splitInterval), metricProbeTime: metricProbeTime, metricIngestionTime: metricIngestionTime, - CPUUtilizationQuery: cpuUtilizationQuery, - CPUUtilizationBreachQuery: cpuUtilizationBreachQuery, - PodReadyLatencyQuery: podReadyLatencyQuery, + CPUUtilizationQuery: (*CPUUtilizationQuery)(compositeQuery), + CPUUtilizationBreachQuery: (*CPUUtilizationBreachQuery)(compositeQuery), + PodReadyLatencyQuery: (*PodReadyLatencyQuery)(compositeQuery), logger: logger}, nil } @@ -210,7 +191,7 @@ func (ps *PrometheusScraper) GetAverageCPUUtilizationByWorkload(namespace string ctx, cancel := context.WithTimeout(context.Background(), ps.queryTimeout) defer cancel() - query := ps.CPUUtilizationQuery.BuildCompositeQuery(namespace, workload) + query := ps.CPUUtilizationQuery.Render(map[string]string{"namespace": namespace, "workload": workload, "workload_type": "deployment"}) var totalDataPoints []DataPoint if ps.api == nil { @@ -325,7 +306,10 @@ func (ps *PrometheusScraper) GetCPUUtilizationBreachDataPoints(namespace, ctx, cancel := context.WithTimeout(context.Background(), ps.queryTimeout) defer cancel() - query := ps.CPUUtilizationBreachQuery.BuildCompositeQuery(namespace, workload, workloadType, redLineUtilization) + query := ps.CPUUtilizationBreachQuery.Render(redLineUtilization, map[string]string{"namespace": namespace, + "workload": workload, "workload_type": "deployment", "owner_kind": workloadType, "owner_name": workload, + "scaletargetref_kind": workloadType, "scaletargetref_name": workload}) + resultChanLength := len(ps.api) + 5 //Added some buffer resultChan := make(chan []DataPoint, resultChanLength) @@ -495,7 +479,7 @@ func (ps *PrometheusScraper) getPodReadyLatencyByWorkload(namespace string, work ctx, cancel := context.WithTimeout(context.Background(), ps.queryTimeout) defer cancel() - query := ps.PodReadyLatencyQuery.BuildCompositeQuery(namespace, workload) + query := ps.PodReadyLatencyQuery.Render(map[string]string{"namespace": namespace, "workload": workload, "workload_type": "deployment"}) podBootstrapTime := 0.0 if ps.api == nil { diff --git a/pkg/metrics/suite_test.go b/pkg/metrics/suite_test.go index 0569918..39a308f 100644 --- a/pkg/metrics/suite_test.go +++ b/pkg/metrics/suite_test.go @@ -18,13 +18,14 @@ package metrics import ( "fmt" + "net/http" + "testing" + "time" + "github.com/prometheus/client_golang/api" v1 "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" - "net/http" - "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -89,16 +90,6 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) - utilizationMetric := "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate" - podOwnerMetric := "namespace_workload_pod:kube_pod_owner:relabel" - resourceLimitMetric := "cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits" - readyReplicasMetric := "kube_replicaset_status_ready_replicas" - replicaSetOwnerMetric := "kube_replicaset_owner" - hpaMaxReplicasMetric := "kube_horizontalpodautoscaler_spec_max_replicas" - hpaOwnerInfoMetric := "kube_horizontalpodautoscaler_info" - podCreatedTimeMetric := "kube_pod_created" - podReadyTimeMetric := "alm_kube_pod_ready_time" - api := v1.NewAPI(client) api1 := v1.NewAPI(client1) metricIngestionTime := 15.0 @@ -113,25 +104,36 @@ var _ = BeforeSuite(func() { address: "http://localhost:8080", }) + cpuUtilizationMetric := "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate" + podOwnerMetric := "namespace_workload_pod:kube_pod_owner:relabel" + resourceLimitMetric := "cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits" + readyReplicasMetric := "kube_replicaset_status_ready_replicas" + replicaSetOwnerMetric := "kube_replicaset_owner" + hpaMaxReplicasMetric := "kube_horizontalpodautoscaler_spec_max_replicas" + hpaOwnerInfoMetric := "kube_horizontalpodautoscaler_info" + podCreatedTimeMetric := "kube_pod_created" + podReadyTimeMetric := "alm_kube_pod_ready_time" + + compositeQuery := NewCompositeQueryBuilder(). + WithQuery("pod_ready_time_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(podReadyTimeMetric).WithLabelKeys([]string{"namespace"}).Build())). + WithQuery("pod_created_time_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(podCreatedTimeMetric).WithLabelKeys([]string{"namespace"}).Build())). + WithQuery("pod_owner_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(podOwnerMetric).WithLabelKeys([]string{"namespace", "workload", "workload_type"}).Build())). + WithQuery("cpu_utilization_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(cpuUtilizationMetric).WithLabelKeys([]string{"namespace"}).Build())). + WithQuery("resource_limit_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(resourceLimitMetric).WithLabelKeys([]string{"namespace"}).Build())). + WithQuery("ready_replicas_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(readyReplicasMetric).WithLabelKeys([]string{"namespace"}).Build())). + WithQuery("replicaset_owner_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(replicaSetOwnerMetric).WithLabelKeys([]string{"namespace", "owner_kind", "owner_name"}).Build())). + WithQuery("hpa_max_replicas_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(hpaMaxReplicasMetric).WithLabelKeys([]string{"namespace"}).Build())). + WithQuery("hpa_owner_info_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(hpaOwnerInfoMetric).WithLabelKeys([]string{"namespace", "scaletargetref_kind", "scaletargetref_name"}).Build())). + Build() + scraper = &PrometheusScraper{api: v1Api, - metricRegistry: &MetricNameRegistry{ - utilizationMetric: utilizationMetric, - podOwnerMetric: podOwnerMetric, - resourceLimitMetric: resourceLimitMetric, - readyReplicasMetric: readyReplicasMetric, - replicaSetOwnerMetric: replicaSetOwnerMetric, - hpaMaxReplicasMetric: hpaMaxReplicasMetric, - hpaOwnerInfoMetric: hpaOwnerInfoMetric, - podCreatedTimeMetric: podCreatedTimeMetric, - podReadyTimeMetric: podReadyTimeMetric, - }, queryTimeout: 30 * time.Second, rangeQuerySplitter: NewRangeQuerySplitter(1 * time.Second), metricIngestionTime: metricIngestionTime, metricProbeTime: metricProbeTime, - CPUUtilizationQuery: NewCPUUtilizationQuery(), - CPUUtilizationBreachQuery: NewCPUUtilizationBreachQuery(), - PodReadyLatencyQuery: NewPodReadyLatencyQuery(), + CPUUtilizationQuery: (*CPUUtilizationQuery)(compositeQuery), + CPUUtilizationBreachQuery: (*CPUUtilizationBreachQuery)(compositeQuery), + PodReadyLatencyQuery: (*PodReadyLatencyQuery)(compositeQuery), } go func() { From 0c07dd6d683a035278832e8058b0f7cb71548094 Mon Sep 17 00:00:00 2001 From: deefreak Date: Tue, 19 Dec 2023 12:08:56 +0530 Subject: [PATCH 4/4] refactored p8s queries implementation --- pkg/metrics/queries.go | 32 +++++++++++++------------- pkg/metrics/scraper.go | 47 +++++++++++++++++++++------------------ pkg/metrics/suite_test.go | 26 +++------------------- 3 files changed, 44 insertions(+), 61 deletions(-) diff --git a/pkg/metrics/queries.go b/pkg/metrics/queries.go index 825f235..8b743be 100644 --- a/pkg/metrics/queries.go +++ b/pkg/metrics/queries.go @@ -35,15 +35,15 @@ type CPUUtilizationQuery CompositeQuery type CPUUtilizationBreachQuery CompositeQuery type PodReadyLatencyQuery CompositeQuery -func (qb *CPUUtilizationQuery) Render(m map[string]string) string { +func (qb *CPUUtilizationQuery) Render(labels map[string]string) string { return fmt.Sprintf("sum(%s * on (namespace,pod) group_left(workload, workload_type)"+ "%s) by(namespace, workload, workload_type)", - qb.queries["cpu_utilization_metric"].Render(m), - qb.queries["pod_owner_metric"].Render(m)) + qb.queries["cpu_utilization_metric"].Render(labels), + qb.queries["pod_owner_metric"].Render(labels)) } -func (qb *CPUUtilizationBreachQuery) Render(redLineUtilization float64, m map[string]string) string { +func (qb *CPUUtilizationBreachQuery) Render(redLineUtilization float64, labels map[string]string) string { return fmt.Sprintf("(sum(%s * on(namespace,pod) group_left(workload, workload_type) "+ "%s) by (namespace, workload, workload_type)/ on (namespace, workload, workload_type) "+ @@ -56,26 +56,26 @@ func (qb *CPUUtilizationBreachQuery) Render(redLineUtilization float64, m map[st "group_left(owner_kind, owner_name) label_replace(label_replace(%s,\"owner_kind\", \"$1\", "+ "\"scaletargetref_kind\", \"(.*)\"), \"owner_name\", \"$1\", \"scaletargetref_name\", \"(.*)\")),"+ "\"workload\", \"$1\", \"owner_name\", \"(.*)\")", - qb.queries["cpu_utilization_metric"].Render(m), - qb.queries["pod_owner_metric"].Render(m), - qb.queries["resource_limit_metric"].Render(m), - qb.queries["pod_owner_metric"].Render(m), + qb.queries["cpu_utilization_metric"].Render(labels), + qb.queries["pod_owner_metric"].Render(labels), + qb.queries["resource_limit_metric"].Render(labels), + qb.queries["pod_owner_metric"].Render(labels), redLineUtilization, - qb.queries["ready_replicas_metric"].Render(m), - qb.queries["replicaset_owner_metric"].Render(m), - qb.queries["hpa_max_replicas_metric"].Render(m), - qb.queries["hpa_owner_info_metric"].Render(m)) + qb.queries["ready_replicas_metric"].Render(labels), + qb.queries["replicaset_owner_metric"].Render(labels), + qb.queries["hpa_max_replicas_metric"].Render(labels), + qb.queries["hpa_owner_info_metric"].Render(labels)) } -func (qb *PodReadyLatencyQuery) Render(m map[string]string) string { +func (qb *PodReadyLatencyQuery) Render(labels map[string]string) string { return fmt.Sprintf("quantile(0.5,(%s - on (namespace,pod) (%s)) "+ "* on (namespace,pod) group_left(workload, workload_type)"+ "(%s))", - qb.queries["pod_ready_time_metric"].Render(m), - qb.queries["pod_created_time_metric"].Render(m), - qb.queries["pod_owner_metric"].Render(m)) + qb.queries["pod_ready_time_metric"].Render(labels), + qb.queries["pod_created_time_metric"].Render(labels), + qb.queries["pod_owner_metric"].Render(labels)) } func ValidateQuery(query string) bool { diff --git a/pkg/metrics/scraper.go b/pkg/metrics/scraper.go index 7d3c060..6cb36a1 100644 --- a/pkg/metrics/scraper.go +++ b/pkg/metrics/scraper.go @@ -121,6 +121,30 @@ type PrometheusInstance struct { address string } +func NewPrometheusCompositeQueries() *CompositeQuery { + cpuUtilizationMetric := "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate" + podOwnerMetric := "namespace_workload_pod:kube_pod_owner:relabel" + resourceLimitMetric := "cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits" + readyReplicasMetric := "kube_replicaset_status_ready_replicas" + replicaSetOwnerMetric := "kube_replicaset_owner" + hpaMaxReplicasMetric := "kube_horizontalpodautoscaler_spec_max_replicas" + hpaOwnerInfoMetric := "kube_horizontalpodautoscaler_info" + podCreatedTimeMetric := "kube_pod_created" + podReadyTimeMetric := "kube_pod_status_ready_time" + + return NewCompositeQueryBuilder(). + WithQuery("pod_ready_time_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(podReadyTimeMetric).WithLabelKeys([]string{"namespace"}).Build())). + WithQuery("pod_created_time_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(podCreatedTimeMetric).WithLabelKeys([]string{"namespace"}).Build())). + WithQuery("pod_owner_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(podOwnerMetric).WithLabelKeys([]string{"namespace", "workload", "workload_type"}).Build())). + WithQuery("cpu_utilization_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(cpuUtilizationMetric).WithLabelKeys([]string{"namespace"}).Build())). + WithQuery("resource_limit_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(resourceLimitMetric).WithLabelKeys([]string{"namespace"}).Build())). + WithQuery("ready_replicas_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(readyReplicasMetric).WithLabelKeys([]string{"namespace"}).Build())). + WithQuery("replicaset_owner_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(replicaSetOwnerMetric).WithLabelKeys([]string{"namespace", "owner_kind", "owner_name"}).Build())). + WithQuery("hpa_max_replicas_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(hpaMaxReplicasMetric).WithLabelKeys([]string{"namespace"}).Build())). + WithQuery("hpa_owner_info_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(hpaOwnerInfoMetric).WithLabelKeys([]string{"namespace", "scaletargetref_kind", "scaletargetref_name"}).Build())). + Build() +} + // NewPrometheusScraper returns a new PrometheusScraper instance. func NewPrometheusScraper(apiUrls []string, @@ -147,27 +171,7 @@ func NewPrometheusScraper(apiUrls []string, }) } - cpuUtilizationMetric := "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate" - podOwnerMetric := "namespace_workload_pod:kube_pod_owner:relabel" - resourceLimitMetric := "cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits" - readyReplicasMetric := "kube_replicaset_status_ready_replicas" - replicaSetOwnerMetric := "kube_replicaset_owner" - hpaMaxReplicasMetric := "kube_horizontalpodautoscaler_spec_max_replicas" - hpaOwnerInfoMetric := "kube_horizontalpodautoscaler_info" - podCreatedTimeMetric := "kube_pod_created" - podReadyTimeMetric := "alm_kube_pod_ready_time" - - compositeQuery := NewCompositeQueryBuilder(). - WithQuery("pod_ready_time_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(podReadyTimeMetric).WithLabelKeys([]string{"namespace"}).Build())). - WithQuery("pod_created_time_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(podCreatedTimeMetric).WithLabelKeys([]string{"namespace"}).Build())). - WithQuery("pod_owner_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(podOwnerMetric).WithLabelKeys([]string{"namespace", "workload", "workload_type"}).Build())). - WithQuery("cpu_utilization_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(cpuUtilizationMetric).WithLabelKeys([]string{"namespace"}).Build())). - WithQuery("resource_limit_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(resourceLimitMetric).WithLabelKeys([]string{"namespace"}).Build())). - WithQuery("ready_replicas_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(readyReplicasMetric).WithLabelKeys([]string{"namespace"}).Build())). - WithQuery("replicaset_owner_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(replicaSetOwnerMetric).WithLabelKeys([]string{"namespace", "owner_kind", "owner_name"}).Build())). - WithQuery("hpa_max_replicas_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(hpaMaxReplicasMetric).WithLabelKeys([]string{"namespace"}).Build())). - WithQuery("hpa_owner_info_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(hpaOwnerInfoMetric).WithLabelKeys([]string{"namespace", "scaletargetref_kind", "scaletargetref_name"}).Build())). - Build() + compositeQuery := NewPrometheusCompositeQueries() return &PrometheusScraper{api: prometheusInstances, queryTimeout: timeout, @@ -310,7 +314,6 @@ func (ps *PrometheusScraper) GetCPUUtilizationBreachDataPoints(namespace, "workload": workload, "workload_type": "deployment", "owner_kind": workloadType, "owner_name": workload, "scaletargetref_kind": workloadType, "scaletargetref_name": workload}) - resultChanLength := len(ps.api) + 5 //Added some buffer resultChan := make(chan []DataPoint, resultChanLength) var wg sync.WaitGroup diff --git a/pkg/metrics/suite_test.go b/pkg/metrics/suite_test.go index 39a308f..5d56407 100644 --- a/pkg/metrics/suite_test.go +++ b/pkg/metrics/suite_test.go @@ -104,27 +104,7 @@ var _ = BeforeSuite(func() { address: "http://localhost:8080", }) - cpuUtilizationMetric := "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate" - podOwnerMetric := "namespace_workload_pod:kube_pod_owner:relabel" - resourceLimitMetric := "cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits" - readyReplicasMetric := "kube_replicaset_status_ready_replicas" - replicaSetOwnerMetric := "kube_replicaset_owner" - hpaMaxReplicasMetric := "kube_horizontalpodautoscaler_spec_max_replicas" - hpaOwnerInfoMetric := "kube_horizontalpodautoscaler_info" - podCreatedTimeMetric := "kube_pod_created" - podReadyTimeMetric := "alm_kube_pod_ready_time" - - compositeQuery := NewCompositeQueryBuilder(). - WithQuery("pod_ready_time_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(podReadyTimeMetric).WithLabelKeys([]string{"namespace"}).Build())). - WithQuery("pod_created_time_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(podCreatedTimeMetric).WithLabelKeys([]string{"namespace"}).Build())). - WithQuery("pod_owner_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(podOwnerMetric).WithLabelKeys([]string{"namespace", "workload", "workload_type"}).Build())). - WithQuery("cpu_utilization_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(cpuUtilizationMetric).WithLabelKeys([]string{"namespace"}).Build())). - WithQuery("resource_limit_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(resourceLimitMetric).WithLabelKeys([]string{"namespace"}).Build())). - WithQuery("ready_replicas_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(readyReplicasMetric).WithLabelKeys([]string{"namespace"}).Build())). - WithQuery("replicaset_owner_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(replicaSetOwnerMetric).WithLabelKeys([]string{"namespace", "owner_kind", "owner_name"}).Build())). - WithQuery("hpa_max_replicas_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(hpaMaxReplicasMetric).WithLabelKeys([]string{"namespace"}).Build())). - WithQuery("hpa_owner_info_metric", (*QueryComponent)(NewQueryComponentBuilder().WithMetric(hpaOwnerInfoMetric).WithLabelKeys([]string{"namespace", "scaletargetref_kind", "scaletargetref_name"}).Build())). - Build() + compositeQuery := NewPrometheusCompositeQueries() scraper = &PrometheusScraper{api: v1Api, queryTimeout: 30 * time.Second, @@ -210,7 +190,7 @@ func registerMetrics() { }, []string{"namespace", "pod"}) podReadyTimeMetric = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "alm_kube_pod_ready_time", + Name: "kube_pod_status_ready_time", Help: "Test metric pod ready", }, []string{"namespace", "pod"}) @@ -262,7 +242,7 @@ func registerMetrics1() { }, []string{"namespace", "pod"}) podReadyTimeMetric1 = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "alm_kube_pod_ready_time", + Name: "kube_pod_status_ready_time", Help: "Test metric pod ready", }, []string{"namespace", "pod"})