test(e2e): add e2e test for latency metrics in advanced scenario

test/e2e/jobs/jobs.go

@@ -7,6 +7,7 @@ import (
 	"github.com/microsoft/retina/test/e2e/framework/types"
 	"github.com/microsoft/retina/test/e2e/scenarios/dns"
 	"github.com/microsoft/retina/test/e2e/scenarios/drop"
+	"github.com/microsoft/retina/test/e2e/scenarios/latency"
 	tcp "github.com/microsoft/retina/test/e2e/scenarios/tcp"
 )
 
@@ -181,5 +182,7 @@ func UpgradeAndTestRetinaAdvancedMetrics(kubeConfigFilePath, chartPath, valuesFi
 		job.AddScenario(dns.ValidateAdvancedDNSMetrics(scenario.name, scenario.req, scenario.resp, kubeConfigFilePath))
 	}
 
+	job.AddScenario(latency.ValidateLatencyMetric())
+
 	return job
 }

test/e2e/scenarios/dns/scenarios.go

@@ -64,6 +64,22 @@ func ValidateBasicDNSMetrics(scenarioName string, req *RequestValidationParams,
 				Duration: sleepDelay,
 			},
 		},
+		{
+			Step: &kubernetes.ExecInPod{
+				PodName:      podName,
+				PodNamespace: "kube-system",
+				Command:      req.Command,
+			},
+			Opts: &types.StepOptions{
+				ExpectError:               req.ExpectError,
+				SkipSavingParamatersToJob: true,
+			},
+		},
+		{
+			Step: &types.Sleep{
+				Duration: sleepDelay,
+			},
+		},
 		{
 			Step: &kubernetes.PortForward{
 				Namespace:             "kube-system",
@@ -167,7 +183,7 @@ func ValidateAdvancedDNSMetrics(scenarioName string, req *RequestValidationParam
 			},
 		},
 		{
-			Step: &ValidateAdvanceDNSRequestMetrics{
+			Step: &ValidateAdvancedDNSRequestMetrics{
 				Namespace:          "kube-system",
 				NumResponse:        req.NumResponse,
 				PodName:            podName,

test/e2e/scenarios/dns/advance.go → test/e2e/scenarios/dns/validate-advanced-dns-metric.go

@@ -17,7 +17,7 @@ var (
 	dnsAdvResponseCountMetricName = "networkobservability_adv_dns_response_count"
 )
 
-type ValidateAdvanceDNSRequestMetrics struct {
+type ValidateAdvancedDNSRequestMetrics struct {
 	Namespace    string
 	NumResponse  string
 	PodName      string
@@ -29,15 +29,15 @@ type ValidateAdvanceDNSRequestMetrics struct {
 	KubeConfigFilePath string
 }
 
-func (v *ValidateAdvanceDNSRequestMetrics) Run() error {
+func (v *ValidateAdvancedDNSRequestMetrics) Run() error {
 	metricsEndpoint := fmt.Sprintf("http://localhost:%d/metrics", common.RetinaPort)
 	// Get Pod IP address
 	podIP, err := kubernetes.GetPodIP(v.KubeConfigFilePath, v.Namespace, v.PodName)
 	if err != nil {
 		return errors.Wrapf(err, "failed to get pod IP address")
 	}
 
-	validateAdvanceDNSRequestMetrics := map[string]string{
+	validateAdvancedDNSRequestMetrics := map[string]string{
 		"ip":            podIP,
 		"namespace":     v.Namespace,
 		"num_response":  v.NumResponse,
@@ -50,7 +50,7 @@ func (v *ValidateAdvanceDNSRequestMetrics) Run() error {
 		"workload_name": v.WorkloadName,
 	}
 
-	err = prom.CheckMetric(metricsEndpoint, dnsAdvRequestCountMetricName, validateAdvanceDNSRequestMetrics)
+	err = prom.CheckMetric(metricsEndpoint, dnsAdvRequestCountMetricName, validateAdvancedDNSRequestMetrics)
 	if err != nil {
 		return errors.Wrapf(err, "failed to verify advance dns request metrics %s", dnsAdvRequestCountMetricName)
 	}
@@ -59,11 +59,11 @@ func (v *ValidateAdvanceDNSRequestMetrics) Run() error {
 	return nil
 }
 
-func (v *ValidateAdvanceDNSRequestMetrics) Prevalidate() error {
+func (v *ValidateAdvancedDNSRequestMetrics) Prevalidate() error {
 	return nil
 }
 
-func (v *ValidateAdvanceDNSRequestMetrics) Stop() error {
+func (v *ValidateAdvancedDNSRequestMetrics) Stop() error {
 	return nil
 }
 

test/e2e/scenarios/latency/scenario.go

@@ -0,0 +1,47 @@
+package latency
+
+import (
+	"time"
+
+	"github.com/microsoft/retina/test/e2e/framework/kubernetes"
+	"github.com/microsoft/retina/test/e2e/framework/types"
+)
+
+const sleepDelay = 5 * time.Second
+
+func ValidateLatencyMetric() *types.Scenario {
+	name := "Latency Metrics"
+	steps := []*types.StepWrapper{
+		{
+			Step: &types.Sleep{
+				Duration: sleepDelay,
+			},
+		},
+		{
+			Step: &kubernetes.PortForward{
+				Namespace:             "kube-system",
+				LabelSelector:         "k8s-app=retina",
+				LocalPort:             "10093",
+				RemotePort:            "10093",
+				Endpoint:              "metrics",
+				OptionalLabelAffinity: "k8s-app=retina",
+			},
+			Opts: &types.StepOptions{
+				SkipSavingParamatersToJob: true,
+				RunInBackgroundWithID:     "latency-port-forward",
+			},
+		},
+		{
+			Step: &ValidateAPIServerLatencyMetric{},
+			Opts: &types.StepOptions{
+				SkipSavingParamatersToJob: true,
+			},
+		},
+		{
+			Step: &types.Stop{
+				BackgroundID: "latency-port-forward",
+			},
+		},
+	}
+	return types.NewScenario(name, steps...)
+}

test/e2e/scenarios/latency/validate-latency-metric.go

@@ -0,0 +1,35 @@
+package latency
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/microsoft/retina/test/e2e/common"
+	prom "github.com/microsoft/retina/test/e2e/framework/prometheus"
+	"github.com/pkg/errors"
+)
+
+var latencyBucketMetricName = "networkobservability_adv_node_apiserver_tcp_handshake_latency"
+
+type ValidateAPIServerLatencyMetric struct{}
+
+func (v *ValidateAPIServerLatencyMetric) Prevalidate() error {
+	return nil
+}
+
+func (v *ValidateAPIServerLatencyMetric) Run() error {
+	promAddress := fmt.Sprintf("http://localhost:%d/metrics", common.RetinaPort)
+
+	metric := map[string]string{}
+	err := prom.CheckMetric(promAddress, latencyBucketMetricName, metric)
+	if err != nil {
+		return errors.Wrapf(err, "failed to verify latency metrics %s", latencyBucketMetricName)
+	}
+
+	log.Printf("found metrics matching %s\n", latencyBucketMetricName)
+	return nil
+}
+
+func (v *ValidateAPIServerLatencyMetric) Stop() error {
+	return nil
+}