Merge branch 'main' into dependabot/pip/dbus-python-1.3.2

.github/workflows/linters.yaml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v3
-    - uses: rojopolis/spellcheck-github-actions@0.30.0
+    - uses: rojopolis/spellcheck-github-actions@0.33.0
       name: Spellcheck
       with:
         config_path: .github/configs/spellcheck.yml

README.md

@@ -9,7 +9,7 @@ can choose the APT cluster `d430` node.
 
 ## Create a cluster
 
-First, configure `script/setup.cfg`. You can specify there which vHive branch to use, loader branch, operation mode
+First, configure `script/setup/setup.cfg`. You can specify there which vHive branch to use, loader branch, operation mode
 (sandbox type), maximum number of pods per node, and the Github token. All these configurations are mandatory.
 We currently support the following modes: containerd (`container`), Firecracker (`firecracker`), and Firecracker with
 snapshots (`firecracker_snapshots`).
@@ -21,12 +21,13 @@ API server certificate.
 * To create a multi-node cluster, specify the node addresses as the arguments and run the following command:
 
 ```bash
-$ bash ./scripts/setup/create_multinode.sh <master_node@IP> <worker_node@IP> ...
+$ bash ./scripts/setup/create_multinode.sh <master_node@IP> <loader_node@IP> <worker_node@IP> ...
 ```
 
-The loader should be running on a separate node that is part of the Kubernetes cluster. Do not collocate master and
-worker node components where the loader is located for performance reasons. Make sure you taint the node where loader is
-located prior to running any experiment.
+This command will create the following setup: control plane is placed on master node, loader node is used for running
+loader and monitoring pods (mostly, Prometheus, if enabled in setup config), workers are used purely for working pods. In
+this setup, neither control plane nor workers are affected by loader and monitoring, creating more reliable measurements
+of performance.
 
 * Single-node cluster (experimental)
 

config/requirements_dev.txt

@@ -19,7 +19,7 @@ pssh==2.3.1
 PyGObject==3.44.1
 pymacaroons==0.13.0
 PyNaCl==1.5.0
-pyOpenSSL==19.0.0
+pyOpenSSL==23.2.0
 pyparsing==3.0.7
 python-apt==2.0.0+ubuntu0.20.4.3
 python-dateutil==2.8.2

go.mod

@@ -11,7 +11,7 @@ require (
 	golang.org/x/sys v0.8.0
 	gonum.org/v1/gonum v0.13.0
 	gonum.org/v1/plot v0.13.0
-	google.golang.org/grpc v1.55.0
+	google.golang.org/grpc v1.56.1
 )
 
 require github.com/stretchr/testify v1.8.0
@@ -43,8 +43,8 @@ require (
 	go.opentelemetry.io/otel/metric v0.20.0 // indirect
 	go.opentelemetry.io/otel/sdk v0.20.0 // indirect
 	go.opentelemetry.io/otel/trace v0.20.0 // indirect
-	golang.org/x/net v0.8.0 // indirect
+	golang.org/x/net v0.9.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
-	google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect
+	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
 )

go.sum

@@ -14,7 +14,7 @@ cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
 cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
-cloud.google.com/go/compute v1.18.0 h1:FEigFqoDbys2cvFkZ9Fjq4gnHBP55anJ0yQyau2f9oY=
+cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY=
 cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
@@ -108,6 +108,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE=
 github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU=
@@ -251,6 +252,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/envoyproxy/protoc-gen-validate v0.10.1 h1:c0g45+xCJhdgFGw7a5QAfdS4byAbud7miNWJ1WwEVf8=
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
@@ -743,14 +745,14 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
-golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
+golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw=
+golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -833,7 +835,7 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw=
+golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -940,8 +942,8 @@ google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfG
 google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
 google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA=
-google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=
+google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A=
+google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU=
 google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
@@ -960,8 +962,8 @@ google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv
 google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
 google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
-google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag=
-google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8=
+google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ=
+google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

pkg/driver/trace_driver.go

@@ -85,8 +85,8 @@ func (d *Driver) CreateMetricsScrapper(interval time.Duration,
 
 	return func() {
 		signalReady.Done()
-		clusterUsageRecords := make(chan interface{}, 100)
 		knStatRecords := make(chan interface{}, 100)
+		scaleRecords := make(chan interface{}, 100)
 		writerDone := sync.WaitGroup{}
 
 		clusterUsageFile, err := os.Create(d.outputFilename("cluster_usage"))
@@ -96,6 +96,9 @@ func (d *Driver) CreateMetricsScrapper(interval time.Duration,
 		writerDone.Add(1)
 		go d.runCSVWriter(knStatRecords, d.outputFilename("kn_stats"), &writerDone)
 
+		writerDone.Add(1)
+		go d.runCSVWriter(scaleRecords, d.outputFilename("deployment_scale"), &writerDone)
+
 		for {
 			select {
 			case <-timer.C:
@@ -111,12 +114,19 @@ func (d *Driver) CreateMetricsScrapper(interval time.Duration,
 				_, err = clusterUsageFile.WriteString("\n")
 				common.Check(err)
 
+				recScale := mc.ScrapeDeploymentScales()
+				timestamp := time.Now().UnixMicro()
+				for _, rec := range recScale {
+					rec.Timestamp = timestamp
+					scaleRecords <- rec
+				}
+
 				recKnative := mc.ScrapeKnStats()
 				recKnative.Timestamp = time.Now().UnixMicro()
 				knStatRecords <- recKnative
 			case <-finishCh:
-				close(clusterUsageRecords)
 				close(knStatRecords)
+				close(scaleRecords)
 
 				writerDone.Wait()
 				allRecordsWritten.Done()

pkg/metric/collect.go

@@ -3,16 +3,12 @@ package metric
 import (
 	"encoding/json"
 	"os/exec"
-	"time"
 
 	log "github.com/sirupsen/logrus"
 )
 
-func ScrapeDeploymentScales() []ScaleRecord {
-	cmd := exec.Command(
-		"python3",
-		"pkg/metric/scrape_scales.py",
-	)
+func ScrapeDeploymentScales() []DeploymentScale {
+	cmd := exec.Command("python3", "pkg/metric/scrape_scales.py")
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		log.Warn("Fail to scrape deployment scales: ", err)
@@ -24,17 +20,7 @@ func ScrapeDeploymentScales() []ScaleRecord {
 		log.Warn("Fail to parse deployment scales: ", string(out[:]), err)
 	}
 
-	timestamp := time.Now().UnixMicro()
-	records := []ScaleRecord{}
-	for _, result := range results {
-		records = append(records, ScaleRecord{
-			Timestamp:    timestamp,
-			Deployment:   result.Deployment,
-			DesiredScale: result.DesiredScale,
-			ActualScale:  result.ActualScale,
-		})
-	}
-	return records
+	return results
 }
 
 func ScrapeKnStats() KnStats {

pkg/metric/record.go

@@ -86,18 +86,15 @@ type ExecutionRecord struct {
 	ColdStartCount int `csv:"coldstart_count"`*/
 }
 
-type ScaleRecord struct {
-	Timestamp    int64  `csv:"timestamp"`
-	Deployment   string `csv:"deployment"`
-	DesiredScale int    `csv:"desired_scale"`
-	ActualScale  int    `csv:"actual_scale"`
-}
-
 type DeploymentScale struct {
-	Timestamp    int64  `csv:"timestamp"`
-	Deployment   string `csv:"deployment"`
-	DesiredScale int    `csv:"desired_scale"`
-	ActualScale  int    `csv:"actual_scale"`
+	Timestamp       int64   `csv:"timestamp" json:"timestamp"`
+	Function        string  `csv:"function" json:"function"`
+	DesiredPods     int     `csv:"desired_pods" json:"desired_pods"`
+	RunningPods     int     `csv:"running_pods" json:"running_pods"`
+	UnreadyPods     int     `csv:"unready_pods" json:"unready_pods"`
+	PendingPods     int     `csv:"pending_pods" json:"pending_pods"`
+	TerminatingPods int     `csv:"terminating_pods" json:"terminating_pods"`
+	ActivatorQueue  float64 `csv:"activator_queue" json:"activator_queue"`
 }
 
 type KnStats struct {
@@ -142,6 +139,8 @@ type ClusterUsage struct {
 	PodCpu          []string  `csv:"pod_cpu" json:"pod_cpu"`
 	PodMemory       []string  `csv:"pod_memory" json:"pod_mem"`
 	Pods            []int     `csv:"pods" json:"pods"`
+	LoaderCpu       float64   `csv:"loader_cpu" json:"loader_cpu"`
+	LoaderMem       float64   `csv:"loader_mem" json:"loader_mem"`
 }
 
 type AdfResult struct {

pkg/metric/scrape_infra.py

@@ -3,22 +3,22 @@
 import sys
 import os
 
-loader_total_cores = 8
+prometheus_ip = os.popen("kubectl get svc -n monitoring | grep prometheus-kube-prometheus-prometheus | awk '{print $3}'").read().strip().split('\n')[0]
 
 def get_promql_query(query):
     def promql_query():
-        return "tools/bin/promql --no-headers --host 'http://localhost:9090' '" + query + "' | grep . | sort | awk '{print $2}'"
+        return "tools/bin/promql --no-headers --host 'http://" + prometheus_ip + ":9090' '" + query + "' | grep . | LC_COLLATE=C sort | awk '{print $2}'"
     return promql_query
 
 if __name__ == "__main__":
     cmd_get_loader_pct = ['bash', 'scripts/metrics/get_loader_cpu_pct.sh']
     cmd_get_abs_vals = ['bash', 'scripts/metrics/get_node_stats_abs.sh']
     cmd_get_pcts = ['bash', 'scripts/metrics/get_node_stats_percent.sh']
     cmd_get_pod_abs_vals = ['bash', 'scripts/metrics/get_pod_stats_abs.sh']
-    query_mem_req = 'sum(kube_pod_container_resource_requests{resource="memory"} and on(container, pod) (kube_pod_container_status_running==1)) by (node)'
-    query_mem_lim = 'sum(kube_pod_container_resource_limits{resource="memory"} and on(container, pod) (kube_pod_container_status_running==1)) by (node)'
-    query_cpu_req = 'sum(kube_pod_container_resource_requests{resource="cpu"} and on(container, pod) (kube_pod_container_status_running==1)) by (node)'
-    query_cpu_lim = 'sum(kube_pod_container_resource_limits{resource="cpu"} and on(container, pod) (kube_pod_container_status_running==1)) by (node)'
+    query_mem_req = 'sum(kube_pod_container_resource_requests{resource="memory"} and on(container, pod) (kube_pod_container_status_running==1) or on(node) (kube_node_info*0)) by (node)'
+    query_mem_lim = 'sum(kube_pod_container_resource_limits{resource="memory"} and on(container, pod) (kube_pod_container_status_running==1) or on(node) (kube_node_info*0)) by (node)'
+    query_cpu_req = 'sum(kube_pod_container_resource_requests{resource="cpu"} and on(container, pod) (kube_pod_container_status_running==1) or on(node) (kube_node_info*0)) by (node)'
+    query_cpu_lim = 'sum(kube_pod_container_resource_limits{resource="cpu"} and on(container, pod) (kube_pod_container_status_running==1) or on(node) (kube_node_info*0)) by (node)'
     query_pod_count = 'count(kube_pod_info and on(pod) max(kube_pod_container_status_running==1) by (pod)) by(node)'
 
     result = {
@@ -42,12 +42,13 @@ def promql_query():
         "pod_cpu": [],
         "pod_mem": [],
         "pods": [],
+        "loader_cpu": 0,
+        "loader_mem": 0,
     }
 
-    loader_cpu_pct, loader_mem_pct = list(
+    result["loader_cpu"], result["loader_mem"] = list(
         map(float, subprocess.check_output(cmd_get_loader_pct).decode("utf-8").strip().split())
     )
-    loader_cpu_pct /= loader_total_cores #* Normalise to [0, 100]
 
     abs_out = subprocess.check_output(cmd_get_abs_vals).decode("utf-8")[:-1].split('\n')
     pcts_out = subprocess.check_output(cmd_get_pcts).decode("utf-8").split('\n')
@@ -66,8 +67,6 @@ def promql_query():
         if is_master:
             # Record master node.
             result['master_cpu_pct'], result['master_mem_pct'] = list(map(float, pcts[:-1].split('%')))
-            result['master_cpu_pct'] = max(0, result['master_cpu_pct'] - loader_cpu_pct)
-            result['master_mem_pct'] = max(0, result['master_mem_pct'] - loader_mem_pct)
             result['master_mem_req'] = float(mem_r)
             result['master_mem_lim'] = float(mem_l)
             result['master_cpu_req'] = float(cpu_r)

pkg/metric/scrape_kn.py

@@ -1,9 +1,11 @@
 import os
 import json
 
+prometheus_ip = os.popen("kubectl get svc -n monitoring | grep prometheus-kube-prometheus-prometheus | awk '{print $3}'").read().strip().split('\n')[0]
+
 def get_promql_query(query):
     def promql_query():
-        return "tools/bin/promql --no-headers --host 'http://localhost:9090' '" + query + "' | awk '{print $1}'"
+        return "tools/bin/promql --no-headers --host 'http://" + prometheus_ip + ":9090' '" + query + "' | awk '{print $1}'"
     return promql_query
 
 if __name__ == "__main__":