Merge branch 'main' into dependabot/pip/pyopenssl-23.2.0

README.md

@@ -9,7 +9,7 @@ can choose the APT cluster `d430` node.
 
 ## Create a cluster
 
-First, configure `script/setup.cfg`. You can specify there which vHive branch to use, loader branch, operation mode
+First, configure `script/setup/setup.cfg`. You can specify there which vHive branch to use, loader branch, operation mode
 (sandbox type), maximum number of pods per node, and the Github token. All these configurations are mandatory.
 We currently support the following modes: containerd (`container`), Firecracker (`firecracker`), and Firecracker with
 snapshots (`firecracker_snapshots`).
@@ -21,12 +21,13 @@ API server certificate.
 * To create a multi-node cluster, specify the node addresses as the arguments and run the following command:
 
 ```bash
-$ bash ./scripts/setup/create_multinode.sh <master_node@IP> <worker_node@IP> ...
+$ bash ./scripts/setup/create_multinode.sh <master_node@IP> <loader_node@IP> <worker_node@IP> ...
 ```
 
-The loader should be running on a separate node that is part of the Kubernetes cluster. Do not collocate master and
-worker node components where the loader is located for performance reasons. Make sure you taint the node where loader is
-located prior to running any experiment.
+This command will create the following setup: control plane is placed on master node, loader node is used for running
+loader and monitoring pods (mostly, Prometheus, if enabled in setup config), workers are used purely for working pods. In
+this setup, neither control plane nor workers are affected by loader and monitoring, creating more reliable measurements
+of performance.
 
 * Single-node cluster (experimental)
 

pkg/metric/record.go

@@ -142,6 +142,8 @@ type ClusterUsage struct {
 	PodCpu          []string  `csv:"pod_cpu" json:"pod_cpu"`
 	PodMemory       []string  `csv:"pod_memory" json:"pod_mem"`
 	Pods            []int     `csv:"pods" json:"pods"`
+	LoaderCpu       float64   `csv:"loader_cpu" json:"loader_cpu"`
+	LoaderMem       float64   `csv:"loader_mem" json:"loader_mem"`
 }
 
 type AdfResult struct {

pkg/metric/scrape_infra.py

@@ -3,11 +3,11 @@
 import sys
 import os
 
-loader_total_cores = 8
+prometheus_ip = os.popen("kubectl get svc -n monitoring | grep prometheus-kube-prometheus-prometheus | awk '{print $3}'").read().strip().split('\n')[0]
 
 def get_promql_query(query):
     def promql_query():
-        return "tools/bin/promql --no-headers --host 'http://localhost:9090' '" + query + "' | grep . | sort | awk '{print $2}'"
+        return "tools/bin/promql --no-headers --host 'http://" + prometheus_ip + ":9090' '" + query + "' | grep . | sort | awk '{print $2}'"
     return promql_query
 
 if __name__ == "__main__":
@@ -42,12 +42,13 @@ def promql_query():
         "pod_cpu": [],
         "pod_mem": [],
         "pods": [],
+        "loader_cpu": 0,
+        "loader_mem": 0,
     }
 
-    loader_cpu_pct, loader_mem_pct = list(
+    result["loader_cpu"], result["loader_mem"] = list(
         map(float, subprocess.check_output(cmd_get_loader_pct).decode("utf-8").strip().split())
     )
-    loader_cpu_pct /= loader_total_cores #* Normalise to [0, 100]
 
     abs_out = subprocess.check_output(cmd_get_abs_vals).decode("utf-8")[:-1].split('\n')
     pcts_out = subprocess.check_output(cmd_get_pcts).decode("utf-8").split('\n')
@@ -66,8 +67,6 @@ def promql_query():
         if is_master:
             # Record master node.
             result['master_cpu_pct'], result['master_mem_pct'] = list(map(float, pcts[:-1].split('%')))
-            result['master_cpu_pct'] = max(0, result['master_cpu_pct'] - loader_cpu_pct)
-            result['master_mem_pct'] = max(0, result['master_mem_pct'] - loader_mem_pct)
             result['master_mem_req'] = float(mem_r)
             result['master_mem_lim'] = float(mem_l)
             result['master_cpu_req'] = float(cpu_r)

pkg/metric/scrape_kn.py

@@ -1,9 +1,11 @@
 import os
 import json
 
+prometheus_ip = os.popen("kubectl get svc -n monitoring | grep prometheus-kube-prometheus-prometheus | awk '{print $3}'").read().strip().split('\n')[0]
+
 def get_promql_query(query):
     def promql_query():
-        return "tools/bin/promql --no-headers --host 'http://localhost:9090' '" + query + "' | awk '{print $1}'"
+        return "tools/bin/promql --no-headers --host 'http://" + prometheus_ip + ":9090' '" + query + "' | awk '{print $1}'"
     return promql_query
 
 if __name__ == "__main__":

scripts/setup/create_multinode.sh

@@ -95,6 +95,14 @@ function setup_master() {
     LOGIN_TOKEN=${LOGIN_TOKEN//[$'\t\r\n']}
 }
 
+function setup_loader() {
+    echo "Setting up loader/monitoring node: $1"
+
+    server_exec "$1" 'wget -q https://go.dev/dl/go1.20.5.linux-amd64.tar.gz >/dev/null'
+    server_exec "$1" 'sudo rm -rf /usr/local/go && sudo tar -C /usr/local/ -xzf go1.20.5.linux-amd64.tar.gz >/dev/null'
+    server_exec "$1" 'echo "export PATH=$PATH:/usr/local/go/bin" >> .profile'
+}
+
 function setup_vhive_firecracker_daemon() {
     node=$1
 
@@ -232,6 +240,7 @@ function clone_loader_on_workers() {
     shift # make argument list only contain worker nodes (drops master node)
 
     setup_master
+    setup_loader $1
     setup_workers "$@"
 
     if [ $PODS_PER_NODE -gt 240 ]; then
@@ -249,15 +258,14 @@ function clone_loader_on_workers() {
     server_exec $MASTER_NODE 'cd loader; bash scripts/setup/patch_init_scale.sh'
 
     source $DIR/label.sh
-    if [[ "$DEPLOY_PROMETHEUS" == true ]]; then
 
-        # # Force placement of metrics collectors and instrumentation on the master node
-        label_workers $MASTER_NODE
-        label_master $MASTER_NODE
-        $DIR/expose_infra_metrics.sh $MASTER_NODE
-    else
-        label_all_workers $MASTER_NODE
-    fi
+    # Force placement of metrics collectors and instrumentation on the loader node and control plane on master
+    label_nodes $MASTER_NODE $1 # loader node is second on the list, becoming first after arg shift
+
     # patch knative to accept nodeselector
     server_exec $MASTER_NODE "cd loader; kubectl patch configmap config-features -n knative-serving -p '{\"data\": {\"kubernetes.podspec-nodeselector\": \"enabled\"}}'"
+
+    if [[ "$DEPLOY_PROMETHEUS" == true ]]; then
+        $DIR/expose_infra_metrics.sh $MASTER_NODE
+    fi
 }

scripts/setup/label.sh

@@ -11,8 +11,11 @@ server_exec() {
   ssh -oStrictHostKeyChecking=no -p 22 $1 $2;
 }
 
-label_master() {
+label_nodes() {
   MASTER_NODE=$1
+  LOADER_NODE=$2
+  LOADER_NODE_NAME="$(server_exec "$LOADER_NODE" hostname)"
+  echo $LOADER_NODE_NAME
 
   server_exec $MASTER_NODE 'kubectl get nodes' > tmp
   sed -i '1d' tmp
@@ -21,46 +24,15 @@ label_master() {
     NODE=$(echo $LINE | cut -d ' ' -f 1)
     TYPE=$(echo $LINE | cut -d ' ' -f 3)
 
+    echo "Label ${NODE}"
     if [[ $TYPE == *"master"* ]]; then
-      echo "Label ${NODE}"
+      server_exec $MASTER_NODE "kubectl label nodes ${NODE} loader-nodetype=master" < /dev/null
+    elif [[ $NODE == $LOADER_NODE_NAME ]]; then
       server_exec $MASTER_NODE "kubectl label nodes ${NODE} loader-nodetype=monitoring" < /dev/null
-    fi
-  done < tmp
-
-  rm tmp
-}
-
-label_workers() {
-  MASTER_NODE=$1
-
-  server_exec $MASTER_NODE 'kubectl get nodes' > tmp
-  sed -i '1d' tmp
-
-  while read LINE; do
-    NODE=$(echo $LINE | cut -d ' ' -f 1)
-    TYPE=$(echo $LINE | cut -d ' ' -f 3)
-
-    if [[ $TYPE != *"master"* ]]; then
-      echo "Label ${NODE}"
+    else
       server_exec $MASTER_NODE "kubectl label nodes ${NODE} loader-nodetype=worker" < /dev/null
     fi
   done < tmp
 
   rm tmp
 }
-
-label_all_workers() {
-  MASTER_NODE=$1
-
-  server_exec $MASTER_NODE 'kubectl get nodes' > tmp
-  sed -i '1d' tmp
-
-  while read LINE; do
-    NODE=$(echo $LINE | cut -d ' ' -f 1)
-
-    echo "Label ${NODE}"
-    server_exec $MASTER_NODE "kubectl label nodes ${NODE} loader-nodetype=worker" < /dev/null
-  done < tmp
-
-  rm tmp
-}