Move load balancer node configuration to switches.json (#17)

* Use a json config as argument for controller Previously, the p4info, bmv2_json and multi_switch arguments were passed to the controller as command line arguments and the switch configuration was hardcoded. This commit makes it possible to configure any number of switches in the configuration file, and also to customize the parameters of individual switches. * Format all json runtime files * Add comment to the controller Makefile target * Add the lb_nodes field to the switches JSON config The lb_nodes field is a list of the nodes that the load balancer routes traffic to. This field consists of a list of dictionaries, where each dictionary contains the following fields: - ip: the IP address of the node - mac: the MAC address of the node - port: the switch port to use to route traffic to the node The controller uses this field to program the load balancer switch with proper match-action table entries. The controller tracks state of the table entries to allow updates after a container migration.
stano45 · Aug 10, 2024 · 66a1075 · 66a1075
1 parent 159ab56
commit 66a1075
Show file tree

Hide file tree

Showing 9 changed files with 61 additions and 117 deletions.
diff --git a/controller/controller.py b/controller/controller.py
@@ -49,6 +49,9 @@ def main(config_file_path):
             raise Exception(
                 "No master switch specified in the configuration file."
             )
+        lb_nodes = master_config.get("lb_nodes", None)
+        if not lb_nodes:
+            raise Exception("No load balancer IP addresses specified.")
 
         master_controller = SwitchController(
             p4info_file_path=master_config["p4info_file_path"],
@@ -59,8 +62,11 @@ def main(config_file_path):
             proto_dump_file=master_config["proto_dump_file"],
             initial_table_rules_file=master_config["runtime_file"],
         )
+
         global nodeManager
-        nodeManager = NodeManager(master_controller)
+        nodeManager = NodeManager(
+            switch_controller=master_controller, lb_nodes=lb_nodes
+        )
 
     except KeyboardInterrupt:
         print("Shutting down.")

diff --git a/controller/node_manager.py b/controller/node_manager.py
@@ -2,13 +2,29 @@
 
 
 class NodeManager(object):
-    def __init__(self, switch_controller: SwitchController):
+    def __init__(self, switch_controller: SwitchController, lb_nodes):
         self.switch_controller = switch_controller
         # ipv4 -> ecmp_select_id
-        self.node_map = {
-            "10.0.2.2": 1,
-            "10.0.3.3": 2,
-        }
+        self.node_map = {}
+        for i, node in enumerate(lb_nodes):
+            # 0 is reserved for 10.0.1.1 (client), the rest for nodes
+            ip = node["ip"]
+            mac = node["mac"]
+            port = node["port"]
+            self.node_map[ip] = i + 1
+            self.switch_controller.insertEcmpNhopEntry(
+                ecmp_select=i + 1,
+                dmac=mac,
+                ipv4=ip,
+                port=port,
+                update_type="INSERT",
+            )
+        self.switch_controller.insertEcmpGroupEntry(
+            matchDstAddr=["10.0.1.10", 32],
+            ecmp_base=1,
+            ecmp_count=len(lb_nodes),
+        )
+        self.switch_controller.readTableRules()
 
     def updateNode(self, old_ip, new_ip, dest_mac, egress_port):
         if new_ip in self.node_map:
@@ -19,7 +35,7 @@ def updateNode(self, old_ip, new_ip, dest_mac, egress_port):
 
         ecmp_select_id = self.node_map.pop(old_ip)
 
-        self.switch_controller.upsertEcmpNhopEntry(
+        self.switch_controller.insertEcmpNhopEntry(
             ecmp_select=ecmp_select_id,
             dmac=dest_mac,
             ipv4=new_ip,

diff --git a/controller/switch_controller.py b/controller/switch_controller.py
@@ -67,7 +67,22 @@ def __init__(
     def __del__(self):
         ShutdownAllSwitchConnections()
 
-    def upsertEcmpNhopEntry(
+    def insertEcmpGroupEntry(
+        self, matchDstAddr, ecmp_base, ecmp_count, update_type="INSERT"
+    ):
+        table_entry = self.p4info_helper.buildTableEntry(
+            table_name="MyIngress.ecmp_group",
+            match_fields={"hdr.ipv4.dstAddr": matchDstAddr},
+            action_name="MyIngress.set_ecmp_select",
+            action_params={"ecmp_base": ecmp_base, "ecmp_count": ecmp_count},
+        )
+        self.sw.WriteTableEntry(table_entry, update_type=update_type)
+        print(
+            f"Updated the 'ecmp_group' table on "
+            f"{self.sw.name=} with {matchDstAddr=}, {ecmp_base=}, {ecmp_count=}"
+        )
+
+    def insertEcmpNhopEntry(
         self, ecmp_select, dmac, ipv4, port, update_type="INSERT"
     ):
         table_entry = self.p4info_helper.buildTableEntry(
@@ -93,7 +108,7 @@ def deleteEcmpNhopEntry(self, ecmp_select):
             f"{self.sw.name=} with {ecmp_select=}"
         )
 
-    def upsertSendFrameEntry(self, egress_port, smac):
+    def insertSendFrameEntry(self, egress_port, smac):
         table_entry = self.p4info_helper.buildTableEntry(
             table_name="MyEgress.send_frame",
             match_fields={"standard_metadata.egress_port": egress_port},

diff --git a/examples/host_containers/config/s1-runtime.json b/examples/host_containers/config/s1-runtime.json
@@ -19,17 +19,6 @@
         "new_src": "10.0.1.10"
       }
     },
-    {
-      "table": "MyIngress.ecmp_group",
-      "match": {
-        "hdr.ipv4.dstAddr": ["10.0.1.10", 32]
-      },
-      "action_name": "MyIngress.set_ecmp_select",
-      "action_params": {
-        "ecmp_base": 1,
-        "ecmp_count": 2
-      }
-    },
     {
       "table": "MyIngress.ecmp_nhop",
       "match": {
@@ -42,30 +31,6 @@
         "port": 1
       }
     },
-    {
-      "table": "MyIngress.ecmp_nhop",
-      "match": {
-        "meta.ecmp_select": 1
-      },
-      "action_name": "MyIngress.set_nhop",
-      "action_params": {
-        "nhop_dmac": "08:00:00:00:02:02",
-        "nhop_ipv4": "10.0.2.2",
-        "port": 2
-      }
-    },
-    {
-      "table": "MyIngress.ecmp_nhop",
-      "match": {
-        "meta.ecmp_select": 2
-      },
-      "action_name": "MyIngress.set_nhop",
-      "action_params": {
-        "nhop_dmac": "08:00:00:00:03:03",
-        "nhop_ipv4": "10.0.3.3",
-        "port": 3
-      }
-    },
     {
       "table": "MyEgress.send_frame",
       "match": {

diff --git a/examples/host_containers/config/switches.json b/examples/host_containers/config/switches.json
@@ -7,7 +7,11 @@
     "runtime_file": "./config/s1-runtime.json",
     "p4info_file_path": "../../load_balancer/build/load_balance.p4.p4info.txt",
     "bmv2_file_path": "../../load_balancer/build/load_balance.json",
-    "master": true
+    "master": true,
+    "lb_nodes": [
+      { "ip": "10.0.2.2", "mac": "08:00:00:00:02:02", "port": 2 },
+      { "ip": "10.0.3.3", "mac": "08:00:00:00:03:03", "port": 3 }
+    ]
   },
   {
     "id": 1,

diff --git a/examples/process_migration/config/s1-runtime.json b/examples/process_migration/config/s1-runtime.json
@@ -19,17 +19,6 @@
         "new_src": "10.0.1.10"
       }
     },
-    {
-      "table": "MyIngress.ecmp_group",
-      "match": {
-        "hdr.ipv4.dstAddr": ["10.0.1.10", 32]
-      },
-      "action_name": "MyIngress.set_ecmp_select",
-      "action_params": {
-        "ecmp_base": 1,
-        "ecmp_count": 2
-      }
-    },
     {
       "table": "MyIngress.ecmp_nhop",
       "match": {
@@ -42,30 +31,6 @@
         "port": 1
       }
     },
-    {
-      "table": "MyIngress.ecmp_nhop",
-      "match": {
-        "meta.ecmp_select": 1
-      },
-      "action_name": "MyIngress.set_nhop",
-      "action_params": {
-        "nhop_dmac": "08:00:00:00:02:02",
-        "nhop_ipv4": "10.0.2.2",
-        "port": 2
-      }
-    },
-    {
-      "table": "MyIngress.ecmp_nhop",
-      "match": {
-        "meta.ecmp_select": 2
-      },
-      "action_name": "MyIngress.set_nhop",
-      "action_params": {
-        "nhop_dmac": "08:00:00:00:03:03",
-        "nhop_ipv4": "10.0.3.3",
-        "port": 3
-      }
-    },
     {
       "table": "MyEgress.send_frame",
       "match": {

diff --git a/examples/process_migration/config/switches.json b/examples/process_migration/config/switches.json
@@ -7,7 +7,11 @@
     "runtime_file": "./config/s1-runtime.json",
     "p4info_file_path": "../../load_balancer/build/load_balance.p4.p4info.txt",
     "bmv2_file_path": "../../load_balancer/build/load_balance.json",
-    "master": true
+    "master": true,
+    "lb_nodes": [
+      { "ip": "10.0.2.2", "mac": "08:00:00:00:02:02", "port": 2 },
+      { "ip": "10.0.3.3", "mac": "08:00:00:00:03:03", "port": 3 }
+    ]
   },
   {
     "id": 1,

diff --git a/examples/switch_container/config/s1-runtime.json b/examples/switch_container/config/s1-runtime.json
@@ -19,17 +19,6 @@
         "new_src": "10.0.1.10"
       }
     },
-    {
-      "table": "MyIngress.ecmp_group",
-      "match": {
-        "hdr.ipv4.dstAddr": ["10.0.1.10", 32]
-      },
-      "action_name": "MyIngress.set_ecmp_select",
-      "action_params": {
-        "ecmp_base": 1,
-        "ecmp_count": 2
-      }
-    },
     {
       "table": "MyIngress.ecmp_nhop",
       "match": {
@@ -42,30 +31,6 @@
         "port": 1
       }
     },
-    {
-      "table": "MyIngress.ecmp_nhop",
-      "match": {
-        "meta.ecmp_select": 1
-      },
-      "action_name": "MyIngress.set_nhop",
-      "action_params": {
-        "nhop_dmac": "08:00:00:00:02:02",
-        "nhop_ipv4": "10.0.2.2",
-        "port": 2
-      }
-    },
-    {
-      "table": "MyIngress.ecmp_nhop",
-      "match": {
-        "meta.ecmp_select": 2
-      },
-      "action_name": "MyIngress.set_nhop",
-      "action_params": {
-        "nhop_dmac": "08:00:00:00:03:03",
-        "nhop_ipv4": "10.0.3.3",
-        "port": 3
-      }
-    },
     {
       "table": "MyEgress.send_frame",
       "match": {

diff --git a/examples/switch_container/config/switches.json b/examples/switch_container/config/switches.json
@@ -7,6 +7,10 @@
     "runtime_file": "config/s1-runtime.json",
     "p4info_file_path": "../../load_balancer/build/load_balance.p4.p4info.txt",
     "bmv2_file_path": "../../load_balancer/build/load_balance.json",
-    "master": true
+    "master": true,
+    "lb_nodes": [
+      { "ip": "10.0.2.2", "mac": "08:00:00:00:02:02", "port": 2 },
+      { "ip": "10.0.3.3", "mac": "08:00:00:00:03:03", "port": 3 }
+    ]
   }
 ]