From 66a10758067c21c04a9f02990a1a5c85a2d1e84d Mon Sep 17 00:00:00 2001 From: Stanislav Kosorin Date: Sat, 10 Aug 2024 20:11:22 +0200 Subject: [PATCH] Move load balancer node configuration to switches.json (#17) * Use a json config as argument for controller Previously, the p4info, bmv2_json and multi_switch arguments were passed to the controller as command line arguments and the switch configuration was hardcoded. This commit makes it possible to configure any number of switches in the configuration file, and also to customize the parameters of individual switches. * Format all json runtime files * Add comment to the controller Makefile target * Add the lb_nodes field to the switches JSON config The lb_nodes field is a list of the nodes that the load balancer routes traffic to. This field consists of a list of dictionaries, where each dictionary contains the following fields: - ip: the IP address of the node - mac: the MAC address of the node - port: the switch port to use to route traffic to the node The controller uses this field to program the load balancer switch with proper match-action table entries. The controller tracks state of the table entries to allow updates after a container migration. --- controller/controller.py | 8 ++++- controller/node_manager.py | 28 +++++++++++---- controller/switch_controller.py | 19 ++++++++-- .../host_containers/config/s1-runtime.json | 35 ------------------- examples/host_containers/config/switches.json | 6 +++- .../process_migration/config/s1-runtime.json | 35 ------------------- .../process_migration/config/switches.json | 6 +++- .../switch_container/config/s1-runtime.json | 35 ------------------- .../switch_container/config/switches.json | 6 +++- 9 files changed, 61 insertions(+), 117 deletions(-) diff --git a/controller/controller.py b/controller/controller.py index 26a9077..4bbb60f 100755 --- a/controller/controller.py +++ b/controller/controller.py @@ -49,6 +49,9 @@ def main(config_file_path): raise Exception( "No master switch specified in the configuration file." ) + lb_nodes = master_config.get("lb_nodes", None) + if not lb_nodes: + raise Exception("No load balancer IP addresses specified.") master_controller = SwitchController( p4info_file_path=master_config["p4info_file_path"], @@ -59,8 +62,11 @@ def main(config_file_path): proto_dump_file=master_config["proto_dump_file"], initial_table_rules_file=master_config["runtime_file"], ) + global nodeManager - nodeManager = NodeManager(master_controller) + nodeManager = NodeManager( + switch_controller=master_controller, lb_nodes=lb_nodes + ) except KeyboardInterrupt: print("Shutting down.") diff --git a/controller/node_manager.py b/controller/node_manager.py index 62305ff..764ff2c 100644 --- a/controller/node_manager.py +++ b/controller/node_manager.py @@ -2,13 +2,29 @@ class NodeManager(object): - def __init__(self, switch_controller: SwitchController): + def __init__(self, switch_controller: SwitchController, lb_nodes): self.switch_controller = switch_controller # ipv4 -> ecmp_select_id - self.node_map = { - "10.0.2.2": 1, - "10.0.3.3": 2, - } + self.node_map = {} + for i, node in enumerate(lb_nodes): + # 0 is reserved for 10.0.1.1 (client), the rest for nodes + ip = node["ip"] + mac = node["mac"] + port = node["port"] + self.node_map[ip] = i + 1 + self.switch_controller.insertEcmpNhopEntry( + ecmp_select=i + 1, + dmac=mac, + ipv4=ip, + port=port, + update_type="INSERT", + ) + self.switch_controller.insertEcmpGroupEntry( + matchDstAddr=["10.0.1.10", 32], + ecmp_base=1, + ecmp_count=len(lb_nodes), + ) + self.switch_controller.readTableRules() def updateNode(self, old_ip, new_ip, dest_mac, egress_port): if new_ip in self.node_map: @@ -19,7 +35,7 @@ def updateNode(self, old_ip, new_ip, dest_mac, egress_port): ecmp_select_id = self.node_map.pop(old_ip) - self.switch_controller.upsertEcmpNhopEntry( + self.switch_controller.insertEcmpNhopEntry( ecmp_select=ecmp_select_id, dmac=dest_mac, ipv4=new_ip, diff --git a/controller/switch_controller.py b/controller/switch_controller.py index fe3e742..a2f3a91 100644 --- a/controller/switch_controller.py +++ b/controller/switch_controller.py @@ -67,7 +67,22 @@ def __init__( def __del__(self): ShutdownAllSwitchConnections() - def upsertEcmpNhopEntry( + def insertEcmpGroupEntry( + self, matchDstAddr, ecmp_base, ecmp_count, update_type="INSERT" + ): + table_entry = self.p4info_helper.buildTableEntry( + table_name="MyIngress.ecmp_group", + match_fields={"hdr.ipv4.dstAddr": matchDstAddr}, + action_name="MyIngress.set_ecmp_select", + action_params={"ecmp_base": ecmp_base, "ecmp_count": ecmp_count}, + ) + self.sw.WriteTableEntry(table_entry, update_type=update_type) + print( + f"Updated the 'ecmp_group' table on " + f"{self.sw.name=} with {matchDstAddr=}, {ecmp_base=}, {ecmp_count=}" + ) + + def insertEcmpNhopEntry( self, ecmp_select, dmac, ipv4, port, update_type="INSERT" ): table_entry = self.p4info_helper.buildTableEntry( @@ -93,7 +108,7 @@ def deleteEcmpNhopEntry(self, ecmp_select): f"{self.sw.name=} with {ecmp_select=}" ) - def upsertSendFrameEntry(self, egress_port, smac): + def insertSendFrameEntry(self, egress_port, smac): table_entry = self.p4info_helper.buildTableEntry( table_name="MyEgress.send_frame", match_fields={"standard_metadata.egress_port": egress_port}, diff --git a/examples/host_containers/config/s1-runtime.json b/examples/host_containers/config/s1-runtime.json index 733810a..6f95aec 100644 --- a/examples/host_containers/config/s1-runtime.json +++ b/examples/host_containers/config/s1-runtime.json @@ -19,17 +19,6 @@ "new_src": "10.0.1.10" } }, - { - "table": "MyIngress.ecmp_group", - "match": { - "hdr.ipv4.dstAddr": ["10.0.1.10", 32] - }, - "action_name": "MyIngress.set_ecmp_select", - "action_params": { - "ecmp_base": 1, - "ecmp_count": 2 - } - }, { "table": "MyIngress.ecmp_nhop", "match": { @@ -42,30 +31,6 @@ "port": 1 } }, - { - "table": "MyIngress.ecmp_nhop", - "match": { - "meta.ecmp_select": 1 - }, - "action_name": "MyIngress.set_nhop", - "action_params": { - "nhop_dmac": "08:00:00:00:02:02", - "nhop_ipv4": "10.0.2.2", - "port": 2 - } - }, - { - "table": "MyIngress.ecmp_nhop", - "match": { - "meta.ecmp_select": 2 - }, - "action_name": "MyIngress.set_nhop", - "action_params": { - "nhop_dmac": "08:00:00:00:03:03", - "nhop_ipv4": "10.0.3.3", - "port": 3 - } - }, { "table": "MyEgress.send_frame", "match": { diff --git a/examples/host_containers/config/switches.json b/examples/host_containers/config/switches.json index dd78560..d5e992f 100644 --- a/examples/host_containers/config/switches.json +++ b/examples/host_containers/config/switches.json @@ -7,7 +7,11 @@ "runtime_file": "./config/s1-runtime.json", "p4info_file_path": "../../load_balancer/build/load_balance.p4.p4info.txt", "bmv2_file_path": "../../load_balancer/build/load_balance.json", - "master": true + "master": true, + "lb_nodes": [ + { "ip": "10.0.2.2", "mac": "08:00:00:00:02:02", "port": 2 }, + { "ip": "10.0.3.3", "mac": "08:00:00:00:03:03", "port": 3 } + ] }, { "id": 1, diff --git a/examples/process_migration/config/s1-runtime.json b/examples/process_migration/config/s1-runtime.json index 733810a..6f95aec 100644 --- a/examples/process_migration/config/s1-runtime.json +++ b/examples/process_migration/config/s1-runtime.json @@ -19,17 +19,6 @@ "new_src": "10.0.1.10" } }, - { - "table": "MyIngress.ecmp_group", - "match": { - "hdr.ipv4.dstAddr": ["10.0.1.10", 32] - }, - "action_name": "MyIngress.set_ecmp_select", - "action_params": { - "ecmp_base": 1, - "ecmp_count": 2 - } - }, { "table": "MyIngress.ecmp_nhop", "match": { @@ -42,30 +31,6 @@ "port": 1 } }, - { - "table": "MyIngress.ecmp_nhop", - "match": { - "meta.ecmp_select": 1 - }, - "action_name": "MyIngress.set_nhop", - "action_params": { - "nhop_dmac": "08:00:00:00:02:02", - "nhop_ipv4": "10.0.2.2", - "port": 2 - } - }, - { - "table": "MyIngress.ecmp_nhop", - "match": { - "meta.ecmp_select": 2 - }, - "action_name": "MyIngress.set_nhop", - "action_params": { - "nhop_dmac": "08:00:00:00:03:03", - "nhop_ipv4": "10.0.3.3", - "port": 3 - } - }, { "table": "MyEgress.send_frame", "match": { diff --git a/examples/process_migration/config/switches.json b/examples/process_migration/config/switches.json index dd78560..d5e992f 100644 --- a/examples/process_migration/config/switches.json +++ b/examples/process_migration/config/switches.json @@ -7,7 +7,11 @@ "runtime_file": "./config/s1-runtime.json", "p4info_file_path": "../../load_balancer/build/load_balance.p4.p4info.txt", "bmv2_file_path": "../../load_balancer/build/load_balance.json", - "master": true + "master": true, + "lb_nodes": [ + { "ip": "10.0.2.2", "mac": "08:00:00:00:02:02", "port": 2 }, + { "ip": "10.0.3.3", "mac": "08:00:00:00:03:03", "port": 3 } + ] }, { "id": 1, diff --git a/examples/switch_container/config/s1-runtime.json b/examples/switch_container/config/s1-runtime.json index 733810a..6f95aec 100644 --- a/examples/switch_container/config/s1-runtime.json +++ b/examples/switch_container/config/s1-runtime.json @@ -19,17 +19,6 @@ "new_src": "10.0.1.10" } }, - { - "table": "MyIngress.ecmp_group", - "match": { - "hdr.ipv4.dstAddr": ["10.0.1.10", 32] - }, - "action_name": "MyIngress.set_ecmp_select", - "action_params": { - "ecmp_base": 1, - "ecmp_count": 2 - } - }, { "table": "MyIngress.ecmp_nhop", "match": { @@ -42,30 +31,6 @@ "port": 1 } }, - { - "table": "MyIngress.ecmp_nhop", - "match": { - "meta.ecmp_select": 1 - }, - "action_name": "MyIngress.set_nhop", - "action_params": { - "nhop_dmac": "08:00:00:00:02:02", - "nhop_ipv4": "10.0.2.2", - "port": 2 - } - }, - { - "table": "MyIngress.ecmp_nhop", - "match": { - "meta.ecmp_select": 2 - }, - "action_name": "MyIngress.set_nhop", - "action_params": { - "nhop_dmac": "08:00:00:00:03:03", - "nhop_ipv4": "10.0.3.3", - "port": 3 - } - }, { "table": "MyEgress.send_frame", "match": { diff --git a/examples/switch_container/config/switches.json b/examples/switch_container/config/switches.json index 5b3d72a..6684bac 100644 --- a/examples/switch_container/config/switches.json +++ b/examples/switch_container/config/switches.json @@ -7,6 +7,10 @@ "runtime_file": "config/s1-runtime.json", "p4info_file_path": "../../load_balancer/build/load_balance.p4.p4info.txt", "bmv2_file_path": "../../load_balancer/build/load_balance.json", - "master": true + "master": true, + "lb_nodes": [ + { "ip": "10.0.2.2", "mac": "08:00:00:00:02:02", "port": 2 }, + { "ip": "10.0.3.3", "mac": "08:00:00:00:03:03", "port": 3 } + ] } ]