Multus ipv6 on vsphere (#11102)

Signed-off-by: Shay Rozen <[email protected]>
red-hat-storage · Jan 9, 2025 · 628f60b · 628f60b
1 parent 4406407
commit 628f60b
Show file tree

Hide file tree

Showing 10 changed files with 167 additions and 14 deletions.
diff --git a/conf/deployment/vsphere/upi_1az_rhcos_multus_public_cluster_vsan_3m_3w_ipv6.yaml b/conf/deployment/vsphere/upi_1az_rhcos_multus_public_cluster_vsan_3m_3w_ipv6.yaml
@@ -0,0 +1,24 @@
+---
+# This config is suppose to work on most of DCs we have.
+DEPLOYMENT:
+  allow_lower_instance_requirements: false
+  ipv6: true
+ENV_DATA:
+  platform: "vsphere"
+  deployment_type: "upi"
+  worker_replicas: 3
+  master_replicas: 3
+  worker_num_cpus: "16"
+  master_num_cpus: "4"
+  master_memory: "16384"
+  compute_memory: "65536"
+  fio_storageutilization_min_mbps: 10.0
+  is_multus_enabled: true
+  multus_public_net_interface: "ens224"
+  multus_cluster_net_interface: "ens224"
+  multus_create_public_net: true
+  multus_create_cluster_net: true
+  multus_public_net_namespace: "default"
+REPORTING:
+  polarion:
+    deployment_id: "OCS-6317"
diff --git a/conf/ocsci/vsphere_upi_enable_multus_ipv6_single_stack.yaml b/conf/ocsci/vsphere_upi_enable_multus_ipv6_single_stack.yaml
@@ -0,0 +1,13 @@
+# This configuration file is to enable vsphere upi ipv6 single stack
+DEPLOYMENT:
+  ipv6: true
+ENV_DATA:
+  is_multus_enabled: true
+  multus_public_net_interface: "ens224"
+  multus_cluster_net_interface: "ens224"
+  multus_create_public_net: true
+  multus_create_cluster_net: true
+  multus_public_net_namespace: "default"
+REPORTING:
+  polarion:
+    deployment_id: "OCS-6137"
diff --git a/ocs_ci/deployment/deployment.py b/ocs_ci/deployment/deployment.py
@@ -1144,8 +1144,11 @@ def deploy_ocs_via_operator(self, image=None):
                         node_obj.exec_oc_debug_cmd(node=node, cmd_list=[ip_link_cmd])
 
             if create_public_net:
+                nad_to_load = constants.MULTUS_PUBLIC_NET_YAML
                 logger.info("Creating Multus public network")
-                public_net_data = templating.load_yaml(constants.MULTUS_PUBLIC_NET_YAML)
+                if config.DEPLOYMENT.get("ipv6"):
+                    nad_to_load = constants.MULTUS_PUBLIC_NET_IPV6_YAML
+                public_net_data = templating.load_yaml(nad_to_load)
                 public_net_data["metadata"]["name"] = config.ENV_DATA.get(
                     "multus_public_net_name"
                 )
@@ -1157,9 +1160,14 @@ def deploy_ocs_via_operator(self, image=None):
                 public_net_config_dict["master"] = config.ENV_DATA.get(
                     "multus_public_net_interface"
                 )
-                public_net_config_dict["ipam"]["range"] = config.ENV_DATA.get(
-                    "multus_public_net_range"
-                )
+                if not config.DEPLOYMENT.get("ipv6"):
+                    public_net_config_dict["ipam"]["range"] = config.ENV_DATA.get(
+                        "multus_public_net_range"
+                    )
+                else:
+                    public_net_config_dict["ipam"]["range"] = config.ENV_DATA.get(
+                        "multus_public_ipv6_net_range"
+                    )
                 public_net_config_dict["type"] = config.ENV_DATA.get(
                     "multus_public_net_type"
                 )
@@ -1175,6 +1183,10 @@ def deploy_ocs_via_operator(self, image=None):
 
             if create_cluster_net:
                 logger.info("Creating Multus cluster network")
+                if config.DEPLOYMENT.get("ipv6"):
+                    constants.MULTUS_CLUSTER_NET_YAML = (
+                        constants.MULTUS_CLUSTER_NET_IPV6_YAML
+                    )
                 cluster_net_data = templating.load_yaml(
                     constants.MULTUS_CLUSTER_NET_YAML
                 )
@@ -1189,12 +1201,14 @@ def deploy_ocs_via_operator(self, image=None):
                 cluster_net_config_dict["master"] = config.ENV_DATA.get(
                     "multus_cluster_net_interface"
                 )
-                cluster_net_config_dict["ipam"]["range"] = config.ENV_DATA.get(
-                    "multus_cluster_net_range"
-                )
-                cluster_net_config_dict["type"] = config.ENV_DATA.get(
-                    "multus_cluster_net_type"
-                )
+                if not config.DEPLOYMENT.get("ipv6"):
+                    cluster_net_config_dict["ipam"]["range"] = config.ENV_DATA.get(
+                        "multus_cluster_net_range"
+                    )
+                else:
+                    cluster_net_config_dict["ipam"]["range"] = config.ENV_DATA.get(
+                        "multus_cluster_ipv6_net_range"
+                    )
                 cluster_net_config_dict["mode"] = config.ENV_DATA.get(
                     "multus_cluster_net_mode"
                 )

diff --git a/ocs_ci/framework/conf/default_config.yaml b/ocs_ci/framework/conf/default_config.yaml
@@ -264,13 +264,15 @@ ENV_DATA:
   multus_public_net_namespace: "openshift-storage"
   multus_public_net_interface: "br-ex"
   multus_public_net_range: "192.168.20.0/24"
+  multus_public_ipv6_net_range: "fd01:db8:1:1::/96"
   multus_public_net_type: "macvlan"
   multus_public_net_mode: "bridge"
   multus_create_cluster_net: true
   multus_cluster_net_name: "private-net"
   multus_cluster_net_namespace: "openshift-storage"
   multus_cluster_net_interface: "br-ex"
   multus_cluster_net_range: "192.168.30.0/24"
+  multus_cluster_ipv6_net_range: "fd01:db8:2:1::/96"
   multus_cluster_net_type: "macvlan"
   multus_cluster_net_mode: "bridge"
   multus_destination_route: "192.168.252.0/24"

diff --git a/ocs_ci/helpers/helpers.py b/ocs_ci/helpers/helpers.py
@@ -4870,6 +4870,8 @@ def add_route_public_nad():
     ]
     nad_config_dict_string = json.dumps(nad_config_dict)
     logger.info("Creating Multus public network")
+    if config.DEPLOYMENT.get("ipv6"):
+        constants.MULTUS_PUBLIC_NET_YAML = constants.MULTUS_PUBLIC_NET_IPV6_YAML
     public_net_data = templating.load_yaml(constants.MULTUS_PUBLIC_NET_YAML)
     public_net_data["metadata"]["name"] = config.ENV_DATA.get("multus_public_net_name")
     public_net_data["metadata"]["namespace"] = config.ENV_DATA.get(
@@ -4959,6 +4961,15 @@ def configure_node_network_configuration_policy_on_all_worker_nodes():
     # This function require changes for compact mode
     logger.info("Configure NodeNetworkConfigurationPolicy on all worker nodes")
     worker_node_names = get_worker_nodes()
+    ip_version = "ipv4"
+    if (
+        config.DEPLOYMENT.get("ipv6")
+        and config.ENV_DATA.get("platform") == constants.VSPHERE_PLATFORM
+    ):
+        constants.NODE_NETWORK_CONFIGURATION_POLICY = (
+            constants.NODE_NETWORK_CONFIGURATION_POLICY_IPV6
+        )
+        ip_version = "ipv6"
     interface_num = 0
     for worker_node_name in worker_node_names:
         node_network_configuration_policy = templating.load_yaml(
@@ -5001,13 +5012,13 @@ def configure_node_network_configuration_policy_on_all_worker_nodes():
             ] = f"ceph-public-net-shim-{worker_node_name}"
             shim_default_ip = node_network_configuration_policy["spec"]["desiredState"][
                 "interfaces"
-            ][0]["ipv4"]["address"][0]["ip"]
+            ][0][ip_version]["address"][0]["ip"]
 
             shim_ip = str(ipaddress.ip_address(shim_default_ip) + interface_num)
             interface_num += 1
 
             node_network_configuration_policy["spec"]["desiredState"]["interfaces"][0][
-                "ipv4"
+                ip_version
             ]["address"][0]["ip"] = shim_ip
 
             node_network_configuration_policy["spec"]["desiredState"]["interfaces"][0][

diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py
@@ -1047,6 +1047,18 @@
 NODE_NETWORK_CONFIGURATION_POLICY = os.path.join(
     TEMPLATE_DEPLOYMENT_DIR, "node_network_configuration_policy.yaml"
 )
+
+MULTUS_PUBLIC_NET_IPV6_YAML = os.path.join(
+    TEMPLATE_DEPLOYMENT_DIR, "multus-public-net-ipv6.yaml"
+)
+MULTUS_CLUSTER_NET_IPV6_YAML = os.path.join(
+    TEMPLATE_DEPLOYMENT_DIR, "multus-cluster-net-ipv6.yaml"
+)
+NODE_NETWORK_CONFIGURATION_POLICY_IPV6 = os.path.join(
+    TEMPLATE_DEPLOYMENT_DIR, "node_network_configuration_policy_ipv6.yaml"
+)
+
+
 NETWORK_ATTACHEMENT_DEFINITION = "network-attachment-definitions.k8s.cni.cncf.io"
 VSPHERE_MULTUS_INTERFACE = "ens224"
 

diff --git a/ocs_ci/ocs/resources/storage_cluster.py b/ocs_ci/ocs/resources/storage_cluster.py
@@ -1983,8 +1983,17 @@ def verify_multus_network():
         )
         mds_map = ceph_fs_dump_data["filesystems"][0]["mdsmap"]
         for _, gid_data in mds_map["info"].items():
-            ip = gid_data["addr"].split(":")[0]
-            range = config.ENV_DATA["multus_public_net_range"]
+            if not config.DEPLOYMENT.get("ipv6"):
+                ip = gid_data["addr"].split(":")[0]
+                range = config.ENV_DATA["multus_public_net_range"]
+
+            else:
+                gid_dt_ip = gid_data["addr"]
+                pattern = r"^\[([\da-f:]+)\]"
+                match = re.search(pattern, gid_dt_ip, re.IGNORECASE)
+                ip = match.group(1)
+                range = config.ENV_DATA["multus_public_ipv6_net_range"]
+
             assert ipaddress.ip_address(ip) in ipaddress.ip_network(range)
 
     log.info("Verifying StorageCluster multus network data")

diff --git a/ocs_ci/templates/ocs-deployment/multus-cluster-net-ipv6.yaml b/ocs_ci/templates/ocs-deployment/multus-cluster-net-ipv6.yaml
@@ -0,0 +1,18 @@
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: cluster-net
+  namespace: openshift-storage
+  labels: {}
+  annotations: {}
+spec:
+  config: '{
+      "cniVersion": "0.3.1",
+      "type": "macvlan",
+      "master": "br-ex",
+      "mode": "bridge",
+      "ipam": {
+            "type": "whereabouts",
+            "range": "fd01:db8:2:1::/96"
+      }
+  }'
diff --git a/ocs_ci/templates/ocs-deployment/multus-public-net-ipv6.yaml b/ocs_ci/templates/ocs-deployment/multus-public-net-ipv6.yaml
@@ -0,0 +1,19 @@
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: public-net
+  namespace: openshift-storage
+  labels: {}
+  annotations: {}
+spec:
+  config: '{
+      "cniVersion": "0.3.1",
+      "type": "macvlan",
+      "master": "br-ex",
+      "mode": "bridge",
+      "ipam": {
+            "type": "whereabouts",
+            "range": "fd01:db8:1:1::/96",
+            "routes": [{"dst": "fd01:db8:3:1::/96"}]
+      }
+  }'
diff --git a/ocs_ci/templates/ocs-deployment/node_network_configuration_policy_ipv6.yaml b/ocs_ci/templates/ocs-deployment/node_network_configuration_policy_ipv6.yaml
@@ -0,0 +1,31 @@
+apiVersion: nmstate.io/v1
+kind: NodeNetworkConfigurationPolicy
+metadata:
+  name: ceph-public-net-shim-worker-node
+  namespace: openshift-storage
+spec:
+  nodeSelector:
+    node-role.kubernetes.io/worker: ""
+    kubernetes.io/hostname: worker-node
+  desiredState:
+    interfaces:
+      - name: odf-pub-shim
+        description: Shim interface used to connect host to OpenShift Data Foundation public Multus network
+        type: mac-vlan
+        state: up
+        mac-vlan:
+          base-iface: enp1s0f1
+          mode: bridge
+          promiscuous: true
+        ipv6:
+          enabled: true
+          dhcp: false
+          address:
+            - ip: fd01:db8:3:1::1 # STATIC IP FOR worker node
+              prefix-length: 96
+    routes:
+      config:
+        - destination: fd01:db8:1:1::/96
+          next-hop-interface: odf-pub-shim
+        - destination: fd01:db8:2:1::/96
+          next-hop-interface: odf-pub-shim