Add IBM Cloud custom network

Signed-off-by: Petr Balogh <[email protected]>

conf/README.md

@@ -322,6 +322,9 @@ higher priority).
 * `odf_provider_mode_deployment` - True if you would like to enable provider mode deployment.
 * `client_subcription_image` - ODF subscription image details for the storageclients.
 * `channel_to_client_subscription` - Channel value for the odf subscription image for storageclients.
+* `custom_vpc` - Applicable only for IMB Cloud IPI deployment where we want to create custom VPC and networking
+  with specific Address prefixes to prevent /18 CIDR to be used.
+* `ip_prefix` - Applicable only for IMB Cloud IPI deployment when custom_vpc, if not specified: 27 prefix will be used.
 
 #### UPGRADE
 

conf/deployment/ibmcloud/ipi_1az_us_east1_rhcos_custom_vpc_3m_3w.yaml

@@ -0,0 +1,28 @@
+---
+# This conifg is for IBM Cloud IPI deployment with custom VPC where we create
+# VPC and network stuff on by own in ocs-ci in us-east (Washington DC).
+DEPLOYMENT:
+  openshift_install_timeout: 4800
+  allow_lower_instance_requirements: false
+ENV_DATA:
+  platform: 'ibm_cloud'
+  deployment_type: 'ipi'
+  region: 'us-east'
+  base_domain: 'ibmcloud2.qe.rh-ocs.com'
+  worker_instance_type: 'bx2-16x64'
+  master_instance_type: 'bx2-4x16'
+  worker_availability_zones:
+    - 'us-east-1'
+  master_availability_zones:
+    - 'us-east-1'
+  worker_replicas: 3
+  master_replicas: 3
+  custom_vpc: true
+
+# The following values need to be set in a separate config and passed to ocs-ci in
+# order to deploy OCP/OCS cluster on IBM Cloud using IPI.
+#
+#AUTH:
+#  ibmcloud:
+#    account_id: ACCOUNT ID PLACEHOLDER
+#    api_key: IBM CLOUD API KEY PLACEHOLDER

conf/deployment/ibmcloud/ipi_1az_us_south1_rhcos_custom_vpc_3m_3w.yaml

@@ -0,0 +1,28 @@
+---
+# This conifg is for IBM Cloud IPI deployment with custom VPC where we create
+# VPC and network stuff on by own in ocs-ci in us-south (Dalas).
+DEPLOYMENT:
+  openshift_install_timeout: 4800
+  allow_lower_instance_requirements: false
+ENV_DATA:
+  platform: 'ibm_cloud'
+  deployment_type: 'ipi'
+  region: 'us-south'
+  base_domain: 'ibmcloud2.qe.rh-ocs.com'
+  worker_instance_type: 'bx2-16x64'
+  master_instance_type: 'bx2-4x16'
+  worker_availability_zones:
+    - 'us-south-1'
+  master_availability_zones:
+    - 'us-south-1'
+  worker_replicas: 3
+  master_replicas: 3
+  custom_vpc: true
+
+# The following values need to be set in a separate config and passed to ocs-ci in
+# order to deploy OCP/OCS cluster on IBM Cloud using IPI.
+#
+#AUTH:
+#  ibmcloud:
+#    account_id: ACCOUNT ID PLACEHOLDER
+#    api_key: IBM CLOUD API KEY PLACEHOLDER

ocs_ci/deployment/ibmcloud.py

@@ -27,6 +27,7 @@
 from ocs_ci.utility.deployment import get_ocp_release_image_from_installer
 from ocs_ci.utility.retry import retry
 from ocs_ci.utility.utils import (
+    get_random_str,
     exec_cmd,
     get_infra_id_from_openshift_install_state,
 )
@@ -173,6 +174,8 @@ def deploy_ocp(self, log_cli_level="DEBUG"):
                     f"Switching region to {other_region} due to lack of load balancers"
                 )
                 ibmcloud.set_region(other_region)
+        if config.ENV_DATA.get("custom_vpc"):
+            self.prepare_custom_vpc_and_network()
         self.ocp_deployment = self.OCPDeployment()
         self.ocp_deployment.deploy_prereq()
 
@@ -499,3 +502,45 @@ def get_load_balancers_count(self, region=None):
             f"Current load balancers count in region {region} is {load_balancers_count}"
         )
         return load_balancers_count
+
+    def prepare_custom_vpc_and_network(self):
+        cluster_id = get_random_str(size=5)
+        config.ENV_DATA["cluster_id"] = cluster_id
+        cluster_name = f"{config.ENV_DATA['cluster_name']}-{cluster_id}"
+        resource_group = cluster_name
+        vpc_name = f"{cluster_name}-vpc"
+        worker_zones = config.ENV_DATA["worker_availability_zones"]
+        master_zones = config.ENV_DATA["master_availability_zones"]
+        ip_prefix = config.ENV_DATA.get("ip_prefix", 27)
+        region = config.ENV_DATA["region"]
+        zones = set(worker_zones + master_zones)
+        ip_prefixes_and_subnets = {}
+        ibmcloud.create_resource_group(resource_group)
+        ibmcloud.create_vpc(vpc_name, resource_group)
+        for zone in zones:
+            ibm_cloud_subnets = constants.IBM_CLOUD_SUBNETS[region]
+            ip_prefixes_and_subnets[zone] = ibmcloud.find_free_network_subnets(
+                ibm_cloud_subnets[zone], ip_prefix
+            )
+        for zone, ip_prefix_and_subnets in ip_prefixes_and_subnets.items():
+            gateway_name = f"{cluster_name}-public-gateway-{zone}"
+            address_prefix, subnet_split1, subnet_split2 = ip_prefix_and_subnets
+            ibmcloud.create_address_prefix(
+                f"{cluster_name}-{zone}", vpc_name, zone, address_prefix
+            )
+            ibmcloud.create_public_gateway(gateway_name, vpc_name, zone, resource_group)
+            for subnet_type, subnet in (
+                ("control-plane", subnet_split1),
+                ("compute", subnet_split2),
+            ):
+                subnet_type_key = f"{subnet_type.replace('-', '_')}_subnets"
+                if not config.ENV_DATA.get(subnet_type_key):
+                    config.ENV_DATA[subnet_type_key] = []
+                subnet_name = f"{cluster_name}-subnet-{subnet_type}-{zone}"
+                config.ENV_DATA[subnet_type_key].append(subnet_name)
+                ibmcloud.create_subnet(
+                    subnet_name, vpc_name, zone, subnet, resource_group
+                )
+                ibmcloud.attach_subnet_to_public_gateway(
+                    subnet_name, gateway_name, vpc_name
+                )

ocs_ci/ocs/constants.py

@@ -1423,6 +1423,21 @@
     HCI_VSPHERE,
 ]
 
+IBM_CLOUD_SUBNETS = {
+    # Washington D.C.
+    "us-east": {
+        "us-east-1": "10.241.0.0/18",
+        "us-east-2": "10.241.64.0/18",
+        "us-east-3": "10.241.128.0/18",
+    },
+    # Dalas
+    "us-south": {
+        "us-south-1": "10.240.0.0/18",
+        "us-south-2": "10.240.64.0/18",
+        "us-south-3": "10.240.128.0/18",
+    },
+}
+
 HCI_PC_OR_MS_PLATFORM = MANAGED_SERVICE_PLATFORMS + HCI_PROVIDER_CLIENT_PLATFORMS
 # AWS i3 worker instance for LSO
 AWS_LSO_WORKER_INSTANCE = "i3en.2xlarge"

ocs_ci/templates/ocp-deployment/install-config-ibm_cloud-ipi.yaml.j2

@@ -47,5 +47,18 @@ fips: {{ fips }}
 platform:
   ibmcloud:
     region: {{ region | default('us-south') }}
+{% if custom_vpc %}
+    resourceGroupName: {{ cluster_name }}-{{ cluster_id }}
+    networkResourceGroupName: {{ cluster_name }}-{{ cluster_id }}
+    vpcName: {{ cluster_name }}-{{ cluster_id }}-vpc
+    controlPlaneSubnets:
+{% for control_plane_subnet in control_plane_subnets %}
+      - {{ control_plane_subnet }}
+{% endfor %}
+    computeSubnets:
+{% for compute_subnet in compute_subnets %}
+      - {{ compute_subnet }}
+{% endfor %}
+{% endif %}
 publish: External
 pullSecret: ''

ocs_ci/utility/ibmcloud.py

@@ -10,6 +10,7 @@
 import re
 import requests
 import time
+import ipaddress
 from copy import copy
 from json import JSONDecodeError
 from ocs_ci.framework import config
@@ -727,3 +728,148 @@ def cleanup_policies_and_service_ids(cluster_name, get_infra_id_from_metadata=Tr
         delete_account_policy(policy["id"], api_token)
     for service_id in service_ids:
         delete_service_id(service_id["id"])
+
+
+def create_resource_group(resource_group):
+    """
+    Create resource group.
+
+    Args:
+        resource_group (str): resource group name
+
+    """
+    run_ibmcloud_cmd(f"ibmcloud resource group-create {resource_group}")
+
+
+def create_vpc(cluster_name, resource_group):
+    """
+    Create VPC.
+
+    Args:
+        cluster_name (str): cluster name
+        resource_group (str): resource group name
+
+    """
+    run_ibmcloud_cmd(
+        f"ibmcloud is vpc-create {cluster_name} --address-prefix-management manual"
+        f" --resource-group-name {resource_group}"
+    )
+
+
+def get_used_subnets():
+    """
+    Get currently used subnets in IBM Cloud
+
+    Returns:
+        list: subnets
+    """
+    subnets_data = json.loads(run_ibmcloud_cmd("ibmcloud is subnets --output json"))
+    return [subnet["ipv4_cidr_block"] for subnet in subnets_data]
+
+
+def create_address_prefix(prefix_name, vpc, zone, cidr):
+    """
+    Create address prefix in VPC.
+
+    Args:
+        prefix_name (str): address prefix name to create
+        vpc (str): VPC name
+        zone (str): zone name
+        cidr (str): CIDR for address prefix
+
+    """
+    run_ibmcloud_cmd(
+        f"ibmcloud is vpc-address-prefix-create {prefix_name} {vpc} {zone} {cidr}"
+    )
+
+
+def create_subnet(subnet_name, vpc, zone, cidr, resource_group):
+    """
+    Create subnet in VPC.
+
+    Args:
+        subnet_name (str): address prefix name to create
+        vpc (str): VPC name
+        zone (str): zone name
+        cidr (str): CIDR for address prefix
+        resource_group (str): resource group name
+
+    """
+    run_ibmcloud_cmd(
+        f"ibmcloud is subnet-create {subnet_name} {vpc} --zone {zone} --ipv4-cidr-block {cidr}"
+        f" --resource-group-name {resource_group}"
+    )
+
+
+def create_public_gateway(gateway_name, vpc, zone, resource_group):
+    """
+    Create public gateway in VPC.
+
+    Args:
+        gateway_name (str): public gateway name
+        vpc (str): VPC name
+        zone (str): zone name
+        resource_group (str): resource group name
+
+    """
+    run_ibmcloud_cmd(
+        f"ibmcloud is public-gateway-create {gateway_name} {vpc} {zone} --resource-group-name {resource_group}"
+    )
+
+
+def attach_subnet_to_public_gateway(subnet_name, gateway_name, vpc):
+    """
+    Attach subnet to public gateway.
+
+    Args:
+        subnet_name (str): subnet name to attach to public gateway
+        gateway_name (str): public gateway name
+        vpc (str): VPC name
+
+    """
+    run_ibmcloud_cmd(
+        f"ibmcloud is subnet-update {subnet_name} --pgw {gateway_name} --vpc {vpc}"
+    )
+
+
+def find_free_network_subnets(subnet_cidr, network_prefix=27):
+    """
+    This function will look for currently used subnet, and will try to find one which
+    is not occupied by any other VPC.
+
+    Args:
+        subnet_cidr (str): subnet CIDR in which range to look for free subnet (e.g. 10.240.0.0/18)
+        network_prefix (int): subnet prefix to look for
+
+    Returns:
+        tuple: (network_with_prefix, network_split_1, network_split2), where
+            network_with_prefix - is network CIDR which we are looking for.
+            network_split1 - is first CIDR split of network_with_prefix
+            network_split2 - is second CIDR split of network_with_prefix
+    """
+    network = ipaddress.ip_network(subnet_cidr)
+
+    # Get all possible /network_prefix+1 networks within the /network_prefix network
+    main_subnets = list(network.subnets(new_prefix=network_prefix))
+    split_subnets = list(network.subnets(new_prefix=network_prefix + 1))
+    zipped_subnets = [
+        (main_subnets[i], split_subnets[2 * i], split_subnets[2 * i + 1])
+        for i in range(len(main_subnets))
+    ]
+
+    for possible_subnets in zipped_subnets:
+        is_free = True
+        list_of_subnets = get_used_subnets()
+        for subnet in list_of_subnets:
+            for possible_subnet in possible_subnets:
+                tested_network = ipaddress.ip_network(subnet)
+                is_subnet = possible_subnet.subnet_of(tested_network)
+                if is_subnet:
+                    logger.debug(
+                        f"Subnet {possible_subnet} is subnet of {tested_network} skipping it!"
+                    )
+                    is_free = False
+                    break
+        if is_free:
+            logger.info(f"Free set of subnets found: {possible_subnets}")
+            return possible_subnets